Skip to content

Commit 8be6fd4

Browse files
myasermyaser
authored
Merge pull request #6852 from zalando-incubator/karpenter-update
Karpenter update
2 parents e0ec987 + 7ebbf1d commit 8be6fd4

20 files changed

+1773
-1523
lines changed

‎cluster/cluster.yaml

Lines changed: 242 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -422,7 +422,7 @@ Resources:
422422
# SHA-1 sum of the root certificate in the trust chain for the certificate
423423
# use to serve the open id discovery document.
424424
- "9e99a48a9960b14926bb7f3b02e22da2b0ab7280"
425-
WorkerIAMRole:
425+
WorkerIAMRole: # role used by worker nodes, including karpenter ones
426426
Properties:
427427
AssumeRolePolicyDocument:
428428
Statement:
@@ -552,7 +552,14 @@ Resources:
552552
RoleName: "{{.Cluster.LocalID}}-app-autoscaler"
553553
Type: 'AWS::IAM::Role'
554554
{{- if eq .Cluster.ConfigItems.karpenter_pools_enabled "true"}}
555-
KarpenterIAMRole:
555+
KarpenterNodeInstanceProfile:
556+
Type: "AWS::IAM::InstanceProfile"
557+
Properties:
558+
InstanceProfileName: "{{ .Cluster.ID | awsValidID }}-WorkerKarpenter-InstanceProfile"
559+
Path: "/"
560+
Roles:
561+
- !Ref WorkerIAMRole
562+
KarpenterIAMRole: # role for the karpenter controller
556563
Properties:
557564
AssumeRolePolicyDocument:
558565
Statement:
@@ -563,70 +570,250 @@ Resources:
563570
- 'sts:AssumeRoleWithWebIdentity'
564571
Condition:
565572
StringEquals:
573+
"{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:aud": "sts.amazonaws.com"
566574
"{{ .Cluster.LocalID }}.{{ .Values.hosted_zone }}:sub": "system:serviceaccount:kube-system:karpenter"
567575
Version: 2012-10-17
568576
Path: /
569577
Policies:
570-
- PolicyDocument:
571-
Statement:
572-
- Effect: Allow
573-
Resource: "*"
574-
Action:
575-
# Write Operations
576-
- ec2:CreateLaunchTemplate
577-
- ec2:CreateFleet
578-
- ec2:RunInstances
579-
- ec2:CreateTags
580-
- ec2:TerminateInstances
581-
- ec2:DeleteLaunchTemplate
582-
# Read Operations
583-
- ec2:DescribeLaunchTemplates
584-
- ec2:DescribeInstances
585-
- ec2:DescribeSecurityGroups
586-
- ec2:DescribeSubnets
587-
- ec2:DescribeImages
588-
- ec2:DescribeInstanceTypes
589-
- ec2:DescribeInstanceTypeOfferings
590-
- ec2:DescribeAvailabilityZones
591-
- ec2:DescribeSpotPriceHistory
592-
- ssm:GetParameter
593-
- pricing:GetProducts
594-
- Effect: Allow
595-
Action: iam:PassRole
596-
Resource: !Sub "arn:aws:iam::${AWS::AccountId}:role/{{.Cluster.LocalID}}-worker"
597-
Sid: PassNodeIAMRole
598-
- Effect: Allow
599-
Action:
600-
# Write Operations
601-
- sqs:DeleteMessage
602-
# Read Operations
603-
- sqs:GetQueueUrl
604-
- sqs:GetQueueAttributes
605-
- sqs:ReceiveMessage
606-
Resource: !GetAtt KarpenterInterruptionQueue.Arn
607-
- Action: ec2:TerminateInstances
608-
Condition:
609-
StringLike:
610-
ec2:ResourceTag/karpenter.sh/provisioner-name: "*"
611-
Effect: Allow
612-
Resource: "*"
613-
Sid: ConditionalEC2Termination
614-
Version: 2012-10-17
615-
PolicyName: root
578+
- PolicyName: "KarpenterControllerPolicy-{{ .Cluster.ID | awsValidID }}"
579+
# The PolicyDocument must be in JSON string format because we use a StringEquals condition that uses an interpolated
580+
# value in one of its key parameters which isn't natively supported by CloudFormation
581+
PolicyDocument: !Sub |
582+
{
583+
"Version": "2012-10-17",
584+
"Statement": [
585+
{
586+
"Sid": "AllowScopedEC2InstanceActions",
587+
"Effect": "Allow",
588+
"Resource": [
589+
"arn:${AWS::Partition}:ec2:${AWS::Region}::image/*",
590+
"arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*",
591+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*",
592+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*",
593+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*"
594+
],
595+
"Action": [
596+
"ec2:RunInstances",
597+
"ec2:CreateFleet"
598+
]
599+
},
600+
{
601+
"Sid": "AllowScopedEC2InstanceActionsWithTags",
602+
"Effect": "Allow",
603+
"Resource": [
604+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:fleet/*",
605+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*",
606+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*",
607+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*",
608+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*",
609+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*"
610+
],
611+
"Action": [
612+
"ec2:RunInstances",
613+
"ec2:CreateFleet",
614+
"ec2:CreateLaunchTemplate"
615+
],
616+
"Condition": {
617+
"StringEquals": {
618+
"aws:RequestTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned"
619+
},
620+
"StringLike": {
621+
"aws:RequestTag/karpenter.sh/nodepool": "*"
622+
}
623+
}
624+
},
625+
{
626+
"Sid": "AllowScopedResourceCreationTagging",
627+
"Effect": "Allow",
628+
"Resource": [
629+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:fleet/*",
630+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*",
631+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*",
632+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*",
633+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*",
634+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*"
635+
],
636+
"Action": "ec2:CreateTags",
637+
"Condition": {
638+
"StringEquals": {
639+
"aws:RequestTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned",
640+
"ec2:CreateAction": [
641+
"RunInstances",
642+
"CreateFleet",
643+
"CreateLaunchTemplate"
644+
]
645+
},
646+
"StringLike": {
647+
"aws:RequestTag/karpenter.sh/nodepool": "*"
648+
}
649+
}
650+
},
651+
{
652+
"Sid": "AllowScopedResourceTagging",
653+
"Effect": "Allow",
654+
"Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*",
655+
"Action": "ec2:CreateTags",
656+
"Condition": {
657+
"StringEquals": {
658+
"aws:ResourceTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned"
659+
},
660+
"StringLike": {
661+
"aws:ResourceTag/karpenter.sh/nodepool": "*"
662+
},
663+
"ForAllValues:StringEquals": {
664+
"aws:TagKeys": [
665+
"karpenter.sh/nodeclaim",
666+
"Name"
667+
]
668+
}
669+
}
670+
},
671+
{
672+
"Sid": "AllowScopedDeletion",
673+
"Effect": "Allow",
674+
"Resource": [
675+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*",
676+
"arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*"
677+
],
678+
"Action": [
679+
"ec2:TerminateInstances",
680+
"ec2:DeleteLaunchTemplate"
681+
],
682+
"Condition": {
683+
"StringEquals": {
684+
"aws:ResourceTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned"
685+
},
686+
"StringLike": {
687+
"aws:ResourceTag/karpenter.sh/nodepool": "*"
688+
}
689+
}
690+
},
691+
{
692+
"Sid": "AllowRegionalReadActions",
693+
"Effect": "Allow",
694+
"Resource": "*",
695+
"Action": [
696+
"ec2:DescribeAvailabilityZones",
697+
"ec2:DescribeImages",
698+
"ec2:DescribeInstances",
699+
"ec2:DescribeInstanceTypeOfferings",
700+
"ec2:DescribeInstanceTypes",
701+
"ec2:DescribeLaunchTemplates",
702+
"ec2:DescribeSecurityGroups",
703+
"ec2:DescribeSpotPriceHistory",
704+
"ec2:DescribeSubnets"
705+
],
706+
"Condition": {
707+
"StringEquals": {
708+
"aws:RequestedRegion": "${AWS::Region}"
709+
}
710+
}
711+
},
712+
{
713+
"Sid": "AllowSSMReadActions",
714+
"Effect": "Allow",
715+
"Resource": "arn:${AWS::Partition}:ssm:${AWS::Region}::parameter/aws/service/*",
716+
"Action": "ssm:GetParameter"
717+
},
718+
{
719+
"Sid": "AllowPricingReadActions",
720+
"Effect": "Allow",
721+
"Resource": "*",
722+
"Action": "pricing:GetProducts"
723+
},
724+
{
725+
"Sid": "AllowInterruptionQueueActions",
726+
"Effect": "Allow",
727+
"Resource": "${KarpenterInterruptionQueue.Arn}",
728+
"Action": [
729+
"sqs:DeleteMessage",
730+
"sqs:GetQueueUrl",
731+
"sqs:ReceiveMessage"
732+
]
733+
},
734+
{
735+
"Sid": "AllowPassingInstanceRole",
736+
"Effect": "Allow",
737+
"Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/{{.Cluster.LocalID}}-worker",
738+
"Action": "iam:PassRole",
739+
"Condition": {
740+
"StringEquals": {
741+
"iam:PassedToService": "ec2.amazonaws.com"
742+
}
743+
}
744+
},
745+
{
746+
"Sid": "AllowScopedInstanceProfileCreationActions",
747+
"Effect": "Allow",
748+
"Resource": "*",
749+
"Action": [
750+
"iam:CreateInstanceProfile"
751+
],
752+
"Condition": {
753+
"StringEquals": {
754+
"aws:RequestTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned",
755+
"aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}"
756+
},
757+
"StringLike": {
758+
"aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*"
759+
}
760+
}
761+
},
762+
{
763+
"Sid": "AllowScopedInstanceProfileTagActions",
764+
"Effect": "Allow",
765+
"Resource": "*",
766+
"Action": [
767+
"iam:TagInstanceProfile"
768+
],
769+
"Condition": {
770+
"StringEquals": {
771+
"aws:ResourceTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned",
772+
"aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}",
773+
"aws:RequestTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned",
774+
"aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}"
775+
},
776+
"StringLike": {
777+
"aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*",
778+
"aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*"
779+
}
780+
}
781+
},
782+
{
783+
"Sid": "AllowScopedInstanceProfileActions",
784+
"Effect": "Allow",
785+
"Resource": "*",
786+
"Action": [
787+
"iam:AddRoleToInstanceProfile",
788+
"iam:RemoveRoleFromInstanceProfile",
789+
"iam:DeleteInstanceProfile"
790+
],
791+
"Condition": {
792+
"StringEquals": {
793+
"aws:ResourceTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned",
794+
"aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}"
795+
},
796+
"StringLike": {
797+
"aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*"
798+
}
799+
}
800+
},
801+
{
802+
"Sid": "AllowInstanceProfileReadActions",
803+
"Effect": "Allow",
804+
"Resource": "*",
805+
"Action": "iam:GetInstanceProfile"
806+
}
807+
]
808+
}
616809
RoleName: "{{.Cluster.LocalID}}-app-karpenter"
617810
Type: 'AWS::IAM::Role'
618-
KarpenterNodeInstanceProfile:
619-
Type: "AWS::IAM::InstanceProfile"
620-
Properties:
621-
InstanceProfileName: "{{ .Cluster.ID | awsValidID }}-WorkerKarpenter-InstanceProfile"
622-
Path: "/"
623-
Roles:
624-
- !Ref WorkerIAMRole
625811
KarpenterInterruptionQueue:
626812
Type: AWS::SQS::Queue
627813
Properties:
628814
QueueName: !Sub "{{.Cluster.LocalID}}-karpenter-interruption-queue"
629815
MessageRetentionPeriod: 300
816+
SqsManagedSseEnabled: true
630817
KarpenterInterruptionQueuePolicy:
631818
Type: AWS::SQS::QueuePolicy
632819
Properties:

0 commit comments

Comments
 (0)