diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 69c03af907..46fb56e4ff 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -120,8 +120,13 @@ No modules. | [aws_sqs_queue.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | | [aws_sqs_queue_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_iam_policy_document.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.controller_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.controller_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.controller_eks_integration](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.controller_iam_integration](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.controller_interruption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.controller_node_lifecycle](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.controller_resource_discovery](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.node_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index af0e947e76..cffad5fffb 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -31,6 +31,24 @@ locals { locals { create_iam_role = var.create && var.create_iam_role + + # Split controller policies to avoid exceeding the 6,144 character IAM policy size limit + # See: https://github.com/terraform-aws-modules/terraform-aws-eks/issues/3512 + # See: https://github.com/aws/karpenter-provider-aws/pull/8690 + controller_policies = local.create_iam_role ? merge( + { + NodeLifecycle = data.aws_iam_policy_document.controller_node_lifecycle[0].json + IAMIntegration = data.aws_iam_policy_document.controller_iam_integration[0].json + EKSIntegration = data.aws_iam_policy_document.controller_eks_integration[0].json + ResourceDiscovery = data.aws_iam_policy_document.controller_resource_discovery[0].json + }, + local.enable_spot_termination ? { + Interruption = data.aws_iam_policy_document.controller_interruption[0].json + } : {}, + var.iam_policy_statements != null ? { + Additional = data.aws_iam_policy_document.controller_additional[0].json + } : {} + ) : {} } data "aws_iam_policy_document" "controller_assume_role" { @@ -71,31 +89,31 @@ resource "aws_iam_role" "controller" { } resource "aws_iam_role_policy" "controller" { - count = local.create_iam_role && var.enable_inline_policy ? 1 : 0 + for_each = var.enable_inline_policy ? local.controller_policies : {} - name = var.iam_policy_use_name_prefix ? null : var.iam_policy_name - name_prefix = var.iam_policy_use_name_prefix ? "${var.iam_policy_name}-" : null + name = var.iam_policy_use_name_prefix ? null : "${var.iam_policy_name}${each.key}" + name_prefix = var.iam_policy_use_name_prefix ? "${var.iam_policy_name}${each.key}-" : null role = aws_iam_role.controller[0].name - policy = data.aws_iam_policy_document.controller[0].json + policy = each.value } resource "aws_iam_policy" "controller" { - count = local.create_iam_role && !var.enable_inline_policy ? 1 : 0 + for_each = !var.enable_inline_policy ? local.controller_policies : {} - name = var.iam_policy_use_name_prefix ? null : var.iam_policy_name - name_prefix = var.iam_policy_use_name_prefix ? "${var.iam_policy_name}-" : null + name = var.iam_policy_use_name_prefix ? null : "${var.iam_policy_name}${each.key}" + name_prefix = var.iam_policy_use_name_prefix ? "${var.iam_policy_name}${each.key}-" : null path = var.iam_policy_path - description = var.iam_policy_description - policy = data.aws_iam_policy_document.controller[0].json + description = "${var.iam_policy_description} - ${each.key}" + policy = each.value tags = var.tags } resource "aws_iam_role_policy_attachment" "controller" { - count = local.create_iam_role && !var.enable_inline_policy ? 1 : 0 + for_each = !var.enable_inline_policy ? local.controller_policies : {} role = aws_iam_role.controller[0].name - policy_arn = aws_iam_policy.controller[0].arn + policy_arn = aws_iam_policy.controller[each.key].arn } resource "aws_iam_role_policy_attachment" "controller_additional" { diff --git a/modules/karpenter/migrations.tf b/modules/karpenter/migrations.tf index b40040f330..630b779195 100644 --- a/modules/karpenter/migrations.tf +++ b/modules/karpenter/migrations.tf @@ -75,3 +75,21 @@ moved { from = aws_iam_role_policy_attachment.node["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"] to = aws_iam_role_policy_attachment.node["AmazonEKS_CNI_Policy"] } + +# Controller IAM managed policy - the old single policy migrates to NodeLifecycle +# since it contains the largest set of statements +moved { + from = aws_iam_policy.controller[0] + to = aws_iam_policy.controller["NodeLifecycle"] +} + +moved { + from = aws_iam_role_policy_attachment.controller[0] + to = aws_iam_role_policy_attachment.controller["NodeLifecycle"] +} + +# Controller IAM inline policy +moved { + from = aws_iam_role_policy.controller[0] + to = aws_iam_role_policy.controller["NodeLifecycle"] +} diff --git a/modules/karpenter/policy.tf b/modules/karpenter/policy.tf index 34937f36eb..1cee2f82ca 100644 --- a/modules/karpenter/policy.tf +++ b/modules/karpenter/policy.tf @@ -1,4 +1,8 @@ -data "aws_iam_policy_document" "controller" { +################################################################################ +# Node Lifecycle Policy +################################################################################ + +data "aws_iam_policy_document" "controller_node_lifecycle" { count = local.create_iam_role ? 1 : 0 statement { @@ -176,55 +180,14 @@ data "aws_iam_policy_document" "controller" { values = ["*"] } } +} - statement { - sid = "AllowRegionalReadActions" - resources = ["*"] - actions = [ - "ec2:DescribeCapacityReservations", - "ec2:DescribeAvailabilityZones", - "ec2:DescribeImages", - "ec2:DescribeInstances", - "ec2:DescribeInstanceTypeOfferings", - "ec2:DescribeInstanceTypes", - "ec2:DescribeLaunchTemplates", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSpotPriceHistory", - "ec2:DescribeSubnets" - ] - - condition { - test = "StringEquals" - variable = "aws:RequestedRegion" - values = [local.region] - } - } - - statement { - sid = "AllowSSMReadActions" - resources = coalescelist(var.ami_id_ssm_parameter_arns, ["arn:${local.partition}:ssm:${local.region}::parameter/aws/service/*"]) - actions = ["ssm:GetParameter"] - } - - statement { - sid = "AllowPricingReadActions" - resources = ["*"] - actions = ["pricing:GetProducts"] - } - - dynamic "statement" { - for_each = local.enable_spot_termination ? [1] : [] +################################################################################ +# IAM Integration Policy +################################################################################ - content { - sid = "AllowInterruptionQueueActions" - resources = [try(aws_sqs_queue.this[0].arn, null)] - actions = [ - "sqs:DeleteMessage", - "sqs:GetQueueUrl", - "sqs:ReceiveMessage" - ] - } - } +data "aws_iam_policy_document" "controller_iam_integration" { + count = local.create_iam_role ? 1 : 0 statement { sid = "AllowPassingInstanceRole" @@ -343,6 +306,63 @@ data "aws_iam_policy_document" "controller" { values = ["*"] } } +} + +################################################################################ +# EKS Integration Policy +################################################################################ + +data "aws_iam_policy_document" "controller_eks_integration" { + count = local.create_iam_role ? 1 : 0 + + statement { + sid = "AllowAPIServerEndpointDiscovery" + resources = ["arn:${local.partition}:eks:${local.region}:${local.account_id}:cluster/${var.cluster_name}"] + actions = ["eks:DescribeCluster"] + } +} + +################################################################################ +# Resource Discovery Policy +################################################################################ + +data "aws_iam_policy_document" "controller_resource_discovery" { + count = local.create_iam_role ? 1 : 0 + + statement { + sid = "AllowRegionalReadActions" + resources = ["*"] + actions = [ + "ec2:DescribeCapacityReservations", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" + ] + + condition { + test = "StringEquals" + variable = "aws:RequestedRegion" + values = [local.region] + } + } + + statement { + sid = "AllowSSMReadActions" + resources = coalescelist(var.ami_id_ssm_parameter_arns, ["arn:${local.partition}:ssm:${local.region}::parameter/aws/service/*"]) + actions = ["ssm:GetParameter"] + } + + statement { + sid = "AllowPricingReadActions" + resources = ["*"] + actions = ["pricing:GetProducts"] + } statement { sid = "AllowInstanceProfileReadActions" @@ -355,12 +375,32 @@ data "aws_iam_policy_document" "controller" { resources = ["*"] actions = ["iam:ListInstanceProfiles"] } +} + +################################################################################ +# Interruption Policy +################################################################################ + +data "aws_iam_policy_document" "controller_interruption" { + count = local.create_iam_role && local.enable_spot_termination ? 1 : 0 statement { - sid = "AllowAPIServerEndpointDiscovery" - resources = ["arn:${local.partition}:eks:${local.region}:${local.account_id}:cluster/${var.cluster_name}"] - actions = ["eks:DescribeCluster"] + sid = "AllowInterruptionQueueActions" + resources = [aws_sqs_queue.this[0].arn] + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] } +} + +################################################################################ +# Additional Custom Policy Statements +################################################################################ + +data "aws_iam_policy_document" "controller_additional" { + count = local.create_iam_role && var.iam_policy_statements != null ? 1 : 0 dynamic "statement" { for_each = var.iam_policy_statements != null ? var.iam_policy_statements : []