diff --git a/modules/common/k8s_callback/k8s_standard/1.0/facets.yaml b/modules/common/k8s_callback/k8s_standard/1.0/facets.yaml index b4ba2894..6f57aaa4 100644 --- a/modules/common/k8s_callback/k8s_standard/1.0/facets.yaml +++ b/modules/common/k8s_callback/k8s_standard/1.0/facets.yaml @@ -5,6 +5,9 @@ description: Creates ServiceAccount with admin role and registers OVH K8s creden with Facets control plane clouds: - kubernetes +- aws +- azure +- gcp inputs: kubernetes_details: type: '@facets/kubernetes-details' diff --git a/modules/karpenter/1.0/facets.yaml b/modules/karpenter/1.0/facets.yaml new file mode 100644 index 00000000..c91586b3 --- /dev/null +++ b/modules/karpenter/1.0/facets.yaml @@ -0,0 +1,89 @@ +clouds: + - aws +description: | + Deploys Karpenter controller and necessary infrastructure for EKS clusters. + This module installs the Karpenter controller, IAM roles, and sets up interruption handling. + Use this module once per cluster, then use kubernetes_node_pool/karpenter for creating NodePools. +flavor: default +inputs: + cloud_account: + description: The AWS Cloud Account for EC2 instance provisioning + displayName: Cloud Account + optional: false + providers: + - aws + type: '@facets/aws_cloud_account' + kubernetes_details: + description: The EKS cluster where Karpenter will be deployed + displayName: Kubernetes Cluster + optional: false + providers: + - kubernetes + - helm + type: '@facets/kubernetes-details' + network_details: + default: + resource_name: default + resource_type: network + displayName: Network + type: '@facets/aws-vpc-details' +intent: karpenter +outputs: + default: + mock: + attributes: + controller_role_arn: mock-string-value + interruption_queue_name: mock-string-value + karpenter_namespace: mock-string-value + karpenter_service_account: mock-string-value + karpenter_version: mock-string-value + node_instance_profile_name: mock-string-value + node_role_arn: mock-string-value + secrets: [] + interfaces: {} + type: '@facets/karpenter-details' +sample: + disabled: false + flavor: default + kind: karpenter + spec: + karpenter_version: 1.0.1 + karpenter_replicas: 2 + interruption_handling: true + version: "1.0" +spec: + description: Configure Karpenter controller for your EKS cluster + properties: + karpenter_version: + default: 1.0.1 + description: Version of Karpenter to deploy (should match your Kubernetes version) + title: Karpenter Version + type: string + x-ui-overrides-only: true + karpenter_replicas: + default: 2 + description: Number of Karpenter controller replicas for high availability + maximum: 5 + minimum: 1 + title: Controller Replicas + type: integer + interruption_handling: + default: true + description: Enable interruption handling for Spot instances + title: Enable Interruption Handling + type: boolean + tags: + description: Additional AWS tags to apply to Karpenter resources + title: Resource Tags + type: object + x-ui-yaml-editor: true + required: + - karpenter_version + title: Karpenter Controller Configuration + type: object + x-ui-order: + - karpenter_version + - karpenter_replicas + - interruption_handling + - tags +version: "1.0" diff --git a/modules/karpenter/1.0/main.tf b/modules/karpenter/1.0/main.tf new file mode 100644 index 00000000..c0ec51bc --- /dev/null +++ b/modules/karpenter/1.0/main.tf @@ -0,0 +1,375 @@ +locals { + karpenter_namespace = "kube-system" + karpenter_service_account = "karpenter" + cluster_name = var.inputs.kubernetes_details.attributes.cluster_name + oidc_provider_arn = var.inputs.kubernetes_details.attributes.oidc_provider_arn + oidc_provider = var.inputs.kubernetes_details.attributes.oidc_provider + aws_region = var.inputs.cloud_account.attributes.aws_region + node_security_group_id = var.inputs.kubernetes_details.attributes.node_security_group_id + + # Merge environment tags with instance tags + instance_tags = merge( + var.environment.cloud_tags, + lookup(var.instance.spec, "tags", {}), + { + "facets:instance_name" = var.instance_name + "facets:environment" = var.environment.name + "facets:component" = "karpenter" + } + ) +} + +# IAM Role for Karpenter Controller +resource "aws_iam_role" "karpenter_controller" { + name = "karpenter-controller-${local.cluster_name}" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Federated = local.oidc_provider_arn + } + Action = "sts:AssumeRoleWithWebIdentity" + Condition = { + StringEquals = { + "${local.oidc_provider}:aud" = "sts.amazonaws.com" + "${local.oidc_provider}:sub" = "system:serviceaccount:${local.karpenter_namespace}:${local.karpenter_service_account}" + } + } + } + ] + }) + + tags = local.instance_tags +} + +# IAM Policy for Karpenter Controller +resource "aws_iam_policy" "karpenter_controller" { + name = "karpenter-controller-${local.cluster_name}" + description = "IAM policy for Karpenter controller to manage EC2 instances" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "Karpenter" + Effect = "Allow" + Action = [ + "ssm:GetParameter", + "ec2:DescribeImages", + "ec2:RunInstances", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeAvailabilityZones", + "ec2:DeleteLaunchTemplate", + "ec2:CreateTags", + "ec2:CreateLaunchTemplate", + "ec2:CreateFleet", + "ec2:DescribeSpotPriceHistory", + "pricing:GetProducts" + ] + Resource = "*" + }, + { + Sid = "ConditionalEC2Termination" + Effect = "Allow" + Action = [ + "ec2:TerminateInstances" + ] + Condition = { + StringLike = { + "ec2:ResourceTag/karpenter.sh/nodepool" = "*" + } + } + Resource = "*" + }, + { + Sid = "PassNodeIAMRole" + Effect = "Allow" + Action = "iam:PassRole" + Resource = aws_iam_role.karpenter_node.arn + }, + { + Sid = "EKSClusterEndpointLookup" + Effect = "Allow" + Action = "eks:DescribeCluster" + Resource = var.inputs.kubernetes_details.attributes.cluster_arn + }, + { + Sid = "AllowScopedInstanceProfileCreationActions" + Effect = "Allow" + Action = [ + "iam:CreateInstanceProfile" + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:RequestedRegion" = local.aws_region + } + StringLike = { + "aws:RequestTag/kubernetes.io/cluster/${local.cluster_name}" = "owned" + "aws:RequestTag/topology.kubernetes.io/region" = "*" + } + } + }, + { + Sid = "AllowScopedInstanceProfileTagActions" + Effect = "Allow" + Action = [ + "iam:TagInstanceProfile" + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:ResourceTag/kubernetes.io/cluster/${local.cluster_name}" = "owned" + "aws:ResourceTag/topology.kubernetes.io/region" = local.aws_region + "aws:RequestedRegion" = local.aws_region + } + } + }, + { + Sid = "AllowScopedInstanceProfileActions" + Effect = "Allow" + Action = [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:ResourceTag/kubernetes.io/cluster/${local.cluster_name}" = "owned" + "aws:ResourceTag/topology.kubernetes.io/region" = local.aws_region + } + } + }, + { + Sid = "AllowInstanceProfileReadActions" + Effect = "Allow" + Action = "iam:GetInstanceProfile" + Resource = "*" + } + ] + }) + + tags = local.instance_tags +} + +# Attach policy to controller role +resource "aws_iam_role_policy_attachment" "karpenter_controller" { + role = aws_iam_role.karpenter_controller.name + policy_arn = aws_iam_policy.karpenter_controller.arn +} + +# IAM Role for Karpenter Nodes +resource "aws_iam_role" "karpenter_node" { + name = "karpenter-node-${local.cluster_name}" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = "ec2.amazonaws.com" + } + Action = "sts:AssumeRole" + } + ] + }) + + tags = local.instance_tags +} + +# Attach required policies to node role +resource "aws_iam_role_policy_attachment" "karpenter_node_policies" { + for_each = toset([ + "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + ]) + + role = aws_iam_role.karpenter_node.name + policy_arn = each.value +} + +# Create instance profile for nodes +resource "aws_iam_instance_profile" "karpenter_node" { + name = "karpenter-node-${local.cluster_name}" + role = aws_iam_role.karpenter_node.name + + tags = local.instance_tags +} + +# Deploy Karpenter via Helm +resource "helm_release" "karpenter" { + namespace = local.karpenter_namespace + create_namespace = false + name = "karpenter" + repository = "oci://public.ecr.aws/karpenter" + chart = "karpenter" + version = var.instance.spec.karpenter_version + wait = true + timeout = 600 + + values = [ + yamlencode({ + serviceAccount = { + name = local.karpenter_service_account + annotations = { + "eks.amazonaws.com/role-arn" = aws_iam_role.karpenter_controller.arn + } + } + settings = { + clusterName = local.cluster_name + clusterEndpoint = var.inputs.kubernetes_details.attributes.cluster_endpoint + interruptionQueue = var.instance.spec.interruption_handling ? aws_sqs_queue.karpenter_interruption[0].name : "" + } + replicas = lookup(var.instance.spec, "karpenter_replicas", 2) + controller = { + resources = { + requests = { + cpu = "100m" + memory = "256Mi" + } + limits = { + cpu = "1000m" + memory = "1Gi" + } + } + } + }) + ] + + depends_on = [ + aws_iam_role_policy_attachment.karpenter_controller + ] +} + +# SQS Queue for Spot Interruption Handling (optional) +resource "aws_sqs_queue" "karpenter_interruption" { + count = var.instance.spec.interruption_handling ? 1 : 0 + + name = "karpenter-${local.cluster_name}" + message_retention_seconds = 300 + sqs_managed_sse_enabled = true + + tags = local.instance_tags +} + +resource "aws_sqs_queue_policy" "karpenter_interruption" { + count = var.instance.spec.interruption_handling ? 1 : 0 + + queue_url = aws_sqs_queue.karpenter_interruption[0].id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AllowEventsToSendMessages" + Effect = "Allow" + Principal = { + Service = [ + "events.amazonaws.com", + "sqs.amazonaws.com" + ] + } + Action = "sqs:SendMessage" + Resource = aws_sqs_queue.karpenter_interruption[0].arn + } + ] + }) +} + +# EventBridge rules for interruption handling +resource "aws_cloudwatch_event_rule" "karpenter_interruption" { + for_each = var.instance.spec.interruption_handling ? { + scheduled_change = { + event_pattern = jsonencode({ + source = ["aws.health"] + detail-type = ["AWS Health Event"] + }) + } + spot_interruption = { + event_pattern = jsonencode({ + source = ["aws.ec2"] + detail-type = ["EC2 Spot Instance Interruption Warning"] + }) + } + rebalance = { + event_pattern = jsonencode({ + source = ["aws.ec2"] + detail-type = ["EC2 Instance Rebalance Recommendation"] + }) + } + instance_state_change = { + event_pattern = jsonencode({ + source = ["aws.ec2"] + detail-type = ["EC2 Instance State-change Notification"] + }) + } + } : {} + + name = "karpenter-${local.cluster_name}-${each.key}" + description = "Karpenter interruption handling for ${each.key}" + + event_pattern = each.value.event_pattern + + tags = local.instance_tags +} + +resource "aws_cloudwatch_event_target" "karpenter_interruption" { + for_each = var.instance.spec.interruption_handling ? { + scheduled_change = "scheduled_change" + spot_interruption = "spot_interruption" + rebalance = "rebalance" + instance_state_change = "instance_state_change" + } : {} + + rule = aws_cloudwatch_event_rule.karpenter_interruption[each.key].name + target_id = "KarpenterInterruptionQueue" + arn = aws_sqs_queue.karpenter_interruption[0].arn +} + +# Add Karpenter node role to aws-auth ConfigMap using EKS access entry +# This allows Karpenter-provisioned nodes to join the cluster +resource "aws_eks_access_entry" "karpenter_node" { + cluster_name = local.cluster_name + principal_arn = aws_iam_role.karpenter_node.arn + type = "EC2_LINUX" + + # Prevent updates that fail due to AWS API restrictions on system:nodes group + lifecycle { + ignore_changes = [ + kubernetes_groups, + user_name + ] + } + + depends_on = [ + aws_iam_role.karpenter_node + ] +} + +# Tag subnets for Karpenter discovery +resource "aws_ec2_tag" "karpenter_subnet_discovery" { + for_each = toset(var.inputs.network_details.attributes.private_subnet_ids) + + resource_id = each.value + key = "karpenter.sh/discovery" + value = local.cluster_name +} + +# Tag security group for Karpenter discovery +resource "aws_ec2_tag" "karpenter_sg_discovery" { + resource_id = local.node_security_group_id + key = "karpenter.sh/discovery" + value = local.cluster_name +} diff --git a/modules/karpenter/1.0/outputs.tf b/modules/karpenter/1.0/outputs.tf new file mode 100644 index 00000000..d5da8ac8 --- /dev/null +++ b/modules/karpenter/1.0/outputs.tf @@ -0,0 +1,20 @@ +locals { + output_attributes = { + # Karpenter controller details + karpenter_namespace = local.karpenter_namespace + karpenter_service_account = local.karpenter_service_account + karpenter_version = var.instance.spec.karpenter_version + + # IAM details + controller_role_arn = aws_iam_role.karpenter_controller.arn + node_role_arn = aws_iam_role.karpenter_node.arn + node_instance_profile_name = aws_iam_instance_profile.karpenter_node.name + + # Interruption handling + interruption_queue_name = var.instance.spec.interruption_handling ? aws_sqs_queue.karpenter_interruption[0].name : "" + + secrets = [] + } + + output_interfaces = {} +} diff --git a/modules/karpenter/1.0/variables.tf b/modules/karpenter/1.0/variables.tf new file mode 100644 index 00000000..fe4347a3 --- /dev/null +++ b/modules/karpenter/1.0/variables.tf @@ -0,0 +1,78 @@ +variable "instance" { + type = object({ + kind = string + flavor = string + version = string + spec = object({ + # Karpenter installation settings + karpenter_version = string + karpenter_replicas = optional(number, 2) + interruption_handling = optional(bool, true) + + # Tags + tags = optional(map(string), {}) + }) + }) + + validation { + condition = can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", var.instance.spec.karpenter_version)) + error_message = "Karpenter version must be in semantic version format (e.g., 1.0.1)" + } +} + +variable "instance_name" { + type = string + description = "Unique architectural name from blueprint" +} + +variable "environment" { + type = object({ + name = string + unique_name = string + namespace = optional(string) + cloud_tags = optional(map(string), {}) + }) + description = "Environment context including name and cloud tags" +} + +variable "inputs" { + type = object({ + cloud_account = object({ + attributes = object({ + aws_region = string + aws_account_id = string + aws_iam_role = string + external_id = optional(string) + session_name = optional(string) + }) + }) + kubernetes_details = object({ + attributes = object({ + cluster_endpoint = string + cluster_ca_certificate = string + cluster_name = string + cluster_version = string + cluster_arn = string + cluster_id = string + oidc_issuer_url = string + oidc_provider = string + oidc_provider_arn = string + node_security_group_id = string + kubernetes_provider_exec = object({ + api_version = string + command = string + args = list(string) + }) + }) + }) + network_details = object({ + attributes = object({ + vpc_id = string + private_subnet_ids = list(string) + public_subnet_ids = list(string) + database_subnet_ids = optional(list(string), []) + }) + }) + }) + description = "Inputs from dependent modules" +} diff --git a/modules/karpenter/1.0/versions.tf b/modules/karpenter/1.0/versions.tf new file mode 100644 index 00000000..7117131f --- /dev/null +++ b/modules/karpenter/1.0/versions.tf @@ -0,0 +1,3 @@ +terraform { + required_version = ">= 1.0" +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/README.md b/modules/kubernetes_cluster/eks_standard/1.0/README.md new file mode 100644 index 00000000..49b80080 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/README.md @@ -0,0 +1,278 @@ +# EKS Standard Module + +## Overview + +This Facets module creates an Amazon EKS (Elastic Kubernetes Service) cluster with managed node groups using the official [terraform-aws-eks](https://github.com/terraform-aws-modules/terraform-aws-eks) module. + +**Important:** This is the `eks_standard` flavor which does **NOT** support EKS Auto Mode. EKS Auto Mode is explicitly disabled in this module. For Auto Mode support, use a separate `eks_auto` flavor module. + +## Module Details + +- **Intent:** `kubernetes_cluster` +- **Flavor:** `eks_standard` +- **Version:** `1.0` +- **Cloud:** AWS + +## Features + +### Core Capabilities + +- ✅ **Managed Node Groups**: Full support for EKS managed node groups with customizable configurations +- ✅ **Multiple Node Groups**: Support for multiple node groups with different instance types and configurations +- ✅ **Auto Scaling**: Configurable min/max/desired capacity per node group +- ✅ **Spot and On-Demand**: Support for both SPOT and ON_DEMAND capacity types +- ✅ **EKS Add-ons**: Managed add-ons for VPC CNI, Kube Proxy, and CoreDNS +- ✅ **Secrets Encryption**: Optional KMS-based encryption for Kubernetes secrets +- ✅ **Provider Exposure**: Exposes Kubernetes and Helm providers for dependent modules +- ✅ **Endpoint Access Control**: Configurable public/private API endpoint access + +### Explicitly Disabled + +- ❌ **EKS Auto Mode**: This flavor does NOT support EKS Auto Mode + +## Dependencies (Inputs) + +This module requires the following inputs: + +### 1. AWS Cloud Account +- **Type:** `@facets/aws_cloud_account` +- **Provides:** AWS provider configuration +- **Required Fields:** + - `aws_region` + - `aws_account_id` + - `aws_iam_role` + +### 2. VPC Network +- **Type:** `@facets/aws_vpc` +- **Provides:** VPC and subnet configuration +- **Required Fields:** + - `vpc_id` + - `private_subnet_ids` (list) + - `public_subnet_ids` (optional list) + +## Outputs + +This module exposes the following outputs: + +### Attributes +- `cluster_id` - The EKS cluster ID +- `cluster_arn` - The EKS cluster ARN +- `cluster_name` - The EKS cluster name +- `cluster_version` - The Kubernetes version +- `cluster_endpoint` - The EKS cluster API endpoint +- `cluster_ca_certificate` - The cluster CA certificate (base64 decoded) +- `cluster_token` - Authentication token (marked as secret) +- `cluster_security_group_id` - The cluster security group ID +- `node_security_group_id` - The node security group ID +- `oidc_provider_arn` - The OIDC provider ARN for IRSA +- `cluster_iam_role_arn` - The cluster IAM role ARN +- `cluster_primary_security_group_id` - The primary security group ID + +### Providers Exposed +- **Kubernetes Provider** (hashicorp/kubernetes v2.23.0) +- **Helm Provider** (hashicorp/helm v2.11.0) + +## Configuration Schema + +### Basic Configuration + +```yaml +spec: + cluster_version: "1.30" # Kubernetes version: 1.28, 1.29, 1.30, 1.31 + cluster_endpoint_public_access: true # Enable public API access + cluster_endpoint_private_access: true # Enable private API access + enable_cluster_encryption: false # Enable KMS secrets encryption +``` + +### Managed Node Groups + +Configure one or more managed node groups using a map structure: + +```yaml +spec: + managed_node_groups: + general: # Node group name + instance_types: ["t3.medium", "t3.large"] + min_size: 1 + max_size: 10 + desired_size: 3 + capacity_type: "ON_DEMAND" # or "SPOT" + disk_size: 50 + labels: + workload-type: general + environment: production + taints: + dedicated: general # Applied with NoSchedule effect + + spot-workers: # Another node group + instance_types: ["t3.xlarge"] + min_size: 0 + max_size: 20 + desired_size: 5 + capacity_type: "SPOT" + disk_size: 100 + labels: + workload-type: batch +``` + +### Cluster Add-ons + +```yaml +spec: + cluster_addons: + vpc_cni: + enabled: true + version: "latest" # or specific version like "v1.12.0-eksbuild.1" + kube_proxy: + enabled: true + version: "latest" + coredns: + enabled: true + version: "latest" +``` + +### Cluster Tags + +```yaml +spec: + cluster_tags: + Team: platform + CostCenter: engineering +``` + +## Example Blueprint Resource + +```yaml +kind: kubernetes_cluster +flavor: eks_standard +version: "1.0" +metadata: + name: my-eks-cluster +disabled: false +spec: + cluster_version: "1.30" + cluster_endpoint_public_access: true + cluster_endpoint_private_access: true + enable_cluster_encryption: true + + cluster_addons: + vpc_cni: + enabled: true + version: "latest" + kube_proxy: + enabled: true + version: "latest" + coredns: + enabled: true + version: "latest" + + managed_node_groups: + system: + instance_types: ["t3.medium"] + min_size: 2 + max_size: 4 + desired_size: 2 + capacity_type: "ON_DEMAND" + disk_size: 50 + labels: + node-type: system + + application: + instance_types: ["t3.large", "t3.xlarge"] + min_size: 3 + max_size: 20 + desired_size: 5 + capacity_type: "ON_DEMAND" + disk_size: 100 + labels: + node-type: application + + cluster_tags: + Environment: production + ManagedBy: facets +``` + +## Important Notes + +### 1. EKS Auto Mode +**This module explicitly does NOT support EKS Auto Mode.** The module is designed for the standard EKS deployment model with managed node groups. If you need EKS Auto Mode capabilities, you should create a separate module with flavor `eks_auto`. + +### 2. Node Group Deployment +- All node groups are deployed in the **private subnets** from the VPC input +- Node groups are managed by AWS EKS (fully managed lifecycle) +- Each node group can have different instance types, scaling configurations, and Kubernetes labels/taints + +### 3. Security +- The module creates a KMS key for secrets encryption when `enable_cluster_encryption: true` +- The KMS key has automatic rotation enabled +- The cluster authentication token is marked as a secret in outputs + +### 4. Networking +- The cluster is deployed across the subnets provided by the VPC network input +- Both public and private subnets (if provided) are used for the control plane +- Worker nodes are always deployed in private subnets + +### 5. Taints +- Taints specified in the node group configuration are applied with the `NoSchedule` effect +- Format: `key: value` pairs in the YAML + +## Module Structure + +``` +eks_standard_module/ +├── facets.yaml # Module metadata and schema +├── variables.tf # Input variable definitions +├── main.tf # Main Terraform resources +├── outputs.tf # Output definitions (locals only) +├── versions.tf # Terraform version constraints +└── README.md # This file +``` + +## Deployment Workflow + +1. **Upload Module** (Preview stage for testing): + ```bash + raptor create iac-module -f ./eks_standard_module --auto-create + ``` + +2. **Test in Testing Project**: + - Add a resource using this module to your blueprint + - Run a plan to validate configuration + - Deploy to test environment + +3. **Publish** (when ready): + ```bash + raptor publish iac-module kubernetes_cluster/eks_standard/1.0 + ``` + +## Design Decisions + +### Why Maps Instead of Arrays for Node Groups? +The module uses maps (patternProperties) for node groups instead of arrays because: +- **Deep merge support**: Environment overrides can modify individual node groups without repeating the entire configuration +- **Stable identifiers**: Node group names serve as keys for clear identification +- **Better maintainability**: Easier to override specific node groups per environment + +### Why No Provider Blocks? +Following Facets conventions: +- Providers are passed from input modules (cloud_account) +- Provider versions are defined in output type schemas +- This ensures consistent provider configuration across modules + +### Why Local Outputs? +Facets automatically extracts outputs from `local.output_attributes` and `local.output_interfaces`: +- No Terraform `output` blocks needed +- Consistent with Facets platform patterns +- Separates infrastructure outputs (attributes) from network endpoints (interfaces) + +## Terraform Compatibility + +- **Terraform Version**: >= 1.5.0, < 2.0.0 +- **terraform-aws-eks Module**: ~> 20.0 + +## Support + +For issues or questions about this module: +1. Check the Facets platform documentation +2. Review the terraform-aws-eks module documentation: https://github.com/terraform-aws-modules/terraform-aws-eks +3. Contact your platform team diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/main.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/main.tf new file mode 100644 index 00000000..05701fe1 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/main.tf @@ -0,0 +1,903 @@ +data "aws_partition" "current" { + count = local.create ? 1 : 0 +} +data "aws_caller_identity" "current" { + count = local.create ? 1 : 0 +} + +data "aws_iam_session_context" "current" { + count = local.create ? 1 : 0 + + # This data source provides information on the IAM source role of an STS assumed role + # For non-role ARNs, this data source simply passes the ARN through issuer ARN + # Ref https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2327#issuecomment-1355581682 + # Ref https://github.com/hashicorp/terraform-provider-aws/issues/28381 + arn = try(data.aws_caller_identity.current[0].arn, "") +} + +locals { + create = var.create && var.putin_khuylo + + partition = try(data.aws_partition.current[0].partition, "") + + cluster_role = try(aws_iam_role.this[0].arn, var.iam_role_arn) + + create_outposts_local_cluster = length(var.outpost_config) > 0 + enable_cluster_encryption_config = length(var.cluster_encryption_config) > 0 && !local.create_outposts_local_cluster + + auto_mode_enabled = try(var.cluster_compute_config.enabled, false) +} + +################################################################################ +# Cluster +################################################################################ + +resource "aws_eks_cluster" "this" { + count = local.create ? 1 : 0 + + name = var.cluster_name + role_arn = local.cluster_role + version = var.cluster_version + enabled_cluster_log_types = var.cluster_enabled_log_types + bootstrap_self_managed_addons = local.auto_mode_enabled ? coalesce(var.bootstrap_self_managed_addons, false) : var.bootstrap_self_managed_addons + # force_update_version = var.cluster_force_update_version + + access_config { + authentication_mode = var.authentication_mode + + # See access entries below - this is a one time operation from the EKS API. + # Instead, we are hardcoding this to false and if users wish to achieve this + # same functionality, we will do that through an access entry which can be + # enabled or disabled at any time of their choosing using the variable + # var.enable_cluster_creator_admin_permissions + bootstrap_cluster_creator_admin_permissions = false + } + + dynamic "compute_config" { + for_each = length(var.cluster_compute_config) > 0 ? [var.cluster_compute_config] : [] + + content { + enabled = local.auto_mode_enabled + node_pools = local.auto_mode_enabled ? try(compute_config.value.node_pools, []) : null + node_role_arn = local.auto_mode_enabled && length(try(compute_config.value.node_pools, [])) > 0 ? try(compute_config.value.node_role_arn, aws_iam_role.eks_auto[0].arn, null) : null + } + } + + vpc_config { + security_group_ids = compact(distinct(concat(var.cluster_additional_security_group_ids, [local.cluster_security_group_id]))) + subnet_ids = coalescelist(var.control_plane_subnet_ids, var.subnet_ids) + endpoint_private_access = var.cluster_endpoint_private_access + endpoint_public_access = var.cluster_endpoint_public_access + public_access_cidrs = var.cluster_endpoint_public_access_cidrs + } + + dynamic "kubernetes_network_config" { + # Not valid on Outposts + for_each = local.create_outposts_local_cluster ? [] : [1] + + content { + dynamic "elastic_load_balancing" { + for_each = local.auto_mode_enabled ? [1] : [] + + content { + enabled = local.auto_mode_enabled + } + } + + ip_family = var.cluster_ip_family + service_ipv4_cidr = var.cluster_service_ipv4_cidr + service_ipv6_cidr = var.cluster_service_ipv6_cidr + } + } + + dynamic "outpost_config" { + for_each = local.create_outposts_local_cluster ? [var.outpost_config] : [] + + content { + control_plane_instance_type = outpost_config.value.control_plane_instance_type + outpost_arns = outpost_config.value.outpost_arns + } + } + + dynamic "encryption_config" { + # Not available on Outposts + for_each = local.enable_cluster_encryption_config ? [var.cluster_encryption_config] : [] + + content { + provider { + key_arn = var.create_kms_key ? module.kms.key_arn : encryption_config.value.provider_key_arn + } + resources = encryption_config.value.resources + } + } + + dynamic "remote_network_config" { + # Not valid on Outposts + for_each = length(var.cluster_remote_network_config) > 0 && !local.create_outposts_local_cluster ? [var.cluster_remote_network_config] : [] + + content { + dynamic "remote_node_networks" { + for_each = [remote_network_config.value.remote_node_networks] + + content { + cidrs = remote_node_networks.value.cidrs + } + } + + dynamic "remote_pod_networks" { + for_each = try([remote_network_config.value.remote_pod_networks], []) + + content { + cidrs = remote_pod_networks.value.cidrs + } + } + } + } + + dynamic "storage_config" { + for_each = local.auto_mode_enabled ? [1] : [] + + content { + block_storage { + enabled = local.auto_mode_enabled + } + } + } + + dynamic "upgrade_policy" { + for_each = length(var.cluster_upgrade_policy) > 0 ? [var.cluster_upgrade_policy] : [] + + content { + support_type = try(upgrade_policy.value.support_type, null) + } + } + + dynamic "zonal_shift_config" { + for_each = length(var.cluster_zonal_shift_config) > 0 ? [var.cluster_zonal_shift_config] : [] + + content { + enabled = try(zonal_shift_config.value.enabled, null) + } + } + + tags = merge( + { terraform-aws-modules = "eks" }, + var.tags, + var.cluster_tags, + ) + + timeouts { + create = try(var.cluster_timeouts.create, null) + update = try(var.cluster_timeouts.update, null) + delete = try(var.cluster_timeouts.delete, null) + } + + depends_on = [ + aws_iam_role_policy_attachment.this, + aws_security_group_rule.cluster, + aws_security_group_rule.node, + aws_cloudwatch_log_group.this, + aws_iam_policy.cni_ipv6_policy, + ] + + lifecycle { + ignore_changes = [ + access_config[0].bootstrap_cluster_creator_admin_permissions + ] + } +} + +resource "aws_ec2_tag" "cluster_primary_security_group" { + # This should not affect the name of the cluster primary security group + # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2006 + # Ref: https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2008 + for_each = { for k, v in merge(var.tags, var.cluster_tags) : + k => v if local.create && k != "Name" && var.create_cluster_primary_security_group_tags + } + + resource_id = aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id + key = each.key + value = each.value +} + +resource "aws_cloudwatch_log_group" "this" { + count = local.create && var.create_cloudwatch_log_group ? 1 : 0 + + name = "/aws/eks/${var.cluster_name}/cluster" + retention_in_days = var.cloudwatch_log_group_retention_in_days + kms_key_id = var.cloudwatch_log_group_kms_key_id + log_group_class = var.cloudwatch_log_group_class + + tags = merge( + var.tags, + var.cloudwatch_log_group_tags, + { Name = "/aws/eks/${var.cluster_name}/cluster" } + ) +} + +################################################################################ +# Access Entry +################################################################################ + +locals { + # This replaces the one time logic from the EKS API with something that can be + # better controlled by users through Terraform + bootstrap_cluster_creator_admin_permissions = { + cluster_creator = { + principal_arn = try(data.aws_iam_session_context.current[0].issuer_arn, "") + type = "STANDARD" + + policy_associations = { + admin = { + policy_arn = "arn:${local.partition}:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" + access_scope = { + type = "cluster" + } + } + } + } + } + + # Merge the bootstrap behavior with the entries that users provide + merged_access_entries = merge( + { for k, v in local.bootstrap_cluster_creator_admin_permissions : k => v if var.enable_cluster_creator_admin_permissions }, + var.access_entries, + ) + + # Flatten out entries and policy associations so users can specify the policy + # associations within a single entry + flattened_access_entries = flatten([ + for entry_key, entry_val in local.merged_access_entries : [ + for pol_key, pol_val in lookup(entry_val, "policy_associations", {}) : + merge( + { + principal_arn = entry_val.principal_arn + entry_key = entry_key + pol_key = pol_key + }, + { for k, v in { + association_policy_arn = pol_val.policy_arn + association_access_scope_type = pol_val.access_scope.type + association_access_scope_namespaces = lookup(pol_val.access_scope, "namespaces", []) + } : k => v if !contains(["EC2_LINUX", "EC2_WINDOWS", "FARGATE_LINUX", "HYBRID_LINUX"], lookup(entry_val, "type", "STANDARD")) }, + ) + ] + ]) +} + +resource "aws_eks_access_entry" "this" { + for_each = { for k, v in local.merged_access_entries : k => v if local.create } + + cluster_name = aws_eks_cluster.this[0].id + kubernetes_groups = try(each.value.kubernetes_groups, null) + principal_arn = each.value.principal_arn + type = try(each.value.type, "STANDARD") + user_name = try(each.value.user_name, null) + + tags = merge(var.tags, try(each.value.tags, {})) +} + +resource "aws_eks_access_policy_association" "this" { + for_each = { for k, v in local.flattened_access_entries : "${v.entry_key}_${v.pol_key}" => v if local.create } + + access_scope { + namespaces = try(each.value.association_access_scope_namespaces, []) + type = each.value.association_access_scope_type + } + + cluster_name = aws_eks_cluster.this[0].id + + policy_arn = each.value.association_policy_arn + principal_arn = each.value.principal_arn + + depends_on = [ + aws_eks_access_entry.this, + ] +} + +################################################################################ +# KMS Key +################################################################################ + +module "kms" { + source = "./modules/kms" + create = local.create && var.create_kms_key && local.enable_cluster_encryption_config # not valid on Outposts + + description = coalesce(var.kms_key_description, "${var.cluster_name} cluster encryption key") + key_usage = "ENCRYPT_DECRYPT" + deletion_window_in_days = var.kms_key_deletion_window_in_days + enable_key_rotation = var.enable_kms_key_rotation + + # Policy + enable_default_policy = var.kms_key_enable_default_policy + key_owners = var.kms_key_owners + key_administrators = coalescelist(var.kms_key_administrators, [try(data.aws_iam_session_context.current[0].issuer_arn, "")]) + key_users = concat([local.cluster_role], var.kms_key_users) + key_service_users = var.kms_key_service_users + source_policy_documents = var.kms_key_source_policy_documents + override_policy_documents = var.kms_key_override_policy_documents + + # Aliases + aliases = var.kms_key_aliases + computed_aliases = { + # Computed since users can pass in computed values for cluster name such as random provider resources + cluster = { name = "eks/${var.cluster_name}" } + } + + tags = merge( + { terraform-aws-modules = "eks" }, + var.tags, + ) +} + +################################################################################ +# Cluster Security Group +# Defaults follow https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html +################################################################################ + +locals { + cluster_sg_name = coalesce(var.cluster_security_group_name, "${var.cluster_name}-cluster") + create_cluster_sg = local.create && var.create_cluster_security_group + + cluster_security_group_id = local.create_cluster_sg ? aws_security_group.cluster[0].id : var.cluster_security_group_id + + # Do not add rules to node security group if the module is not creating it + cluster_security_group_rules = { for k, v in { + ingress_nodes_443 = { + description = "Node groups to cluster API" + protocol = "tcp" + from_port = 443 + to_port = 443 + type = "ingress" + source_node_security_group = true + } + } : k => v if local.create_node_sg } +} + +resource "aws_security_group" "cluster" { + count = local.create_cluster_sg ? 1 : 0 + + name = var.cluster_security_group_use_name_prefix ? null : local.cluster_sg_name + name_prefix = var.cluster_security_group_use_name_prefix ? "${local.cluster_sg_name}${var.prefix_separator}" : null + description = var.cluster_security_group_description + vpc_id = var.vpc_id + + tags = merge( + var.tags, + { "Name" = local.cluster_sg_name }, + var.cluster_security_group_tags + ) + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_security_group_rule" "cluster" { + for_each = { for k, v in merge( + local.cluster_security_group_rules, + var.cluster_security_group_additional_rules + ) : k => v if local.create_cluster_sg } + + # Required + security_group_id = aws_security_group.cluster[0].id + protocol = each.value.protocol + from_port = each.value.from_port + to_port = each.value.to_port + type = each.value.type + + # Optional + description = lookup(each.value, "description", null) + cidr_blocks = lookup(each.value, "cidr_blocks", null) + ipv6_cidr_blocks = lookup(each.value, "ipv6_cidr_blocks", null) + prefix_list_ids = lookup(each.value, "prefix_list_ids", null) + self = lookup(each.value, "self", null) + source_security_group_id = try(each.value.source_node_security_group, false) ? local.node_security_group_id : lookup(each.value, "source_security_group_id", null) +} + +################################################################################ +# IRSA +# Note - this is different from EKS identity provider +################################################################################ + +locals { + # Not available on outposts + create_oidc_provider = local.create && var.enable_irsa && !local.create_outposts_local_cluster + + oidc_root_ca_thumbprint = local.create_oidc_provider && var.include_oidc_root_ca_thumbprint ? [data.tls_certificate.this[0].certificates[0].sha1_fingerprint] : [] +} + +data "tls_certificate" "this" { + # Not available on outposts + count = local.create_oidc_provider && var.include_oidc_root_ca_thumbprint ? 1 : 0 + + url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer +} + +resource "aws_iam_openid_connect_provider" "oidc_provider" { + # Not available on outposts + count = local.create_oidc_provider ? 1 : 0 + + client_id_list = distinct(compact(concat(["sts.amazonaws.com"], var.openid_connect_audiences))) + thumbprint_list = concat(local.oidc_root_ca_thumbprint, var.custom_oidc_thumbprints) + url = aws_eks_cluster.this[0].identity[0].oidc[0].issuer + + tags = merge( + { Name = "${var.cluster_name}-eks-irsa" }, + var.tags + ) +} + +################################################################################ +# IAM Role +################################################################################ + +locals { + create_iam_role = local.create && var.create_iam_role + iam_role_name = coalesce(var.iam_role_name, "${var.cluster_name}-cluster") + iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" + + cluster_encryption_policy_name = coalesce(var.cluster_encryption_policy_name, "${local.iam_role_name}-ClusterEncryption") + + # Standard EKS cluster + eks_standard_iam_role_policies = { for k, v in { + AmazonEKSClusterPolicy = "${local.iam_role_policy_prefix}/AmazonEKSClusterPolicy", + } : k => v if !local.create_outposts_local_cluster && !local.auto_mode_enabled } + + # EKS cluster with EKS auto mode enabled + eks_auto_mode_iam_role_policies = { for k, v in { + AmazonEKSClusterPolicy = "${local.iam_role_policy_prefix}/AmazonEKSClusterPolicy" + AmazonEKSComputePolicy = "${local.iam_role_policy_prefix}/AmazonEKSComputePolicy" + AmazonEKSBlockStoragePolicy = "${local.iam_role_policy_prefix}/AmazonEKSBlockStoragePolicy" + AmazonEKSLoadBalancingPolicy = "${local.iam_role_policy_prefix}/AmazonEKSLoadBalancingPolicy" + AmazonEKSNetworkingPolicy = "${local.iam_role_policy_prefix}/AmazonEKSNetworkingPolicy" + } : k => v if !local.create_outposts_local_cluster && local.auto_mode_enabled } + + # EKS local cluster on Outposts + eks_outpost_iam_role_policies = { for k, v in { + AmazonEKSClusterPolicy = "${local.iam_role_policy_prefix}/AmazonEKSLocalOutpostClusterPolicy" + } : k => v if local.create_outposts_local_cluster && !local.auto_mode_enabled } + + # Security groups for pods + eks_sgpp_iam_role_policies = { for k, v in { + AmazonEKSVPCResourceController = "${local.iam_role_policy_prefix}/AmazonEKSVPCResourceController" + } : k => v if var.enable_security_groups_for_pods && !local.create_outposts_local_cluster && !local.auto_mode_enabled } +} + +data "aws_iam_policy_document" "assume_role_policy" { + count = local.create && var.create_iam_role ? 1 : 0 + + statement { + sid = "EKSClusterAssumeRole" + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + type = "Service" + identifiers = ["eks.amazonaws.com"] + } + + dynamic "principals" { + for_each = local.create_outposts_local_cluster ? [1] : [] + + content { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } + } +} + +resource "aws_iam_role" "this" { + count = local.create_iam_role ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + # name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}${var.prefix_separator}" : null + path = var.iam_role_path + description = var.iam_role_description + + assume_role_policy = data.aws_iam_policy_document.assume_role_policy[0].json + permissions_boundary = var.iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.iam_role_tags) +} + +# Policies attached ref https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html +resource "aws_iam_role_policy_attachment" "this" { + for_each = { for k, v in merge( + local.eks_standard_iam_role_policies, + local.eks_auto_mode_iam_role_policies, + local.eks_outpost_iam_role_policies, + local.eks_sgpp_iam_role_policies, + ) : k => v if local.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +resource "aws_iam_role_policy_attachment" "additional" { + for_each = { for k, v in var.iam_role_additional_policies : k => v if local.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +# Using separate attachment due to `The "for_each" value depends on resource attributes that cannot be determined until apply` +resource "aws_iam_role_policy_attachment" "cluster_encryption" { + # Encryption config not available on Outposts + count = local.create_iam_role && var.attach_cluster_encryption_policy && local.enable_cluster_encryption_config ? 1 : 0 + + policy_arn = aws_iam_policy.cluster_encryption[0].arn + role = aws_iam_role.this[0].name +} + +resource "aws_iam_policy" "cluster_encryption" { + # Encryption config not available on Outposts + count = local.create_iam_role && var.attach_cluster_encryption_policy && local.enable_cluster_encryption_config ? 1 : 0 + + name = var.cluster_encryption_policy_use_name_prefix ? null : local.cluster_encryption_policy_name + name_prefix = var.cluster_encryption_policy_use_name_prefix ? local.cluster_encryption_policy_name : null + description = var.cluster_encryption_policy_description + path = var.cluster_encryption_policy_path + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ListGrants", + "kms:DescribeKey", + ] + Effect = "Allow" + Resource = var.create_kms_key ? module.kms.key_arn : var.cluster_encryption_config.provider_key_arn + }, + ] + }) + + tags = merge(var.tags, var.cluster_encryption_policy_tags) +} + +data "aws_iam_policy_document" "custom" { + count = local.create_iam_role && var.enable_auto_mode_custom_tags ? 1 : 0 + + dynamic "statement" { + for_each = var.enable_auto_mode_custom_tags ? [1] : [] + + content { + sid = "Compute" + actions = [ + "ec2:CreateFleet", + "ec2:RunInstances", + "ec2:CreateLaunchTemplate", + ] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/eks:kubernetes-node-class-name" + values = ["*"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/eks:kubernetes-node-pool-name" + values = ["*"] + } + } + } + + dynamic "statement" { + for_each = var.enable_auto_mode_custom_tags ? [1] : [] + + content { + sid = "Storage" + actions = [ + "ec2:CreateVolume", + "ec2:CreateSnapshot", + ] + resources = [ + "arn:${local.partition}:ec2:*:*:volume/*", + "arn:${local.partition}:ec2:*:*:snapshot/*", + ] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + } + } + + dynamic "statement" { + for_each = var.enable_auto_mode_custom_tags ? [1] : [] + + content { + sid = "Networking" + actions = ["ec2:CreateNetworkInterface"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:kubernetes-cni-node-name" + values = ["*"] + } + } + } + + dynamic "statement" { + for_each = var.enable_auto_mode_custom_tags ? [1] : [] + + content { + sid = "LoadBalancer" + actions = [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateRule", + "ec2:CreateSecurityGroup", + ] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + } + } + + dynamic "statement" { + for_each = var.enable_auto_mode_custom_tags ? [1] : [] + + content { + sid = "ShieldProtection" + actions = ["shield:CreateProtection"] + resources = ["*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + } + } + + dynamic "statement" { + for_each = var.enable_auto_mode_custom_tags ? [1] : [] + + content { + sid = "ShieldTagResource" + actions = ["shield:TagResource"] + resources = ["arn:${local.partition}:shield::*:protection/*"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = ["$${aws:PrincipalTag/eks:eks-cluster-name}"] + } + } + } +} + +resource "aws_iam_policy" "custom" { + count = local.create_iam_role && var.enable_auto_mode_custom_tags ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description + + policy = data.aws_iam_policy_document.custom[0].json + + tags = merge(var.tags, var.iam_role_tags) +} + +resource "aws_iam_role_policy_attachment" "custom" { + count = local.create_iam_role && var.enable_auto_mode_custom_tags ? 1 : 0 + + policy_arn = aws_iam_policy.custom[0].arn + role = aws_iam_role.this[0].name +} + +################################################################################ +# EKS Addons +################################################################################ + +locals { + # TODO - Set to `NONE` on next breaking change when default addons are disabled + resolve_conflicts_on_create_default = coalesce(var.bootstrap_self_managed_addons, true) ? "OVERWRITE" : "NONE" +} + +data "aws_eks_addon_version" "this" { + for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster } + + addon_name = try(each.value.name, each.key) + kubernetes_version = coalesce(var.cluster_version, aws_eks_cluster.this[0].version) + # TODO - Set default fallback to `true` on next breaking change + most_recent = try(each.value.most_recent, null) +} + +resource "aws_eks_addon" "this" { + # Not supported on outposts + for_each = { for k, v in var.cluster_addons : k => v if !try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster } + + cluster_name = aws_eks_cluster.this[0].id + addon_name = try(each.value.name, each.key) + + addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) + configuration_values = try(each.value.configuration_values, null) + + dynamic "pod_identity_association" { + for_each = try(each.value.pod_identity_association, []) + + content { + role_arn = pod_identity_association.value.role_arn + service_account = pod_identity_association.value.service_account + } + } + + preserve = try(each.value.preserve, true) + # TODO - Set to `NONE` on next breaking change when default addons are disabled + resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, local.resolve_conflicts_on_create_default) + resolve_conflicts_on_update = try(each.value.resolve_conflicts_on_update, "OVERWRITE") + service_account_role_arn = try(each.value.service_account_role_arn, null) + + timeouts { + create = try(each.value.timeouts.create, var.cluster_addons_timeouts.create, null) + update = try(each.value.timeouts.update, var.cluster_addons_timeouts.update, null) + delete = try(each.value.timeouts.delete, var.cluster_addons_timeouts.delete, null) + } + + depends_on = [ + module.fargate_profile, + module.eks_managed_node_group, + module.self_managed_node_group, + ] + + tags = merge(var.tags, try(each.value.tags, {})) +} + + +resource "aws_eks_addon" "before_compute" { + # Not supported on outposts + for_each = { for k, v in var.cluster_addons : k => v if try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster } + + cluster_name = aws_eks_cluster.this[0].id + addon_name = try(each.value.name, each.key) + + addon_version = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version) + configuration_values = try(each.value.configuration_values, null) + + dynamic "pod_identity_association" { + for_each = try(each.value.pod_identity_association, []) + + content { + role_arn = pod_identity_association.value.role_arn + service_account = pod_identity_association.value.service_account + } + } + + preserve = try(each.value.preserve, true) + # TODO - Set to `NONE` on next breaking change when default addons are disabled + resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, local.resolve_conflicts_on_create_default) + resolve_conflicts_on_update = try(each.value.resolve_conflicts_on_update, "OVERWRITE") + service_account_role_arn = try(each.value.service_account_role_arn, null) + + timeouts { + create = try(each.value.timeouts.create, var.cluster_addons_timeouts.create, null) + update = try(each.value.timeouts.update, var.cluster_addons_timeouts.update, null) + delete = try(each.value.timeouts.delete, var.cluster_addons_timeouts.delete, null) + } + + tags = merge(var.tags, try(each.value.tags, {})) +} + +################################################################################ +# EKS Identity Provider +# Note - this is different from IRSA +################################################################################ + +locals { + # Maintain current behavior for <= 1.29, remove default for >= 1.30 + # `null` will return the latest Kubernetes version from the EKS API, which at time of writing is 1.30 + # https://github.com/kubernetes/kubernetes/pull/123561 + # TODO - remove on next breaking change in conjunction with issuer URL change below + idpc_backwards_compat_version = contains(["1.21", "1.22", "1.23", "1.24", "1.25", "1.26", "1.27", "1.28", "1.29"], coalesce(var.cluster_version, "1.30")) + idpc_issuer_url = local.idpc_backwards_compat_version ? try(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, null) : null +} + +resource "aws_eks_identity_provider_config" "this" { + for_each = { for k, v in var.cluster_identity_providers : k => v if local.create && !local.create_outposts_local_cluster } + + cluster_name = aws_eks_cluster.this[0].id + + oidc { + client_id = each.value.client_id + groups_claim = lookup(each.value, "groups_claim", null) + groups_prefix = lookup(each.value, "groups_prefix", null) + identity_provider_config_name = try(each.value.identity_provider_config_name, each.key) + # TODO - make argument explicitly required on next breaking change + issuer_url = try(each.value.issuer_url, local.idpc_issuer_url) + required_claims = lookup(each.value, "required_claims", null) + username_claim = lookup(each.value, "username_claim", null) + username_prefix = lookup(each.value, "username_prefix", null) + } + + tags = merge(var.tags, try(each.value.tags, {})) +} + +################################################################################ +# EKS Auto Node IAM Role +################################################################################ + +locals { + create_node_iam_role = local.create && var.create_node_iam_role && local.auto_mode_enabled + node_iam_role_name = coalesce(var.node_iam_role_name, "${var.cluster_name}-eks-auto") +} + +data "aws_iam_policy_document" "node_assume_role_policy" { + count = local.create_node_iam_role ? 1 : 0 + + statement { + sid = "EKSAutoNodeAssumeRole" + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "eks_auto" { + count = local.create_node_iam_role ? 1 : 0 + + name = var.node_iam_role_use_name_prefix ? null : local.node_iam_role_name + # name_prefix = var.node_iam_role_use_name_prefix ? "${local.node_iam_role_name}-" : null + path = var.node_iam_role_path + description = var.node_iam_role_description + + assume_role_policy = data.aws_iam_policy_document.node_assume_role_policy[0].json + permissions_boundary = var.node_iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.node_iam_role_tags) +} + +# Policies attached ref https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html +resource "aws_iam_role_policy_attachment" "eks_auto" { + for_each = { for k, v in { + AmazonEKSWorkerNodeMinimalPolicy = "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodeMinimalPolicy", + AmazonEC2ContainerRegistryPullOnly = "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryPullOnly", + } : k => v if local.create_node_iam_role } + + policy_arn = each.value + role = aws_iam_role.eks_auto[0].name +} + +resource "aws_iam_role_policy_attachment" "eks_auto_additional" { + for_each = { for k, v in var.node_iam_role_additional_policies : k => v if local.create_node_iam_role } + + policy_arn = each.value + role = aws_iam_role.eks_auto[0].name +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/_user_data/README.md b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/_user_data/README.md new file mode 100644 index 00000000..185335b1 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/_user_data/README.md @@ -0,0 +1,64 @@ +# User Data Module + +Configuration in this directory renders the appropriate user data for the given inputs. See [`docs/user_data.md`](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/docs/user_data.md) for more info. + +See [`tests/user-data/`](https://github.com/terraform-aws-modules/terraform-aws-eks/tree/master/tests/user-data) for various tests cases using this module. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [cloudinit](#requirement\_cloudinit) | >= 2.0 | +| [null](#requirement\_null) | >= 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [cloudinit](#provider\_cloudinit) | >= 2.0 | +| [null](#provider\_null) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [null_resource.validate_cluster_service_cidr](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [cloudinit_config.al2023_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | +| [cloudinit_config.linux_eks_managed_node_group](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_cluster\_dns\_ips](#input\_additional\_cluster\_dns\_ips) | Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*` | `list(string)` | `[]` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `null` | no | +| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | +| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | +| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | +| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `""` | no | +| [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | +| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | [Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | +| [create](#input\_create) | Determines whether to create user-data or not | `bool` | `true` | no | +| [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template | `bool` | `false` | no | +| [is\_eks\_managed\_node\_group](#input\_is\_eks\_managed\_node\_group) | Determines whether the user data is used on nodes in an EKS managed node group. Used to determine if user data will be appended or not | `bool` | `true` | no | +| [platform](#input\_platform) | [DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | `string` | `"linux"` | no | +| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | +| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | +| [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [platform](#output\_platform) | [DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023, or `windows | +| [user\_data](#output\_user\_data) | Base64 encoded user data rendered for the provided inputs | + diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/_user_data/main.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/_user_data/main.tf new file mode 100644 index 00000000..29d5c92d --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/_user_data/main.tf @@ -0,0 +1,148 @@ +# The `cluster_service_cidr` is required when `create == true` +# This is a hacky way to make that logic work, otherwise Terraform always wants a value +# and supplying any old value like `""` or `null` is not valid and will silently +# fail to join nodes to the cluster +resource "null_resource" "validate_cluster_service_cidr" { + lifecycle { + precondition { + # The length 6 is currently arbitrary, but it's a safe bet that the CIDR will be longer than that + # The main point is that a value needs to be provided when `create = true` + condition = var.create ? length(local.cluster_service_cidr) > 6 : true + error_message = "`cluster_service_cidr` is required when `create = true`." + } + } +} + +locals { + # Converts AMI type into user data type that represents the underlying format (bash, toml, PS1, nodeadm) + # TODO - platform will be removed in v21.0 and only `ami_type` will be valid + ami_type_to_user_data_type = { + AL2_x86_64 = "linux" + AL2_x86_64_GPU = "linux" + AL2_ARM_64 = "linux" + BOTTLEROCKET_ARM_64 = "bottlerocket" + BOTTLEROCKET_x86_64 = "bottlerocket" + BOTTLEROCKET_ARM_64_FIPS = "bottlerocket" + BOTTLEROCKET_x86_64_FIPS = "bottlerocket" + BOTTLEROCKET_ARM_64_NVIDIA = "bottlerocket" + BOTTLEROCKET_x86_64_NVIDIA = "bottlerocket" + WINDOWS_CORE_2019_x86_64 = "windows" + WINDOWS_FULL_2019_x86_64 = "windows" + WINDOWS_CORE_2022_x86_64 = "windows" + WINDOWS_FULL_2022_x86_64 = "windows" + AL2023_x86_64_STANDARD = "al2023" + AL2023_ARM_64_STANDARD = "al2023" + AL2023_x86_64_NEURON = "al2023" + AL2023_x86_64_NVIDIA = "al2023" + } + # Try to use `ami_type` first, but fall back to current, default behavior + # TODO - will be removed in v21.0 + user_data_type = try(local.ami_type_to_user_data_type[var.ami_type], var.platform) + + template_path = { + al2023 = "${path.module}/../../templates/al2023_user_data.tpl" + bottlerocket = "${path.module}/../../templates/bottlerocket_user_data.tpl" + linux = "${path.module}/../../templates/linux_user_data.tpl" + windows = "${path.module}/../../templates/windows_user_data.tpl" + } + + cluster_service_cidr = try(coalesce(var.cluster_service_ipv4_cidr, var.cluster_service_cidr), "") + cluster_dns_ips = flatten(concat([try(cidrhost(local.cluster_service_cidr, 10), "")], var.additional_cluster_dns_ips)) + + user_data = base64encode(templatefile( + coalesce(var.user_data_template_path, local.template_path[local.user_data_type]), + { + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami + enable_bootstrap_user_data = var.enable_bootstrap_user_data + + # Required to bootstrap node + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + + cluster_service_cidr = local.cluster_service_cidr + cluster_ip_family = var.cluster_ip_family + + # Bottlerocket + cluster_dns_ips = "[${join(", ", formatlist("\"%s\"", local.cluster_dns_ips))}]" + + # Optional + bootstrap_extra_args = var.bootstrap_extra_args + pre_bootstrap_user_data = var.pre_bootstrap_user_data + post_bootstrap_user_data = var.post_bootstrap_user_data + } + )) + + user_data_type_to_rendered = { + al2023 = { + user_data = var.create ? try(data.cloudinit_config.al2023_eks_managed_node_group[0].rendered, local.user_data) : "" + } + bottlerocket = { + user_data = var.create && local.user_data_type == "bottlerocket" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.bootstrap_extra_args != "") ? local.user_data : "" + } + linux = { + user_data = var.create ? try(data.cloudinit_config.linux_eks_managed_node_group[0].rendered, local.user_data) : "" + } + windows = { + user_data = var.create && local.user_data_type == "windows" && (var.enable_bootstrap_user_data || var.user_data_template_path != "" || var.pre_bootstrap_user_data != "") ? local.user_data : "" + } + } +} + +# https://github.com/aws/containers-roadmap/issues/596#issuecomment-675097667 +# Managed node group data must in MIME multi-part archive format, +# as by default, EKS will merge the bootstrapping command required for nodes to join the +# cluster with your user data. If you use a custom AMI in your launch template, +# this merging will NOT happen and you are responsible for nodes joining the cluster. +# See docs for more details -> https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-user-data + +data "cloudinit_config" "linux_eks_managed_node_group" { + count = var.create && local.user_data_type == "linux" && var.is_eks_managed_node_group && !var.enable_bootstrap_user_data && var.pre_bootstrap_user_data != "" && var.user_data_template_path == "" ? 1 : 0 + + base64_encode = true + gzip = false + boundary = "//" + + # Prepend to existing user data supplied by AWS EKS + part { + content = var.pre_bootstrap_user_data + content_type = "text/x-shellscript" + } +} + +# Scenarios: +# +# 1. Do nothing - provide nothing +# 2. Prepend stuff on EKS MNG (before EKS MNG adds its bit at the end) +# 3. Own all of the stuff on self-MNG or EKS MNG w/ custom AMI + +locals { + nodeadm_cloudinit = var.enable_bootstrap_user_data ? concat( + var.cloudinit_pre_nodeadm, + [{ + content_type = "application/node.eks.aws" + content = base64decode(local.user_data) + }], + var.cloudinit_post_nodeadm + ) : var.cloudinit_pre_nodeadm +} + +data "cloudinit_config" "al2023_eks_managed_node_group" { + count = var.create && local.user_data_type == "al2023" && length(local.nodeadm_cloudinit) > 0 ? 1 : 0 + + base64_encode = true + gzip = false + boundary = "MIMEBOUNDARY" + + dynamic "part" { + # Using the index is fine in this context since any change in user data will be a replacement + for_each = { for i, v in local.nodeadm_cloudinit : i => v } + + content { + content = part.value.content + content_type = try(part.value.content_type, null) + filename = try(part.value.filename, null) + merge_type = try(part.value.merge_type, null) + } + } +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/_user_data/outputs.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/_user_data/outputs.tf new file mode 100644 index 00000000..7bebb3f2 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/_user_data/outputs.tf @@ -0,0 +1,9 @@ +output "user_data" { + description = "Base64 encoded user data rendered for the provided inputs" + value = try(local.user_data_type_to_rendered[local.user_data_type].user_data, null) +} + +output "platform" { + description = "[DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023, or `windows`" + value = local.user_data_type +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/_user_data/variables.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/_user_data/variables.tf new file mode 100644 index 00000000..d5a1ef1b --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/_user_data/variables.tf @@ -0,0 +1,118 @@ +variable "create" { + description = "Determines whether to create user-data or not" + type = bool + default = true +} + +variable "platform" { + description = "[DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" + type = string + default = "linux" +} + +variable "ami_type" { + description = "Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values" + type = string + default = null +} + +variable "enable_bootstrap_user_data" { + description = "Determines whether the bootstrap configurations are populated within the user data template" + type = bool + default = false +} + +variable "is_eks_managed_node_group" { + description = "Determines whether the user data is used on nodes in an EKS managed node group. Used to determine if user data will be appended or not" + type = bool + default = true +} + +variable "cluster_name" { + description = "Name of the EKS cluster" + type = string + default = "" +} + +variable "cluster_endpoint" { + description = "Endpoint of associated EKS cluster" + type = string + default = "" +} + +variable "cluster_auth_base64" { + description = "Base64 encoded CA of associated EKS cluster" + type = string + default = "" +} + +variable "cluster_service_cidr" { + description = "The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself" + type = string + default = "" +} + +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" + type = string + default = "ipv4" +} + +variable "additional_cluster_dns_ips" { + description = "Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*`" + type = list(string) + default = [] +} + +# TODO - remove at next breaking change +variable "cluster_service_ipv4_cidr" { + description = "[Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" + type = string + default = null +} + +variable "pre_bootstrap_user_data" { + description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" + type = string + default = "" +} + +variable "post_bootstrap_user_data" { + description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" + type = string + default = "" +} + +variable "bootstrap_extra_args" { + description = "Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" + type = string + default = "" +} + +variable "user_data_template_path" { + description = "Path to a local, custom user data template file to use when rendering user data" + type = string + default = "" +} + +variable "cloudinit_pre_nodeadm" { + description = "Array of cloud-init document parts that are created before the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} + +variable "cloudinit_post_nodeadm" { + description = "Array of cloud-init document parts that are created after the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/aws-auth/README.md b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/aws-auth/README.md new file mode 100644 index 00000000..d4b97f14 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/aws-auth/README.md @@ -0,0 +1,81 @@ +# `aws-auth` Module + +Configuration in this directory creates/updates the `aws-auth` ConfigMap. + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws//modules/aws-auth" + version = "~> 20.0" + + manage_aws_auth_configmap = true + + aws_auth_roles = [ + { + rolearn = "arn:aws:iam::66666666666:role/role1" + username = "role1" + groups = ["system:masters"] + }, + ] + + aws_auth_users = [ + { + userarn = "arn:aws:iam::66666666666:user/user1" + username = "user1" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::66666666666:user/user2" + username = "user2" + groups = ["system:masters"] + }, + ] + + aws_auth_accounts = [ + "777777777777", + "888888888888", + ] +} +``` + +## Usage + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [kubernetes](#requirement\_kubernetes) | >= 2.20 | + +## Providers + +| Name | Version | +|------|---------| +| [kubernetes](#provider\_kubernetes) | >= 2.20 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | +| [kubernetes_config_map_v1_data.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1_data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [aws\_auth\_accounts](#input\_aws\_auth\_accounts) | List of account maps to add to the aws-auth configmap | `list(any)` | `[]` | no | +| [aws\_auth\_roles](#input\_aws\_auth\_roles) | List of role maps to add to the aws-auth configmap | `list(any)` | `[]` | no | +| [aws\_auth\_users](#input\_aws\_auth\_users) | List of user maps to add to the aws-auth configmap | `list(any)` | `[]` | no | +| [create](#input\_create) | Controls if resources should be created (affects all resources) | `bool` | `true` | no | +| [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | +| [manage\_aws\_auth\_configmap](#input\_manage\_aws\_auth\_configmap) | Determines whether to manage the aws-auth configmap | `bool` | `true` | no | + +## Outputs + +No outputs. + diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/aws-auth/main.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/aws-auth/main.tf new file mode 100644 index 00000000..2f7e9694 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/aws-auth/main.tf @@ -0,0 +1,47 @@ + +################################################################################ +# aws-auth configmap +################################################################################ + +locals { + aws_auth_configmap_data = { + mapRoles = yamlencode(var.aws_auth_roles) + mapUsers = yamlencode(var.aws_auth_users) + mapAccounts = yamlencode(var.aws_auth_accounts) + } +} + +resource "kubernetes_config_map" "aws_auth" { + count = var.create && var.create_aws_auth_configmap ? 1 : 0 + + metadata { + name = "aws-auth" + namespace = "kube-system" + } + + data = local.aws_auth_configmap_data + + lifecycle { + # We are ignoring the data here since we will manage it with the resource below + # This is only intended to be used in scenarios where the configmap does not exist + ignore_changes = [data, metadata[0].labels, metadata[0].annotations] + } +} + +resource "kubernetes_config_map_v1_data" "aws_auth" { + count = var.create && var.manage_aws_auth_configmap ? 1 : 0 + + force = true + + metadata { + name = "aws-auth" + namespace = "kube-system" + } + + data = local.aws_auth_configmap_data + + depends_on = [ + # Required for instances where the configmap does not exist yet to avoid race condition + kubernetes_config_map.aws_auth, + ] +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/aws-auth/outputs.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/aws-auth/outputs.tf new file mode 100644 index 00000000..e69de29b diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/aws-auth/variables.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/aws-auth/variables.tf new file mode 100644 index 00000000..3aaeb023 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/aws-auth/variables.tf @@ -0,0 +1,39 @@ +variable "create" { + description = "Controls if resources should be created (affects all resources)" + type = bool + default = true +} + +################################################################################ +# aws-auth ConfigMap +################################################################################ + +variable "create_aws_auth_configmap" { + description = "Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap`" + type = bool + default = false +} + +variable "manage_aws_auth_configmap" { + description = "Determines whether to manage the aws-auth configmap" + type = bool + default = true +} + +variable "aws_auth_roles" { + description = "List of role maps to add to the aws-auth configmap" + type = list(any) + default = [] +} + +variable "aws_auth_users" { + description = "List of user maps to add to the aws-auth configmap" + type = list(any) + default = [] +} + +variable "aws_auth_accounts" { + description = "List of account maps to add to the aws-auth configmap" + type = list(any) + default = [] +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/README.md b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/README.md new file mode 100644 index 00000000..b658c644 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/README.md @@ -0,0 +1,217 @@ +# EKS Managed Node Group Module + +Configuration in this directory creates an EKS Managed Node Group along with an IAM role, security group, and launch template + +## Usage + +```hcl +module "eks_managed_node_group" { + source = "terraform-aws-modules/eks/aws//modules/eks-managed-node-group" + + name = "separate-eks-mng" + cluster_name = "my-cluster" + cluster_version = "1.31" + + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + + // The following variables are necessary if you decide to use the module outside of the parent EKS module context. + // Without it, the security groups of the nodes are empty and thus won't join the cluster. + cluster_primary_security_group_id = module.eks.cluster_primary_security_group_id + vpc_security_group_ids = [module.eks.node_security_group_id] + + // Note: `disk_size`, and `remote_access` can only be set when using the EKS managed node group default launch template + // This module defaults to providing a custom launch template to allow for custom security groups, tag propagation, etc. + // use_custom_launch_template = false + // disk_size = 50 + // + // # Remote access cannot be specified with a launch template + // remote_access = { + // ec2_ssh_key = module.key_pair.key_pair_name + // source_security_group_ids = [aws_security_group.remote_access.id] + // } + + min_size = 1 + max_size = 10 + desired_size = 1 + + instance_types = ["t3.large"] + capacity_type = "SPOT" + + labels = { + Environment = "test" + GithubRepo = "terraform-aws-eks" + GithubOrg = "terraform-aws-modules" + } + + taints = { + dedicated = { + key = "dedicated" + value = "gpuGroup" + effect = "NO_SCHEDULE" + } + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [aws](#requirement\_aws) | >= 5.95 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 5.95 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [user\_data](#module\_user\_data) | ../_user_data | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_autoscaling_schedule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_schedule) | resource | +| [aws_eks_node_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_placement_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/placement_group) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_ec2_instance_type.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | +| [aws_ec2_instance_type_offerings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | +| [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_ssm_parameter.ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | +| [aws_subnets.placement_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [ami\_id](#input\_ami\_id) | The AMI from which to launch the instance. If not supplied, EKS will use its own default image | `string` | `""` | no | +| [ami\_release\_version](#input\_ami\_release\_version) | The AMI version. Defaults to latest AMI release version for the given Kubernetes version and AMI type | `string` | `null` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `null` | no | +| [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI | `any` | `{}` | no | +| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | +| [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | +| [capacity\_type](#input\_capacity\_type) | Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT` | `string` | `"ON_DEMAND"` | no | +| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | +| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | +| [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `null` | no | +| [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | +| [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | +| [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | [Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | +| [cluster\_version](#input\_cluster\_version) | Kubernetes version. Defaults to EKS Cluster Kubernetes version | `string` | `null` | no | +| [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | +| [create](#input\_create) | Determines whether to create EKS managed node group or not | `bool` | `true` | no | +| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_iam\_role\_policy](#input\_create\_iam\_role\_policy) | Determines whether an IAM role policy is created or not | `bool` | `true` | no | +| [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | +| [create\_placement\_group](#input\_create\_placement\_group) | Determines whether a placement group is created & used by the node group | `bool` | `false` | no | +| [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | +| [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | +| [desired\_size](#input\_desired\_size) | Desired number of instances/nodes | `number` | `1` | no | +| [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | +| [disk\_size](#input\_disk\_size) | Disk size in GiB for nodes. Defaults to `20`. Only valid when `use_custom_launch_template` = `false` | `number` | `null` | no | +| [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance(s) will be EBS-optimized | `bool` | `null` | no | +| [efa\_indices](#input\_efa\_indices) | The indices of the network interfaces that should be EFA-enabled. Only valid when `enable_efa_support` = `true` | `list(number)` |
[
0
]
| no | +| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | +| [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | +| [enable\_bootstrap\_user\_data](#input\_enable\_bootstrap\_user\_data) | Determines whether the bootstrap configurations are populated within the user data template. Only valid when using a custom AMI via `ami_id` | `bool` | `false` | no | +| [enable\_efa\_only](#input\_enable\_efa\_only) | Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later | `bool` | `false` | no | +| [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | +| [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | +| [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | +| [force\_update\_version](#input\_force\_update\_version) | Force version update if existing pods are unable to be drained due to a pod disruption budget issue | `bool` | `null` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | +| [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the node group. Required if `create_iam_role` is set to `false` | `string` | `null` | no | +| [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | +| [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `{}` | no | +| [instance\_types](#input\_instance\_types) | Set of instance types associated with the EKS Node Group. Defaults to `["t3.medium"]` | `list(string)` | `null` | no | +| [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | +| [key\_name](#input\_key\_name) | The key name that should be used for the instance(s) | `string` | `null` | no | +| [labels](#input\_labels) | Key-value map of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument. Other Kubernetes labels applied to the EKS Node Group will not be managed | `map(string)` | `null` | no | +| [launch\_template\_default\_version](#input\_launch\_template\_default\_version) | Default version of the launch template | `string` | `null` | no | +| [launch\_template\_description](#input\_launch\_template\_description) | Description of the launch template | `string` | `null` | no | +| [launch\_template\_id](#input\_launch\_template\_id) | The ID of an existing launch template to use. Required when `create_launch_template` = `false` and `use_custom_launch_template` = `true` | `string` | `""` | no | +| [launch\_template\_name](#input\_launch\_template\_name) | Name of launch template to be created | `string` | `null` | no | +| [launch\_template\_tags](#input\_launch\_template\_tags) | A map of additional tags to add to the tag\_specifications of launch template created | `map(string)` | `{}` | no | +| [launch\_template\_use\_name\_prefix](#input\_launch\_template\_use\_name\_prefix) | Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix | `bool` | `true` | no | +| [launch\_template\_version](#input\_launch\_template\_version) | Launch template version number. The default is `$Default` | `string` | `null` | no | +| [license\_specifications](#input\_license\_specifications) | A map of license specifications to associate with | `any` | `{}` | no | +| [maintenance\_options](#input\_maintenance\_options) | The maintenance options for the instance | `any` | `{}` | no | +| [max\_size](#input\_max\_size) | Maximum number of instances/nodes | `number` | `3` | no | +| [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | +| [min\_size](#input\_min\_size) | Minimum number of instances/nodes | `number` | `0` | no | +| [name](#input\_name) | Name of the EKS managed node group | `string` | `""` | no | +| [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | +| [node\_repair\_config](#input\_node\_repair\_config) | The node auto repair configuration for the node group |
object({
enabled = optional(bool, true)
})
| `null` | no | +| [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | +| [placement\_group\_az](#input\_placement\_group\_az) | Availability zone where placement group is created (ex. `eu-west-1c`) | `string` | `null` | no | +| [placement\_group\_strategy](#input\_placement\_group\_strategy) | The placement group strategy | `string` | `"cluster"` | no | +| [platform](#input\_platform) | [DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | `string` | `"linux"` | no | +| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | +| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | +| [private\_dns\_name\_options](#input\_private\_dns\_name\_options) | The options for the instance hostname. The default values are inherited from the subnet | `map(string)` | `{}` | no | +| [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | +| [remote\_access](#input\_remote\_access) | Configuration block with remote access settings. Only valid when `use_custom_launch_template` = `false` | `any` | `{}` | no | +| [schedules](#input\_schedules) | Map of autoscaling group schedule to create | `map(any)` | `{}` | no | +| [subnet\_ids](#input\_subnet\_ids) | Identifiers of EC2 Subnets to associate with the EKS Node Group. These subnets must have the following resource tag: `kubernetes.io/cluster/CLUSTER_NAME` | `list(string)` | `null` | no | +| [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface"
]
| no | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | +| [taints](#input\_taints) | The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group | `any` | `{}` | no | +| [timeouts](#input\_timeouts) | Create, update, and delete timeout configurations for the node group | `map(string)` | `{}` | no | +| [update\_config](#input\_update\_config) | Configuration block of settings for max unavailable resources during node group updates | `map(string)` |
{
"max_unavailable_percentage": 33
}
| no | +| [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update the launch templates default version on each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | +| [use\_custom\_launch\_template](#input\_use\_custom\_launch\_template) | Determines whether to use a custom launch template or not. If set to `false`, EKS will use its own default launch template | `bool` | `true` | no | +| [use\_latest\_ami\_release\_version](#input\_use\_latest\_ami\_release\_version) | Determines whether to use the latest AMI release version for the given `ami_type` (except for `CUSTOM`). Note: `ami_type` and `cluster_version` must be supplied in order to enable this feature | `bool` | `false` | no | +| [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix | `bool` | `true` | no | +| [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | +| [vpc\_security\_group\_ids](#input\_vpc\_security\_group\_ids) | A list of security group IDs to associate | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [autoscaling\_group\_schedule\_arns](#output\_autoscaling\_group\_schedule\_arns) | ARNs of autoscaling group schedules | +| [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | +| [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [launch\_template\_arn](#output\_launch\_template\_arn) | The ARN of the launch template | +| [launch\_template\_id](#output\_launch\_template\_id) | The ID of the launch template | +| [launch\_template\_latest\_version](#output\_launch\_template\_latest\_version) | The latest version of the launch template | +| [launch\_template\_name](#output\_launch\_template\_name) | The name of the launch template | +| [node\_group\_arn](#output\_node\_group\_arn) | Amazon Resource Name (ARN) of the EKS Node Group | +| [node\_group\_autoscaling\_group\_names](#output\_node\_group\_autoscaling\_group\_names) | List of the autoscaling group names | +| [node\_group\_id](#output\_node\_group\_id) | EKS Cluster name and EKS Node Group name separated by a colon (`:`) | +| [node\_group\_labels](#output\_node\_group\_labels) | Map of labels applied to the node group | +| [node\_group\_resources](#output\_node\_group\_resources) | List of objects containing information about underlying resources | +| [node\_group\_status](#output\_node\_group\_status) | Status of the EKS Node Group | +| [node\_group\_taints](#output\_node\_group\_taints) | List of objects containing information about taints applied to the node group | +| [platform](#output\_platform) | [DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | + diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/main.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/main.tf new file mode 100644 index 00000000..7da5fb70 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/main.tf @@ -0,0 +1,695 @@ +data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} + +################################################################################ +# User Data +################################################################################ + +module "user_data" { + source = "../_user_data" + + create = var.create + platform = var.platform + ami_type = var.ami_type + + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + cluster_ip_family = var.cluster_ip_family + cluster_service_cidr = try(coalesce(var.cluster_service_cidr, var.cluster_service_ipv4_cidr), "") + + enable_bootstrap_user_data = var.enable_bootstrap_user_data + pre_bootstrap_user_data = var.pre_bootstrap_user_data + post_bootstrap_user_data = var.post_bootstrap_user_data + bootstrap_extra_args = var.bootstrap_extra_args + user_data_template_path = var.user_data_template_path + + cloudinit_pre_nodeadm = var.cloudinit_pre_nodeadm + cloudinit_post_nodeadm = var.cloudinit_post_nodeadm +} + +################################################################################ +# EFA Support +################################################################################ + +data "aws_ec2_instance_type" "this" { + count = var.create && var.enable_efa_support ? 1 : 0 + + instance_type = local.efa_instance_type +} + +locals { + enable_efa_support = var.create && var.enable_efa_support + + efa_instance_type = try(element(var.instance_types, 0), "") + num_network_cards = try(data.aws_ec2_instance_type.this[0].maximum_network_cards, 0) + + # Primary network interface must be EFA, remaining can be EFA or EFA-only + efa_network_interfaces = [ + for i in range(local.num_network_cards) : { + associate_public_ip_address = false + delete_on_termination = true + device_index = i == 0 ? 0 : 1 + network_card_index = i + interface_type = var.enable_efa_only ? contains(concat([0], var.efa_indices), i) ? "efa" : "efa-only" : "efa" + } + ] + + network_interfaces = local.enable_efa_support ? local.efa_network_interfaces : var.network_interfaces +} + +################################################################################ +# Launch template +################################################################################ + +locals { + launch_template_name = coalesce(var.launch_template_name, "${var.name}-eks-node-group") + security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) + + placement = local.create_placement_group ? { group_name = aws_placement_group.this[0].name } : var.placement +} + +resource "aws_launch_template" "this" { + count = var.create && var.create_launch_template && var.use_custom_launch_template ? 1 : 0 + + dynamic "block_device_mappings" { + for_each = var.block_device_mappings + + content { + device_name = try(block_device_mappings.value.device_name, null) + + dynamic "ebs" { + for_each = try([block_device_mappings.value.ebs], []) + + content { + delete_on_termination = try(ebs.value.delete_on_termination, null) + encrypted = try(ebs.value.encrypted, null) + iops = try(ebs.value.iops, null) + kms_key_id = try(ebs.value.kms_key_id, null) + snapshot_id = try(ebs.value.snapshot_id, null) + throughput = try(ebs.value.throughput, null) + volume_size = try(ebs.value.volume_size, null) + volume_type = try(ebs.value.volume_type, null) + } + } + + no_device = try(block_device_mappings.value.no_device, null) + virtual_name = try(block_device_mappings.value.virtual_name, null) + } + } + + dynamic "capacity_reservation_specification" { + for_each = length(var.capacity_reservation_specification) > 0 ? [var.capacity_reservation_specification] : [] + + content { + capacity_reservation_preference = try(capacity_reservation_specification.value.capacity_reservation_preference, null) + + dynamic "capacity_reservation_target" { + for_each = try([capacity_reservation_specification.value.capacity_reservation_target], []) + + content { + capacity_reservation_id = try(capacity_reservation_target.value.capacity_reservation_id, null) + capacity_reservation_resource_group_arn = try(capacity_reservation_target.value.capacity_reservation_resource_group_arn, null) + } + } + } + } + + dynamic "cpu_options" { + for_each = length(var.cpu_options) > 0 ? [var.cpu_options] : [] + + content { + core_count = try(cpu_options.value.core_count, null) + threads_per_core = try(cpu_options.value.threads_per_core, null) + } + } + + dynamic "credit_specification" { + for_each = length(var.credit_specification) > 0 ? [var.credit_specification] : [] + + content { + cpu_credits = try(credit_specification.value.cpu_credits, null) + } + } + + default_version = var.launch_template_default_version + description = var.launch_template_description + disable_api_termination = var.disable_api_termination + ebs_optimized = var.ebs_optimized + + + dynamic "enclave_options" { + for_each = length(var.enclave_options) > 0 ? [var.enclave_options] : [] + + content { + enabled = enclave_options.value.enabled + } + } + + # Set on EKS managed node group, will fail if set here + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics + # dynamic "hibernation_options" { + # for_each = length(var.hibernation_options) > 0 ? [var.hibernation_options] : [] + + # content { + # configured = hibernation_options.value.configured + # } + # } + + # Set on EKS managed node group, will fail if set here + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics + # dynamic "iam_instance_profile" { + # for_each = [var.iam_instance_profile] + # content { + # name = lookup(var.iam_instance_profile, "name", null) + # arn = lookup(var.iam_instance_profile, "arn", null) + # } + # } + + image_id = var.ami_id + # Set on EKS managed node group, will fail if set here + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics + # instance_initiated_shutdown_behavior = var.instance_initiated_shutdown_behavior + + dynamic "instance_market_options" { + for_each = length(var.instance_market_options) > 0 ? [var.instance_market_options] : [] + + content { + market_type = try(instance_market_options.value.market_type, null) + + dynamic "spot_options" { + for_each = try([instance_market_options.value.spot_options], []) + + content { + block_duration_minutes = try(spot_options.value.block_duration_minutes, null) + instance_interruption_behavior = try(spot_options.value.instance_interruption_behavior, null) + max_price = try(spot_options.value.max_price, null) + spot_instance_type = try(spot_options.value.spot_instance_type, null) + valid_until = try(spot_options.value.valid_until, null) + } + } + } + } + + # Instance type(s) are generally set on the node group, + # except when a ML capacity block reseravtion is used + instance_type = var.capacity_type == "CAPACITY_BLOCK" ? element(var.instance_types, 0) : null + kernel_id = var.kernel_id + key_name = var.key_name + + dynamic "license_specification" { + for_each = length(var.license_specifications) > 0 ? var.license_specifications : {} + + content { + license_configuration_arn = license_specification.value.license_configuration_arn + } + } + + dynamic "maintenance_options" { + for_each = length(var.maintenance_options) > 0 ? [var.maintenance_options] : [] + + content { + auto_recovery = try(maintenance_options.value.auto_recovery, null) + } + } + + dynamic "metadata_options" { + for_each = length(var.metadata_options) > 0 ? [var.metadata_options] : [] + + content { + http_endpoint = try(metadata_options.value.http_endpoint, null) + http_protocol_ipv6 = try(metadata_options.value.http_protocol_ipv6, null) + http_put_response_hop_limit = try(metadata_options.value.http_put_response_hop_limit, null) + http_tokens = try(metadata_options.value.http_tokens, null) + instance_metadata_tags = try(metadata_options.value.instance_metadata_tags, null) + } + } + + dynamic "monitoring" { + for_each = var.enable_monitoring ? [1] : [] + + content { + enabled = var.enable_monitoring + } + } + + name = var.launch_template_use_name_prefix ? null : local.launch_template_name + name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name}-" : null + + dynamic "network_interfaces" { + for_each = local.network_interfaces + + content { + associate_carrier_ip_address = try(network_interfaces.value.associate_carrier_ip_address, null) + associate_public_ip_address = try(network_interfaces.value.associate_public_ip_address, null) + delete_on_termination = try(network_interfaces.value.delete_on_termination, null) + description = try(network_interfaces.value.description, null) + device_index = try(network_interfaces.value.device_index, null) + interface_type = try(network_interfaces.value.interface_type, null) + ipv4_address_count = try(network_interfaces.value.ipv4_address_count, null) + ipv4_addresses = try(network_interfaces.value.ipv4_addresses, []) + ipv4_prefix_count = try(network_interfaces.value.ipv4_prefix_count, null) + ipv4_prefixes = try(network_interfaces.value.ipv4_prefixes, null) + ipv6_address_count = try(network_interfaces.value.ipv6_address_count, null) + ipv6_addresses = try(network_interfaces.value.ipv6_addresses, []) + ipv6_prefix_count = try(network_interfaces.value.ipv6_prefix_count, null) + ipv6_prefixes = try(network_interfaces.value.ipv6_prefixes, []) + network_card_index = try(network_interfaces.value.network_card_index, null) + network_interface_id = try(network_interfaces.value.network_interface_id, null) + primary_ipv6 = try(network_interfaces.value.primary_ipv6, null) + private_ip_address = try(network_interfaces.value.private_ip_address, null) + # Ref: https://github.com/hashicorp/terraform-provider-aws/issues/4570 + security_groups = compact(concat(try(network_interfaces.value.security_groups, []), local.security_group_ids)) + # Set on EKS managed node group, will fail if set here + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-basics + # subnet_id = try(network_interfaces.value.subnet_id, null) + } + } + + dynamic "placement" { + for_each = length(local.placement) > 0 ? [local.placement] : [] + + content { + affinity = try(placement.value.affinity, null) + availability_zone = lookup(placement.value, "availability_zone", null) + group_name = lookup(placement.value, "group_name", null) + host_id = lookup(placement.value, "host_id", null) + host_resource_group_arn = lookup(placement.value, "host_resource_group_arn", null) + partition_number = try(placement.value.partition_number, null) + spread_domain = try(placement.value.spread_domain, null) + tenancy = try(placement.value.tenancy, null) + } + } + + dynamic "private_dns_name_options" { + for_each = length(var.private_dns_name_options) > 0 ? [var.private_dns_name_options] : [] + + content { + enable_resource_name_dns_aaaa_record = try(private_dns_name_options.value.enable_resource_name_dns_aaaa_record, null) + enable_resource_name_dns_a_record = try(private_dns_name_options.value.enable_resource_name_dns_a_record, null) + hostname_type = try(private_dns_name_options.value.hostname_type, null) + } + } + + ram_disk_id = var.ram_disk_id + + dynamic "tag_specifications" { + for_each = toset(var.tag_specifications) + + content { + resource_type = tag_specifications.key + tags = merge(var.tags, { Name = var.name }, var.launch_template_tags) + } + } + + update_default_version = var.update_launch_template_default_version + user_data = module.user_data.user_data + vpc_security_group_ids = length(local.network_interfaces) > 0 ? [] : local.security_group_ids + + tags = merge( + var.tags, + var.launch_template_tags, + ) + + # Prevent premature access of policies by pods that + # require permissions on create/destroy that depend on nodes + depends_on = [ + aws_iam_role_policy_attachment.this, + aws_iam_role_policy_attachment.additional, + ] + + lifecycle { + create_before_destroy = true + } +} + +################################################################################ +# AMI SSM Parameter +################################################################################ + +locals { + # Just to ensure templating doesn't fail when values are not provided + ssm_cluster_version = var.cluster_version != null ? var.cluster_version : "" + ssm_ami_type = var.ami_type != null ? var.ami_type : "" + + # Map the AMI type to the respective SSM param path + ssm_ami_type_to_ssm_param = { + AL2_x86_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2/recommended/release_version" + AL2_x86_64_GPU = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-gpu/recommended/release_version" + AL2_ARM_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-arm64/recommended/release_version" + CUSTOM = "NONE" + BOTTLEROCKET_ARM_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/arm64/latest/image_version" + BOTTLEROCKET_x86_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/x86_64/latest/image_version" + BOTTLEROCKET_ARM_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-fips/arm64/latest/image_version" + BOTTLEROCKET_x86_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-fips/x86_64/latest/image_version" + BOTTLEROCKET_ARM_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/arm64/latest/image_version" + BOTTLEROCKET_x86_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/x86_64/latest/image_version" + WINDOWS_CORE_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-${local.ssm_cluster_version}" + WINDOWS_FULL_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-${local.ssm_cluster_version}" + WINDOWS_CORE_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-EKS_Optimized-${local.ssm_cluster_version}" + WINDOWS_FULL_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-${local.ssm_cluster_version}" + AL2023_x86_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/standard/recommended/release_version" + AL2023_ARM_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/standard/recommended/release_version" + AL2023_x86_64_NEURON = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/neuron/recommended/release_version" + AL2023_x86_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/nvidia/recommended/release_version" + } + + # The Windows SSM params currently do not have a release version, so we have to get the full output JSON blob and parse out the release version + windows_latest_ami_release_version = var.create && var.use_latest_ami_release_version && startswith(local.ssm_ami_type, "WINDOWS") ? nonsensitive(jsondecode(data.aws_ssm_parameter.ami[0].value)["release_version"]) : null + # Based on the steps above, try to get an AMI release version - if not, `null` is returned + latest_ami_release_version = startswith(local.ssm_ami_type, "WINDOWS") ? local.windows_latest_ami_release_version : try(nonsensitive(data.aws_ssm_parameter.ami[0].value), null) +} + +data "aws_ssm_parameter" "ami" { + count = var.create && var.use_latest_ami_release_version ? 1 : 0 + + name = local.ssm_ami_type_to_ssm_param[var.ami_type] +} + +################################################################################ +# Node Group +################################################################################ + +locals { + launch_template_id = var.create && var.create_launch_template ? try(aws_launch_template.this[0].id, null) : var.launch_template_id + # Change order to allow users to set version priority before using defaults + launch_template_version = coalesce(var.launch_template_version, try(aws_launch_template.this[0].default_version, "$Default")) +} + +resource "aws_eks_node_group" "this" { + count = var.create ? 1 : 0 + + # Required + cluster_name = var.cluster_name + node_role_arn = var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn + subnet_ids = local.create_placement_group ? data.aws_subnets.placement_group[0].ids : var.subnet_ids + + scaling_config { + min_size = var.min_size + max_size = var.max_size + desired_size = var.desired_size + } + + # Optional + node_group_name = var.use_name_prefix ? null : var.name + node_group_name_prefix = var.use_name_prefix ? "${var.name}-" : null + + # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami + ami_type = var.ami_id != "" ? null : var.ami_type + release_version = var.ami_id != "" ? null : var.use_latest_ami_release_version ? local.latest_ami_release_version : var.ami_release_version + version = var.ami_id != "" ? null : var.cluster_version + + capacity_type = var.capacity_type + disk_size = var.use_custom_launch_template ? null : var.disk_size # if using a custom LT, set disk size on custom LT or else it will error here + force_update_version = var.force_update_version + # ML capacity block reservation requires instance type to be set on the launch template + instance_types = var.capacity_type == "CAPACITY_BLOCK" ? null : var.instance_types + labels = var.labels + + dynamic "launch_template" { + for_each = var.use_custom_launch_template ? [1] : [] + + content { + id = local.launch_template_id + version = local.launch_template_version + } + } + + dynamic "remote_access" { + for_each = length(var.remote_access) > 0 ? [var.remote_access] : [] + + content { + ec2_ssh_key = try(remote_access.value.ec2_ssh_key, null) + source_security_group_ids = try(remote_access.value.source_security_group_ids, []) + } + } + + dynamic "taint" { + for_each = var.taints + + content { + key = taint.value.key + value = try(taint.value.value, null) + effect = taint.value.effect + } + } + + dynamic "update_config" { + for_each = length(var.update_config) > 0 ? [var.update_config] : [] + + content { + max_unavailable_percentage = try(update_config.value.max_unavailable_percentage, null) + max_unavailable = try(update_config.value.max_unavailable, null) + } + } + + dynamic "node_repair_config" { + for_each = var.node_repair_config != null ? [var.node_repair_config] : [] + + content { + enabled = node_repair_config.value.enabled + } + } + + timeouts { + create = lookup(var.timeouts, "create", null) + update = lookup(var.timeouts, "update", null) + delete = lookup(var.timeouts, "delete", null) + } + + lifecycle { + create_before_destroy = true + ignore_changes = [ + scaling_config[0].desired_size, + ] + } + + tags = merge( + var.tags, + { Name = var.name } + ) +} + +################################################################################ +# IAM Role +################################################################################ + +locals { + create_iam_role = var.create && var.create_iam_role + + iam_role_name = coalesce(var.iam_role_name, "${var.name}-eks-node-group") + iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + + ipv4_cni_policy = { for k, v in { + AmazonEKS_CNI_Policy = "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" } + ipv6_cni_policy = { for k, v in { + AmazonEKS_CNI_IPv6_Policy = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" + } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" } +} + +data "aws_iam_policy_document" "assume_role_policy" { + count = local.create_iam_role ? 1 : 0 + + statement { + sid = "EKSNodeAssumeRole" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "this" { + count = local.create_iam_role ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description + + assume_role_policy = data.aws_iam_policy_document.assume_role_policy[0].json + permissions_boundary = var.iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.iam_role_tags) +} + +# Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group +resource "aws_iam_role_policy_attachment" "this" { + for_each = { for k, v in merge( + { + AmazonEKSWorkerNodePolicy = "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" + AmazonEC2ContainerRegistryReadOnly = "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" + }, + local.ipv4_cni_policy, + local.ipv6_cni_policy + ) : k => v if local.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +resource "aws_iam_role_policy_attachment" "additional" { + for_each = { for k, v in var.iam_role_additional_policies : k => v if local.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +################################################################################ +# IAM Role Policy +################################################################################ + +locals { + create_iam_role_policy = local.create_iam_role && var.create_iam_role_policy && length(var.iam_role_policy_statements) > 0 +} + +data "aws_iam_policy_document" "role" { + count = local.create_iam_role_policy ? 1 : 0 + + dynamic "statement" { + for_each = var.iam_role_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +resource "aws_iam_role_policy" "this" { + count = local.create_iam_role_policy ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + policy = data.aws_iam_policy_document.role[0].json + role = aws_iam_role.this[0].id +} + +################################################################################ +# Placement Group +################################################################################ + +locals { + create_placement_group = var.create && (local.enable_efa_support || var.create_placement_group) +} + +resource "aws_placement_group" "this" { + count = local.create_placement_group ? 1 : 0 + + name = "${var.cluster_name}-${var.name}" + strategy = var.placement_group_strategy + + tags = var.tags +} + +################################################################################ +# Instance AZ Lookup + +# Instances usually used in placement groups w/ EFA are only available in +# select availability zones. These data sources will cross reference the availability +# zones supported by the instance type with the subnets provided to ensure only +# AZs/subnets that are supported are used. +################################################################################ + +# Find the availability zones supported by the instance type +# TODO - remove at next breaking change +# Force users to be explicit about which AZ to use when using placement groups, +# with or without EFA support +data "aws_ec2_instance_type_offerings" "this" { + count = local.enable_efa_support ? 1 : 0 + + filter { + name = "instance-type" + values = [local.efa_instance_type] + } + + location_type = "availability-zone-id" +} + +# Reverse the lookup to find one of the subnets provided based on the availability +# availability zone ID of the queried instance type (supported) +data "aws_subnets" "placement_group" { + count = local.create_placement_group ? 1 : 0 + + filter { + name = "subnet-id" + values = var.subnet_ids + } + + # The data source can lookup the first available AZ or you can specify an AZ (next filter) + dynamic "filter" { + for_each = var.enable_efa_support && var.placement_group_az == null ? [1] : [] + + content { + name = "availability-zone-id" + values = data.aws_ec2_instance_type_offerings.this[0].locations + } + } + + dynamic "filter" { + for_each = var.placement_group_az != null ? [var.placement_group_az] : [] + + content { + name = "availability-zone" + values = [filter.value] + } + } +} + +################################################################################ +# Autoscaling Group Schedule +################################################################################ + +resource "aws_autoscaling_schedule" "this" { + for_each = { for k, v in var.schedules : k => v if var.create && var.create_schedule } + + scheduled_action_name = each.key + autoscaling_group_name = aws_eks_node_group.this[0].resources[0].autoscaling_groups[0].name + + min_size = try(each.value.min_size, -1) + max_size = try(each.value.max_size, -1) + desired_capacity = try(each.value.desired_size, -1) + start_time = try(each.value.start_time, null) + end_time = try(each.value.end_time, null) + time_zone = try(each.value.time_zone, null) + + # [Minute] [Hour] [Day_of_Month] [Month_of_Year] [Day_of_Week] + # Cron examples: https://crontab.guru/examples.html + recurrence = try(each.value.recurrence, null) +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/migrations.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/migrations.tf new file mode 100644 index 00000000..5d51a720 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/migrations.tf @@ -0,0 +1,20 @@ +################################################################################ +# Migrations: v20.7 -> v20.8 +################################################################################ + +# Node IAM role policy attachment +# Commercial partition only - `moved` does now allow multiple moves to same target +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"] + to = aws_iam_role_policy_attachment.this["AmazonEKSWorkerNodePolicy"] +} + +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"] + to = aws_iam_role_policy_attachment.this["AmazonEC2ContainerRegistryReadOnly"] +} + +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"] + to = aws_iam_role_policy_attachment.this["AmazonEKS_CNI_Policy"] +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/outputs.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/outputs.tf new file mode 100644 index 00000000..8cab6e2c --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/outputs.tf @@ -0,0 +1,99 @@ +################################################################################ +# Launch template +################################################################################ + +output "launch_template_id" { + description = "The ID of the launch template" + value = try(aws_launch_template.this[0].id, null) +} + +output "launch_template_arn" { + description = "The ARN of the launch template" + value = try(aws_launch_template.this[0].arn, null) +} + +output "launch_template_latest_version" { + description = "The latest version of the launch template" + value = try(aws_launch_template.this[0].latest_version, null) +} + +output "launch_template_name" { + description = "The name of the launch template" + value = try(aws_launch_template.this[0].name, null) +} + +################################################################################ +# Node Group +################################################################################ + +output "node_group_arn" { + description = "Amazon Resource Name (ARN) of the EKS Node Group" + value = try(aws_eks_node_group.this[0].arn, null) +} + +output "node_group_id" { + description = "EKS Cluster name and EKS Node Group name separated by a colon (`:`)" + value = try(aws_eks_node_group.this[0].id, null) +} + +output "node_group_resources" { + description = "List of objects containing information about underlying resources" + value = try(aws_eks_node_group.this[0].resources, null) +} + +output "node_group_autoscaling_group_names" { + description = "List of the autoscaling group names" + value = try(flatten(aws_eks_node_group.this[0].resources[*].autoscaling_groups[*].name), []) +} + +output "node_group_status" { + description = "Status of the EKS Node Group" + value = try(aws_eks_node_group.this[0].status, null) +} + +output "node_group_labels" { + description = "Map of labels applied to the node group" + value = try(aws_eks_node_group.this[0].labels, {}) +} + +output "node_group_taints" { + description = "List of objects containing information about taints applied to the node group" + value = try(aws_eks_node_group.this[0].taint, []) +} + +################################################################################ +# Autoscaling Group Schedule +################################################################################ + +output "autoscaling_group_schedule_arns" { + description = "ARNs of autoscaling group schedules" + value = { for k, v in aws_autoscaling_schedule.this : k => v.arn } +} + +################################################################################ +# IAM Role +################################################################################ + +output "iam_role_name" { + description = "The name of the IAM role" + value = try(aws_iam_role.this[0].name, null) +} + +output "iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = try(aws_iam_role.this[0].arn, var.iam_role_arn) +} + +output "iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.this[0].unique_id, null) +} + +################################################################################ +# Additional +################################################################################ + +output "platform" { + description = "[DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" + value = module.user_data.platform +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/variables.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/variables.tf new file mode 100644 index 00000000..1137bacf --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/eks-managed-node-group/variables.tf @@ -0,0 +1,575 @@ +variable "create" { + description = "Determines whether to create EKS managed node group or not" + type = bool + default = true +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + +variable "platform" { + description = "[DEPRECATED - use `ami_type` instead. Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" + type = string + default = "linux" +} + +################################################################################ +# User Data +################################################################################ + +variable "enable_bootstrap_user_data" { + description = "Determines whether the bootstrap configurations are populated within the user data template. Only valid when using a custom AMI via `ami_id`" + type = bool + default = false +} + +variable "cluster_name" { + description = "Name of associated EKS cluster" + type = string + default = null +} + +variable "cluster_endpoint" { + description = "Endpoint of associated EKS cluster" + type = string + default = "" +} + +variable "cluster_auth_base64" { + description = "Base64 encoded CA of associated EKS cluster" + type = string + default = "" +} + +variable "cluster_service_cidr" { + description = "The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself" + type = string + default = "" +} + +# TODO - remove at next breaking change +variable "cluster_service_ipv4_cidr" { + description = "[Deprecated] The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" + type = string + default = null +} + +variable "pre_bootstrap_user_data" { + description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" + type = string + default = "" +} + +variable "post_bootstrap_user_data" { + description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" + type = string + default = "" +} + +variable "bootstrap_extra_args" { + description = "Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" + type = string + default = "" +} + +variable "user_data_template_path" { + description = "Path to a local, custom user data template file to use when rendering user data" + type = string + default = "" +} + +variable "cloudinit_pre_nodeadm" { + description = "Array of cloud-init document parts that are created before the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} + +variable "cloudinit_post_nodeadm" { + description = "Array of cloud-init document parts that are created after the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} + +################################################################################ +# Launch template +################################################################################ + +variable "create_launch_template" { + description = "Determines whether to create a launch template or not. If set to `false`, EKS will use its own default launch template" + type = bool + default = true +} + +variable "use_custom_launch_template" { + description = "Determines whether to use a custom launch template or not. If set to `false`, EKS will use its own default launch template" + type = bool + default = true +} + +variable "launch_template_id" { + description = "The ID of an existing launch template to use. Required when `create_launch_template` = `false` and `use_custom_launch_template` = `true`" + type = string + default = "" +} + +variable "launch_template_name" { + description = "Name of launch template to be created" + type = string + default = null +} + +variable "launch_template_use_name_prefix" { + description = "Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix" + type = bool + default = true +} + +variable "launch_template_description" { + description = "Description of the launch template" + type = string + default = null +} + +variable "ebs_optimized" { + description = "If true, the launched EC2 instance(s) will be EBS-optimized" + type = bool + default = null +} + +variable "ami_id" { + description = "The AMI from which to launch the instance. If not supplied, EKS will use its own default image" + type = string + default = "" +} + +variable "key_name" { + description = "The key name that should be used for the instance(s)" + type = string + default = null +} + +variable "vpc_security_group_ids" { + description = "A list of security group IDs to associate" + type = list(string) + default = [] +} + +variable "cluster_primary_security_group_id" { + description = "The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service" + type = string + default = null +} + +variable "launch_template_default_version" { + description = "Default version of the launch template" + type = string + default = null +} + +variable "update_launch_template_default_version" { + description = "Whether to update the launch templates default version on each update. Conflicts with `launch_template_default_version`" + type = bool + default = true +} + +variable "disable_api_termination" { + description = "If true, enables EC2 instance termination protection" + type = bool + default = null +} + +variable "kernel_id" { + description = "The kernel ID" + type = string + default = null +} + +variable "ram_disk_id" { + description = "The ID of the ram disk" + type = string + default = null +} + +variable "block_device_mappings" { + description = "Specify volumes to attach to the instance besides the volumes specified by the AMI" + type = any + default = {} +} + +variable "capacity_reservation_specification" { + description = "Targeting for EC2 capacity reservations" + type = any + default = {} +} + +variable "cpu_options" { + description = "The CPU options for the instance" + type = map(string) + default = {} +} + +variable "credit_specification" { + description = "Customize the credit specification of the instance" + type = map(string) + default = {} +} + + + +variable "enclave_options" { + description = "Enable Nitro Enclaves on launched instances" + type = map(string) + default = {} +} + +variable "instance_market_options" { + description = "The market (purchasing) option for the instance" + type = any + default = {} +} + +variable "maintenance_options" { + description = "The maintenance options for the instance" + type = any + default = {} +} + +variable "license_specifications" { + description = "A map of license specifications to associate with" + type = any + default = {} +} + +variable "metadata_options" { + description = "Customize the metadata options for the instance" + type = map(string) + default = { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 2 + } +} + +# TODO - make this false by default at next breaking change +variable "enable_monitoring" { + description = "Enables/disables detailed monitoring" + type = bool + default = true +} + +variable "enable_efa_support" { + description = "Determines whether to enable Elastic Fabric Adapter (EFA) support" + type = bool + default = false +} + +# TODO - make this true by default at next breaking change (remove variable, only pass indices) +variable "enable_efa_only" { + description = "Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later" + type = bool + default = false +} + +variable "efa_indices" { + description = "The indices of the network interfaces that should be EFA-enabled. Only valid when `enable_efa_support` = `true`" + type = list(number) + default = [0] +} + +variable "network_interfaces" { + description = "Customize network interfaces to be attached at instance boot time" + type = list(any) + default = [] +} + +variable "placement" { + description = "The placement of the instance" + type = map(string) + default = {} +} + +variable "create_placement_group" { + description = "Determines whether a placement group is created & used by the node group" + type = bool + default = false +} + +# TODO - remove at next breaking change +variable "placement_group_strategy" { + description = "The placement group strategy" + type = string + default = "cluster" +} + +variable "private_dns_name_options" { + description = "The options for the instance hostname. The default values are inherited from the subnet" + type = map(string) + default = {} +} + +variable "launch_template_tags" { + description = "A map of additional tags to add to the tag_specifications of launch template created" + type = map(string) + default = {} +} + +variable "tag_specifications" { + description = "The tags to apply to the resources during launch" + type = list(string) + default = ["instance", "volume", "network-interface"] +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +variable "subnet_ids" { + description = "Identifiers of EC2 Subnets to associate with the EKS Node Group. These subnets must have the following resource tag: `kubernetes.io/cluster/CLUSTER_NAME`" + type = list(string) + default = null +} + +variable "placement_group_az" { + description = "Availability zone where placement group is created (ex. `eu-west-1c`)" + type = string + default = null +} + +variable "min_size" { + description = "Minimum number of instances/nodes" + type = number + default = 0 +} + +variable "max_size" { + description = "Maximum number of instances/nodes" + type = number + default = 3 +} + +variable "desired_size" { + description = "Desired number of instances/nodes" + type = number + default = 1 +} + +variable "name" { + description = "Name of the EKS managed node group" + type = string + default = "" +} + +variable "use_name_prefix" { + description = "Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix" + type = bool + default = true +} + +variable "ami_type" { + description = "Type of Amazon Machine Image (AMI) associated with the EKS Node Group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values" + type = string + default = null +} + +variable "ami_release_version" { + description = "The AMI version. Defaults to latest AMI release version for the given Kubernetes version and AMI type" + type = string + default = null +} + +variable "use_latest_ami_release_version" { + description = "Determines whether to use the latest AMI release version for the given `ami_type` (except for `CUSTOM`). Note: `ami_type` and `cluster_version` must be supplied in order to enable this feature" + type = bool + default = false +} + +variable "capacity_type" { + description = "Type of capacity associated with the EKS Node Group. Valid values: `ON_DEMAND`, `SPOT`" + type = string + default = "ON_DEMAND" +} + +variable "disk_size" { + description = "Disk size in GiB for nodes. Defaults to `20`. Only valid when `use_custom_launch_template` = `false`" + type = number + default = null +} + +variable "force_update_version" { + description = "Force version update if existing pods are unable to be drained due to a pod disruption budget issue" + type = bool + default = null +} + +variable "instance_types" { + description = "Set of instance types associated with the EKS Node Group. Defaults to `[\"t3.medium\"]`" + type = list(string) + default = null +} + +variable "labels" { + description = "Key-value map of Kubernetes labels. Only labels that are applied with the EKS API are managed by this argument. Other Kubernetes labels applied to the EKS Node Group will not be managed" + type = map(string) + default = null +} + +variable "cluster_version" { + description = "Kubernetes version. Defaults to EKS Cluster Kubernetes version" + type = string + default = null +} + +variable "launch_template_version" { + description = "Launch template version number. The default is `$Default`" + type = string + default = null +} + +variable "remote_access" { + description = "Configuration block with remote access settings. Only valid when `use_custom_launch_template` = `false`" + type = any + default = {} +} + +variable "taints" { + description = "The Kubernetes taints to be applied to the nodes in the node group. Maximum of 50 taints per node group" + type = any + default = {} +} + +variable "update_config" { + description = "Configuration block of settings for max unavailable resources during node group updates" + type = map(string) + default = { + max_unavailable_percentage = 33 + } +} + +variable "node_repair_config" { + description = "The node auto repair configuration for the node group" + type = object({ + enabled = optional(bool, true) + }) + default = null +} + +variable "timeouts" { + description = "Create, update, and delete timeout configurations for the node group" + type = map(string) + default = {} +} + +################################################################################ +# IAM Role +################################################################################ + +variable "create_iam_role" { + description = "Determines whether an IAM role is created or to use an existing IAM role" + type = bool + default = true +} + +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" + type = string + default = "ipv4" +} + +variable "iam_role_arn" { + description = "Existing IAM role ARN for the node group. Required if `create_iam_role` is set to `false`" + type = string + default = null +} + +variable "iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = null +} + +variable "iam_role_use_name_prefix" { + description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "iam_role_path" { + description = "IAM role path" + type = string + default = null +} + +variable "iam_role_description" { + description = "Description of the role" + type = string + default = null +} + +variable "iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the IAM role" + type = string + default = null +} + +variable "iam_role_attach_cni_policy" { + description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" + type = bool + default = true +} + +variable "iam_role_additional_policies" { + description = "Additional policies to be added to the IAM role" + type = map(string) + default = {} +} + +variable "iam_role_tags" { + description = "A map of additional tags to add to the IAM role created" + type = map(string) + default = {} +} + +################################################################################ +# IAM Role Policy +################################################################################ + +variable "create_iam_role_policy" { + description = "Determines whether an IAM role policy is created or not" + type = bool + default = true +} + +variable "iam_role_policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + +################################################################################ +# Autoscaling Group Schedule +################################################################################ + +variable "create_schedule" { + description = "Determines whether to create autoscaling group schedule or not" + type = bool + default = true +} + +variable "schedules" { + description = "Map of autoscaling group schedule to create" + type = map(any) + default = {} +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/README.md b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/README.md new file mode 100644 index 00000000..d6ab4ed1 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/README.md @@ -0,0 +1,95 @@ +# EKS Fargate Profile Module + +Configuration in this directory creates a Fargate EKS Profile + +## Usage + +```hcl +module "fargate_profile" { + source = "terraform-aws-modules/eks/aws//modules/fargate-profile" + + name = "separate-fargate-profile" + cluster_name = "my-cluster" + + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + selectors = [{ + namespace = "kube-system" + }] + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [aws](#requirement\_aws) | >= 5.95 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 5.95 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_eks_fargate_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_fargate_profile) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | +| [cluster\_name](#input\_cluster\_name) | Name of the EKS cluster | `string` | `null` | no | +| [create](#input\_create) | Determines whether to create Fargate profile or not | `bool` | `true` | no | +| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_iam\_role\_policy](#input\_create\_iam\_role\_policy) | Determines whether an IAM role policy is created or not | `bool` | `true` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | +| [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the Fargate profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | +| [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `""` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | +| [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [name](#input\_name) | Name of the EKS Fargate Profile | `string` | `""` | no | +| [selectors](#input\_selectors) | Configuration block(s) for selecting Kubernetes Pods to execute with this Fargate Profile | `any` | `[]` | no | +| [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs for the EKS Fargate Profile | `list(string)` | `[]` | no | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | +| [timeouts](#input\_timeouts) | Create and delete timeout configurations for the Fargate Profile | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [fargate\_profile\_arn](#output\_fargate\_profile\_arn) | Amazon Resource Name (ARN) of the EKS Fargate Profile | +| [fargate\_profile\_id](#output\_fargate\_profile\_id) | EKS Cluster name and EKS Fargate Profile name separated by a colon (`:`) | +| [fargate\_profile\_pod\_execution\_role\_arn](#output\_fargate\_profile\_pod\_execution\_role\_arn) | Amazon Resource Name (ARN) of the EKS Fargate Profile Pod execution role ARN | +| [fargate\_profile\_status](#output\_fargate\_profile\_status) | Status of the EKS Fargate Profile | +| [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | +| [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | + diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/main.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/main.tf new file mode 100644 index 00000000..ffbbfb82 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/main.tf @@ -0,0 +1,173 @@ +data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + +locals { + create_iam_role = var.create && var.create_iam_role + + iam_role_name = coalesce(var.iam_role_name, var.name, "fargate-profile") + iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + + ipv4_cni_policy = { for k, v in { + AmazonEKS_CNI_Policy = "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" } + ipv6_cni_policy = { for k, v in { + AmazonEKS_CNI_IPv6_Policy = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" + } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" } +} + +################################################################################ +# IAM Role +################################################################################ + +data "aws_iam_policy_document" "assume_role_policy" { + count = local.create_iam_role ? 1 : 0 + + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["eks-fargate-pods.amazonaws.com"] + } + + condition { + test = "ArnLike" + variable = "aws:SourceArn" + + values = [ + "arn:${data.aws_partition.current.partition}:eks:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:fargateprofile/${var.cluster_name}/*", + ] + } + } +} + +resource "aws_iam_role" "this" { + count = local.create_iam_role ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description + + assume_role_policy = data.aws_iam_policy_document.assume_role_policy[0].json + permissions_boundary = var.iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.iam_role_tags) +} + +resource "aws_iam_role_policy_attachment" "this" { + for_each = { for k, v in merge( + { + AmazonEKSFargatePodExecutionRolePolicy = "${local.iam_role_policy_prefix}/AmazonEKSFargatePodExecutionRolePolicy" + }, + local.ipv4_cni_policy, + local.ipv6_cni_policy + ) : k => v if local.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +resource "aws_iam_role_policy_attachment" "additional" { + for_each = { for k, v in var.iam_role_additional_policies : k => v if local.create_iam_role } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +################################################################################ +# IAM Role Policy +################################################################################ + +locals { + create_iam_role_policy = local.create_iam_role && var.create_iam_role_policy && length(var.iam_role_policy_statements) > 0 +} + +data "aws_iam_policy_document" "role" { + count = local.create_iam_role_policy ? 1 : 0 + + dynamic "statement" { + for_each = var.iam_role_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +resource "aws_iam_role_policy" "this" { + count = local.create_iam_role_policy ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + policy = data.aws_iam_policy_document.role[0].json + role = aws_iam_role.this[0].id +} + +################################################################################ +# Fargate Profile +################################################################################ + +resource "aws_eks_fargate_profile" "this" { + count = var.create ? 1 : 0 + + cluster_name = var.cluster_name + fargate_profile_name = var.name + pod_execution_role_arn = var.create_iam_role ? aws_iam_role.this[0].arn : var.iam_role_arn + subnet_ids = var.subnet_ids + + dynamic "selector" { + for_each = var.selectors + + content { + namespace = selector.value.namespace + labels = lookup(selector.value, "labels", {}) + } + } + + dynamic "timeouts" { + for_each = [var.timeouts] + content { + create = lookup(var.timeouts, "create", null) + delete = lookup(var.timeouts, "delete", null) + } + } + + tags = var.tags +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/migrations.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/migrations.tf new file mode 100644 index 00000000..02494f68 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/migrations.tf @@ -0,0 +1,15 @@ +################################################################################ +# Migrations: v20.8 -> v20.9 +################################################################################ + +# Node IAM role policy attachment +# Commercial partition only - `moved` does now allow multiple moves to same target +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy"] + to = aws_iam_role_policy_attachment.this["AmazonEKSFargatePodExecutionRolePolicy"] +} + +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"] + to = aws_iam_role_policy_attachment.this["AmazonEKS_CNI_Policy"] +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/outputs.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/outputs.tf new file mode 100644 index 00000000..96763bfb --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/outputs.tf @@ -0,0 +1,42 @@ +################################################################################ +# IAM Role +################################################################################ + +output "iam_role_name" { + description = "The name of the IAM role" + value = try(aws_iam_role.this[0].name, null) +} + +output "iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = try(aws_iam_role.this[0].arn, var.iam_role_arn) +} + +output "iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.this[0].unique_id, null) +} + +################################################################################ +# Fargate Profile +################################################################################ + +output "fargate_profile_arn" { + description = "Amazon Resource Name (ARN) of the EKS Fargate Profile" + value = try(aws_eks_fargate_profile.this[0].arn, null) +} + +output "fargate_profile_id" { + description = "EKS Cluster name and EKS Fargate Profile name separated by a colon (`:`)" + value = try(aws_eks_fargate_profile.this[0].id, null) +} + +output "fargate_profile_status" { + description = "Status of the EKS Fargate Profile" + value = try(aws_eks_fargate_profile.this[0].status, null) +} + +output "fargate_profile_pod_execution_role_arn" { + description = "Amazon Resource Name (ARN) of the EKS Fargate Profile Pod execution role ARN" + value = try(aws_eks_fargate_profile.this[0].pod_execution_role_arn, null) +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/variables.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/variables.tf new file mode 100644 index 00000000..3e37b8c1 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/fargate-profile/variables.tf @@ -0,0 +1,131 @@ +variable "create" { + description = "Determines whether to create Fargate profile or not" + type = bool + default = true +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + +################################################################################ +# IAM Role +################################################################################ + +variable "create_iam_role" { + description = "Determines whether an IAM role is created or to use an existing IAM role" + type = bool + default = true +} + +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" + type = string + default = "ipv4" +} + +variable "iam_role_arn" { + description = "Existing IAM role ARN for the Fargate profile. Required if `create_iam_role` is set to `false`" + type = string + default = null +} + +variable "iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = "" +} + +variable "iam_role_use_name_prefix" { + description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "iam_role_path" { + description = "IAM role path" + type = string + default = null +} + +variable "iam_role_description" { + description = "Description of the role" + type = string + default = null +} + +variable "iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the IAM role" + type = string + default = null +} + +variable "iam_role_attach_cni_policy" { + description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" + type = bool + default = true +} + +variable "iam_role_additional_policies" { + description = "Additional policies to be added to the IAM role" + type = map(string) + default = {} +} + +variable "iam_role_tags" { + description = "A map of additional tags to add to the IAM role created" + type = map(string) + default = {} +} + +################################################################################ +# IAM Role Policy +################################################################################ + +variable "create_iam_role_policy" { + description = "Determines whether an IAM role policy is created or not" + type = bool + default = true +} + +variable "iam_role_policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + +################################################################################ +# Fargate Profile +################################################################################ + +variable "cluster_name" { + description = "Name of the EKS cluster" + type = string + default = null +} + +variable "name" { + description = "Name of the EKS Fargate Profile" + type = string + default = "" +} + +variable "subnet_ids" { + description = "A list of subnet IDs for the EKS Fargate Profile" + type = list(string) + default = [] +} + +variable "selectors" { + description = "Configuration block(s) for selecting Kubernetes Pods to execute with this Fargate Profile" + type = any + default = [] +} + +variable "timeouts" { + description = "Create and delete timeout configurations for the Fargate Profile" + type = map(string) + default = {} +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/hybrid-node-role/README.md b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/hybrid-node-role/README.md new file mode 100644 index 00000000..50c5a05e --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/hybrid-node-role/README.md @@ -0,0 +1,159 @@ +# EKS Hybrid Node Role Module + +Terraform module which creates IAM role and policy resources for Amazon EKS Hybrid Node(s). + +## Usage + +EKS Hybrid nodes use the AWS IAM Authenticator and temporary IAM credentials provisioned by AWS SSM or AWS IAM Roles Anywhere to authenticate with the EKS cluster. This module supports both SSM and IAM Roles Anywhere based IAM permissions. + +### SSM + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + + ... + access_entries = { + hybrid-node-role = { + principal_arn = module.eks_hybrid_node_role.arn + type = "HYBRID_LINUX" + } + } +} + +module "eks_hybrid_node_role" { + source = "terraform-aws-modules/eks/aws//modules/hybrid-node-role" + + name = "hybrid" + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + +### IAM Roles Anywhere + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + + ... + access_entries = { + hybrid-node-role = { + principal_arn = module.eks_hybrid_node_role.arn + type = "HYBRID_LINUX" + } + } +} + +module "eks_hybrid_node_role" { + source = "terraform-aws-modules/eks/aws//modules/hybrid-node-role" + + name = "hybrid-ira" + + enable_ira = true + + ira_trust_anchor_source_type = "CERTIFICATE_BUNDLE" + ira_trust_anchor_x509_certificate_data = <<-EOT + MIIFMzCCAxugAwIBAgIRAMnVXU7ncv/+Cl16eJbZ9hswDQYJKoZIhvcNAQELBQAw + ... + MGx/BMRkrNUVcg3xA0lhECo/olodCkmZo5/mjybbjFQwJzDSKFoW + EOT + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [aws](#requirement\_aws) | >= 5.95 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 5.95 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_policy.intermediate](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.intermediate](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.intermediate](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_rolesanywhere_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/rolesanywhere_profile) | resource | +| [aws_rolesanywhere_trust_anchor.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/rolesanywhere_trust_anchor) | resource | +| [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.intermediate](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.intermediate_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cluster\_arns](#input\_cluster\_arns) | List of EKS cluster ARNs to allow the node to describe | `list(string)` |
[
"*"
]
| no | +| [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | +| [description](#input\_description) | IAM role description | `string` | `"EKS Hybrid Node IAM role"` | no | +| [enable\_ira](#input\_enable\_ira) | Enables IAM Roles Anywhere based IAM permissions on the node | `bool` | `false` | no | +| [enable\_pod\_identity](#input\_enable\_pod\_identity) | Enables EKS Pod Identity based IAM permissions on the node | `bool` | `true` | no | +| [intermediate\_policy\_name](#input\_intermediate\_policy\_name) | Name of the IAM policy | `string` | `null` | no | +| [intermediate\_policy\_statements](#input\_intermediate\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [intermediate\_policy\_use\_name\_prefix](#input\_intermediate\_policy\_use\_name\_prefix) | Determines whether the name of the IAM policy (`intermediate_policy_name`) is used as a prefix | `bool` | `true` | no | +| [intermediate\_role\_description](#input\_intermediate\_role\_description) | IAM role description | `string` | `"EKS Hybrid Node IAM Roles Anywhere intermediate IAM role"` | no | +| [intermediate\_role\_name](#input\_intermediate\_role\_name) | Name of the IAM role | `string` | `null` | no | +| [intermediate\_role\_path](#input\_intermediate\_role\_path) | Path of the IAM role | `string` | `"/"` | no | +| [intermediate\_role\_policies](#input\_intermediate\_role\_policies) | Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format | `map(string)` | `{}` | no | +| [intermediate\_role\_use\_name\_prefix](#input\_intermediate\_role\_use\_name\_prefix) | Determines whether the name of the IAM role (`intermediate_role_name`) is used as a prefix | `bool` | `true` | no | +| [ira\_profile\_duration\_seconds](#input\_ira\_profile\_duration\_seconds) | The number of seconds the vended session credentials are valid for. Defaults to `3600` | `number` | `null` | no | +| [ira\_profile\_managed\_policy\_arns](#input\_ira\_profile\_managed\_policy\_arns) | A list of managed policy ARNs that apply to the vended session credentials | `list(string)` | `[]` | no | +| [ira\_profile\_name](#input\_ira\_profile\_name) | Name of the Roles Anywhere profile | `string` | `null` | no | +| [ira\_profile\_require\_instance\_properties](#input\_ira\_profile\_require\_instance\_properties) | Specifies whether instance properties are required in [CreateSession](https://docs.aws.amazon.com/rolesanywhere/latest/APIReference/API_CreateSession.html) requests with this profile | `bool` | `null` | no | +| [ira\_profile\_session\_policy](#input\_ira\_profile\_session\_policy) | A session policy that applies to the trust boundary of the vended session credentials | `string` | `null` | no | +| [ira\_trust\_anchor\_acm\_pca\_arn](#input\_ira\_trust\_anchor\_acm\_pca\_arn) | The ARN of the ACM PCA that issued the trust anchor certificate | `string` | `null` | no | +| [ira\_trust\_anchor\_name](#input\_ira\_trust\_anchor\_name) | Name of the Roles Anywhere trust anchor | `string` | `null` | no | +| [ira\_trust\_anchor\_notification\_settings](#input\_ira\_trust\_anchor\_notification\_settings) | Notification settings for the trust anchor | `any` | `[]` | no | +| [ira\_trust\_anchor\_source\_type](#input\_ira\_trust\_anchor\_source\_type) | The source type of the trust anchor | `string` | `null` | no | +| [ira\_trust\_anchor\_x509\_certificate\_data](#input\_ira\_trust\_anchor\_x509\_certificate\_data) | The X.509 certificate data of the trust anchor | `string` | `null` | no | +| [max\_session\_duration](#input\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | +| [name](#input\_name) | Name of the IAM role | `string` | `"EKSHybridNode"` | no | +| [path](#input\_path) | Path of the IAM role | `string` | `"/"` | no | +| [permissions\_boundary\_arn](#input\_permissions\_boundary\_arn) | Permissions boundary ARN to use for the IAM role | `string` | `null` | no | +| [policies](#input\_policies) | Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format | `map(string)` | `{}` | no | +| [policy\_description](#input\_policy\_description) | IAM policy description | `string` | `"EKS Hybrid Node IAM role policy"` | no | +| [policy\_name](#input\_policy\_name) | Name of the IAM policy | `string` | `"EKSHybridNode"` | no | +| [policy\_path](#input\_policy\_path) | Path of the IAM policy | `string` | `"/"` | no | +| [policy\_statements](#input\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [policy\_use\_name\_prefix](#input\_policy\_use\_name\_prefix) | Determines whether the name of the IAM policy (`policy_name`) is used as a prefix | `bool` | `true` | no | +| [tags](#input\_tags) | A map of additional tags to add the the IAM role | `map(any)` | `{}` | no | +| [trust\_anchor\_arns](#input\_trust\_anchor\_arns) | List of IAM Roles Anywhere trust anchor ARNs. Required if `enable_ira` is set to `true` | `list(string)` | `[]` | no | +| [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether the name of the IAM role (`name`) is used as a prefix | `bool` | `true` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [arn](#output\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | +| [intermediate\_role\_arn](#output\_intermediate\_role\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | +| [intermediate\_role\_name](#output\_intermediate\_role\_name) | The name of the node IAM role | +| [intermediate\_role\_unique\_id](#output\_intermediate\_role\_unique\_id) | Stable and unique string identifying the node IAM role | +| [name](#output\_name) | The name of the node IAM role | +| [unique\_id](#output\_unique\_id) | Stable and unique string identifying the node IAM role | + diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/hybrid-node-role/main.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/hybrid-node-role/main.tf new file mode 100644 index 00000000..5a2473f2 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/hybrid-node-role/main.tf @@ -0,0 +1,369 @@ +data "aws_partition" "current" { + count = var.create ? 1 : 0 +} + +locals { + partition = try(data.aws_partition.current[0].partition, "aws") +} + +################################################################################ +# Node IAM Role +################################################################################ + +data "aws_iam_policy_document" "assume_role" { + count = var.create ? 1 : 0 + + # SSM + dynamic "statement" { + for_each = var.enable_ira ? [] : [1] + + content { + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + type = "Service" + identifiers = ["ssm.amazonaws.com"] + } + } + } + + # IAM Roles Anywhere + dynamic "statement" { + for_each = var.enable_ira ? [1] : [] + + content { + actions = [ + "sts:TagSession", + "sts:SetSourceIdentity", + ] + + principals { + type = "AWS" + identifiers = [aws_iam_role.intermediate[0].arn] + } + } + } + + dynamic "statement" { + for_each = var.enable_ira ? [1] : [] + + content { + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + type = "AWS" + identifiers = [aws_iam_role.intermediate[0].arn] + } + + condition { + test = "StringEquals" + variable = "sts:RoleSessionName" + values = ["$${aws:PrincipalTag/x509Subject/CN}"] + } + } + } +} + +resource "aws_iam_role" "this" { + count = var.create ? 1 : 0 + + name = var.use_name_prefix ? null : var.name + name_prefix = var.use_name_prefix ? "${var.name}-" : null + path = var.path + description = var.description + + assume_role_policy = data.aws_iam_policy_document.assume_role[0].json + max_session_duration = var.max_session_duration + permissions_boundary = var.permissions_boundary_arn + force_detach_policies = true + + tags = var.tags +} + +################################################################################ +# Node IAM Role Policy +################################################################################ + +data "aws_iam_policy_document" "this" { + count = var.create ? 1 : 0 + + statement { + actions = [ + "ssm:DeregisterManagedInstance", + "ssm:DescribeInstanceInformation", + ] + + resources = ["*"] + } + + statement { + actions = ["eks:DescribeCluster"] + resources = var.cluster_arns + } + + dynamic "statement" { + for_each = var.enable_pod_identity ? [1] : [] + + content { + actions = ["eks-auth:AssumeRoleForPodIdentity"] + resources = ["*"] + } + } + + dynamic "statement" { + for_each = var.policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +resource "aws_iam_policy" "this" { + count = var.create ? 1 : 0 + + name = var.policy_use_name_prefix ? null : var.policy_name + name_prefix = var.policy_use_name_prefix ? "${var.policy_name}-" : null + path = var.policy_path + description = var.policy_description + policy = data.aws_iam_policy_document.this[0].json + + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "this" { + for_each = { for k, v in merge( + { + node = try(aws_iam_policy.this[0].arn, null) + AmazonSSMManagedInstanceCore = "arn:${local.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" + AmazonEC2ContainerRegistryPullOnly = "arn:${local.partition}:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly" + }, + var.policies + ) : k => v if var.create } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +################################################################################ +# Roles Anywhere Profile +################################################################################ + +locals { + enable_ira = var.create && var.enable_ira +} + +resource "aws_rolesanywhere_profile" "this" { + count = local.enable_ira ? 1 : 0 + + duration_seconds = var.ira_profile_duration_seconds + managed_policy_arns = var.ira_profile_managed_policy_arns + name = try(coalesce(var.ira_profile_name, var.name), null) + require_instance_properties = var.ira_profile_require_instance_properties + role_arns = [aws_iam_role.intermediate[0].arn] + session_policy = var.ira_profile_session_policy + + tags = var.tags +} + +################################################################################ +# Roles Anywhere Trust Anchor +################################################################################ + +resource "aws_rolesanywhere_trust_anchor" "this" { + count = local.enable_ira ? 1 : 0 + + name = try(coalesce(var.ira_trust_anchor_name, var.name), null) + + dynamic "notification_settings" { + for_each = var.ira_trust_anchor_notification_settings + + content { + channel = try(notification_settings.value.channel, null) + enabled = try(notification_settings.value.enabled, null) + event = try(notification_settings.value.event, null) + threshold = try(notification_settings.value.threshold, null) + } + } + + source { + source_data { + acm_pca_arn = var.ira_trust_anchor_acm_pca_arn + x509_certificate_data = var.ira_trust_anchor_x509_certificate_data + } + source_type = var.ira_trust_anchor_source_type + } + + tags = var.tags +} + +################################################################################ +# Intermediate IAM Role +################################################################################ + +data "aws_iam_policy_document" "intermediate_assume_role" { + count = local.enable_ira ? 1 : 0 + + statement { + actions = [ + "sts:AssumeRole", + "sts:TagSession", + "sts:SetSourceIdentity", + ] + + principals { + type = "Service" + identifiers = ["rolesanywhere.amazonaws.com"] + } + + condition { + test = "ArnEquals" + variable = "aws:SourceArn" + values = concat(var.trust_anchor_arns, aws_rolesanywhere_trust_anchor.this[*].arn) + } + } +} + +locals { + intermediate_role_use_name_prefix = coalesce(var.intermediate_role_use_name_prefix, var.use_name_prefix) + intermediate_role_name = coalesce(var.intermediate_role_name, "${var.name}-inter") +} + +resource "aws_iam_role" "intermediate" { + count = local.enable_ira ? 1 : 0 + + name = local.intermediate_role_use_name_prefix ? null : local.intermediate_role_name + name_prefix = local.intermediate_role_use_name_prefix ? "${local.intermediate_role_name}-" : null + path = coalesce(var.intermediate_role_path, var.path) + description = var.intermediate_role_description + + assume_role_policy = data.aws_iam_policy_document.intermediate_assume_role[0].json + max_session_duration = var.max_session_duration + permissions_boundary = var.permissions_boundary_arn + force_detach_policies = true + + tags = var.tags +} + +################################################################################ +# Intermediate IAM Role Policy +################################################################################ + +data "aws_iam_policy_document" "intermediate" { + count = local.enable_ira ? 1 : 0 + + statement { + actions = ["eks:DescribeCluster"] + resources = var.cluster_arns + } + + dynamic "statement" { + for_each = var.intermediate_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +locals { + intermediate_policy_use_name_prefix = coalesce(var.intermediate_policy_use_name_prefix, var.policy_use_name_prefix) + intermediate_policy_name = coalesce(var.intermediate_policy_name, var.policy_name) +} + +resource "aws_iam_policy" "intermediate" { + count = local.enable_ira ? 1 : 0 + + name = local.intermediate_policy_use_name_prefix ? null : local.intermediate_policy_name + name_prefix = local.intermediate_policy_use_name_prefix ? "${local.intermediate_policy_name}-" : null + path = var.policy_path + description = var.policy_description + policy = data.aws_iam_policy_document.intermediate[0].json + + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "intermediate" { + for_each = { for k, v in merge( + { + intermediate = try(aws_iam_policy.intermediate[0].arn, null) + AmazonEC2ContainerRegistryPullOnly = "arn:${local.partition}:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly" + }, + var.intermediate_role_policies + ) : k => v if local.enable_ira } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/hybrid-node-role/outputs.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/hybrid-node-role/outputs.tf new file mode 100644 index 00000000..dc4e26e5 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/hybrid-node-role/outputs.tf @@ -0,0 +1,37 @@ +################################################################################ +# Node IAM Role +################################################################################ + +output "name" { + description = "The name of the node IAM role" + value = try(aws_iam_role.this[0].name, null) +} + +output "arn" { + description = "The Amazon Resource Name (ARN) specifying the node IAM role" + value = try(aws_iam_role.this[0].arn, null) +} + +output "unique_id" { + description = "Stable and unique string identifying the node IAM role" + value = try(aws_iam_role.this[0].unique_id, null) +} + +################################################################################ +# Intermedaite IAM Role +################################################################################ + +output "intermediate_role_name" { + description = "The name of the node IAM role" + value = try(aws_iam_role.intermediate[0].name, null) +} + +output "intermediate_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the node IAM role" + value = try(aws_iam_role.intermediate[0].arn, null) +} + +output "intermediate_role_unique_id" { + description = "Stable and unique string identifying the node IAM role" + value = try(aws_iam_role.intermediate[0].unique_id, null) +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/hybrid-node-role/variables.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/hybrid-node-role/variables.tf new file mode 100644 index 00000000..6e8d8046 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/hybrid-node-role/variables.tf @@ -0,0 +1,239 @@ +variable "create" { + description = "Controls if resources should be created (affects nearly all resources)" + type = bool + default = true +} + +################################################################################ +# Node IAM Role +################################################################################ + +variable "name" { + description = "Name of the IAM role" + type = string + default = "EKSHybridNode" +} + +variable "use_name_prefix" { + description = "Determines whether the name of the IAM role (`name`) is used as a prefix" + type = bool + default = true +} + +variable "path" { + description = "Path of the IAM role" + type = string + default = "/" +} + +variable "description" { + description = "IAM role description" + type = string + default = "EKS Hybrid Node IAM role" +} + +variable "max_session_duration" { + description = "Maximum API session duration in seconds between 3600 and 43200" + type = number + default = null +} + +variable "permissions_boundary_arn" { + description = "Permissions boundary ARN to use for the IAM role" + type = string + default = null +} + +variable "tags" { + description = "A map of additional tags to add the the IAM role" + type = map(any) + default = {} +} + +variable "enable_ira" { + description = "Enables IAM Roles Anywhere based IAM permissions on the node" + type = bool + default = false +} + +variable "trust_anchor_arns" { + description = "List of IAM Roles Anywhere trust anchor ARNs. Required if `enable_ira` is set to `true`" + type = list(string) + default = [] +} + +################################################################################ +# Node IAM Role Policy +################################################################################ + +variable "policy_name" { + description = "Name of the IAM policy" + type = string + default = "EKSHybridNode" +} + +variable "policy_use_name_prefix" { + description = "Determines whether the name of the IAM policy (`policy_name`) is used as a prefix" + type = bool + default = true +} + +variable "policy_path" { + description = "Path of the IAM policy" + type = string + default = "/" +} + +variable "policy_description" { + description = "IAM policy description" + type = string + default = "EKS Hybrid Node IAM role policy" +} + +variable "policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + +variable "policies" { + description = "Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format" + type = map(string) + default = {} +} + +variable "cluster_arns" { + description = "List of EKS cluster ARNs to allow the node to describe" + type = list(string) + default = ["*"] +} + +variable "enable_pod_identity" { + description = "Enables EKS Pod Identity based IAM permissions on the node" + type = bool + default = true +} + +################################################################################ +# IAM Roles Anywhere Profile +################################################################################ + +variable "ira_profile_name" { + description = "Name of the Roles Anywhere profile" + type = string + default = null +} + +variable "ira_profile_duration_seconds" { + description = "The number of seconds the vended session credentials are valid for. Defaults to `3600`" + type = number + default = null +} + +variable "ira_profile_managed_policy_arns" { + description = "A list of managed policy ARNs that apply to the vended session credentials" + type = list(string) + default = [] +} + +variable "ira_profile_require_instance_properties" { + description = "Specifies whether instance properties are required in [CreateSession](https://docs.aws.amazon.com/rolesanywhere/latest/APIReference/API_CreateSession.html) requests with this profile" + type = bool + default = null +} + +variable "ira_profile_session_policy" { + description = "A session policy that applies to the trust boundary of the vended session credentials" + type = string + default = null +} + +################################################################################ +# Roles Anywhere Trust Anchor +################################################################################ + +variable "ira_trust_anchor_name" { + description = "Name of the Roles Anywhere trust anchor" + type = string + default = null +} + +variable "ira_trust_anchor_notification_settings" { + description = "Notification settings for the trust anchor" + type = any + default = [] +} + +variable "ira_trust_anchor_acm_pca_arn" { + description = "The ARN of the ACM PCA that issued the trust anchor certificate" + type = string + default = null +} + +variable "ira_trust_anchor_x509_certificate_data" { + description = "The X.509 certificate data of the trust anchor" + type = string + default = null +} + +variable "ira_trust_anchor_source_type" { + description = "The source type of the trust anchor" + type = string + default = null +} + +################################################################################ +# Intermediate IAM Role +################################################################################ + +variable "intermediate_role_name" { + description = "Name of the IAM role" + type = string + default = null +} + +variable "intermediate_role_use_name_prefix" { + description = "Determines whether the name of the IAM role (`intermediate_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "intermediate_role_path" { + description = "Path of the IAM role" + type = string + default = "/" +} + +variable "intermediate_role_description" { + description = "IAM role description" + type = string + default = "EKS Hybrid Node IAM Roles Anywhere intermediate IAM role" +} + +################################################################################ +# Intermediate IAM Role Policy +################################################################################ + +variable "intermediate_policy_name" { + description = "Name of the IAM policy" + type = string + default = null +} + +variable "intermediate_policy_use_name_prefix" { + description = "Determines whether the name of the IAM policy (`intermediate_policy_name`) is used as a prefix" + type = bool + default = true +} + +variable "intermediate_policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + +variable "intermediate_role_policies" { + description = "Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format" + type = map(string) + default = {} +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/README.md b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/README.md new file mode 100644 index 00000000..4523024f --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/README.md @@ -0,0 +1,203 @@ +# Karpenter Module + +Configuration in this directory creates the AWS resources required by Karpenter + +## Usage + +### All Resources (Default) + +In the following example, the Karpenter module will create: +- An IAM role for use with Pod Identity and a scoped IAM policy for the Karpenter controller +- A Pod Identity association to grant Karpenter controller access provided by the IAM Role +- A Node IAM role that Karpenter will use to create an Instance Profile for the nodes to receive IAM permissions +- An access entry for the Node IAM role to allow nodes to join the cluster +- SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc. + +```hcl +module "eks" { + source = "terraform-aws-modules/eks/aws" + + ... +} + +module "karpenter" { + source = "terraform-aws-modules/eks/aws//modules/karpenter" + + cluster_name = module.eks.cluster_name + + # Attach additional IAM policies to the Karpenter node IAM role + node_iam_role_additional_policies = { + AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" + } + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + +### Re-Use Existing Node IAM Role + +In the following example, the Karpenter module will create: +- An IAM role for use with Pod Identity and a scoped IAM policy for the Karpenter controller +- SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc. + +In this scenario, Karpenter will re-use an existing Node IAM role from the EKS managed node group which already has the necessary access entry permissions: + +```hcl +module "eks" { + source = "terraform-aws-modules/eks" + + # Shown just for connection between cluster and Karpenter sub-module below + eks_managed_node_groups = { + initial = { + instance_types = ["t3.medium"] + + min_size = 1 + max_size = 3 + desired_size = 1 + } + } + ... +} + +module "karpenter" { + source = "terraform-aws-modules/eks/aws//modules/karpenter" + + cluster_name = module.eks.cluster_name + + create_node_iam_role = false + node_iam_role_arn = module.eks.eks_managed_node_groups["initial"].iam_role_arn + + # Since the node group role will already have an access entry + create_access_entry = false + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [aws](#requirement\_aws) | >= 5.95 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 5.95 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_cloudwatch_event_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | +| [aws_cloudwatch_event_target.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_eks_access_entry.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | +| [aws_eks_pod_identity_association.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_pod_identity_association) | resource | +| [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_policy.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.controller_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.node_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_sqs_queue.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | +| [aws_sqs_queue_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.controller_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.node_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.v033](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.v1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [access\_entry\_type](#input\_access\_entry\_type) | Type of the access entry. `EC2_LINUX`, `FARGATE_LINUX`, or `EC2_WINDOWS`; defaults to `EC2_LINUX` | `string` | `"EC2_LINUX"` | no | +| [ami\_id\_ssm\_parameter\_arns](#input\_ami\_id\_ssm\_parameter\_arns) | List of SSM Parameter ARNs that Karpenter controller is allowed read access (for retrieving AMI IDs) | `list(string)` | `[]` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. Note: If `ipv6` is specified, the `AmazonEKS_CNI_IPv6_Policy` must exist in the account. This policy is created by the EKS module with `create_cni_ipv6_iam_policy = true` | `string` | `"ipv4"` | no | +| [cluster\_name](#input\_cluster\_name) | The name of the EKS cluster | `string` | `""` | no | +| [create](#input\_create) | Controls if resources should be created (affects nearly all resources) | `bool` | `true` | no | +| [create\_access\_entry](#input\_create\_access\_entry) | Determines whether an access entry is created for the IAM role used by the node IAM role | `bool` | `true` | no | +| [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created | `bool` | `true` | no | +| [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `false` | no | +| [create\_node\_iam\_role](#input\_create\_node\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_pod\_identity\_association](#input\_create\_pod\_identity\_association) | Determines whether to create pod identity association | `bool` | `false` | no | +| [enable\_irsa](#input\_enable\_irsa) | Determines whether to enable support for IAM role for service accounts | `bool` | `false` | no | +| [enable\_pod\_identity](#input\_enable\_pod\_identity) | Determines whether to enable support for EKS pod identity | `bool` | `true` | no | +| [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | +| [enable\_v1\_permissions](#input\_enable\_v1\_permissions) | Determines whether to enable permissions suitable for v1+ (`true`) or for v0.33.x-v0.37.x (`false`) | `bool` | `false` | no | +| [iam\_policy\_description](#input\_iam\_policy\_description) | IAM policy description | `string` | `"Karpenter controller IAM policy"` | no | +| [iam\_policy\_name](#input\_iam\_policy\_name) | Name of the IAM policy | `string` | `"KarpenterController"` | no | +| [iam\_policy\_path](#input\_iam\_policy\_path) | Path of the IAM policy | `string` | `"/"` | no | +| [iam\_policy\_statements](#input\_iam\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [iam\_policy\_use\_name\_prefix](#input\_iam\_policy\_use\_name\_prefix) | Determines whether the name of the IAM policy (`iam_policy_name`) is used as a prefix | `bool` | `true` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | IAM role description | `string` | `"Karpenter controller IAM role"` | no | +| [iam\_role\_max\_session\_duration](#input\_iam\_role\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name of the IAM role | `string` | `"KarpenterController"` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | Path of the IAM role | `string` | `"/"` | no | +| [iam\_role\_permissions\_boundary\_arn](#input\_iam\_role\_permissions\_boundary\_arn) | Permissions boundary ARN to use for the IAM role | `string` | `null` | no | +| [iam\_role\_policies](#input\_iam\_role\_policies) | Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format | `map(string)` | `{}` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add the the IAM role | `map(any)` | `{}` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether the name of the IAM role (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [irsa\_assume\_role\_condition\_test](#input\_irsa\_assume\_role\_condition\_test) | Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role | `string` | `"StringEquals"` | no | +| [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` |
[
"karpenter:karpenter"
]
| no | +| [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no | +| [namespace](#input\_namespace) | Namespace to associate with the Karpenter Pod Identity | `string` | `"kube-system"` | no | +| [node\_iam\_role\_additional\_policies](#input\_node\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | +| [node\_iam\_role\_arn](#input\_node\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | +| [node\_iam\_role\_attach\_cni\_policy](#input\_node\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | +| [node\_iam\_role\_description](#input\_node\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [node\_iam\_role\_max\_session\_duration](#input\_node\_iam\_role\_max\_session\_duration) | Maximum API session duration in seconds between 3600 and 43200 | `number` | `null` | no | +| [node\_iam\_role\_name](#input\_node\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | +| [node\_iam\_role\_path](#input\_node\_iam\_role\_path) | IAM role path | `string` | `"/"` | no | +| [node\_iam\_role\_permissions\_boundary](#input\_node\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [node\_iam\_role\_tags](#input\_node\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | +| [node\_iam\_role\_use\_name\_prefix](#input\_node\_iam\_role\_use\_name\_prefix) | Determines whether the Node IAM role name (`node_iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [queue\_kms\_data\_key\_reuse\_period\_seconds](#input\_queue\_kms\_data\_key\_reuse\_period\_seconds) | The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again | `number` | `null` | no | +| [queue\_kms\_master\_key\_id](#input\_queue\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no | +| [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | +| [queue\_name](#input\_queue\_name) | Name of the SQS queue | `string` | `null` | no | +| [rule\_name\_prefix](#input\_rule\_name\_prefix) | Prefix used for all event bridge rules | `string` | `"Karpenter"` | no | +| [service\_account](#input\_service\_account) | Service account to associate with the Karpenter Pod Identity | `string` | `"karpenter"` | no | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [event\_rules](#output\_event\_rules) | Map of the event rules created and their attributes | +| [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the controller IAM role | +| [iam\_role\_name](#output\_iam\_role\_name) | The name of the controller IAM role | +| [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the controller IAM role | +| [instance\_profile\_arn](#output\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile | +| [instance\_profile\_id](#output\_instance\_profile\_id) | Instance profile's ID | +| [instance\_profile\_name](#output\_instance\_profile\_name) | Name of the instance profile | +| [instance\_profile\_unique](#output\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | +| [namespace](#output\_namespace) | Namespace associated with the Karpenter Pod Identity | +| [node\_access\_entry\_arn](#output\_node\_access\_entry\_arn) | Amazon Resource Name (ARN) of the node Access Entry | +| [node\_iam\_role\_arn](#output\_node\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | +| [node\_iam\_role\_name](#output\_node\_iam\_role\_name) | The name of the node IAM role | +| [node\_iam\_role\_unique\_id](#output\_node\_iam\_role\_unique\_id) | Stable and unique string identifying the node IAM role | +| [queue\_arn](#output\_queue\_arn) | The ARN of the SQS queue | +| [queue\_name](#output\_queue\_name) | The name of the created Amazon SQS queue | +| [queue\_url](#output\_queue\_url) | The URL for the created Amazon SQS queue | +| [service\_account](#output\_service\_account) | Service Account associated with the Karpenter Pod Identity | + diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/main.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/main.tf new file mode 100644 index 00000000..d03dfa49 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/main.tf @@ -0,0 +1,372 @@ +data "aws_region" "current" {} +data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} + +locals { + account_id = data.aws_caller_identity.current.account_id + dns_suffix = data.aws_partition.current.dns_suffix + partition = data.aws_partition.current.partition + region = data.aws_region.current.name +} + +################################################################################ +# Karpenter controller IAM Role +################################################################################ + +locals { + create_iam_role = var.create && var.create_iam_role + irsa_oidc_provider_url = replace(var.irsa_oidc_provider_arn, "/^(.*provider/)/", "") +} + +data "aws_iam_policy_document" "controller_assume_role" { + count = local.create_iam_role ? 1 : 0 + + # Pod Identity + dynamic "statement" { + for_each = var.enable_pod_identity ? [1] : [] + + content { + actions = [ + "sts:AssumeRole", + "sts:TagSession", + ] + + principals { + type = "Service" + identifiers = ["pods.eks.amazonaws.com"] + } + } + } + + # IAM Roles for Service Accounts (IRSA) + dynamic "statement" { + for_each = var.enable_irsa ? [1] : [] + + content { + actions = ["sts:AssumeRoleWithWebIdentity"] + + principals { + type = "Federated" + identifiers = [var.irsa_oidc_provider_arn] + } + + condition { + test = var.irsa_assume_role_condition_test + variable = "${local.irsa_oidc_provider_url}:sub" + values = [for sa in var.irsa_namespace_service_accounts : "system:serviceaccount:${sa}"] + } + + # https://aws.amazon.com/premiumsupport/knowledge-center/eks-troubleshoot-oidc-and-irsa/?nc1=h_ls + condition { + test = var.irsa_assume_role_condition_test + variable = "${local.irsa_oidc_provider_url}:aud" + values = ["sts.amazonaws.com"] + } + } + } +} + +resource "aws_iam_role" "controller" { + count = local.create_iam_role ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : var.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${var.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description + + assume_role_policy = data.aws_iam_policy_document.controller_assume_role[0].json + max_session_duration = var.iam_role_max_session_duration + permissions_boundary = var.iam_role_permissions_boundary_arn + force_detach_policies = true + + tags = merge(var.tags, var.iam_role_tags) +} + +data "aws_iam_policy_document" "controller" { + count = local.create_iam_role ? 1 : 0 + + source_policy_documents = var.enable_v1_permissions ? [data.aws_iam_policy_document.v1[0].json] : [data.aws_iam_policy_document.v033[0].json] +} + +resource "aws_iam_policy" "controller" { + count = local.create_iam_role ? 1 : 0 + + name = var.iam_policy_use_name_prefix ? null : var.iam_policy_name + name_prefix = var.iam_policy_use_name_prefix ? "${var.iam_policy_name}-" : null + path = var.iam_policy_path + description = var.iam_policy_description + policy = data.aws_iam_policy_document.controller[0].json + + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "controller" { + count = local.create_iam_role ? 1 : 0 + + role = aws_iam_role.controller[0].name + policy_arn = aws_iam_policy.controller[0].arn +} + +resource "aws_iam_role_policy_attachment" "controller_additional" { + for_each = { for k, v in var.iam_role_policies : k => v if local.create_iam_role } + + role = aws_iam_role.controller[0].name + policy_arn = each.value +} + +################################################################################ +# Pod Identity Association +################################################################################ + +resource "aws_eks_pod_identity_association" "karpenter" { + count = local.create_iam_role && var.enable_pod_identity && var.create_pod_identity_association ? 1 : 0 + + cluster_name = var.cluster_name + namespace = var.namespace + service_account = var.service_account + role_arn = aws_iam_role.controller[0].arn + + tags = var.tags +} + +################################################################################ +# Node Termination Queue +################################################################################ + +locals { + enable_spot_termination = var.create && var.enable_spot_termination + + queue_name = coalesce(var.queue_name, "Karpenter-${var.cluster_name}") +} + +resource "aws_sqs_queue" "this" { + count = local.enable_spot_termination ? 1 : 0 + + name = local.queue_name + message_retention_seconds = 300 + sqs_managed_sse_enabled = var.queue_managed_sse_enabled ? var.queue_managed_sse_enabled : null + kms_master_key_id = var.queue_kms_master_key_id + kms_data_key_reuse_period_seconds = var.queue_kms_data_key_reuse_period_seconds + + tags = var.tags +} + +data "aws_iam_policy_document" "queue" { + count = local.enable_spot_termination ? 1 : 0 + + statement { + sid = "SqsWrite" + actions = ["sqs:SendMessage"] + resources = [aws_sqs_queue.this[0].arn] + + principals { + type = "Service" + identifiers = [ + "events.amazonaws.com", + "sqs.amazonaws.com", + ] + } + } + statement { + sid = "DenyHTTP" + effect = "Deny" + actions = [ + "sqs:*" + ] + resources = [aws_sqs_queue.this[0].arn] + condition { + test = "StringEquals" + variable = "aws:SecureTransport" + values = [ + "false" + ] + } + principals { + type = "*" + identifiers = [ + "*" + ] + } + } +} + +resource "aws_sqs_queue_policy" "this" { + count = local.enable_spot_termination ? 1 : 0 + + queue_url = aws_sqs_queue.this[0].url + policy = data.aws_iam_policy_document.queue[0].json +} + +################################################################################ +# Node Termination Event Rules +################################################################################ + +locals { + events = { + health_event = { + name = "HealthEvent" + description = "Karpenter interrupt - AWS health event" + event_pattern = { + source = ["aws.health"] + detail-type = ["AWS Health Event"] + } + } + spot_interrupt = { + name = "SpotInterrupt" + description = "Karpenter interrupt - EC2 spot instance interruption warning" + event_pattern = { + source = ["aws.ec2"] + detail-type = ["EC2 Spot Instance Interruption Warning"] + } + } + instance_rebalance = { + name = "InstanceRebalance" + description = "Karpenter interrupt - EC2 instance rebalance recommendation" + event_pattern = { + source = ["aws.ec2"] + detail-type = ["EC2 Instance Rebalance Recommendation"] + } + } + instance_state_change = { + name = "InstanceStateChange" + description = "Karpenter interrupt - EC2 instance state-change notification" + event_pattern = { + source = ["aws.ec2"] + detail-type = ["EC2 Instance State-change Notification"] + } + } + } +} + +resource "aws_cloudwatch_event_rule" "this" { + for_each = { for k, v in local.events : k => v if local.enable_spot_termination } + + name_prefix = "${var.rule_name_prefix}${each.value.name}-" + description = each.value.description + event_pattern = jsonencode(each.value.event_pattern) + + tags = merge( + { "ClusterName" : var.cluster_name }, + var.tags, + ) +} + +resource "aws_cloudwatch_event_target" "this" { + for_each = { for k, v in local.events : k => v if local.enable_spot_termination } + + rule = aws_cloudwatch_event_rule.this[each.key].name + target_id = "KarpenterInterruptionQueueTarget" + arn = aws_sqs_queue.this[0].arn +} + +################################################################################ +# Node IAM Role +# This is used by the nodes launched by Karpenter +################################################################################ + +locals { + create_node_iam_role = var.create && var.create_node_iam_role + + node_iam_role_name = coalesce(var.node_iam_role_name, "Karpenter-${var.cluster_name}") + node_iam_role_policy_prefix = "arn:${local.partition}:iam::aws:policy" + + ipv4_cni_policy = { for k, v in { + AmazonEKS_CNI_Policy = "${local.node_iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + } : k => v if var.node_iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" } + ipv6_cni_policy = { for k, v in { + AmazonEKS_CNI_IPv6_Policy = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" + } : k => v if var.node_iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" } +} + +data "aws_iam_policy_document" "node_assume_role" { + count = local.create_node_iam_role ? 1 : 0 + + statement { + sid = "EKSNodeAssumeRole" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.${local.dns_suffix}"] + } + } +} + +resource "aws_iam_role" "node" { + count = local.create_node_iam_role ? 1 : 0 + + name = var.node_iam_role_use_name_prefix ? null : local.node_iam_role_name + name_prefix = var.node_iam_role_use_name_prefix ? "${local.node_iam_role_name}-" : null + path = var.node_iam_role_path + description = var.node_iam_role_description + + assume_role_policy = data.aws_iam_policy_document.node_assume_role[0].json + max_session_duration = var.node_iam_role_max_session_duration + permissions_boundary = var.node_iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.node_iam_role_tags) +} + +# Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group +resource "aws_iam_role_policy_attachment" "node" { + for_each = { for k, v in merge( + { + AmazonEKSWorkerNodePolicy = "${local.node_iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" + AmazonEC2ContainerRegistryReadOnly = "${local.node_iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" + }, + local.ipv4_cni_policy, + local.ipv6_cni_policy + ) : k => v if local.create_node_iam_role } + + policy_arn = each.value + role = aws_iam_role.node[0].name +} + +resource "aws_iam_role_policy_attachment" "node_additional" { + for_each = { for k, v in var.node_iam_role_additional_policies : k => v if local.create_node_iam_role } + + policy_arn = each.value + role = aws_iam_role.node[0].name +} + +################################################################################ +# Access Entry +################################################################################ + +resource "aws_eks_access_entry" "node" { + count = var.create && var.create_access_entry ? 1 : 0 + + cluster_name = var.cluster_name + principal_arn = var.create_node_iam_role ? aws_iam_role.node[0].arn : var.node_iam_role_arn + type = var.access_entry_type + + tags = var.tags + + depends_on = [ + # If we try to add this too quickly, it fails. So .... we wait + aws_sqs_queue_policy.this, + ] +} + +################################################################################ +# Node IAM Instance Profile +# This is used by the nodes launched by Karpenter +# Starting with Karpenter 0.32 this is no longer required as Karpenter will +# create the Instance Profile +################################################################################ + +locals { + external_role_name = try(replace(var.node_iam_role_arn, "/^(.*role/)/", ""), null) +} + +resource "aws_iam_instance_profile" "this" { + count = var.create && var.create_instance_profile ? 1 : 0 + + name = var.node_iam_role_use_name_prefix ? null : local.node_iam_role_name + name_prefix = var.node_iam_role_use_name_prefix ? "${local.node_iam_role_name}-" : null + path = var.node_iam_role_path + role = var.create_node_iam_role ? aws_iam_role.node[0].name : local.external_role_name + + tags = merge(var.tags, var.node_iam_role_tags) +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/migrations.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/migrations.tf new file mode 100644 index 00000000..b40040f3 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/migrations.tf @@ -0,0 +1,77 @@ +################################################################################ +# Migrations: v19.21 -> v20.0 +################################################################################ + +# Node IAM role +moved { + from = aws_iam_role.this + to = aws_iam_role.node +} + +moved { + from = aws_iam_policy.this + to = aws_iam_policy.node +} + +moved { + from = aws_iam_role_policy_attachment.this + to = aws_iam_role_policy_attachment.node +} + +moved { + from = aws_iam_role_policy_attachment.additional + to = aws_iam_role_policy_attachment.node_additional +} + +# Controller IAM role +moved { + from = aws_iam_role.irsa + to = aws_iam_role.controller +} + +moved { + from = aws_iam_policy.irsa + to = aws_iam_policy.controller +} + +moved { + from = aws_iam_role_policy_attachment.irsa + to = aws_iam_role_policy_attachment.controller +} + +moved { + from = aws_iam_role_policy_attachment.irsa_additional + to = aws_iam_role_policy_attachment.controller_additional +} + +# Spelling correction +moved { + from = aws_cloudwatch_event_target.this["spot_interupt"] + to = aws_cloudwatch_event_target.this["spot_interrupt"] +} + +moved { + from = aws_cloudwatch_event_rule.this["spot_interupt"] + to = aws_cloudwatch_event_rule.this["spot_interrupt"] +} + +################################################################################ +# Migrations: v20.7 -> v20.8 +################################################################################ + +# Node IAM role policy attachment +# Commercial partition only - `moved` does now allow multiple moves to same target +moved { + from = aws_iam_role_policy_attachment.node["arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"] + to = aws_iam_role_policy_attachment.node["AmazonEKSWorkerNodePolicy"] +} + +moved { + from = aws_iam_role_policy_attachment.node["arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"] + to = aws_iam_role_policy_attachment.node["AmazonEC2ContainerRegistryReadOnly"] +} + +moved { + from = aws_iam_role_policy_attachment.node["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"] + to = aws_iam_role_policy_attachment.node["AmazonEKS_CNI_Policy"] +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/outputs.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/outputs.tf new file mode 100644 index 00000000..a71d4724 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/outputs.tf @@ -0,0 +1,112 @@ +################################################################################ +# Karpenter controller IAM Role +################################################################################ + +output "iam_role_name" { + description = "The name of the controller IAM role" + value = try(aws_iam_role.controller[0].name, null) +} + +output "iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the controller IAM role" + value = try(aws_iam_role.controller[0].arn, null) +} + +output "iam_role_unique_id" { + description = "Stable and unique string identifying the controller IAM role" + value = try(aws_iam_role.controller[0].unique_id, null) +} + +################################################################################ +# Node Termination Queue +################################################################################ + +output "queue_arn" { + description = "The ARN of the SQS queue" + value = try(aws_sqs_queue.this[0].arn, null) +} + +output "queue_name" { + description = "The name of the created Amazon SQS queue" + value = try(aws_sqs_queue.this[0].name, null) +} + +output "queue_url" { + description = "The URL for the created Amazon SQS queue" + value = try(aws_sqs_queue.this[0].url, null) +} + +################################################################################ +# Node Termination Event Rules +################################################################################ + +output "event_rules" { + description = "Map of the event rules created and their attributes" + value = aws_cloudwatch_event_rule.this +} + +################################################################################ +# Node IAM Role +################################################################################ + +output "node_iam_role_name" { + description = "The name of the node IAM role" + value = try(aws_iam_role.node[0].name, null) +} + +output "node_iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the node IAM role" + value = try(aws_iam_role.node[0].arn, var.node_iam_role_arn) +} + +output "node_iam_role_unique_id" { + description = "Stable and unique string identifying the node IAM role" + value = try(aws_iam_role.node[0].unique_id, null) +} + +################################################################################ +# Access Entry +################################################################################ + +output "node_access_entry_arn" { + description = "Amazon Resource Name (ARN) of the node Access Entry" + value = try(aws_eks_access_entry.node[0].access_entry_arn, null) +} + +################################################################################ +# Node IAM Instance Profile +################################################################################ + +output "instance_profile_arn" { + description = "ARN assigned by AWS to the instance profile" + value = try(aws_iam_instance_profile.this[0].arn, null) +} + +output "instance_profile_id" { + description = "Instance profile's ID" + value = try(aws_iam_instance_profile.this[0].id, null) +} + +output "instance_profile_name" { + description = "Name of the instance profile" + value = try(aws_iam_instance_profile.this[0].name, null) +} + +output "instance_profile_unique" { + description = "Stable and unique string identifying the IAM instance profile" + value = try(aws_iam_instance_profile.this[0].unique_id, null) +} + +################################################################################ +# Pod Identity +################################################################################ + +output "namespace" { + description = "Namespace associated with the Karpenter Pod Identity" + value = var.namespace +} + +output "service_account" { + description = "Service Account associated with the Karpenter Pod Identity" + value = var.service_account +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/policy.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/policy.tf new file mode 100644 index 00000000..26c0b235 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/policy.tf @@ -0,0 +1,750 @@ +################################################################################ +# v0.33.x - v0.37.x Controller IAM Policy +################################################################################ + +data "aws_iam_policy_document" "v033" { + count = local.create_iam_role ? 1 : 0 + + statement { + sid = "AllowScopedEC2InstanceActions" + resources = [ + "arn:${local.partition}:ec2:*::image/*", + "arn:${local.partition}:ec2:*::snapshot/*", + "arn:${local.partition}:ec2:*:*:spot-instances-request/*", + "arn:${local.partition}:ec2:*:*:security-group/*", + "arn:${local.partition}:ec2:*:*:subnet/*", + "arn:${local.partition}:ec2:*:*:launch-template/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + } + + statement { + sid = "AllowScopedEC2InstanceActionsWithTags" + resources = [ + "arn:${local.partition}:ec2:*:*:fleet/*", + "arn:${local.partition}:ec2:*:*:instance/*", + "arn:${local.partition}:ec2:*:*:volume/*", + "arn:${local.partition}:ec2:*:*:network-interface/*", + "arn:${local.partition}:ec2:*:*:launch-template/*", + "arn:${local.partition}:ec2:*:*:spot-instances-request/*", + ] + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate" + ] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceCreationTagging" + resources = [ + "arn:${local.partition}:ec2:*:*:fleet/*", + "arn:${local.partition}:ec2:*:*:instance/*", + "arn:${local.partition}:ec2:*:*:volume/*", + "arn:${local.partition}:ec2:*:*:network-interface/*", + "arn:${local.partition}:ec2:*:*:launch-template/*", + "arn:${local.partition}:ec2:*:*:spot-instances-request/*", + ] + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "ec2:CreateAction" + values = [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate", + ] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceTagging" + resources = ["arn:${local.partition}:ec2:*:*:instance/*"] + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + + condition { + test = "ForAllValues:StringEquals" + variable = "aws:TagKeys" + values = [ + "karpenter.sh/nodeclaim", + "Name", + ] + } + } + + statement { + sid = "AllowScopedDeletion" + resources = [ + "arn:${local.partition}:ec2:*:*:instance/*", + "arn:${local.partition}:ec2:*:*:launch-template/*" + ] + + actions = [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate" + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowRegionalReadActions" + resources = ["*"] + actions = [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" + ] + + condition { + test = "StringEquals" + variable = "aws:RequestedRegion" + values = [local.region] + } + } + + statement { + sid = "AllowSSMReadActions" + resources = coalescelist(var.ami_id_ssm_parameter_arns, ["arn:${local.partition}:ssm:${local.region}::parameter/aws/service/*"]) + actions = ["ssm:GetParameter"] + } + + statement { + sid = "AllowPricingReadActions" + resources = ["*"] + actions = ["pricing:GetProducts"] + } + + dynamic "statement" { + for_each = local.enable_spot_termination ? [1] : [] + + content { + sid = "AllowInterruptionQueueActions" + resources = [try(aws_sqs_queue.this[0].arn, null)] + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] + } + } + + statement { + sid = "AllowPassingInstanceRole" + resources = var.create_node_iam_role ? [aws_iam_role.node[0].arn] : [var.node_iam_role_arn] + actions = ["iam:PassRole"] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["ec2.${local.dns_suffix}"] + } + } + + statement { + sid = "AllowScopedInstanceProfileCreationActions" + resources = ["*"] + actions = ["iam:CreateInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileTagActions" + resources = ["*"] + actions = ["iam:TagInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileActions" + resources = ["*"] + actions = [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowInstanceProfileReadActions" + resources = ["*"] + actions = ["iam:GetInstanceProfile"] + } + + statement { + sid = "AllowAPIServerEndpointDiscovery" + resources = ["arn:${local.partition}:eks:${local.region}:${local.account_id}:cluster/${var.cluster_name}"] + actions = ["eks:DescribeCluster"] + } + + dynamic "statement" { + for_each = var.iam_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +################################################################################ +# v1.0.x Controller IAM Policy +################################################################################ + +data "aws_iam_policy_document" "v1" { + count = local.create_iam_role ? 1 : 0 + + statement { + sid = "AllowScopedEC2InstanceAccessActions" + resources = [ + "arn:${local.partition}:ec2:${local.region}::image/*", + "arn:${local.partition}:ec2:${local.region}::snapshot/*", + "arn:${local.partition}:ec2:${local.region}:*:security-group/*", + "arn:${local.partition}:ec2:${local.region}:*:subnet/*", + "arn:${local.partition}:ec2:${local.region}:*:capacity-reservation/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + } + + statement { + sid = "AllowScopedEC2LaunchTemplateAccessActions" + resources = [ + "arn:${local.partition}:ec2:${local.region}:*:launch-template/*" + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedEC2InstanceActionsWithTags" + resources = [ + "arn:${local.partition}:ec2:${local.region}:*:fleet/*", + "arn:${local.partition}:ec2:${local.region}:*:instance/*", + "arn:${local.partition}:ec2:${local.region}:*:volume/*", + "arn:${local.partition}:ec2:${local.region}:*:network-interface/*", + "arn:${local.partition}:ec2:${local.region}:*:launch-template/*", + "arn:${local.partition}:ec2:${local.region}:*:spot-instances-request/*", + ] + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate" + ] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_name] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceCreationTagging" + resources = [ + "arn:${local.partition}:ec2:${local.region}:*:fleet/*", + "arn:${local.partition}:ec2:${local.region}:*:instance/*", + "arn:${local.partition}:ec2:${local.region}:*:volume/*", + "arn:${local.partition}:ec2:${local.region}:*:network-interface/*", + "arn:${local.partition}:ec2:${local.region}:*:launch-template/*", + "arn:${local.partition}:ec2:${local.region}:*:spot-instances-request/*", + ] + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_name] + } + + condition { + test = "StringEquals" + variable = "ec2:CreateAction" + values = [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate", + ] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceTagging" + resources = ["arn:${local.partition}:ec2:${local.region}:*:instance/*"] + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + + condition { + test = "StringEqualsIfExists" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_name] + } + + condition { + test = "ForAllValues:StringEquals" + variable = "aws:TagKeys" + values = [ + "eks:eks-cluster-name", + "karpenter.sh/nodeclaim", + "Name", + ] + } + } + + statement { + sid = "AllowScopedDeletion" + resources = [ + "arn:${local.partition}:ec2:${local.region}:*:instance/*", + "arn:${local.partition}:ec2:${local.region}:*:launch-template/*" + ] + + actions = [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate" + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowRegionalReadActions" + resources = ["*"] + actions = [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" + ] + + condition { + test = "StringEquals" + variable = "aws:RequestedRegion" + values = [local.region] + } + } + + statement { + sid = "AllowSSMReadActions" + resources = coalescelist(var.ami_id_ssm_parameter_arns, ["arn:${local.partition}:ssm:${local.region}::parameter/aws/service/*"]) + actions = ["ssm:GetParameter"] + } + + statement { + sid = "AllowPricingReadActions" + resources = ["*"] + actions = ["pricing:GetProducts"] + } + + dynamic "statement" { + for_each = local.enable_spot_termination ? [1] : [] + + content { + sid = "AllowInterruptionQueueActions" + resources = [try(aws_sqs_queue.this[0].arn, null)] + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] + } + } + + statement { + sid = "AllowPassingInstanceRole" + resources = var.create_node_iam_role ? [aws_iam_role.node[0].arn] : [var.node_iam_role_arn] + actions = ["iam:PassRole"] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["ec2.${local.dns_suffix}"] + } + } + + statement { + sid = "AllowScopedInstanceProfileCreationActions" + resources = ["arn:${local.partition}:iam::${local.account_id}:instance-profile/*"] + actions = ["iam:CreateInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_name] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileTagActions" + resources = ["arn:${local.partition}:iam::${local.account_id}:instance-profile/*"] + actions = ["iam:TagInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_name] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileActions" + resources = ["arn:${local.partition}:iam::${local.account_id}:instance-profile/*"] + actions = [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [local.region] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowInstanceProfileReadActions" + resources = ["arn:${local.partition}:iam::${local.account_id}:instance-profile/*"] + actions = ["iam:GetInstanceProfile"] + } + + statement { + sid = "AllowAPIServerEndpointDiscovery" + resources = ["arn:${local.partition}:eks:${local.region}:${local.account_id}:cluster/${var.cluster_name}"] + actions = ["eks:DescribeCluster"] + } + + dynamic "statement" { + for_each = var.iam_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/variables.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/variables.tf new file mode 100644 index 00000000..71b2cbdf --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/karpenter/variables.tf @@ -0,0 +1,320 @@ +variable "create" { + description = "Controls if resources should be created (affects nearly all resources)" + type = bool + default = true +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + +variable "cluster_name" { + description = "The name of the EKS cluster" + type = string + default = "" +} + +################################################################################ +# Karpenter controller IAM Role +################################################################################ + +variable "create_iam_role" { + description = "Determines whether an IAM role is created" + type = bool + default = true +} + +variable "iam_role_name" { + description = "Name of the IAM role" + type = string + default = "KarpenterController" +} + +variable "iam_role_use_name_prefix" { + description = "Determines whether the name of the IAM role (`iam_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "iam_role_path" { + description = "Path of the IAM role" + type = string + default = "/" +} + +variable "iam_role_description" { + description = "IAM role description" + type = string + default = "Karpenter controller IAM role" +} + +variable "iam_role_max_session_duration" { + description = "Maximum API session duration in seconds between 3600 and 43200" + type = number + default = null +} + +variable "iam_role_permissions_boundary_arn" { + description = "Permissions boundary ARN to use for the IAM role" + type = string + default = null +} + +variable "iam_role_tags" { + description = "A map of additional tags to add the the IAM role" + type = map(any) + default = {} +} + +variable "iam_policy_name" { + description = "Name of the IAM policy" + type = string + default = "KarpenterController" +} + +variable "iam_policy_use_name_prefix" { + description = "Determines whether the name of the IAM policy (`iam_policy_name`) is used as a prefix" + type = bool + default = true +} + +variable "iam_policy_path" { + description = "Path of the IAM policy" + type = string + default = "/" +} + +variable "iam_policy_description" { + description = "IAM policy description" + type = string + default = "Karpenter controller IAM policy" +} + +variable "iam_policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + +variable "iam_role_policies" { + description = "Policies to attach to the IAM role in `{'static_name' = 'policy_arn'}` format" + type = map(string) + default = {} +} + +variable "ami_id_ssm_parameter_arns" { + description = "List of SSM Parameter ARNs that Karpenter controller is allowed read access (for retrieving AMI IDs)" + type = list(string) + default = [] +} + +variable "enable_pod_identity" { + description = "Determines whether to enable support for EKS pod identity" + type = bool + default = true +} + +# TODO - make v1 permssions the default policy at next breaking change +variable "enable_v1_permissions" { + description = "Determines whether to enable permissions suitable for v1+ (`true`) or for v0.33.x-v0.37.x (`false`)" + type = bool + default = false +} + +################################################################################ +# IAM Role for Service Account (IRSA) +################################################################################ + +variable "enable_irsa" { + description = "Determines whether to enable support for IAM role for service accounts" + type = bool + default = false +} + +variable "irsa_oidc_provider_arn" { + description = "OIDC provider arn used in trust policy for IAM role for service accounts" + type = string + default = "" +} + +variable "irsa_namespace_service_accounts" { + description = "List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts" + type = list(string) + default = ["karpenter:karpenter"] +} + +variable "irsa_assume_role_condition_test" { + description = "Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role" + type = string + default = "StringEquals" +} + +################################################################################ +# Pod Identity Association +################################################################################ +# TODO - Change default to `true` at next breaking change +variable "create_pod_identity_association" { + description = "Determines whether to create pod identity association" + type = bool + default = false +} + +variable "namespace" { + description = "Namespace to associate with the Karpenter Pod Identity" + type = string + default = "kube-system" +} + +variable "service_account" { + description = "Service account to associate with the Karpenter Pod Identity" + type = string + default = "karpenter" +} + +################################################################################ +# Node Termination Queue +################################################################################ + +variable "enable_spot_termination" { + description = "Determines whether to enable native spot termination handling" + type = bool + default = true +} + +variable "queue_name" { + description = "Name of the SQS queue" + type = string + default = null +} + +variable "queue_managed_sse_enabled" { + description = "Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys" + type = bool + default = true +} + +variable "queue_kms_master_key_id" { + description = "The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK" + type = string + default = null +} + +variable "queue_kms_data_key_reuse_period_seconds" { + description = "The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again" + type = number + default = null +} + +################################################################################ +# Node IAM Role +################################################################################ + +variable "create_node_iam_role" { + description = "Determines whether an IAM role is created or to use an existing IAM role" + type = bool + default = true +} + +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. Note: If `ipv6` is specified, the `AmazonEKS_CNI_IPv6_Policy` must exist in the account. This policy is created by the EKS module with `create_cni_ipv6_iam_policy = true`" + type = string + default = "ipv4" +} + +variable "node_iam_role_arn" { + description = "Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false`" + type = string + default = null +} + +variable "node_iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = null +} + +variable "node_iam_role_use_name_prefix" { + description = "Determines whether the Node IAM role name (`node_iam_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "node_iam_role_path" { + description = "IAM role path" + type = string + default = "/" +} + +variable "node_iam_role_description" { + description = "Description of the role" + type = string + default = null +} + +variable "node_iam_role_max_session_duration" { + description = "Maximum API session duration in seconds between 3600 and 43200" + type = number + default = null +} + +variable "node_iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the IAM role" + type = string + default = null +} + +variable "node_iam_role_attach_cni_policy" { + description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" + type = bool + default = true +} + +variable "node_iam_role_additional_policies" { + description = "Additional policies to be added to the IAM role" + type = map(string) + default = {} +} + +variable "node_iam_role_tags" { + description = "A map of additional tags to add to the IAM role created" + type = map(string) + default = {} +} + +################################################################################ +# Access Entry +################################################################################ + +variable "create_access_entry" { + description = "Determines whether an access entry is created for the IAM role used by the node IAM role" + type = bool + default = true +} + +variable "access_entry_type" { + description = "Type of the access entry. `EC2_LINUX`, `FARGATE_LINUX`, or `EC2_WINDOWS`; defaults to `EC2_LINUX`" + type = string + default = "EC2_LINUX" +} + +################################################################################ +# Node IAM Instance Profile +################################################################################ + +variable "create_instance_profile" { + description = "Whether to create an IAM instance profile" + type = bool + default = false +} + +################################################################################ +# Event Bridge Rules +################################################################################ + +variable "rule_name_prefix" { + description = "Prefix used for all event bridge rules" + type = string + default = "Karpenter" +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/kms/main.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/kms/main.tf new file mode 100644 index 00000000..5f3d7bb1 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/kms/main.tf @@ -0,0 +1,484 @@ +data "aws_partition" "current" { + count = var.create ? 1 : 0 +} +data "aws_caller_identity" "current" { + count = var.create ? 1 : 0 +} + +locals { + account_id = try(data.aws_caller_identity.current[0].account_id, "") + partition = try(data.aws_partition.current[0].partition, "") + dns_suffix = try(data.aws_partition.current[0].dns_suffix, "") +} + +################################################################################ +# Key +################################################################################ + +resource "aws_kms_key" "this" { + count = var.create && !var.create_external && !var.create_replica && !var.create_replica_external ? 1 : 0 + + bypass_policy_lockout_safety_check = var.bypass_policy_lockout_safety_check + customer_master_key_spec = var.customer_master_key_spec + custom_key_store_id = var.custom_key_store_id + deletion_window_in_days = var.deletion_window_in_days + description = var.description + enable_key_rotation = var.enable_key_rotation + is_enabled = var.is_enabled + key_usage = var.key_usage + multi_region = var.multi_region + policy = coalesce(var.policy, data.aws_iam_policy_document.this[0].json) + + tags = var.tags +} + +################################################################################ +# External Key +################################################################################ + +resource "aws_kms_external_key" "this" { + count = var.create && var.create_external && !var.create_replica && !var.create_replica_external ? 1 : 0 + + bypass_policy_lockout_safety_check = var.bypass_policy_lockout_safety_check + deletion_window_in_days = var.deletion_window_in_days + description = var.description + enabled = var.is_enabled + key_material_base64 = var.key_material_base64 + multi_region = var.multi_region + policy = coalesce(var.policy, data.aws_iam_policy_document.this[0].json) + valid_to = var.valid_to + + tags = var.tags +} + +################################################################################ +# Replica Key +################################################################################ + +resource "aws_kms_replica_key" "this" { + count = var.create && var.create_replica && !var.create_external && !var.create_replica_external ? 1 : 0 + + bypass_policy_lockout_safety_check = var.bypass_policy_lockout_safety_check + deletion_window_in_days = var.deletion_window_in_days + description = var.description + primary_key_arn = var.primary_key_arn + enabled = var.is_enabled + policy = coalesce(var.policy, data.aws_iam_policy_document.this[0].json) + + tags = var.tags +} + +################################################################################ +# Replica External Key +################################################################################ + +resource "aws_kms_replica_external_key" "this" { + count = var.create && !var.create_replica && !var.create_external && var.create_replica_external ? 1 : 0 + + bypass_policy_lockout_safety_check = var.bypass_policy_lockout_safety_check + deletion_window_in_days = var.deletion_window_in_days + description = var.description + enabled = var.is_enabled + key_material_base64 = var.key_material_base64 + policy = coalesce(var.policy, data.aws_iam_policy_document.this[0].json) + primary_key_arn = var.primary_external_key_arn + valid_to = var.valid_to + + tags = var.tags +} + +################################################################################ +# Policy +################################################################################ + +data "aws_iam_policy_document" "this" { + count = var.create ? 1 : 0 + + source_policy_documents = var.source_policy_documents + override_policy_documents = var.override_policy_documents + + # Default policy - account wide access to all key operations + dynamic "statement" { + for_each = var.enable_default_policy ? [1] : [] + + content { + sid = "Default" + actions = ["kms:*"] + resources = ["*"] + + principals { + type = "AWS" + identifiers = ["arn:${local.partition}:iam::${local.account_id}:root"] + } + } + } + + # Key owner - all key operations + dynamic "statement" { + for_each = length(var.key_owners) > 0 ? [1] : [] + + content { + sid = "KeyOwner" + actions = ["kms:*"] + resources = ["*"] + + principals { + type = "AWS" + identifiers = var.key_owners + } + } + } + + # Key administrators - https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-administrators + dynamic "statement" { + for_each = length(var.key_administrators) > 0 ? [1] : [] + + content { + sid = "KeyAdministration" + actions = [ + "kms:Create*", + "kms:Describe*", + "kms:Enable*", + "kms:List*", + "kms:Put*", + "kms:Update*", + "kms:Revoke*", + "kms:Disable*", + "kms:Get*", + "kms:Delete*", + "kms:TagResource", + "kms:UntagResource", + "kms:ScheduleKeyDeletion", + "kms:CancelKeyDeletion", + "kms:ReplicateKey", + "kms:ImportKeyMaterial" + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = var.key_administrators + } + } + } + + # Key users - https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-users + dynamic "statement" { + for_each = length(var.key_users) > 0 ? [1] : [] + + content { + sid = "KeyUsage" + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey", + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = var.key_users + } + } + } + + # Key service users - https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-service-integration + dynamic "statement" { + for_each = length(var.key_service_users) > 0 ? [1] : [] + + content { + sid = "KeyServiceUsage" + actions = [ + "kms:CreateGrant", + "kms:ListGrants", + "kms:RevokeGrant", + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = var.key_service_users + } + + condition { + test = "Bool" + variable = "kms:GrantIsForAWSResource" + values = [true] + } + } + } + + # Key service roles for autoscaling - https://docs.aws.amazon.com/autoscaling/ec2/userguide/key-policy-requirements-EBS-encryption.html#policy-example-cmk-access + dynamic "statement" { + for_each = length(var.key_service_roles_for_autoscaling) > 0 ? [1] : [] + + content { + sid = "KeyServiceRolesASG" + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey", + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = var.key_service_roles_for_autoscaling + } + } + } + + dynamic "statement" { + for_each = length(var.key_service_roles_for_autoscaling) > 0 ? [1] : [] + + content { + sid = "KeyServiceRolesASGPersistentVol" + actions = [ + "kms:CreateGrant" + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = var.key_service_roles_for_autoscaling + } + + condition { + test = "Bool" + variable = "kms:GrantIsForAWSResource" + values = [true] + } + } + } + + # Key cryptographic operations - https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-users-crypto + dynamic "statement" { + for_each = length(var.key_symmetric_encryption_users) > 0 ? [1] : [] + + content { + sid = "KeySymmetricEncryption" + actions = [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:Encrypt", + "kms:GenerateDataKey*", + "kms:ReEncrypt*", + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = var.key_symmetric_encryption_users + } + } + } + + dynamic "statement" { + for_each = length(var.key_hmac_users) > 0 ? [1] : [] + + content { + sid = "KeyHMAC" + actions = [ + "kms:DescribeKey", + "kms:GenerateMac", + "kms:VerifyMac", + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = var.key_hmac_users + } + } + } + + dynamic "statement" { + for_each = length(var.key_asymmetric_public_encryption_users) > 0 ? [1] : [] + + content { + sid = "KeyAsymmetricPublicEncryption" + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:DescribeKey", + "kms:GetPublicKey", + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = var.key_asymmetric_public_encryption_users + } + } + } + + dynamic "statement" { + for_each = length(var.key_asymmetric_sign_verify_users) > 0 ? [1] : [] + + content { + sid = "KeyAsymmetricSignVerify" + actions = [ + "kms:DescribeKey", + "kms:GetPublicKey", + "kms:Sign", + "kms:Verify", + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = var.key_asymmetric_sign_verify_users + } + } + } + + # https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/access-control-managing-permissions.html#KMS-key-policy-for-DNSSEC + dynamic "statement" { + for_each = var.enable_route53_dnssec ? [1] : [] + + content { + sid = "Route53DnssecService" + actions = [ + "kms:DescribeKey", + "kms:GetPublicKey", + "kms:Sign", + ] + resources = ["*"] + + principals { + type = "Service" + identifiers = ["dnssec-route53.${local.dns_suffix}"] + } + } + } + + # https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/access-control-managing-permissions.html#KMS-key-policy-for-DNSSEC + dynamic "statement" { + for_each = var.enable_route53_dnssec ? [1] : [] + + content { + sid = "Route53DnssecGrant" + actions = ["kms:CreateGrant"] + resources = ["*"] + + principals { + type = "Service" + identifiers = ["dnssec-route53.${local.dns_suffix}"] + } + + condition { + test = "Bool" + variable = "kms:GrantIsForAWSResource" + values = ["true"] + } + + dynamic "condition" { + for_each = var.route53_dnssec_sources + + content { + test = "StringEquals" + variable = "aws:SourceAccount" + values = try(condition.value.account_ids, [local.account_id]) + } + } + + dynamic "condition" { + for_each = var.route53_dnssec_sources + + content { + test = "ArnLike" + variable = "aws:SourceArn" + values = [try(condition.value.hosted_zone_arn, "arn:${local.partition}:route53:::hostedzone/*")] + } + } + } + } + + dynamic "statement" { + for_each = var.key_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +################################################################################ +# Alias +################################################################################ + +locals { + aliases = { for k, v in toset(var.aliases) : k => { name = v } } +} + +resource "aws_kms_alias" "this" { + for_each = { for k, v in merge(local.aliases, var.computed_aliases) : k => v if var.create } + + name = var.aliases_use_name_prefix ? null : "alias/${each.value.name}" + name_prefix = var.aliases_use_name_prefix ? "alias/${each.value.name}-" : null + target_key_id = try(aws_kms_key.this[0].key_id, aws_kms_external_key.this[0].id, aws_kms_replica_key.this[0].key_id, aws_kms_replica_external_key.this[0].key_id) +} + +################################################################################ +# Grant +################################################################################ + +resource "aws_kms_grant" "this" { + for_each = { for k, v in var.grants : k => v if var.create } + + name = try(each.value.name, each.key) + key_id = try(aws_kms_key.this[0].key_id, aws_kms_external_key.this[0].id, aws_kms_replica_key.this[0].key_id, aws_kms_replica_external_key.this[0].key_id) + grantee_principal = each.value.grantee_principal + operations = each.value.operations + + dynamic "constraints" { + for_each = length(lookup(each.value, "constraints", {})) == 0 ? [] : [each.value.constraints] + + content { + encryption_context_equals = try(constraints.value.encryption_context_equals, null) + encryption_context_subset = try(constraints.value.encryption_context_subset, null) + } + } + + retiring_principal = try(each.value.retiring_principal, null) + grant_creation_tokens = try(each.value.grant_creation_tokens, null) + retire_on_delete = try(each.value.retire_on_delete, null) +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/kms/outputs.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/kms/outputs.tf new file mode 100644 index 00000000..5f48c556 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/kms/outputs.tf @@ -0,0 +1,51 @@ +################################################################################ +# Key +################################################################################ + +output "key_arn" { + description = "The Amazon Resource Name (ARN) of the key" + value = try(aws_kms_key.this[0].arn, aws_kms_external_key.this[0].arn, aws_kms_replica_key.this[0].arn, aws_kms_replica_external_key.this[0].arn, null) +} + +output "key_id" { + description = "The globally unique identifier for the key" + value = try(aws_kms_key.this[0].key_id, aws_kms_external_key.this[0].id, aws_kms_replica_key.this[0].key_id, aws_kms_replica_external_key.this[0].key_id, null) +} + +output "key_policy" { + description = "The IAM resource policy set on the key" + value = try(aws_kms_key.this[0].policy, aws_kms_external_key.this[0].policy, aws_kms_replica_key.this[0].policy, aws_kms_replica_external_key.this[0].policy, null) +} + +output "external_key_expiration_model" { + description = "Whether the key material expires. Empty when pending key material import, otherwise `KEY_MATERIAL_EXPIRES` or `KEY_MATERIAL_DOES_NOT_EXPIRE`" + value = try(aws_kms_external_key.this[0].expiration_model, aws_kms_replica_external_key.this[0].expiration_model, null) +} + +output "external_key_state" { + description = "The state of the CMK" + value = try(aws_kms_external_key.this[0].key_state, aws_kms_replica_external_key.this[0].key_state, null) +} + +output "external_key_usage" { + description = "The cryptographic operations for which you can use the CMK" + value = try(aws_kms_external_key.this[0].key_usage, aws_kms_replica_external_key.this[0].key_usage, null) +} + +################################################################################ +# Alias +################################################################################ + +output "aliases" { + description = "A map of aliases created and their attributes" + value = aws_kms_alias.this +} + +################################################################################ +# Grant +################################################################################ + +output "grants" { + description = "A map of grants created and their attributes" + value = aws_kms_grant.this +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/kms/variables.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/kms/variables.tf new file mode 100644 index 00000000..0eb9b90c --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/kms/variables.tf @@ -0,0 +1,247 @@ +variable "create" { + description = "Determines whether resources will be created (affects all resources)" + type = bool + default = true +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + +################################################################################ +# Key +################################################################################ + +variable "create_external" { + description = "Determines whether an external CMK (externally provided material) will be created or a standard CMK (AWS provided material)" + type = bool + default = false +} + +variable "bypass_policy_lockout_safety_check" { + description = "A flag to indicate whether to bypass the key policy lockout safety check. Setting this value to true increases the risk that the KMS key becomes unmanageable" + type = bool + default = null +} + +variable "customer_master_key_spec" { + description = "Specifies whether the key contains a symmetric key or an asymmetric key pair and the encryption algorithms or signing algorithms that the key supports. Valid values: `SYMMETRIC_DEFAULT`, `RSA_2048`, `RSA_3072`, `RSA_4096`, `HMAC_256`, `ECC_NIST_P256`, `ECC_NIST_P384`, `ECC_NIST_P521`, or `ECC_SECG_P256K1`. Defaults to `SYMMETRIC_DEFAULT`" + type = string + default = null +} + +variable "custom_key_store_id" { + description = "ID of the KMS Custom Key Store where the key will be stored instead of KMS (eg CloudHSM)." + type = string + default = null +} + +variable "deletion_window_in_days" { + description = "The waiting period, specified in number of days. After the waiting period ends, AWS KMS deletes the KMS key. If you specify a value, it must be between `7` and `30`, inclusive. If you do not specify a value, it defaults to `30`" + type = number + default = null +} + +variable "description" { + description = "The description of the key as viewed in AWS console" + type = string + default = null +} + +variable "enable_key_rotation" { + description = "Specifies whether key rotation is enabled. Defaults to `true`" + type = bool + default = true +} + +variable "is_enabled" { + description = "Specifies whether the key is enabled. Defaults to `true`" + type = bool + default = null +} + +variable "key_material_base64" { + description = "Base64 encoded 256-bit symmetric encryption key material to import. The CMK is permanently associated with this key material. External key only" + type = string + default = null +} + +variable "key_usage" { + description = "Specifies the intended use of the key. Valid values: `ENCRYPT_DECRYPT` or `SIGN_VERIFY`. Defaults to `ENCRYPT_DECRYPT`" + type = string + default = null +} + +variable "multi_region" { + description = "Indicates whether the KMS key is a multi-Region (`true`) or regional (`false`) key. Defaults to `false`" + type = bool + default = false +} + +variable "policy" { + description = "A valid policy JSON document. Although this is a key policy, not an IAM policy, an `aws_iam_policy_document`, in the form that designates a principal, can be used" + type = string + default = null +} + +variable "valid_to" { + description = "Time at which the imported key material expires. When the key material expires, AWS KMS deletes the key material and the CMK becomes unusable. If not specified, key material does not expire" + type = string + default = null +} + +variable "enable_default_policy" { + description = "Specifies whether to enable the default key policy. Defaults to `true`" + type = bool + default = true +} + +variable "key_owners" { + description = "A list of IAM ARNs for those who will have full key permissions (`kms:*`)" + type = list(string) + default = [] +} + +variable "key_administrators" { + description = "A list of IAM ARNs for [key administrators](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-administrators)" + type = list(string) + default = [] +} + +variable "key_users" { + description = "A list of IAM ARNs for [key users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-users)" + type = list(string) + default = [] +} + +variable "key_service_users" { + description = "A list of IAM ARNs for [key service users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-service-integration)" + type = list(string) + default = [] +} + +variable "key_service_roles_for_autoscaling" { + description = "A list of IAM ARNs for [AWSServiceRoleForAutoScaling roles](https://docs.aws.amazon.com/autoscaling/ec2/userguide/key-policy-requirements-EBS-encryption.html#policy-example-cmk-access)" + type = list(string) + default = [] +} + +variable "key_symmetric_encryption_users" { + description = "A list of IAM ARNs for [key symmetric encryption users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-users-crypto)" + type = list(string) + default = [] +} + +variable "key_hmac_users" { + description = "A list of IAM ARNs for [key HMAC users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-users-crypto)" + type = list(string) + default = [] +} + +variable "key_asymmetric_public_encryption_users" { + description = "A list of IAM ARNs for [key asymmetric public encryption users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-users-crypto)" + type = list(string) + default = [] +} + +variable "key_asymmetric_sign_verify_users" { + description = "A list of IAM ARNs for [key asymmetric sign and verify users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-users-crypto)" + type = list(string) + default = [] +} + +variable "key_statements" { + description = "A map of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) for custom permission usage" + type = any + default = {} +} + +variable "source_policy_documents" { + description = "List of IAM policy documents that are merged together into the exported document. Statements must have unique `sid`s" + type = list(string) + default = [] +} + +variable "override_policy_documents" { + description = "List of IAM policy documents that are merged together into the exported document. In merging, statements with non-blank `sid`s will override statements with the same `sid`" + type = list(string) + default = [] +} + +variable "enable_route53_dnssec" { + description = "Determines whether the KMS policy used for Route53 DNSSEC signing is enabled" + type = bool + default = false +} + +variable "route53_dnssec_sources" { + description = "A list of maps containing `account_ids` and Route53 `hosted_zone_arn` that will be allowed to sign DNSSEC records" + type = list(any) + default = [] +} + +################################################################################ +# Replica Key +################################################################################ + +variable "create_replica" { + description = "Determines whether a replica standard CMK will be created (AWS provided material)" + type = bool + default = false +} + +variable "primary_key_arn" { + description = "The primary key arn of a multi-region replica key" + type = string + default = null +} + +################################################################################ +# Replica External Key +################################################################################ + +variable "create_replica_external" { + description = "Determines whether a replica external CMK will be created (externally provided material)" + type = bool + default = false +} + +variable "primary_external_key_arn" { + description = "The primary external key arn of a multi-region replica external key" + type = string + default = null +} + +################################################################################ +# Alias +################################################################################ + +variable "aliases" { + description = "A list of aliases to create. Note - due to the use of `toset()`, values must be static strings and not computed values" + type = list(string) + default = [] +} + +variable "computed_aliases" { + description = "A map of aliases to create. Values provided via the `name` key of the map can be computed from upstream resources" + type = any + default = {} +} + +variable "aliases_use_name_prefix" { + description = "Determines whether the alias name is used as a prefix" + type = bool + default = false +} + +################################################################################ +# Grant +################################################################################ + +variable "grants" { + description = "A map of grant definitions to create" + type = any + default = {} +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/README.md b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/README.md new file mode 100644 index 00000000..fd6f97ad --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/README.md @@ -0,0 +1,231 @@ +# Self Managed Node Group Module + +Configuration in this directory creates a Self Managed Node Group (AutoScaling Group) along with an IAM role, security group, and launch template + +## Usage + +```hcl +module "self_managed_node_group" { + source = "terraform-aws-modules/eks/aws//modules/self-managed-node-group" + + name = "separate-self-mng" + cluster_name = "my-cluster" + cluster_version = "1.31" + cluster_endpoint = "https://012345678903AB2BAE5D1E0BFE0E2B50.gr7.us-east-1.eks.amazonaws.com" + cluster_auth_base64 = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKbXFqQ1VqNGdGR2w3ZW5PeWthWnZ2RjROOTVOUEZCM2o0cGhVZUsrWGFtN2ZSQnZya0d6OGxKZmZEZWF2b2plTwpQK2xOZFlqdHZncmxCUEpYdHZIZmFzTzYxVzdIZmdWQ2EvamdRM2w3RmkvL1dpQmxFOG9oWUZkdWpjc0s1SXM2CnNkbk5KTTNYUWN2TysrSitkV09NT2ZlNzlsSWdncmdQLzgvRU9CYkw3eUY1aU1hS3lsb1RHL1V3TlhPUWt3ZUcKblBNcjdiUmdkQ1NCZTlXYXowOGdGRmlxV2FOditsTDhsODBTdFZLcWVNVlUxbjQyejVwOVpQRTd4T2l6L0xTNQpYV2lXWkVkT3pMN0xBWGVCS2gzdkhnczFxMkI2d1BKZnZnS1NzWllQRGFpZTloT1NNOUJkNFNPY3JrZTRYSVBOCkVvcXVhMlYrUDRlTWJEQzhMUkVWRDdCdVZDdWdMTldWOTBoL3VJUy9WU2VOcEdUOGVScE5DakszSjc2aFlsWm8KWjNGRG5QWUY0MWpWTHhiOXF0U1ROdEp6amYwWXBEYnFWci9xZzNmQWlxbVorMzd3YWM1eHlqMDZ4cmlaRUgzZgpUM002d2lCUEVHYVlGeWN5TmNYTk5aYW9DWDJVL0N1d2JsUHAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==" + + subnet_ids = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] + + // The following variables are necessary if you decide to use the module outside of the parent EKS module context. + // Without it, the security groups of the nodes are empty and thus won't join the cluster. + vpc_security_group_ids = [ + module.eks.cluster_primary_security_group_id, + module.eks.cluster_security_group_id, + ] + + min_size = 1 + max_size = 10 + desired_size = 1 + + launch_template_name = "separate-self-mng" + instance_type = "m5.large" + + tags = { + Environment = "dev" + Terraform = "true" + } +} +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.2 | +| [aws](#requirement\_aws) | >= 5.95 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 5.95 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [user\_data](#module\_user\_data) | ../_user_data | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_autoscaling_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource | +| [aws_autoscaling_schedule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_schedule) | resource | +| [aws_eks_access_entry.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | +| [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_launch_template.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_placement_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/placement_group) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_ec2_instance_type.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | +| [aws_ec2_instance_type_offerings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | +| [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_ssm_parameter.ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | +| [aws_subnets.placement_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_cluster\_dns\_ips](#input\_additional\_cluster\_dns\_ips) | Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*` | `list(string)` | `[]` | no | +| [ami\_id](#input\_ami\_id) | The AMI from which to launch the instance | `string` | `""` | no | +| [ami\_type](#input\_ami\_type) | Type of Amazon Machine Image (AMI) associated with the node group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values | `string` | `"AL2_x86_64"` | no | +| [autoscaling\_group\_tags](#input\_autoscaling\_group\_tags) | A map of additional tags to add to the autoscaling group created. Tags are applied to the autoscaling group only and are NOT propagated to instances | `map(string)` | `{}` | no | +| [availability\_zones](#input\_availability\_zones) | A list of one or more availability zones for the group. Used for EC2-Classic and default subnets when not specified with `subnet_ids` argument. Conflicts with `subnet_ids` | `list(string)` | `null` | no | +| [block\_device\_mappings](#input\_block\_device\_mappings) | Specify volumes to attach to the instance besides the volumes specified by the AMI | `any` | `{}` | no | +| [bootstrap\_extra\_args](#input\_bootstrap\_extra\_args) | Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data | `string` | `""` | no | +| [capacity\_rebalance](#input\_capacity\_rebalance) | Indicates whether capacity rebalance is enabled | `bool` | `null` | no | +| [capacity\_reservation\_specification](#input\_capacity\_reservation\_specification) | Targeting for EC2 capacity reservations | `any` | `{}` | no | +| [cloudinit\_post\_nodeadm](#input\_cloudinit\_post\_nodeadm) | Array of cloud-init document parts that are created after the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cloudinit\_pre\_nodeadm](#input\_cloudinit\_pre\_nodeadm) | Array of cloud-init document parts that are created before the nodeadm document part |
list(object({
content = string
content_type = optional(string)
filename = optional(string)
merge_type = optional(string)
}))
| `[]` | no | +| [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of associated EKS cluster | `string` | `""` | no | +| [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of associated EKS cluster | `string` | `""` | no | +| [cluster\_ip\_family](#input\_cluster\_ip\_family) | The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6` | `string` | `"ipv4"` | no | +| [cluster\_name](#input\_cluster\_name) | Name of associated EKS cluster | `string` | `""` | no | +| [cluster\_primary\_security\_group\_id](#input\_cluster\_primary\_security\_group\_id) | The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service | `string` | `null` | no | +| [cluster\_service\_cidr](#input\_cluster\_service\_cidr) | The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself | `string` | `""` | no | +| [cluster\_version](#input\_cluster\_version) | Kubernetes cluster version - used to lookup default AMI ID if one is not provided | `string` | `null` | no | +| [context](#input\_context) | Reserved | `string` | `null` | no | +| [cpu\_options](#input\_cpu\_options) | The CPU options for the instance | `map(string)` | `{}` | no | +| [create](#input\_create) | Determines whether to create self managed node group or not | `bool` | `true` | no | +| [create\_access\_entry](#input\_create\_access\_entry) | Determines whether an access entry is created for the IAM role used by the node group | `bool` | `true` | no | +| [create\_autoscaling\_group](#input\_create\_autoscaling\_group) | Determines whether to create autoscaling group or not | `bool` | `true` | no | +| [create\_iam\_instance\_profile](#input\_create\_iam\_instance\_profile) | Determines whether an IAM instance profile is created or to use an existing IAM instance profile | `bool` | `true` | no | +| [create\_iam\_role\_policy](#input\_create\_iam\_role\_policy) | Determines whether an IAM role policy is created or not | `bool` | `true` | no | +| [create\_launch\_template](#input\_create\_launch\_template) | Determines whether to create launch template or not | `bool` | `true` | no | +| [create\_placement\_group](#input\_create\_placement\_group) | Determines whether a placement group is created & used by the node group | `bool` | `false` | no | +| [create\_schedule](#input\_create\_schedule) | Determines whether to create autoscaling group schedule or not | `bool` | `true` | no | +| [credit\_specification](#input\_credit\_specification) | Customize the credit specification of the instance | `map(string)` | `{}` | no | +| [default\_cooldown](#input\_default\_cooldown) | The amount of time, in seconds, after a scaling activity completes before another scaling activity can start | `number` | `null` | no | +| [default\_instance\_warmup](#input\_default\_instance\_warmup) | Amount of time, in seconds, until a newly launched instance can contribute to the Amazon CloudWatch metrics. This delay lets an instance finish initializing before Amazon EC2 Auto Scaling aggregates instance metrics, resulting in more reliable usage data | `number` | `null` | no | +| [delete\_timeout](#input\_delete\_timeout) | Delete timeout to wait for destroying autoscaling group | `string` | `null` | no | +| [desired\_size](#input\_desired\_size) | The number of Amazon EC2 instances that should be running in the autoscaling group | `number` | `1` | no | +| [desired\_size\_type](#input\_desired\_size\_type) | The unit of measurement for the value specified for `desired_size`. Supported for attribute-based instance type selection only. Valid values: `units`, `vcpu`, `memory-mib` | `string` | `null` | no | +| [disable\_api\_termination](#input\_disable\_api\_termination) | If true, enables EC2 instance termination protection | `bool` | `null` | no | +| [ebs\_optimized](#input\_ebs\_optimized) | If true, the launched EC2 instance will be EBS-optimized | `bool` | `null` | no | +| [efa\_indices](#input\_efa\_indices) | The indices of the network interfaces that should be EFA-enabled. Only valid when `enable_efa_support` = `true` | `list(number)` |
[
0
]
| no | +| [elastic\_gpu\_specifications](#input\_elastic\_gpu\_specifications) | The elastic GPU to attach to the instance | `any` | `{}` | no | +| [elastic\_inference\_accelerator](#input\_elastic\_inference\_accelerator) | Configuration block containing an Elastic Inference Accelerator to attach to the instance | `map(string)` | `{}` | no | +| [enable\_efa\_only](#input\_enable\_efa\_only) | Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later | `bool` | `false` | no | +| [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no | +| [enable\_monitoring](#input\_enable\_monitoring) | Enables/disables detailed monitoring | `bool` | `true` | no | +| [enabled\_metrics](#input\_enabled\_metrics) | A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances` | `list(string)` | `[]` | no | +| [enclave\_options](#input\_enclave\_options) | Enable Nitro Enclaves on launched instances | `map(string)` | `{}` | no | +| [force\_delete](#input\_force\_delete) | Allows deleting the Auto Scaling Group without waiting for all instances in the pool to terminate. You can force an Auto Scaling Group to delete even if it's in the process of scaling a resource. Normally, Terraform drains all the instances before deleting the group. This bypasses that behavior and potentially leaves resources dangling | `bool` | `null` | no | +| [force\_delete\_warm\_pool](#input\_force\_delete\_warm\_pool) | Allows deleting the Auto Scaling Group without waiting for all instances in the warm pool to terminate | `bool` | `null` | no | +| [health\_check\_grace\_period](#input\_health\_check\_grace\_period) | Time (in seconds) after instance comes into service before checking health | `number` | `null` | no | +| [health\_check\_type](#input\_health\_check\_type) | `EC2` or `ELB`. Controls how health checking is done | `string` | `null` | no | +| [hibernation\_options](#input\_hibernation\_options) | The hibernation options for the instance | `map(string)` | `{}` | no | +| [iam\_instance\_profile\_arn](#input\_iam\_instance\_profile\_arn) | Amazon Resource Name (ARN) of an existing IAM instance profile that provides permissions for the node group. Required if `create_iam_instance_profile` = `false` | `string` | `null` | no | +| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | +| [iam\_role\_arn](#input\_iam\_role\_arn) | ARN of the IAM role used by the instance profile. Required when `create_access_entry = true` and `create_iam_instance_profile = false` | `string` | `null` | no | +| [iam\_role\_attach\_cni\_policy](#input\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | +| [iam\_role\_description](#input\_iam\_role\_description) | Description of the role | `string` | `null` | no | +| [iam\_role\_name](#input\_iam\_role\_name) | Name to use on IAM role created | `string` | `null` | no | +| [iam\_role\_path](#input\_iam\_role\_path) | IAM role path | `string` | `null` | no | +| [iam\_role\_permissions\_boundary](#input\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the IAM role | `string` | `null` | no | +| [iam\_role\_policy\_statements](#input\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no | +| [iam\_role\_tags](#input\_iam\_role\_tags) | A map of additional tags to add to the IAM role created | `map(string)` | `{}` | no | +| [iam\_role\_use\_name\_prefix](#input\_iam\_role\_use\_name\_prefix) | Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix | `bool` | `true` | no | +| [ignore\_failed\_scaling\_activities](#input\_ignore\_failed\_scaling\_activities) | Whether to ignore failed Auto Scaling scaling activities while waiting for capacity. | `bool` | `null` | no | +| [initial\_lifecycle\_hooks](#input\_initial\_lifecycle\_hooks) | One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource | `list(map(string))` | `[]` | no | +| [instance\_initiated\_shutdown\_behavior](#input\_instance\_initiated\_shutdown\_behavior) | Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`) | `string` | `null` | no | +| [instance\_maintenance\_policy](#input\_instance\_maintenance\_policy) | If this block is configured, add a instance maintenance policy to the specified Auto Scaling group | `any` | `{}` | no | +| [instance\_market\_options](#input\_instance\_market\_options) | The market (purchasing) option for the instance | `any` | `{}` | no | +| [instance\_refresh](#input\_instance\_refresh) | If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated | `any` |
{
"preferences": {
"min_healthy_percentage": 66
},
"strategy": "Rolling"
}
| no | +| [instance\_requirements](#input\_instance\_requirements) | The attribute requirements for the type of instance. If present then `instance_type` cannot be present | `any` | `{}` | no | +| [instance\_type](#input\_instance\_type) | The type of the instance to launch | `string` | `""` | no | +| [kernel\_id](#input\_kernel\_id) | The kernel ID | `string` | `null` | no | +| [key\_name](#input\_key\_name) | The key name that should be used for the instance | `string` | `null` | no | +| [launch\_template\_default\_version](#input\_launch\_template\_default\_version) | Default Version of the launch template | `string` | `null` | no | +| [launch\_template\_description](#input\_launch\_template\_description) | Description of the launch template | `string` | `null` | no | +| [launch\_template\_id](#input\_launch\_template\_id) | The ID of an existing launch template to use. Required when `create_launch_template` = `false` | `string` | `""` | no | +| [launch\_template\_name](#input\_launch\_template\_name) | Name of launch template to be created | `string` | `null` | no | +| [launch\_template\_tags](#input\_launch\_template\_tags) | A map of additional tags to add to the tag\_specifications of launch template created | `map(string)` | `{}` | no | +| [launch\_template\_use\_name\_prefix](#input\_launch\_template\_use\_name\_prefix) | Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix | `bool` | `true` | no | +| [launch\_template\_version](#input\_launch\_template\_version) | Launch template version. Can be version number, `$Latest`, or `$Default` | `string` | `null` | no | +| [license\_specifications](#input\_license\_specifications) | A map of license specifications to associate with | `any` | `{}` | no | +| [maintenance\_options](#input\_maintenance\_options) | The maintenance options for the instance | `any` | `{}` | no | +| [max\_instance\_lifetime](#input\_max\_instance\_lifetime) | The maximum amount of time, in seconds, that an instance can be in service, values must be either equal to 0 or between 604800 and 31536000 seconds | `number` | `null` | no | +| [max\_size](#input\_max\_size) | The maximum size of the autoscaling group | `number` | `3` | no | +| [metadata\_options](#input\_metadata\_options) | Customize the metadata options for the instance | `map(string)` |
{
"http_endpoint": "enabled",
"http_put_response_hop_limit": 2,
"http_tokens": "required"
}
| no | +| [metrics\_granularity](#input\_metrics\_granularity) | The granularity to associate with the metrics to collect. The only valid value is `1Minute` | `string` | `null` | no | +| [min\_elb\_capacity](#input\_min\_elb\_capacity) | Setting this causes Terraform to wait for this number of instances to show up healthy in the ELB only on creation. Updates will not wait on ELB instance number changes | `number` | `null` | no | +| [min\_size](#input\_min\_size) | The minimum size of the autoscaling group | `number` | `0` | no | +| [mixed\_instances\_policy](#input\_mixed\_instances\_policy) | Configuration block containing settings to define launch targets for Auto Scaling groups | `any` | `null` | no | +| [name](#input\_name) | Name of the Self managed Node Group | `string` | `""` | no | +| [network\_interfaces](#input\_network\_interfaces) | Customize network interfaces to be attached at instance boot time | `list(any)` | `[]` | no | +| [placement](#input\_placement) | The placement of the instance | `map(string)` | `{}` | no | +| [placement\_group](#input\_placement\_group) | The name of the placement group into which you'll launch your instances, if any | `string` | `null` | no | +| [placement\_group\_az](#input\_placement\_group\_az) | Availability zone where placement group is created (ex. `eu-west-1c`) | `string` | `null` | no | +| [platform](#input\_platform) | [DEPRECATED - must use `ami_type` instead. Will be removed in `v21.0`] | `string` | `null` | no | +| [post\_bootstrap\_user\_data](#input\_post\_bootstrap\_user\_data) | User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | +| [pre\_bootstrap\_user\_data](#input\_pre\_bootstrap\_user\_data) | User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*` | `string` | `""` | no | +| [private\_dns\_name\_options](#input\_private\_dns\_name\_options) | The options for the instance hostname. The default values are inherited from the subnet | `map(string)` | `{}` | no | +| [protect\_from\_scale\_in](#input\_protect\_from\_scale\_in) | Allows setting instance protection. The autoscaling group will not select instances with this setting for termination during scale in events. | `bool` | `false` | no | +| [ram\_disk\_id](#input\_ram\_disk\_id) | The ID of the ram disk | `string` | `null` | no | +| [schedules](#input\_schedules) | Map of autoscaling group schedule to create | `map(any)` | `{}` | no | +| [service\_linked\_role\_arn](#input\_service\_linked\_role\_arn) | The ARN of the service-linked role that the ASG will use to call other AWS services | `string` | `null` | no | +| [subnet\_ids](#input\_subnet\_ids) | A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones` | `list(string)` | `null` | no | +| [suspended\_processes](#input\_suspended\_processes) | A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly | `list(string)` | `[]` | no | +| [tag\_specifications](#input\_tag\_specifications) | The tags to apply to the resources during launch | `list(string)` |
[
"instance",
"volume",
"network-interface"
]
| no | +| [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | +| [target\_group\_arns](#input\_target\_group\_arns) | A set of `aws_alb_target_group` ARNs, for use with Application or Network Load Balancing | `list(string)` | `[]` | no | +| [termination\_policies](#input\_termination\_policies) | A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default` | `list(string)` | `[]` | no | +| [update\_launch\_template\_default\_version](#input\_update\_launch\_template\_default\_version) | Whether to update Default Version each update. Conflicts with `launch_template_default_version` | `bool` | `true` | no | +| [use\_mixed\_instances\_policy](#input\_use\_mixed\_instances\_policy) | Determines whether to use a mixed instances policy in the autoscaling group or not | `bool` | `false` | no | +| [use\_name\_prefix](#input\_use\_name\_prefix) | Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix | `bool` | `true` | no | +| [user\_data\_template\_path](#input\_user\_data\_template\_path) | Path to a local, custom user data template file to use when rendering user data | `string` | `""` | no | +| [vpc\_security\_group\_ids](#input\_vpc\_security\_group\_ids) | A list of security group IDs to associate | `list(string)` | `[]` | no | +| [wait\_for\_capacity\_timeout](#input\_wait\_for\_capacity\_timeout) | A maximum duration that Terraform should wait for ASG instances to be healthy before timing out. (See also Waiting for Capacity below.) Setting this to '0' causes Terraform to skip all Capacity Waiting behavior. | `string` | `null` | no | +| [wait\_for\_elb\_capacity](#input\_wait\_for\_elb\_capacity) | Setting this will cause Terraform to wait for exactly this number of healthy instances in all attached load balancers on both create and update operations. Takes precedence over `min_elb_capacity` behavior. | `number` | `null` | no | +| [warm\_pool](#input\_warm\_pool) | If this block is configured, add a Warm Pool to the specified Auto Scaling group | `any` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [access\_entry\_arn](#output\_access\_entry\_arn) | Amazon Resource Name (ARN) of the Access Entry | +| [autoscaling\_group\_arn](#output\_autoscaling\_group\_arn) | The ARN for this autoscaling group | +| [autoscaling\_group\_availability\_zones](#output\_autoscaling\_group\_availability\_zones) | The availability zones of the autoscaling group | +| [autoscaling\_group\_default\_cooldown](#output\_autoscaling\_group\_default\_cooldown) | Time between a scaling activity and the succeeding scaling activity | +| [autoscaling\_group\_desired\_capacity](#output\_autoscaling\_group\_desired\_capacity) | The number of Amazon EC2 instances that should be running in the group | +| [autoscaling\_group\_health\_check\_grace\_period](#output\_autoscaling\_group\_health\_check\_grace\_period) | Time after instance comes into service before checking health | +| [autoscaling\_group\_health\_check\_type](#output\_autoscaling\_group\_health\_check\_type) | EC2 or ELB. Controls how health checking is done | +| [autoscaling\_group\_id](#output\_autoscaling\_group\_id) | The autoscaling group id | +| [autoscaling\_group\_max\_size](#output\_autoscaling\_group\_max\_size) | The maximum size of the autoscaling group | +| [autoscaling\_group\_min\_size](#output\_autoscaling\_group\_min\_size) | The minimum size of the autoscaling group | +| [autoscaling\_group\_name](#output\_autoscaling\_group\_name) | The autoscaling group name | +| [autoscaling\_group\_schedule\_arns](#output\_autoscaling\_group\_schedule\_arns) | ARNs of autoscaling group schedules | +| [autoscaling\_group\_vpc\_zone\_identifier](#output\_autoscaling\_group\_vpc\_zone\_identifier) | The VPC zone identifier | +| [iam\_instance\_profile\_arn](#output\_iam\_instance\_profile\_arn) | ARN assigned by AWS to the instance profile | +| [iam\_instance\_profile\_id](#output\_iam\_instance\_profile\_id) | Instance profile's ID | +| [iam\_instance\_profile\_unique](#output\_iam\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | +| [iam\_role\_arn](#output\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the IAM role | +| [iam\_role\_name](#output\_iam\_role\_name) | The name of the IAM role | +| [iam\_role\_unique\_id](#output\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role | +| [image\_id](#output\_image\_id) | ID of the image | +| [launch\_template\_arn](#output\_launch\_template\_arn) | The ARN of the launch template | +| [launch\_template\_id](#output\_launch\_template\_id) | The ID of the launch template | +| [launch\_template\_latest\_version](#output\_launch\_template\_latest\_version) | The latest version of the launch template | +| [launch\_template\_name](#output\_launch\_template\_name) | The name of the launch template | +| [platform](#output\_platform) | [DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows` | +| [user\_data](#output\_user\_data) | Base64 encoded user data | + diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/main.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/main.tf new file mode 100644 index 00000000..132278a1 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/main.tf @@ -0,0 +1,1030 @@ +data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} + +################################################################################ +# AMI SSM Parameter +################################################################################ + +locals { + # Just to ensure templating doesn't fail when values are not provided + ssm_cluster_version = var.cluster_version != null ? var.cluster_version : "" + + # TODO - Temporary stopgap for backwards compatibility until v21.0 + ami_type_to_user_data_type = { + AL2_x86_64 = "linux" + AL2_x86_64_GPU = "linux" + AL2_ARM_64 = "linux" + BOTTLEROCKET_ARM_64 = "bottlerocket" + BOTTLEROCKET_x86_64 = "bottlerocket" + BOTTLEROCKET_ARM_64_FIPS = "bottlerocket" + BOTTLEROCKET_x86_64_FIPS = "bottlerocket" + BOTTLEROCKET_ARM_64_NVIDIA = "bottlerocket" + BOTTLEROCKET_x86_64_NVIDIA = "bottlerocket" + WINDOWS_CORE_2019_x86_64 = "windows" + WINDOWS_FULL_2019_x86_64 = "windows" + WINDOWS_CORE_2022_x86_64 = "windows" + WINDOWS_FULL_2022_x86_64 = "windows" + AL2023_x86_64_STANDARD = "al2023" + AL2023_ARM_64_STANDARD = "al2023" + AL2023_x86_64_NEURON = "al2023" + AL2023_x86_64_NVIDIA = "al2023" + } + + user_data_type = local.ami_type_to_user_data_type[var.ami_type] + + # Map the AMI type to the respective SSM param path + ami_type_to_ssm_param = { + AL2_x86_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2/recommended/image_id" + AL2_x86_64_GPU = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-gpu/recommended/image_id" + AL2_ARM_64 = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2-arm64/recommended/image_id" + BOTTLEROCKET_ARM_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/arm64/latest/image_id" + BOTTLEROCKET_x86_64 = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}/x86_64/latest/image_id" + BOTTLEROCKET_ARM_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-fips/arm64/latest/image_id" + BOTTLEROCKET_x86_64_FIPS = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-fips/x86_64/latest/image_id" + BOTTLEROCKET_ARM_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/arm64/latest/image_id" + BOTTLEROCKET_x86_64_NVIDIA = "/aws/service/bottlerocket/aws-k8s-${local.ssm_cluster_version}-nvidia/x86_64/latest/image_id" + WINDOWS_CORE_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-${local.ssm_cluster_version}/image_id" + WINDOWS_FULL_2019_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-${local.ssm_cluster_version}/image_id" + WINDOWS_CORE_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-EKS_Optimized-${local.ssm_cluster_version}/image_id" + WINDOWS_FULL_2022_x86_64 = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-${local.ssm_cluster_version}/image_id" + AL2023_x86_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/standard/recommended/image_id" + AL2023_ARM_64_STANDARD = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/arm64/standard/recommended/image_id" + AL2023_x86_64_NEURON = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/neuron/recommended/image_id" + AL2023_x86_64_NVIDIA = "/aws/service/eks/optimized-ami/${local.ssm_cluster_version}/amazon-linux-2023/x86_64/nvidia/recommended/image_id" + } +} + +data "aws_ssm_parameter" "ami" { + count = var.create ? 1 : 0 + + name = local.ami_type_to_ssm_param[var.ami_type] +} + +################################################################################ +# User Data +################################################################################ + +module "user_data" { + source = "../_user_data" + + create = var.create + platform = local.user_data_type + ami_type = var.ami_type + is_eks_managed_node_group = false + + cluster_name = var.cluster_name + cluster_endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + cluster_ip_family = var.cluster_ip_family + cluster_service_cidr = var.cluster_service_cidr + additional_cluster_dns_ips = var.additional_cluster_dns_ips + + enable_bootstrap_user_data = true + pre_bootstrap_user_data = var.pre_bootstrap_user_data + post_bootstrap_user_data = var.post_bootstrap_user_data + bootstrap_extra_args = var.bootstrap_extra_args + user_data_template_path = var.user_data_template_path + + cloudinit_pre_nodeadm = var.cloudinit_pre_nodeadm + cloudinit_post_nodeadm = var.cloudinit_post_nodeadm +} + +################################################################################ +# EFA Support +################################################################################ + +data "aws_ec2_instance_type" "this" { + count = var.create && var.enable_efa_support ? 1 : 0 + + instance_type = var.instance_type +} + +locals { + enable_efa_support = var.create && var.enable_efa_support && local.instance_type_provided + + instance_type_provided = var.instance_type != "" + num_network_cards = try(data.aws_ec2_instance_type.this[0].maximum_network_cards, 0) + + # Primary network interface must be EFA, remaining can be EFA or EFA-only + efa_network_interfaces = [ + for i in range(local.num_network_cards) : { + associate_public_ip_address = false + delete_on_termination = true + device_index = i == 0 ? 0 : 1 + network_card_index = i + interface_type = var.enable_efa_only ? contains(concat([0], var.efa_indices), i) ? "efa" : "efa-only" : "efa" + } + ] + + network_interfaces = local.enable_efa_support ? local.efa_network_interfaces : var.network_interfaces +} + +################################################################################ +# Launch template +################################################################################ + +locals { + launch_template_name = coalesce(var.launch_template_name, "${var.name}-node-group") + security_group_ids = compact(concat([var.cluster_primary_security_group_id], var.vpc_security_group_ids)) + + placement = local.enable_efa_support ? { group_name = aws_placement_group.this[0].name } : var.placement +} + +resource "aws_launch_template" "this" { + count = var.create && var.create_launch_template ? 1 : 0 + + dynamic "block_device_mappings" { + for_each = var.block_device_mappings + + content { + device_name = try(block_device_mappings.value.device_name, null) + + dynamic "ebs" { + for_each = try([block_device_mappings.value.ebs], []) + + content { + delete_on_termination = try(ebs.value.delete_on_termination, null) + encrypted = try(ebs.value.encrypted, null) + iops = try(ebs.value.iops, null) + kms_key_id = try(ebs.value.kms_key_id, null) + snapshot_id = try(ebs.value.snapshot_id, null) + throughput = try(ebs.value.throughput, null) + volume_size = try(ebs.value.volume_size, null) + volume_type = try(ebs.value.volume_type, null) + } + } + + no_device = try(block_device_mappings.value.no_device, null) + virtual_name = try(block_device_mappings.value.virtual_name, null) + } + } + + dynamic "capacity_reservation_specification" { + for_each = length(var.capacity_reservation_specification) > 0 ? [var.capacity_reservation_specification] : [] + + content { + capacity_reservation_preference = try(capacity_reservation_specification.value.capacity_reservation_preference, null) + + dynamic "capacity_reservation_target" { + for_each = try([capacity_reservation_specification.value.capacity_reservation_target], []) + + content { + capacity_reservation_id = try(capacity_reservation_target.value.capacity_reservation_id, null) + capacity_reservation_resource_group_arn = try(capacity_reservation_target.value.capacity_reservation_resource_group_arn, null) + } + } + } + } + + dynamic "cpu_options" { + for_each = length(var.cpu_options) > 0 ? [var.cpu_options] : [] + + content { + core_count = try(cpu_options.value.core_count, null) + threads_per_core = try(cpu_options.value.threads_per_core, null) + } + } + + dynamic "credit_specification" { + for_each = length(var.credit_specification) > 0 ? [var.credit_specification] : [] + + content { + cpu_credits = try(credit_specification.value.cpu_credits, null) + } + } + + default_version = var.launch_template_default_version + description = var.launch_template_description + disable_api_termination = var.disable_api_termination + ebs_optimized = var.ebs_optimized + + + dynamic "enclave_options" { + for_each = length(var.enclave_options) > 0 ? [var.enclave_options] : [] + + content { + enabled = enclave_options.value.enabled + } + } + + dynamic "hibernation_options" { + for_each = length(var.hibernation_options) > 0 ? [var.hibernation_options] : [] + + content { + configured = hibernation_options.value.configured + } + } + + iam_instance_profile { + arn = var.create_iam_instance_profile ? aws_iam_instance_profile.this[0].arn : var.iam_instance_profile_arn + } + + image_id = coalesce(var.ami_id, nonsensitive(data.aws_ssm_parameter.ami[0].value)) + instance_initiated_shutdown_behavior = var.instance_initiated_shutdown_behavior + + dynamic "instance_market_options" { + for_each = length(var.instance_market_options) > 0 ? [var.instance_market_options] : [] + + content { + market_type = try(instance_market_options.value.market_type, null) + + dynamic "spot_options" { + for_each = try([instance_market_options.value.spot_options], []) + + content { + block_duration_minutes = try(spot_options.value.block_duration_minutes, null) + instance_interruption_behavior = try(spot_options.value.instance_interruption_behavior, null) + max_price = try(spot_options.value.max_price, null) + spot_instance_type = try(spot_options.value.spot_instance_type, null) + valid_until = try(spot_options.value.valid_until, null) + } + } + } + } + + dynamic "instance_requirements" { + for_each = length(var.instance_requirements) > 0 ? [var.instance_requirements] : [] + + content { + + dynamic "accelerator_count" { + for_each = try([instance_requirements.value.accelerator_count], []) + + content { + max = try(accelerator_count.value.max, null) + min = try(accelerator_count.value.min, null) + } + } + + accelerator_manufacturers = try(instance_requirements.value.accelerator_manufacturers, []) + accelerator_names = try(instance_requirements.value.accelerator_names, []) + + dynamic "accelerator_total_memory_mib" { + for_each = try([instance_requirements.value.accelerator_total_memory_mib], []) + + content { + max = try(accelerator_total_memory_mib.value.max, null) + min = try(accelerator_total_memory_mib.value.min, null) + } + } + + accelerator_types = try(instance_requirements.value.accelerator_types, []) + allowed_instance_types = try(instance_requirements.value.allowed_instance_types, null) + bare_metal = try(instance_requirements.value.bare_metal, null) + + dynamic "baseline_ebs_bandwidth_mbps" { + for_each = try([instance_requirements.value.baseline_ebs_bandwidth_mbps], []) + + content { + max = try(baseline_ebs_bandwidth_mbps.value.max, null) + min = try(baseline_ebs_bandwidth_mbps.value.min, null) + } + } + + burstable_performance = try(instance_requirements.value.burstable_performance, null) + cpu_manufacturers = try(instance_requirements.value.cpu_manufacturers, []) + excluded_instance_types = try(instance_requirements.value.excluded_instance_types, null) + instance_generations = try(instance_requirements.value.instance_generations, []) + local_storage = try(instance_requirements.value.local_storage, null) + local_storage_types = try(instance_requirements.value.local_storage_types, []) + + dynamic "memory_gib_per_vcpu" { + for_each = try([instance_requirements.value.memory_gib_per_vcpu], []) + + content { + max = try(memory_gib_per_vcpu.value.max, null) + min = try(memory_gib_per_vcpu.value.min, null) + } + } + + dynamic "memory_mib" { + for_each = [instance_requirements.value.memory_mib] + + content { + max = try(memory_mib.value.max, null) + min = memory_mib.value.min + } + } + + dynamic "network_bandwidth_gbps" { + for_each = try([instance_requirements.value.network_bandwidth_gbps], []) + + content { + max = try(network_bandwidth_gbps.value.max, null) + min = try(network_bandwidth_gbps.value.min, null) + } + } + + dynamic "network_interface_count" { + for_each = try([instance_requirements.value.network_interface_count], []) + + content { + max = try(network_interface_count.value.max, null) + min = try(network_interface_count.value.min, null) + } + } + + on_demand_max_price_percentage_over_lowest_price = try(instance_requirements.value.on_demand_max_price_percentage_over_lowest_price, null) + require_hibernate_support = try(instance_requirements.value.require_hibernate_support, null) + spot_max_price_percentage_over_lowest_price = try(instance_requirements.value.spot_max_price_percentage_over_lowest_price, null) + + dynamic "total_local_storage_gb" { + for_each = try([instance_requirements.value.total_local_storage_gb], []) + + content { + max = try(total_local_storage_gb.value.max, null) + min = try(total_local_storage_gb.value.min, null) + } + } + + dynamic "vcpu_count" { + for_each = [instance_requirements.value.vcpu_count] + + content { + max = try(vcpu_count.value.max, null) + min = vcpu_count.value.min + } + } + } + } + + instance_type = var.instance_type + kernel_id = var.kernel_id + key_name = var.key_name + + dynamic "license_specification" { + for_each = length(var.license_specifications) > 0 ? var.license_specifications : {} + + content { + license_configuration_arn = license_specification.value.license_configuration_arn + } + } + + dynamic "maintenance_options" { + for_each = length(var.maintenance_options) > 0 ? [var.maintenance_options] : [] + + content { + auto_recovery = try(maintenance_options.value.auto_recovery, null) + } + } + + dynamic "metadata_options" { + for_each = length(var.metadata_options) > 0 ? [var.metadata_options] : [] + + content { + http_endpoint = try(metadata_options.value.http_endpoint, null) + http_protocol_ipv6 = try(metadata_options.value.http_protocol_ipv6, null) + http_put_response_hop_limit = try(metadata_options.value.http_put_response_hop_limit, null) + http_tokens = try(metadata_options.value.http_tokens, null) + instance_metadata_tags = try(metadata_options.value.instance_metadata_tags, null) + } + } + + dynamic "monitoring" { + for_each = var.enable_monitoring ? [1] : [] + + content { + enabled = var.enable_monitoring + } + } + + name = var.launch_template_use_name_prefix ? null : local.launch_template_name + name_prefix = var.launch_template_use_name_prefix ? "${local.launch_template_name}-" : null + + dynamic "network_interfaces" { + for_each = local.network_interfaces + + content { + associate_carrier_ip_address = try(network_interfaces.value.associate_carrier_ip_address, null) + associate_public_ip_address = try(network_interfaces.value.associate_public_ip_address, null) + delete_on_termination = try(network_interfaces.value.delete_on_termination, null) + description = try(network_interfaces.value.description, null) + device_index = try(network_interfaces.value.device_index, null) + interface_type = try(network_interfaces.value.interface_type, null) + ipv4_address_count = try(network_interfaces.value.ipv4_address_count, null) + ipv4_addresses = try(network_interfaces.value.ipv4_addresses, []) + ipv4_prefix_count = try(network_interfaces.value.ipv4_prefix_count, null) + ipv4_prefixes = try(network_interfaces.value.ipv4_prefixes, null) + ipv6_address_count = try(network_interfaces.value.ipv6_address_count, null) + ipv6_addresses = try(network_interfaces.value.ipv6_addresses, []) + ipv6_prefix_count = try(network_interfaces.value.ipv6_prefix_count, null) + ipv6_prefixes = try(network_interfaces.value.ipv6_prefixes, []) + network_card_index = try(network_interfaces.value.network_card_index, null) + network_interface_id = try(network_interfaces.value.network_interface_id, null) + primary_ipv6 = try(network_interfaces.value.primary_ipv6, null) + private_ip_address = try(network_interfaces.value.private_ip_address, null) + # Ref: https://github.com/hashicorp/terraform-provider-aws/issues/4570 + security_groups = compact(concat(try(network_interfaces.value.security_groups, []), local.security_group_ids)) + subnet_id = try(network_interfaces.value.subnet_id, null) + } + } + + dynamic "placement" { + for_each = length(local.placement) > 0 ? [local.placement] : [] + + content { + affinity = try(placement.value.affinity, null) + availability_zone = lookup(placement.value, "availability_zone", null) + group_name = lookup(placement.value, "group_name", null) + host_id = lookup(placement.value, "host_id", null) + host_resource_group_arn = lookup(placement.value, "host_resource_group_arn", null) + partition_number = try(placement.value.partition_number, null) + spread_domain = try(placement.value.spread_domain, null) + tenancy = try(placement.value.tenancy, null) + } + } + + dynamic "private_dns_name_options" { + for_each = length(var.private_dns_name_options) > 0 ? [var.private_dns_name_options] : [] + + content { + enable_resource_name_dns_aaaa_record = try(private_dns_name_options.value.enable_resource_name_dns_aaaa_record, null) + enable_resource_name_dns_a_record = try(private_dns_name_options.value.enable_resource_name_dns_a_record, null) + hostname_type = try(private_dns_name_options.value.hostname_type, null) + } + } + + ram_disk_id = var.ram_disk_id + + dynamic "tag_specifications" { + for_each = toset(var.tag_specifications) + + content { + resource_type = tag_specifications.key + tags = merge(var.tags, { Name = var.name }, var.launch_template_tags) + } + } + + update_default_version = var.update_launch_template_default_version + user_data = module.user_data.user_data + vpc_security_group_ids = length(local.network_interfaces) > 0 ? [] : local.security_group_ids + + tags = var.tags + + # Prevent premature access of policies by pods that + # require permissions on create/destroy that depend on nodes + depends_on = [ + aws_iam_role_policy_attachment.this, + aws_iam_role_policy_attachment.additional, + ] + + lifecycle { + create_before_destroy = true + } +} + +################################################################################ +# Node Group +################################################################################ + +locals { + launch_template_id = var.create && var.create_launch_template ? aws_launch_template.this[0].id : var.launch_template_id + # Change order to allow users to set version priority before using defaults + launch_template_version = coalesce(var.launch_template_version, try(aws_launch_template.this[0].default_version, "$Default")) +} + +resource "aws_autoscaling_group" "this" { + count = var.create && var.create_autoscaling_group ? 1 : 0 + + availability_zones = var.availability_zones + capacity_rebalance = var.capacity_rebalance + context = var.context + default_cooldown = var.default_cooldown + default_instance_warmup = var.default_instance_warmup + desired_capacity = var.desired_size + desired_capacity_type = var.desired_size_type + enabled_metrics = var.enabled_metrics + force_delete = var.force_delete + force_delete_warm_pool = var.force_delete_warm_pool + health_check_grace_period = var.health_check_grace_period + health_check_type = var.health_check_type + + dynamic "initial_lifecycle_hook" { + for_each = var.initial_lifecycle_hooks + + content { + default_result = try(initial_lifecycle_hook.value.default_result, null) + heartbeat_timeout = try(initial_lifecycle_hook.value.heartbeat_timeout, null) + lifecycle_transition = initial_lifecycle_hook.value.lifecycle_transition + name = initial_lifecycle_hook.value.name + notification_metadata = try(initial_lifecycle_hook.value.notification_metadata, null) + notification_target_arn = try(initial_lifecycle_hook.value.notification_target_arn, null) + role_arn = try(initial_lifecycle_hook.value.role_arn, null) + } + } + + dynamic "instance_maintenance_policy" { + for_each = length(var.instance_maintenance_policy) > 0 ? [var.instance_maintenance_policy] : [] + + content { + min_healthy_percentage = instance_maintenance_policy.value.min_healthy_percentage + max_healthy_percentage = instance_maintenance_policy.value.max_healthy_percentage + } + } + + dynamic "instance_refresh" { + for_each = length(var.instance_refresh) > 0 ? [var.instance_refresh] : [] + + content { + dynamic "preferences" { + for_each = try([instance_refresh.value.preferences], []) + + content { + checkpoint_delay = try(preferences.value.checkpoint_delay, null) + checkpoint_percentages = try(preferences.value.checkpoint_percentages, null) + instance_warmup = try(preferences.value.instance_warmup, null) + max_healthy_percentage = try(preferences.value.max_healthy_percentage, null) + min_healthy_percentage = try(preferences.value.min_healthy_percentage, null) + scale_in_protected_instances = try(preferences.value.scale_in_protected_instances, null) + skip_matching = try(preferences.value.skip_matching, null) + standby_instances = try(preferences.value.standby_instances, null) + } + } + + strategy = instance_refresh.value.strategy + triggers = try(instance_refresh.value.triggers, null) + } + } + + dynamic "launch_template" { + for_each = var.use_mixed_instances_policy ? [] : [1] + + content { + id = local.launch_template_id + version = local.launch_template_version + } + } + + max_instance_lifetime = var.max_instance_lifetime + max_size = var.max_size + metrics_granularity = var.metrics_granularity + min_elb_capacity = var.min_elb_capacity + min_size = var.min_size + + ignore_failed_scaling_activities = var.ignore_failed_scaling_activities + + dynamic "mixed_instances_policy" { + for_each = var.use_mixed_instances_policy ? [var.mixed_instances_policy] : [] + + content { + dynamic "instances_distribution" { + for_each = try([mixed_instances_policy.value.instances_distribution], []) + + content { + on_demand_allocation_strategy = try(instances_distribution.value.on_demand_allocation_strategy, null) + on_demand_base_capacity = try(instances_distribution.value.on_demand_base_capacity, null) + on_demand_percentage_above_base_capacity = try(instances_distribution.value.on_demand_percentage_above_base_capacity, null) + spot_allocation_strategy = try(instances_distribution.value.spot_allocation_strategy, null) + spot_instance_pools = try(instances_distribution.value.spot_instance_pools, null) + spot_max_price = try(instances_distribution.value.spot_max_price, null) + } + } + + launch_template { + launch_template_specification { + launch_template_id = local.launch_template_id + version = local.launch_template_version + } + + dynamic "override" { + for_each = try(mixed_instances_policy.value.override, []) + + content { + dynamic "instance_requirements" { + for_each = try([override.value.instance_requirements], []) + + content { + + dynamic "accelerator_count" { + for_each = try([instance_requirements.value.accelerator_count], []) + + content { + max = try(accelerator_count.value.max, null) + min = try(accelerator_count.value.min, null) + } + } + + accelerator_manufacturers = try(instance_requirements.value.accelerator_manufacturers, []) + accelerator_names = try(instance_requirements.value.accelerator_names, []) + + dynamic "accelerator_total_memory_mib" { + for_each = try([instance_requirements.value.accelerator_total_memory_mib], []) + + content { + max = try(accelerator_total_memory_mib.value.max, null) + min = try(accelerator_total_memory_mib.value.min, null) + } + } + + accelerator_types = try(instance_requirements.value.accelerator_types, []) + allowed_instance_types = try(instance_requirements.value.allowed_instance_types, null) + bare_metal = try(instance_requirements.value.bare_metal, null) + + dynamic "baseline_ebs_bandwidth_mbps" { + for_each = try([instance_requirements.value.baseline_ebs_bandwidth_mbps], []) + + content { + max = try(baseline_ebs_bandwidth_mbps.value.max, null) + min = try(baseline_ebs_bandwidth_mbps.value.min, null) + } + } + + burstable_performance = try(instance_requirements.value.burstable_performance, null) + cpu_manufacturers = try(instance_requirements.value.cpu_manufacturers, []) + excluded_instance_types = try(instance_requirements.value.excluded_instance_types, []) + instance_generations = try(instance_requirements.value.instance_generations, []) + local_storage = try(instance_requirements.value.local_storage, null) + local_storage_types = try(instance_requirements.value.local_storage_types, []) + + dynamic "memory_gib_per_vcpu" { + for_each = try([instance_requirements.value.memory_gib_per_vcpu], []) + + content { + max = try(memory_gib_per_vcpu.value.max, null) + min = try(memory_gib_per_vcpu.value.min, null) + } + } + + dynamic "memory_mib" { + for_each = [instance_requirements.value.memory_mib] + + content { + max = try(memory_mib.value.max, null) + min = memory_mib.value.min + } + } + + dynamic "network_interface_count" { + for_each = try([instance_requirements.value.network_interface_count], []) + + content { + max = try(network_interface_count.value.max, null) + min = try(network_interface_count.value.min, null) + } + } + + on_demand_max_price_percentage_over_lowest_price = try(instance_requirements.value.on_demand_max_price_percentage_over_lowest_price, null) + require_hibernate_support = try(instance_requirements.value.require_hibernate_support, null) + spot_max_price_percentage_over_lowest_price = try(instance_requirements.value.spot_max_price_percentage_over_lowest_price, null) + + dynamic "total_local_storage_gb" { + for_each = try([instance_requirements.value.total_local_storage_gb], []) + + content { + max = try(total_local_storage_gb.value.max, null) + min = try(total_local_storage_gb.value.min, null) + } + } + + dynamic "vcpu_count" { + for_each = [instance_requirements.value.vcpu_count] + + content { + max = try(vcpu_count.value.max, null) + min = vcpu_count.value.min + } + } + } + } + + instance_type = try(override.value.instance_type, null) + + dynamic "launch_template_specification" { + for_each = try([override.value.launch_template_specification], []) + + content { + launch_template_id = try(launch_template_specification.value.launch_template_id, null) + version = try(launch_template_specification.value.version, null) + } + } + + weighted_capacity = try(override.value.weighted_capacity, null) + } + } + } + } + } + + name = var.use_name_prefix ? null : var.name + name_prefix = var.use_name_prefix ? "${var.name}-" : null + placement_group = var.placement_group + protect_from_scale_in = var.protect_from_scale_in + service_linked_role_arn = var.service_linked_role_arn + suspended_processes = var.suspended_processes + + dynamic "tag" { + for_each = merge( + { + "Name" = var.name + "kubernetes.io/cluster/${var.cluster_name}" = "owned" + "k8s.io/cluster/${var.cluster_name}" = "owned" + }, + var.tags + ) + + content { + key = tag.key + value = tag.value + propagate_at_launch = true + } + } + + dynamic "tag" { + for_each = var.autoscaling_group_tags + + content { + key = tag.key + value = tag.value + propagate_at_launch = false + } + } + + target_group_arns = var.target_group_arns + termination_policies = var.termination_policies + vpc_zone_identifier = local.enable_efa_support ? data.aws_subnets.placement_group[0].ids : var.subnet_ids + wait_for_capacity_timeout = var.wait_for_capacity_timeout + wait_for_elb_capacity = var.wait_for_elb_capacity + + dynamic "warm_pool" { + for_each = length(var.warm_pool) > 0 ? [var.warm_pool] : [] + + content { + dynamic "instance_reuse_policy" { + for_each = try([warm_pool.value.instance_reuse_policy], []) + + content { + reuse_on_scale_in = try(instance_reuse_policy.value.reuse_on_scale_in, null) + } + } + + max_group_prepared_capacity = try(warm_pool.value.max_group_prepared_capacity, null) + min_size = try(warm_pool.value.min_size, null) + pool_state = try(warm_pool.value.pool_state, null) + } + } + + timeouts { + delete = var.delete_timeout + } + + lifecycle { + create_before_destroy = true + ignore_changes = [ + desired_capacity + ] + } +} + +################################################################################ +# IAM Role +################################################################################ + +locals { + create_iam_instance_profile = var.create && var.create_iam_instance_profile + + iam_role_name = coalesce(var.iam_role_name, "${var.name}-node-group") + iam_role_policy_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + + ipv4_cni_policy = { for k, v in { + AmazonEKS_CNI_Policy = "${local.iam_role_policy_prefix}/AmazonEKS_CNI_Policy" + } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv4" } + ipv6_cni_policy = { for k, v in { + AmazonEKS_CNI_IPv6_Policy = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:policy/AmazonEKS_CNI_IPv6_Policy" + } : k => v if var.iam_role_attach_cni_policy && var.cluster_ip_family == "ipv6" } +} + +data "aws_iam_policy_document" "assume_role_policy" { + count = local.create_iam_instance_profile ? 1 : 0 + + statement { + sid = "EKSNodeAssumeRole" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "this" { + count = local.create_iam_instance_profile ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + description = var.iam_role_description + + assume_role_policy = data.aws_iam_policy_document.assume_role_policy[0].json + permissions_boundary = var.iam_role_permissions_boundary + force_detach_policies = true + + tags = merge(var.tags, var.iam_role_tags) +} + +# Policies attached ref https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group +resource "aws_iam_role_policy_attachment" "this" { + for_each = { for k, v in merge( + { + AmazonEKSWorkerNodePolicy = "${local.iam_role_policy_prefix}/AmazonEKSWorkerNodePolicy" + AmazonEC2ContainerRegistryReadOnly = "${local.iam_role_policy_prefix}/AmazonEC2ContainerRegistryReadOnly" + }, + local.ipv4_cni_policy, + local.ipv6_cni_policy + ) : k => v if local.create_iam_instance_profile } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +resource "aws_iam_role_policy_attachment" "additional" { + for_each = { for k, v in var.iam_role_additional_policies : k => v if local.create_iam_instance_profile } + + policy_arn = each.value + role = aws_iam_role.this[0].name +} + +resource "aws_iam_instance_profile" "this" { + count = local.create_iam_instance_profile ? 1 : 0 + + role = aws_iam_role.this[0].name + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + path = var.iam_role_path + + tags = merge(var.tags, var.iam_role_tags) + + lifecycle { + create_before_destroy = true + } +} + +################################################################################ +# IAM Role Policy +################################################################################ + +locals { + create_iam_role_policy = local.create_iam_instance_profile && var.create_iam_role_policy && length(var.iam_role_policy_statements) > 0 +} + +data "aws_iam_policy_document" "role" { + count = local.create_iam_role_policy ? 1 : 0 + + dynamic "statement" { + for_each = var.iam_role_policy_statements + + content { + sid = try(statement.value.sid, null) + actions = try(statement.value.actions, null) + not_actions = try(statement.value.not_actions, null) + effect = try(statement.value.effect, null) + resources = try(statement.value.resources, null) + not_resources = try(statement.value.not_resources, null) + + dynamic "principals" { + for_each = try(statement.value.principals, []) + + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + + dynamic "not_principals" { + for_each = try(statement.value.not_principals, []) + + content { + type = not_principals.value.type + identifiers = not_principals.value.identifiers + } + } + + dynamic "condition" { + for_each = try(statement.value.conditions, []) + + content { + test = condition.value.test + values = condition.value.values + variable = condition.value.variable + } + } + } + } +} + +resource "aws_iam_role_policy" "this" { + count = local.create_iam_role_policy ? 1 : 0 + + name = var.iam_role_use_name_prefix ? null : local.iam_role_name + name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null + policy = data.aws_iam_policy_document.role[0].json + role = aws_iam_role.this[0].id +} + +################################################################################ +# Placement Group +################################################################################ + +locals { + create_placement_group = var.create && (local.enable_efa_support || var.create_placement_group) +} + +resource "aws_placement_group" "this" { + count = local.create_placement_group ? 1 : 0 + + name = "${var.cluster_name}-${var.name}" + strategy = "cluster" + + tags = var.tags +} + +################################################################################ +# Instance AZ Lookup + +# Instances usually used in placement groups w/ EFA are only available in +# select availability zones. These data sources will cross reference the availability +# zones supported by the instance type with the subnets provided to ensure only +# AZs/subnets that are supported are used. +################################################################################ + +# Find the availability zones supported by the instance type +# TODO - remove at next breaking change +# Force users to be explicit about which AZ to use when using placement groups, +# with or without EFA support +data "aws_ec2_instance_type_offerings" "this" { + count = local.enable_efa_support ? 1 : 0 + + filter { + name = "instance-type" + values = [var.instance_type] + } + + location_type = "availability-zone-id" +} + +# Reverse the lookup to find one of the subnets provided based on the availability +# availability zone ID of the queried instance type (supported) +data "aws_subnets" "placement_group" { + count = local.create_placement_group ? 1 : 0 + + filter { + name = "subnet-id" + values = var.subnet_ids + } + + # The data source can lookup the first available AZ or you can specify an AZ (next filter) + dynamic "filter" { + for_each = local.create_placement_group && var.placement_group_az == null ? [1] : [] + + content { + name = "availability-zone-id" + values = data.aws_ec2_instance_type_offerings.this[0].locations + } + } + + dynamic "filter" { + for_each = var.placement_group_az != null ? [var.placement_group_az] : [] + + content { + name = "availability-zone" + values = [filter.value] + } + } +} + +################################################################################ +# Access Entry +################################################################################ + +resource "aws_eks_access_entry" "this" { + count = var.create && var.create_access_entry ? 1 : 0 + + cluster_name = var.cluster_name + principal_arn = var.create_iam_instance_profile ? aws_iam_role.this[0].arn : var.iam_role_arn + type = local.user_data_type == "windows" ? "EC2_WINDOWS" : "EC2_LINUX" + + tags = var.tags +} + +################################################################################ +# Autoscaling group schedule +################################################################################ + +resource "aws_autoscaling_schedule" "this" { + for_each = { for k, v in var.schedules : k => v if var.create && var.create_schedule } + + scheduled_action_name = each.key + autoscaling_group_name = aws_autoscaling_group.this[0].name + + min_size = try(each.value.min_size, null) + max_size = try(each.value.max_size, null) + desired_capacity = try(each.value.desired_size, null) + start_time = try(each.value.start_time, null) + end_time = try(each.value.end_time, null) + time_zone = try(each.value.time_zone, null) + + # [Minute] [Hour] [Day_of_Month] [Month_of_Year] [Day_of_Week] + # Cron examples: https://crontab.guru/examples.html + recurrence = try(each.value.recurrence, null) +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/migrations.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/migrations.tf new file mode 100644 index 00000000..5d51a720 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/migrations.tf @@ -0,0 +1,20 @@ +################################################################################ +# Migrations: v20.7 -> v20.8 +################################################################################ + +# Node IAM role policy attachment +# Commercial partition only - `moved` does now allow multiple moves to same target +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"] + to = aws_iam_role_policy_attachment.this["AmazonEKSWorkerNodePolicy"] +} + +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"] + to = aws_iam_role_policy_attachment.this["AmazonEC2ContainerRegistryReadOnly"] +} + +moved { + from = aws_iam_role_policy_attachment.this["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"] + to = aws_iam_role_policy_attachment.this["AmazonEKS_CNI_Policy"] +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/outputs.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/outputs.tf new file mode 100644 index 00000000..9607810a --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/outputs.tf @@ -0,0 +1,157 @@ +################################################################################ +# Launch template +################################################################################ + +output "launch_template_id" { + description = "The ID of the launch template" + value = try(aws_launch_template.this[0].id, null) +} + +output "launch_template_arn" { + description = "The ARN of the launch template" + value = try(aws_launch_template.this[0].arn, null) +} + +output "launch_template_latest_version" { + description = "The latest version of the launch template" + value = try(aws_launch_template.this[0].latest_version, null) +} + +output "launch_template_name" { + description = "The name of the launch template" + value = try(aws_launch_template.this[0].name, null) +} + +################################################################################ +# autoscaling group +################################################################################ + +output "autoscaling_group_arn" { + description = "The ARN for this autoscaling group" + value = try(aws_autoscaling_group.this[0].arn, null) +} + +output "autoscaling_group_id" { + description = "The autoscaling group id" + value = try(aws_autoscaling_group.this[0].id, null) +} + +output "autoscaling_group_name" { + description = "The autoscaling group name" + value = try(aws_autoscaling_group.this[0].name, null) +} + +output "autoscaling_group_min_size" { + description = "The minimum size of the autoscaling group" + value = try(aws_autoscaling_group.this[0].min_size, null) +} + +output "autoscaling_group_max_size" { + description = "The maximum size of the autoscaling group" + value = try(aws_autoscaling_group.this[0].max_size, null) +} + +output "autoscaling_group_desired_capacity" { + description = "The number of Amazon EC2 instances that should be running in the group" + value = try(aws_autoscaling_group.this[0].desired_capacity, null) +} + +output "autoscaling_group_default_cooldown" { + description = "Time between a scaling activity and the succeeding scaling activity" + value = try(aws_autoscaling_group.this[0].default_cooldown, null) +} + +output "autoscaling_group_health_check_grace_period" { + description = "Time after instance comes into service before checking health" + value = try(aws_autoscaling_group.this[0].health_check_grace_period, null) +} + +output "autoscaling_group_health_check_type" { + description = "EC2 or ELB. Controls how health checking is done" + value = try(aws_autoscaling_group.this[0].health_check_type, null) +} + +output "autoscaling_group_availability_zones" { + description = "The availability zones of the autoscaling group" + value = try(aws_autoscaling_group.this[0].availability_zones, null) +} + +output "autoscaling_group_vpc_zone_identifier" { + description = "The VPC zone identifier" + value = try(aws_autoscaling_group.this[0].vpc_zone_identifier, null) +} + +################################################################################ +# IAM Role +################################################################################ + +output "iam_role_name" { + description = "The name of the IAM role" + value = try(aws_iam_role.this[0].name, null) +} + +output "iam_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the IAM role" + value = try(aws_iam_role.this[0].arn, null) +} + +output "iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.this[0].unique_id, null) +} + +################################################################################ +# IAM Instance Profile +################################################################################ + +output "iam_instance_profile_arn" { + description = "ARN assigned by AWS to the instance profile" + value = try(aws_iam_instance_profile.this[0].arn, var.iam_instance_profile_arn) +} + +output "iam_instance_profile_id" { + description = "Instance profile's ID" + value = try(aws_iam_instance_profile.this[0].id, null) +} + +output "iam_instance_profile_unique" { + description = "Stable and unique string identifying the IAM instance profile" + value = try(aws_iam_instance_profile.this[0].unique_id, null) +} + +################################################################################ +# Access Entry +################################################################################ + +output "access_entry_arn" { + description = "Amazon Resource Name (ARN) of the Access Entry" + value = try(aws_eks_access_entry.this[0].access_entry_arn, null) +} + +################################################################################ +# Autoscaling Group Schedule +################################################################################ + +output "autoscaling_group_schedule_arns" { + description = "ARNs of autoscaling group schedules" + value = { for k, v in aws_autoscaling_schedule.this : k => v.arn } +} + +################################################################################ +# Additional +################################################################################ + +output "platform" { + description = "[DEPRECATED - Will be removed in `v21.0`] Identifies the OS platform as `bottlerocket`, `linux` (AL2), `al2023`, or `windows`" + value = module.user_data.platform +} + +output "image_id" { + description = "ID of the image" + value = try(aws_launch_template.this[0].image_id, null) +} + +output "user_data" { + description = "Base64 encoded user data" + value = try(module.user_data.user_data, null) +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/variables.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/variables.tf new file mode 100644 index 00000000..dbbc68e6 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/modules/self-managed-node-group/variables.tf @@ -0,0 +1,719 @@ +variable "create" { + description = "Determines whether to create self managed node group or not" + type = bool + default = true +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + +# tflint-ignore: terraform_unused_declarations +variable "platform" { + description = "[DEPRECATED - must use `ami_type` instead. Will be removed in `v21.0`]" + type = string + default = null + + validation { + condition = var.platform == null + error_message = "`platform` is no longer valid due to the number of OS choices. Please provide an [`ami_type`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-amitype) instead." + } +} + +################################################################################ +# User Data +################################################################################ + +variable "cluster_name" { + description = "Name of associated EKS cluster" + type = string + default = "" +} + +variable "cluster_endpoint" { + description = "Endpoint of associated EKS cluster" + type = string + default = "" +} + +variable "cluster_auth_base64" { + description = "Base64 encoded CA of associated EKS cluster" + type = string + default = "" +} + +variable "cluster_service_cidr" { + description = "The CIDR block (IPv4 or IPv6) used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself" + type = string + default = "" +} + +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`" + type = string + default = "ipv4" +} + +variable "additional_cluster_dns_ips" { + description = "Additional DNS IP addresses to use for the cluster. Only used when `ami_type` = `BOTTLEROCKET_*`" + type = list(string) + default = [] +} + +variable "pre_bootstrap_user_data" { + description = "User data that is injected into the user data script ahead of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" + type = string + default = "" +} + +variable "post_bootstrap_user_data" { + description = "User data that is appended to the user data script after of the EKS bootstrap script. Not used when `ami_type` = `BOTTLEROCKET_*`" + type = string + default = "" +} + +variable "bootstrap_extra_args" { + description = "Additional arguments passed to the bootstrap script. When `ami_type` = `BOTTLEROCKET_*`; these are additional [settings](https://github.com/bottlerocket-os/bottlerocket#settings) that are provided to the Bottlerocket user data" + type = string + default = "" +} + +variable "user_data_template_path" { + description = "Path to a local, custom user data template file to use when rendering user data" + type = string + default = "" +} + +variable "cloudinit_pre_nodeadm" { + description = "Array of cloud-init document parts that are created before the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} + +variable "cloudinit_post_nodeadm" { + description = "Array of cloud-init document parts that are created after the nodeadm document part" + type = list(object({ + content = string + content_type = optional(string) + filename = optional(string) + merge_type = optional(string) + })) + default = [] +} + +################################################################################ +# Launch template +################################################################################ + +variable "create_launch_template" { + description = "Determines whether to create launch template or not" + type = bool + default = true +} + +variable "launch_template_id" { + description = "The ID of an existing launch template to use. Required when `create_launch_template` = `false`" + type = string + default = "" +} + +variable "launch_template_name" { + description = "Name of launch template to be created" + type = string + default = null +} + +variable "launch_template_use_name_prefix" { + description = "Determines whether to use `launch_template_name` as is or create a unique name beginning with the `launch_template_name` as the prefix" + type = bool + default = true +} + +variable "launch_template_description" { + description = "Description of the launch template" + type = string + default = null +} + +variable "launch_template_default_version" { + description = "Default Version of the launch template" + type = string + default = null +} + +variable "update_launch_template_default_version" { + description = "Whether to update Default Version each update. Conflicts with `launch_template_default_version`" + type = bool + default = true +} + +variable "disable_api_termination" { + description = "If true, enables EC2 instance termination protection" + type = bool + default = null +} + +variable "instance_initiated_shutdown_behavior" { + description = "Shutdown behavior for the instance. Can be `stop` or `terminate`. (Default: `stop`)" + type = string + default = null +} + +variable "kernel_id" { + description = "The kernel ID" + type = string + default = null +} + +variable "ram_disk_id" { + description = "The ID of the ram disk" + type = string + default = null +} + +variable "block_device_mappings" { + description = "Specify volumes to attach to the instance besides the volumes specified by the AMI" + type = any + default = {} +} + +variable "capacity_reservation_specification" { + description = "Targeting for EC2 capacity reservations" + type = any + default = {} +} + +variable "cpu_options" { + description = "The CPU options for the instance" + type = map(string) + default = {} +} + +variable "credit_specification" { + description = "Customize the credit specification of the instance" + type = map(string) + default = {} +} + +variable "enclave_options" { + description = "Enable Nitro Enclaves on launched instances" + type = map(string) + default = {} +} + +variable "hibernation_options" { + description = "The hibernation options for the instance" + type = map(string) + default = {} +} + +variable "instance_market_options" { + description = "The market (purchasing) option for the instance" + type = any + default = {} +} + +variable "maintenance_options" { + description = "The maintenance options for the instance" + type = any + default = {} +} + +variable "license_specifications" { + description = "A map of license specifications to associate with" + type = any + default = {} +} + +variable "network_interfaces" { + description = "Customize network interfaces to be attached at instance boot time" + type = list(any) + default = [] +} + +variable "placement" { + description = "The placement of the instance" + type = map(string) + default = {} +} + +variable "create_placement_group" { + description = "Determines whether a placement group is created & used by the node group" + type = bool + default = false +} + +variable "private_dns_name_options" { + description = "The options for the instance hostname. The default values are inherited from the subnet" + type = map(string) + default = {} +} + +variable "ebs_optimized" { + description = "If true, the launched EC2 instance will be EBS-optimized" + type = bool + default = null +} + +variable "ami_id" { + description = "The AMI from which to launch the instance" + type = string + default = "" +} + +variable "ami_type" { + description = "Type of Amazon Machine Image (AMI) associated with the node group. See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/APIReference/API_Nodegroup.html#AmazonEKS-Type-Nodegroup-amiType) for valid values" + type = string + default = "AL2_x86_64" +} + +variable "cluster_version" { + description = "Kubernetes cluster version - used to lookup default AMI ID if one is not provided" + type = string + default = null +} + +variable "instance_requirements" { + description = "The attribute requirements for the type of instance. If present then `instance_type` cannot be present" + type = any + default = {} +} + +variable "instance_type" { + description = "The type of the instance to launch" + type = string + default = "" +} + +variable "key_name" { + description = "The key name that should be used for the instance" + type = string + default = null +} + +variable "vpc_security_group_ids" { + description = "A list of security group IDs to associate" + type = list(string) + default = [] +} + +variable "cluster_primary_security_group_id" { + description = "The ID of the EKS cluster primary security group to associate with the instance(s). This is the security group that is automatically created by the EKS service" + type = string + default = null +} + +variable "enable_monitoring" { + description = "Enables/disables detailed monitoring" + type = bool + default = true +} + +variable "enable_efa_support" { + description = "Determines whether to enable Elastic Fabric Adapter (EFA) support" + type = bool + default = false +} + +# TODO - make this true by default at next breaking change (remove variable, only pass indices) +variable "enable_efa_only" { + description = "Determines whether to enable EFA (`false`, default) or EFA and EFA-only (`true`) network interfaces. Note: requires vpc-cni version `v1.18.4` or later" + type = bool + default = false +} + +variable "efa_indices" { + description = "The indices of the network interfaces that should be EFA-enabled. Only valid when `enable_efa_support` = `true`" + type = list(number) + default = [0] +} + +variable "metadata_options" { + description = "Customize the metadata options for the instance" + type = map(string) + default = { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 2 + } +} + +variable "launch_template_tags" { + description = "A map of additional tags to add to the tag_specifications of launch template created" + type = map(string) + default = {} +} + +variable "tag_specifications" { + description = "The tags to apply to the resources during launch" + type = list(string) + default = ["instance", "volume", "network-interface"] +} + +################################################################################ +# Autoscaling group +################################################################################ + +variable "create_autoscaling_group" { + description = "Determines whether to create autoscaling group or not" + type = bool + default = true +} + +variable "name" { + description = "Name of the Self managed Node Group" + type = string + default = "" +} + +variable "use_name_prefix" { + description = "Determines whether to use `name` as is or create a unique name beginning with the `name` as the prefix" + type = bool + default = true +} + +variable "launch_template_version" { + description = "Launch template version. Can be version number, `$Latest`, or `$Default`" + type = string + default = null +} + +variable "availability_zones" { + description = "A list of one or more availability zones for the group. Used for EC2-Classic and default subnets when not specified with `subnet_ids` argument. Conflicts with `subnet_ids`" + type = list(string) + default = null +} + +variable "placement_group_az" { + description = "Availability zone where placement group is created (ex. `eu-west-1c`)" + type = string + default = null +} + +variable "subnet_ids" { + description = "A list of subnet IDs to launch resources in. Subnets automatically determine which availability zones the group will reside. Conflicts with `availability_zones`" + type = list(string) + default = null +} + +variable "min_size" { + description = "The minimum size of the autoscaling group" + type = number + default = 0 +} + +variable "max_size" { + description = "The maximum size of the autoscaling group" + type = number + default = 3 +} + +variable "desired_size" { + description = "The number of Amazon EC2 instances that should be running in the autoscaling group" + type = number + default = 1 +} + +variable "desired_size_type" { + description = "The unit of measurement for the value specified for `desired_size`. Supported for attribute-based instance type selection only. Valid values: `units`, `vcpu`, `memory-mib`" + type = string + default = null +} + +variable "ignore_failed_scaling_activities" { + description = "Whether to ignore failed Auto Scaling scaling activities while waiting for capacity." + type = bool + default = null +} + +variable "context" { + description = "Reserved" + type = string + default = null +} + +variable "capacity_rebalance" { + description = "Indicates whether capacity rebalance is enabled" + type = bool + default = null +} + +variable "min_elb_capacity" { + description = "Setting this causes Terraform to wait for this number of instances to show up healthy in the ELB only on creation. Updates will not wait on ELB instance number changes" + type = number + default = null +} + +variable "wait_for_elb_capacity" { + description = "Setting this will cause Terraform to wait for exactly this number of healthy instances in all attached load balancers on both create and update operations. Takes precedence over `min_elb_capacity` behavior." + type = number + default = null +} + +variable "wait_for_capacity_timeout" { + description = "A maximum duration that Terraform should wait for ASG instances to be healthy before timing out. (See also Waiting for Capacity below.) Setting this to '0' causes Terraform to skip all Capacity Waiting behavior." + type = string + default = null +} + +variable "default_cooldown" { + description = "The amount of time, in seconds, after a scaling activity completes before another scaling activity can start" + type = number + default = null +} + +variable "default_instance_warmup" { + description = "Amount of time, in seconds, until a newly launched instance can contribute to the Amazon CloudWatch metrics. This delay lets an instance finish initializing before Amazon EC2 Auto Scaling aggregates instance metrics, resulting in more reliable usage data" + type = number + default = null +} + +variable "protect_from_scale_in" { + description = "Allows setting instance protection. The autoscaling group will not select instances with this setting for termination during scale in events." + type = bool + default = false +} + +variable "target_group_arns" { + description = "A set of `aws_alb_target_group` ARNs, for use with Application or Network Load Balancing" + type = list(string) + default = [] +} + +variable "placement_group" { + description = "The name of the placement group into which you'll launch your instances, if any" + type = string + default = null +} + +variable "health_check_type" { + description = "`EC2` or `ELB`. Controls how health checking is done" + type = string + default = null +} + +variable "health_check_grace_period" { + description = "Time (in seconds) after instance comes into service before checking health" + type = number + default = null +} + +variable "force_delete" { + description = "Allows deleting the Auto Scaling Group without waiting for all instances in the pool to terminate. You can force an Auto Scaling Group to delete even if it's in the process of scaling a resource. Normally, Terraform drains all the instances before deleting the group. This bypasses that behavior and potentially leaves resources dangling" + type = bool + default = null +} + +variable "force_delete_warm_pool" { + description = "Allows deleting the Auto Scaling Group without waiting for all instances in the warm pool to terminate" + type = bool + default = null +} + +variable "termination_policies" { + description = "A list of policies to decide how the instances in the Auto Scaling Group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `OldestLaunchTemplate`, `AllocationStrategy`, `Default`" + type = list(string) + default = [] +} + +variable "suspended_processes" { + description = "A list of processes to suspend for the Auto Scaling Group. The allowed values are `Launch`, `Terminate`, `HealthCheck`, `ReplaceUnhealthy`, `AZRebalance`, `AlarmNotification`, `ScheduledActions`, `AddToLoadBalancer`. Note that if you suspend either the `Launch` or `Terminate` process types, it can prevent your Auto Scaling Group from functioning properly" + type = list(string) + default = [] +} + +variable "max_instance_lifetime" { + description = "The maximum amount of time, in seconds, that an instance can be in service, values must be either equal to 0 or between 604800 and 31536000 seconds" + type = number + default = null +} + +variable "enabled_metrics" { + description = "A list of metrics to collect. The allowed values are `GroupDesiredCapacity`, `GroupInServiceCapacity`, `GroupPendingCapacity`, `GroupMinSize`, `GroupMaxSize`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupStandbyCapacity`, `GroupTerminatingCapacity`, `GroupTerminatingInstances`, `GroupTotalCapacity`, `GroupTotalInstances`" + type = list(string) + default = [] +} + +variable "metrics_granularity" { + description = "The granularity to associate with the metrics to collect. The only valid value is `1Minute`" + type = string + default = null +} + +variable "service_linked_role_arn" { + description = "The ARN of the service-linked role that the ASG will use to call other AWS services" + type = string + default = null +} + +variable "initial_lifecycle_hooks" { + description = "One or more Lifecycle Hooks to attach to the Auto Scaling Group before instances are launched. The syntax is exactly the same as the separate `aws_autoscaling_lifecycle_hook` resource, without the `autoscaling_group_name` attribute. Please note that this will only work when creating a new Auto Scaling Group. For all other use-cases, please use `aws_autoscaling_lifecycle_hook` resource" + type = list(map(string)) + default = [] +} + +variable "instance_maintenance_policy" { + description = "If this block is configured, add a instance maintenance policy to the specified Auto Scaling group" + type = any + default = {} +} + +variable "instance_refresh" { + description = "If this block is configured, start an Instance Refresh when this Auto Scaling Group is updated" + type = any + default = { + strategy = "Rolling" + preferences = { + min_healthy_percentage = 66 + } + } +} + +variable "use_mixed_instances_policy" { + description = "Determines whether to use a mixed instances policy in the autoscaling group or not" + type = bool + default = false +} + +variable "mixed_instances_policy" { + description = "Configuration block containing settings to define launch targets for Auto Scaling groups" + type = any + default = null +} + +variable "warm_pool" { + description = "If this block is configured, add a Warm Pool to the specified Auto Scaling group" + type = any + default = {} +} + +variable "delete_timeout" { + description = "Delete timeout to wait for destroying autoscaling group" + type = string + default = null +} + +variable "autoscaling_group_tags" { + description = "A map of additional tags to add to the autoscaling group created. Tags are applied to the autoscaling group only and are NOT propagated to instances" + type = map(string) + default = {} +} + +################################################################################ +# IAM Role +################################################################################ + +variable "create_iam_instance_profile" { + description = "Determines whether an IAM instance profile is created or to use an existing IAM instance profile" + type = bool + default = true +} + +variable "iam_instance_profile_arn" { + description = "Amazon Resource Name (ARN) of an existing IAM instance profile that provides permissions for the node group. Required if `create_iam_instance_profile` = `false`" + type = string + default = null +} + +variable "iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = null +} + +variable "iam_role_use_name_prefix" { + description = "Determines whether cluster IAM role name (`iam_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "iam_role_path" { + description = "IAM role path" + type = string + default = null +} + +variable "iam_role_description" { + description = "Description of the role" + type = string + default = null +} + +variable "iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the IAM role" + type = string + default = null +} + +variable "iam_role_attach_cni_policy" { + description = "Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster" + type = bool + default = true +} + +variable "iam_role_additional_policies" { + description = "Additional policies to be added to the IAM role" + type = map(string) + default = {} +} + +variable "iam_role_tags" { + description = "A map of additional tags to add to the IAM role created" + type = map(string) + default = {} +} + +################################################################################ +# IAM Role Policy +################################################################################ + +variable "create_iam_role_policy" { + description = "Determines whether an IAM role policy is created or not" + type = bool + default = true +} + +variable "iam_role_policy_statements" { + description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed" + type = any + default = [] +} + +################################################################################ +# Access Entry +################################################################################ + +variable "create_access_entry" { + description = "Determines whether an access entry is created for the IAM role used by the node group" + type = bool + default = true +} + +variable "iam_role_arn" { + description = "ARN of the IAM role used by the instance profile. Required when `create_access_entry = true` and `create_iam_instance_profile = false`" + type = string + default = null +} + +################################################################################ +# Autoscaling group schedule +################################################################################ + +variable "create_schedule" { + description = "Determines whether to create autoscaling group schedule or not" + type = bool + default = true +} + +variable "schedules" { + description = "Map of autoscaling group schedule to create" + type = map(any) + default = {} +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/node_groups.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/node_groups.tf new file mode 100644 index 00000000..1cdebac7 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/node_groups.tf @@ -0,0 +1,563 @@ +locals { + metadata_options = { + http_endpoint = "enabled" + http_tokens = "required" + http_put_response_hop_limit = 2 + } + + # EKS managed node group + default_update_config = { + max_unavailable_percentage = 33 + } + + # Self-managed node group + default_instance_refresh = { + strategy = "Rolling" + preferences = { + min_healthy_percentage = 66 + } + } + + kubernetes_network_config = try(aws_eks_cluster.this[0].kubernetes_network_config[0], {}) +} + +# This sleep resource is used to provide a timed gap between the cluster creation and the downstream dependencies +# that consume the outputs from here. Any of the values that are used as triggers can be used in dependencies +# to ensure that the downstream resources are created after both the cluster is ready and the sleep time has passed. +# This was primarily added to give addons that need to be configured BEFORE data plane compute resources +# enough time to create and configure themselves before the data plane compute resources are created. +resource "time_sleep" "this" { + count = var.create ? 1 : 0 + + create_duration = var.dataplane_wait_duration + + triggers = { + cluster_name = aws_eks_cluster.this[0].id + cluster_endpoint = aws_eks_cluster.this[0].endpoint + cluster_version = aws_eks_cluster.this[0].version + cluster_service_cidr = var.cluster_ip_family == "ipv6" ? try(local.kubernetes_network_config.service_ipv6_cidr, "") : try(local.kubernetes_network_config.service_ipv4_cidr, "") + + cluster_certificate_authority_data = aws_eks_cluster.this[0].certificate_authority[0].data + } +} + +################################################################################ +# EKS IPV6 CNI Policy +# https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy +################################################################################ + +data "aws_iam_policy_document" "cni_ipv6_policy" { + count = var.create && var.create_cni_ipv6_iam_policy ? 1 : 0 + + statement { + sid = "AssignDescribe" + actions = [ + "ec2:AssignIpv6Addresses", + "ec2:DescribeInstances", + "ec2:DescribeTags", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeInstanceTypes" + ] + resources = ["*"] + } + + statement { + sid = "CreateTags" + actions = ["ec2:CreateTags"] + resources = ["arn:${local.partition}:ec2:*:*:network-interface/*"] + } +} + +# Note - we are keeping this to a minimum in hopes that its soon replaced with an AWS managed policy like `AmazonEKS_CNI_Policy` +resource "aws_iam_policy" "cni_ipv6_policy" { + count = var.create && var.create_cni_ipv6_iam_policy ? 1 : 0 + + # Will cause conflicts if trying to create on multiple clusters but necessary to reference by exact name in sub-modules + name = "AmazonEKS_CNI_IPv6_Policy" + description = "IAM policy for EKS CNI to assign IPV6 addresses" + policy = data.aws_iam_policy_document.cni_ipv6_policy[0].json + + tags = var.tags +} + +################################################################################ +# Node Security Group +# Defaults follow https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html +# Plus NTP/HTTPS (otherwise nodes fail to launch) +################################################################################ + +locals { + node_sg_name = coalesce(var.node_security_group_name, "${var.cluster_name}-node") + create_node_sg = var.create && var.create_node_security_group + + node_security_group_id = local.create_node_sg ? aws_security_group.node[0].id : var.node_security_group_id + + node_security_group_rules = { + ingress_cluster_443 = { + description = "Cluster API to node groups" + protocol = "tcp" + from_port = 443 + to_port = 443 + type = "ingress" + source_cluster_security_group = true + } + ingress_cluster_kubelet = { + description = "Cluster API to node kubelets" + protocol = "tcp" + from_port = 10250 + to_port = 10250 + type = "ingress" + source_cluster_security_group = true + } + ingress_self_coredns_tcp = { + description = "Node to node CoreDNS" + protocol = "tcp" + from_port = 53 + to_port = 53 + type = "ingress" + self = true + } + ingress_self_coredns_udp = { + description = "Node to node CoreDNS UDP" + protocol = "udp" + from_port = 53 + to_port = 53 + type = "ingress" + self = true + } + } + + node_security_group_recommended_rules = { for k, v in { + ingress_nodes_ephemeral = { + description = "Node to node ingress on ephemeral ports" + protocol = "tcp" + from_port = 1025 + to_port = 65535 + type = "ingress" + self = true + } + # metrics-server + ingress_cluster_4443_webhook = { + description = "Cluster API to node 4443/tcp webhook" + protocol = "tcp" + from_port = 4443 + to_port = 4443 + type = "ingress" + source_cluster_security_group = true + } + # prometheus-adapter + ingress_cluster_6443_webhook = { + description = "Cluster API to node 6443/tcp webhook" + protocol = "tcp" + from_port = 6443 + to_port = 6443 + type = "ingress" + source_cluster_security_group = true + } + # Karpenter + ingress_cluster_8443_webhook = { + description = "Cluster API to node 8443/tcp webhook" + protocol = "tcp" + from_port = 8443 + to_port = 8443 + type = "ingress" + source_cluster_security_group = true + } + # ALB controller, NGINX + ingress_cluster_9443_webhook = { + description = "Cluster API to node 9443/tcp webhook" + protocol = "tcp" + from_port = 9443 + to_port = 9443 + type = "ingress" + source_cluster_security_group = true + } + egress_all = { + description = "Allow all egress" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.cluster_ip_family == "ipv6" ? ["::/0"] : null + } + } : k => v if var.node_security_group_enable_recommended_rules } + + efa_security_group_rules = { for k, v in + { + ingress_all_self_efa = { + description = "Node to node EFA" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + egress_all_self_efa = { + description = "Node to node EFA" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + self = true + } + } : k => v if var.enable_efa_support + } +} + +resource "aws_security_group" "node" { + count = local.create_node_sg ? 1 : 0 + + name = var.node_security_group_use_name_prefix ? null : local.node_sg_name + name_prefix = var.node_security_group_use_name_prefix ? "${local.node_sg_name}${var.prefix_separator}" : null + description = var.node_security_group_description + vpc_id = var.vpc_id + + tags = merge( + var.tags, + { + "Name" = local.node_sg_name + "kubernetes.io/cluster/${var.cluster_name}" = "owned" + }, + var.node_security_group_tags + ) + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_security_group_rule" "node" { + for_each = { for k, v in merge( + local.efa_security_group_rules, + local.node_security_group_rules, + local.node_security_group_recommended_rules, + var.node_security_group_additional_rules, + ) : k => v if local.create_node_sg } + + # Required + security_group_id = aws_security_group.node[0].id + protocol = each.value.protocol + from_port = each.value.from_port + to_port = each.value.to_port + type = each.value.type + + # Optional + description = lookup(each.value, "description", null) + cidr_blocks = lookup(each.value, "cidr_blocks", null) + ipv6_cidr_blocks = lookup(each.value, "ipv6_cidr_blocks", null) + prefix_list_ids = lookup(each.value, "prefix_list_ids", []) + self = lookup(each.value, "self", null) + source_security_group_id = try(each.value.source_cluster_security_group, false) ? local.cluster_security_group_id : lookup(each.value, "source_security_group_id", null) +} + +################################################################################ +# Fargate Profile +################################################################################ + +module "fargate_profile" { + source = "./modules/fargate-profile" + + for_each = { for k, v in var.fargate_profiles : k => v if var.create && !local.create_outposts_local_cluster } + + create = try(each.value.create, true) + + # Fargate Profile + cluster_name = time_sleep.this[0].triggers["cluster_name"] + cluster_ip_family = var.cluster_ip_family + name = try(each.value.name, each.key) + subnet_ids = try(each.value.subnet_ids, var.fargate_profile_defaults.subnet_ids, var.subnet_ids) + selectors = try(each.value.selectors, var.fargate_profile_defaults.selectors, []) + timeouts = try(each.value.timeouts, var.fargate_profile_defaults.timeouts, {}) + + # IAM role + create_iam_role = try(each.value.create_iam_role, var.fargate_profile_defaults.create_iam_role, true) + iam_role_arn = try(each.value.iam_role_arn, var.fargate_profile_defaults.iam_role_arn, null) + iam_role_name = try(each.value.iam_role_name, var.fargate_profile_defaults.iam_role_name, null) + iam_role_use_name_prefix = try(each.value.iam_role_use_name_prefix, var.fargate_profile_defaults.iam_role_use_name_prefix, true) + iam_role_path = try(each.value.iam_role_path, var.fargate_profile_defaults.iam_role_path, null) + iam_role_description = try(each.value.iam_role_description, var.fargate_profile_defaults.iam_role_description, "Fargate profile IAM role") + iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.fargate_profile_defaults.iam_role_permissions_boundary, null) + iam_role_tags = try(each.value.iam_role_tags, var.fargate_profile_defaults.iam_role_tags, {}) + iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.fargate_profile_defaults.iam_role_attach_cni_policy, true) + # To better understand why this `lookup()` logic is required, see: + # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 + iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.fargate_profile_defaults, "iam_role_additional_policies", {})) + create_iam_role_policy = try(each.value.create_iam_role_policy, var.fargate_profile_defaults.create_iam_role_policy, true) + iam_role_policy_statements = try(each.value.iam_role_policy_statements, var.fargate_profile_defaults.iam_role_policy_statements, []) + + tags = merge(var.tags, try(each.value.tags, var.fargate_profile_defaults.tags, {})) +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +module "eks_managed_node_group" { + source = "./modules/eks-managed-node-group" + + for_each = { for k, v in var.eks_managed_node_groups : k => v if var.create && !local.create_outposts_local_cluster } + + create = try(each.value.create, true) + + cluster_name = time_sleep.this[0].triggers["cluster_name"] + cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, time_sleep.this[0].triggers["cluster_version"]) + + # EKS Managed Node Group + name = try(each.value.name, each.key) + use_name_prefix = try(each.value.use_name_prefix, var.eks_managed_node_group_defaults.use_name_prefix, true) + + subnet_ids = try(each.value.subnet_ids, var.eks_managed_node_group_defaults.subnet_ids, var.subnet_ids) + + min_size = try(each.value.min_size, var.eks_managed_node_group_defaults.min_size, 1) + max_size = try(each.value.max_size, var.eks_managed_node_group_defaults.max_size, 3) + desired_size = try(each.value.desired_size, var.eks_managed_node_group_defaults.desired_size, 1) + + ami_id = try(each.value.ami_id, var.eks_managed_node_group_defaults.ami_id, "") + ami_type = try(each.value.ami_type, var.eks_managed_node_group_defaults.ami_type, null) + ami_release_version = try(each.value.ami_release_version, var.eks_managed_node_group_defaults.ami_release_version, null) + use_latest_ami_release_version = try(each.value.use_latest_ami_release_version, var.eks_managed_node_group_defaults.use_latest_ami_release_version, false) + + capacity_type = try(each.value.capacity_type, var.eks_managed_node_group_defaults.capacity_type, null) + disk_size = try(each.value.disk_size, var.eks_managed_node_group_defaults.disk_size, null) + force_update_version = try(each.value.force_update_version, var.eks_managed_node_group_defaults.force_update_version, null) + instance_types = try(each.value.instance_types, var.eks_managed_node_group_defaults.instance_types, null) + labels = try(each.value.labels, var.eks_managed_node_group_defaults.labels, null) + node_repair_config = try(each.value.node_repair_config, var.eks_managed_node_group_defaults.node_repair_config, null) + remote_access = try(each.value.remote_access, var.eks_managed_node_group_defaults.remote_access, {}) + taints = try(each.value.taints, var.eks_managed_node_group_defaults.taints, {}) + update_config = try(each.value.update_config, var.eks_managed_node_group_defaults.update_config, local.default_update_config) + timeouts = try(each.value.timeouts, var.eks_managed_node_group_defaults.timeouts, {}) + + # User data + platform = try(each.value.platform, var.eks_managed_node_group_defaults.platform, "linux") + cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") + cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") + cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr + cluster_ip_family = var.cluster_ip_family + cluster_service_cidr = try(time_sleep.this[0].triggers["cluster_service_cidr"], "") + enable_bootstrap_user_data = try(each.value.enable_bootstrap_user_data, var.eks_managed_node_group_defaults.enable_bootstrap_user_data, false) + pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.eks_managed_node_group_defaults.pre_bootstrap_user_data, "") + post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.eks_managed_node_group_defaults.post_bootstrap_user_data, "") + bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.eks_managed_node_group_defaults.bootstrap_extra_args, "") + user_data_template_path = try(each.value.user_data_template_path, var.eks_managed_node_group_defaults.user_data_template_path, "") + cloudinit_pre_nodeadm = try(each.value.cloudinit_pre_nodeadm, var.eks_managed_node_group_defaults.cloudinit_pre_nodeadm, []) + cloudinit_post_nodeadm = try(each.value.cloudinit_post_nodeadm, var.eks_managed_node_group_defaults.cloudinit_post_nodeadm, []) + + # Launch Template + create_launch_template = try(each.value.create_launch_template, var.eks_managed_node_group_defaults.create_launch_template, true) + use_custom_launch_template = try(each.value.use_custom_launch_template, var.eks_managed_node_group_defaults.use_custom_launch_template, true) + launch_template_id = try(each.value.launch_template_id, var.eks_managed_node_group_defaults.launch_template_id, "") + launch_template_name = try(each.value.launch_template_name, var.eks_managed_node_group_defaults.launch_template_name, each.key) + launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.eks_managed_node_group_defaults.launch_template_use_name_prefix, true) + launch_template_version = try(each.value.launch_template_version, var.eks_managed_node_group_defaults.launch_template_version, null) + launch_template_default_version = try(each.value.launch_template_default_version, var.eks_managed_node_group_defaults.launch_template_default_version, null) + update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.eks_managed_node_group_defaults.update_launch_template_default_version, true) + launch_template_description = try(each.value.launch_template_description, var.eks_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} EKS managed node group") + launch_template_tags = try(each.value.launch_template_tags, var.eks_managed_node_group_defaults.launch_template_tags, {}) + tag_specifications = try(each.value.tag_specifications, var.eks_managed_node_group_defaults.tag_specifications, ["instance", "volume", "network-interface"]) + + ebs_optimized = try(each.value.ebs_optimized, var.eks_managed_node_group_defaults.ebs_optimized, null) + key_name = try(each.value.key_name, var.eks_managed_node_group_defaults.key_name, null) + disable_api_termination = try(each.value.disable_api_termination, var.eks_managed_node_group_defaults.disable_api_termination, null) + kernel_id = try(each.value.kernel_id, var.eks_managed_node_group_defaults.kernel_id, null) + ram_disk_id = try(each.value.ram_disk_id, var.eks_managed_node_group_defaults.ram_disk_id, null) + + block_device_mappings = try(each.value.block_device_mappings, var.eks_managed_node_group_defaults.block_device_mappings, {}) + capacity_reservation_specification = try(each.value.capacity_reservation_specification, var.eks_managed_node_group_defaults.capacity_reservation_specification, {}) + cpu_options = try(each.value.cpu_options, var.eks_managed_node_group_defaults.cpu_options, {}) + credit_specification = try(each.value.credit_specification, var.eks_managed_node_group_defaults.credit_specification, {}) + enclave_options = try(each.value.enclave_options, var.eks_managed_node_group_defaults.enclave_options, {}) + instance_market_options = try(each.value.instance_market_options, var.eks_managed_node_group_defaults.instance_market_options, {}) + license_specifications = try(each.value.license_specifications, var.eks_managed_node_group_defaults.license_specifications, {}) + metadata_options = try(each.value.metadata_options, var.eks_managed_node_group_defaults.metadata_options, local.metadata_options) + enable_monitoring = try(each.value.enable_monitoring, var.eks_managed_node_group_defaults.enable_monitoring, true) + enable_efa_support = try(each.value.enable_efa_support, var.eks_managed_node_group_defaults.enable_efa_support, false) + enable_efa_only = try(each.value.enable_efa_only, var.eks_managed_node_group_defaults.enable_efa_only, false) + efa_indices = try(each.value.efa_indices, var.eks_managed_node_group_defaults.efa_indices, [0]) + create_placement_group = try(each.value.create_placement_group, var.eks_managed_node_group_defaults.create_placement_group, false) + placement = try(each.value.placement, var.eks_managed_node_group_defaults.placement, {}) + placement_group_az = try(each.value.placement_group_az, var.eks_managed_node_group_defaults.placement_group_az, null) + placement_group_strategy = try(each.value.placement_group_strategy, var.eks_managed_node_group_defaults.placement_group_strategy, "cluster") + network_interfaces = try(each.value.network_interfaces, var.eks_managed_node_group_defaults.network_interfaces, []) + maintenance_options = try(each.value.maintenance_options, var.eks_managed_node_group_defaults.maintenance_options, {}) + private_dns_name_options = try(each.value.private_dns_name_options, var.eks_managed_node_group_defaults.private_dns_name_options, {}) + + # IAM role + create_iam_role = try(each.value.create_iam_role, var.eks_managed_node_group_defaults.create_iam_role, true) + iam_role_arn = try(each.value.iam_role_arn, var.eks_managed_node_group_defaults.iam_role_arn, null) + iam_role_name = try(each.value.iam_role_name, var.eks_managed_node_group_defaults.iam_role_name, null) + iam_role_use_name_prefix = try(each.value.iam_role_use_name_prefix, var.eks_managed_node_group_defaults.iam_role_use_name_prefix, true) + iam_role_path = try(each.value.iam_role_path, var.eks_managed_node_group_defaults.iam_role_path, null) + iam_role_description = try(each.value.iam_role_description, var.eks_managed_node_group_defaults.iam_role_description, "EKS managed node group IAM role") + iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.eks_managed_node_group_defaults.iam_role_permissions_boundary, null) + iam_role_tags = try(each.value.iam_role_tags, var.eks_managed_node_group_defaults.iam_role_tags, {}) + iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.eks_managed_node_group_defaults.iam_role_attach_cni_policy, true) + # To better understand why this `lookup()` logic is required, see: + # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 + iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.eks_managed_node_group_defaults, "iam_role_additional_policies", {})) + create_iam_role_policy = try(each.value.create_iam_role_policy, var.eks_managed_node_group_defaults.create_iam_role_policy, true) + iam_role_policy_statements = try(each.value.iam_role_policy_statements, var.eks_managed_node_group_defaults.iam_role_policy_statements, []) + + # Autoscaling group schedule + create_schedule = try(each.value.create_schedule, var.eks_managed_node_group_defaults.create_schedule, true) + schedules = try(each.value.schedules, var.eks_managed_node_group_defaults.schedules, {}) + + # Security group + vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.eks_managed_node_group_defaults.vpc_security_group_ids, []))) + cluster_primary_security_group_id = try(each.value.attach_cluster_primary_security_group, var.eks_managed_node_group_defaults.attach_cluster_primary_security_group, false) ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null + + tags = merge(var.tags, try(each.value.tags, var.eks_managed_node_group_defaults.tags, {})) +} + +################################################################################ +# Self Managed Node Group +################################################################################ + +module "self_managed_node_group" { + source = "./modules/self-managed-node-group" + + for_each = { for k, v in var.self_managed_node_groups : k => v if var.create } + + create = try(each.value.create, true) + + cluster_name = time_sleep.this[0].triggers["cluster_name"] + + # Autoscaling Group + create_autoscaling_group = try(each.value.create_autoscaling_group, var.self_managed_node_group_defaults.create_autoscaling_group, true) + + name = try(each.value.name, each.key) + use_name_prefix = try(each.value.use_name_prefix, var.self_managed_node_group_defaults.use_name_prefix, true) + + availability_zones = try(each.value.availability_zones, var.self_managed_node_group_defaults.availability_zones, null) + subnet_ids = try(each.value.subnet_ids, var.self_managed_node_group_defaults.subnet_ids, var.subnet_ids) + + min_size = try(each.value.min_size, var.self_managed_node_group_defaults.min_size, 0) + max_size = try(each.value.max_size, var.self_managed_node_group_defaults.max_size, 3) + desired_size = try(each.value.desired_size, var.self_managed_node_group_defaults.desired_size, 1) + desired_size_type = try(each.value.desired_size_type, var.self_managed_node_group_defaults.desired_size_type, null) + capacity_rebalance = try(each.value.capacity_rebalance, var.self_managed_node_group_defaults.capacity_rebalance, null) + min_elb_capacity = try(each.value.min_elb_capacity, var.self_managed_node_group_defaults.min_elb_capacity, null) + wait_for_elb_capacity = try(each.value.wait_for_elb_capacity, var.self_managed_node_group_defaults.wait_for_elb_capacity, null) + wait_for_capacity_timeout = try(each.value.wait_for_capacity_timeout, var.self_managed_node_group_defaults.wait_for_capacity_timeout, null) + default_cooldown = try(each.value.default_cooldown, var.self_managed_node_group_defaults.default_cooldown, null) + default_instance_warmup = try(each.value.default_instance_warmup, var.self_managed_node_group_defaults.default_instance_warmup, null) + protect_from_scale_in = try(each.value.protect_from_scale_in, var.self_managed_node_group_defaults.protect_from_scale_in, null) + context = try(each.value.context, var.self_managed_node_group_defaults.context, null) + + target_group_arns = try(each.value.target_group_arns, var.self_managed_node_group_defaults.target_group_arns, []) + create_placement_group = try(each.value.create_placement_group, var.self_managed_node_group_defaults.create_placement_group, false) + placement_group = try(each.value.placement_group, var.self_managed_node_group_defaults.placement_group, null) + placement_group_az = try(each.value.placement_group_az, var.self_managed_node_group_defaults.placement_group_az, null) + health_check_type = try(each.value.health_check_type, var.self_managed_node_group_defaults.health_check_type, null) + health_check_grace_period = try(each.value.health_check_grace_period, var.self_managed_node_group_defaults.health_check_grace_period, null) + + ignore_failed_scaling_activities = try(each.value.ignore_failed_scaling_activities, var.self_managed_node_group_defaults.ignore_failed_scaling_activities, null) + + force_delete = try(each.value.force_delete, var.self_managed_node_group_defaults.force_delete, null) + force_delete_warm_pool = try(each.value.force_delete_warm_pool, var.self_managed_node_group_defaults.force_delete_warm_pool, null) + termination_policies = try(each.value.termination_policies, var.self_managed_node_group_defaults.termination_policies, []) + suspended_processes = try(each.value.suspended_processes, var.self_managed_node_group_defaults.suspended_processes, []) + max_instance_lifetime = try(each.value.max_instance_lifetime, var.self_managed_node_group_defaults.max_instance_lifetime, null) + + enabled_metrics = try(each.value.enabled_metrics, var.self_managed_node_group_defaults.enabled_metrics, []) + metrics_granularity = try(each.value.metrics_granularity, var.self_managed_node_group_defaults.metrics_granularity, null) + service_linked_role_arn = try(each.value.service_linked_role_arn, var.self_managed_node_group_defaults.service_linked_role_arn, null) + + initial_lifecycle_hooks = try(each.value.initial_lifecycle_hooks, var.self_managed_node_group_defaults.initial_lifecycle_hooks, []) + instance_maintenance_policy = try(each.value.instance_maintenance_policy, var.self_managed_node_group_defaults.instance_maintenance_policy, {}) + instance_refresh = try(each.value.instance_refresh, var.self_managed_node_group_defaults.instance_refresh, local.default_instance_refresh) + use_mixed_instances_policy = try(each.value.use_mixed_instances_policy, var.self_managed_node_group_defaults.use_mixed_instances_policy, false) + mixed_instances_policy = try(each.value.mixed_instances_policy, var.self_managed_node_group_defaults.mixed_instances_policy, null) + warm_pool = try(each.value.warm_pool, var.self_managed_node_group_defaults.warm_pool, {}) + + delete_timeout = try(each.value.delete_timeout, var.self_managed_node_group_defaults.delete_timeout, null) + autoscaling_group_tags = try(each.value.autoscaling_group_tags, var.self_managed_node_group_defaults.autoscaling_group_tags, {}) + + # User data + platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, null) + # TODO - update this when `var.platform` is removed in v21.0 + ami_type = try(each.value.ami_type, var.self_managed_node_group_defaults.ami_type, "AL2_x86_64") + cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") + cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") + cluster_service_cidr = try(time_sleep.this[0].triggers["cluster_service_cidr"], "") + cluster_ip_family = var.cluster_ip_family + pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.self_managed_node_group_defaults.pre_bootstrap_user_data, "") + post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.self_managed_node_group_defaults.post_bootstrap_user_data, "") + bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.self_managed_node_group_defaults.bootstrap_extra_args, "") + user_data_template_path = try(each.value.user_data_template_path, var.self_managed_node_group_defaults.user_data_template_path, "") + cloudinit_pre_nodeadm = try(each.value.cloudinit_pre_nodeadm, var.self_managed_node_group_defaults.cloudinit_pre_nodeadm, []) + cloudinit_post_nodeadm = try(each.value.cloudinit_post_nodeadm, var.self_managed_node_group_defaults.cloudinit_post_nodeadm, []) + + # Launch Template + create_launch_template = try(each.value.create_launch_template, var.self_managed_node_group_defaults.create_launch_template, true) + launch_template_id = try(each.value.launch_template_id, var.self_managed_node_group_defaults.launch_template_id, "") + launch_template_name = try(each.value.launch_template_name, var.self_managed_node_group_defaults.launch_template_name, each.key) + launch_template_use_name_prefix = try(each.value.launch_template_use_name_prefix, var.self_managed_node_group_defaults.launch_template_use_name_prefix, true) + launch_template_version = try(each.value.launch_template_version, var.self_managed_node_group_defaults.launch_template_version, null) + launch_template_default_version = try(each.value.launch_template_default_version, var.self_managed_node_group_defaults.launch_template_default_version, null) + update_launch_template_default_version = try(each.value.update_launch_template_default_version, var.self_managed_node_group_defaults.update_launch_template_default_version, true) + launch_template_description = try(each.value.launch_template_description, var.self_managed_node_group_defaults.launch_template_description, "Custom launch template for ${try(each.value.name, each.key)} self managed node group") + launch_template_tags = try(each.value.launch_template_tags, var.self_managed_node_group_defaults.launch_template_tags, {}) + tag_specifications = try(each.value.tag_specifications, var.self_managed_node_group_defaults.tag_specifications, ["instance", "volume", "network-interface"]) + + ebs_optimized = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null) + ami_id = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "") + cluster_version = try(each.value.cluster_version, var.self_managed_node_group_defaults.cluster_version, time_sleep.this[0].triggers["cluster_version"]) + instance_type = try(each.value.instance_type, var.self_managed_node_group_defaults.instance_type, "m6i.large") + key_name = try(each.value.key_name, var.self_managed_node_group_defaults.key_name, null) + + disable_api_termination = try(each.value.disable_api_termination, var.self_managed_node_group_defaults.disable_api_termination, null) + instance_initiated_shutdown_behavior = try(each.value.instance_initiated_shutdown_behavior, var.self_managed_node_group_defaults.instance_initiated_shutdown_behavior, null) + kernel_id = try(each.value.kernel_id, var.self_managed_node_group_defaults.kernel_id, null) + ram_disk_id = try(each.value.ram_disk_id, var.self_managed_node_group_defaults.ram_disk_id, null) + + block_device_mappings = try(each.value.block_device_mappings, var.self_managed_node_group_defaults.block_device_mappings, {}) + capacity_reservation_specification = try(each.value.capacity_reservation_specification, var.self_managed_node_group_defaults.capacity_reservation_specification, {}) + cpu_options = try(each.value.cpu_options, var.self_managed_node_group_defaults.cpu_options, {}) + credit_specification = try(each.value.credit_specification, var.self_managed_node_group_defaults.credit_specification, {}) + enclave_options = try(each.value.enclave_options, var.self_managed_node_group_defaults.enclave_options, {}) + hibernation_options = try(each.value.hibernation_options, var.self_managed_node_group_defaults.hibernation_options, {}) + instance_requirements = try(each.value.instance_requirements, var.self_managed_node_group_defaults.instance_requirements, {}) + instance_market_options = try(each.value.instance_market_options, var.self_managed_node_group_defaults.instance_market_options, {}) + license_specifications = try(each.value.license_specifications, var.self_managed_node_group_defaults.license_specifications, {}) + metadata_options = try(each.value.metadata_options, var.self_managed_node_group_defaults.metadata_options, local.metadata_options) + enable_monitoring = try(each.value.enable_monitoring, var.self_managed_node_group_defaults.enable_monitoring, true) + enable_efa_support = try(each.value.enable_efa_support, var.self_managed_node_group_defaults.enable_efa_support, false) + enable_efa_only = try(each.value.enable_efa_only, var.self_managed_node_group_defaults.enable_efa_only, false) + efa_indices = try(each.value.efa_indices, var.self_managed_node_group_defaults.efa_indices, [0]) + network_interfaces = try(each.value.network_interfaces, var.self_managed_node_group_defaults.network_interfaces, []) + placement = try(each.value.placement, var.self_managed_node_group_defaults.placement, {}) + maintenance_options = try(each.value.maintenance_options, var.self_managed_node_group_defaults.maintenance_options, {}) + private_dns_name_options = try(each.value.private_dns_name_options, var.self_managed_node_group_defaults.private_dns_name_options, {}) + + # IAM role + create_iam_instance_profile = try(each.value.create_iam_instance_profile, var.self_managed_node_group_defaults.create_iam_instance_profile, true) + iam_instance_profile_arn = try(each.value.iam_instance_profile_arn, var.self_managed_node_group_defaults.iam_instance_profile_arn, null) + iam_role_name = try(each.value.iam_role_name, var.self_managed_node_group_defaults.iam_role_name, null) + iam_role_use_name_prefix = try(each.value.iam_role_use_name_prefix, var.self_managed_node_group_defaults.iam_role_use_name_prefix, true) + iam_role_path = try(each.value.iam_role_path, var.self_managed_node_group_defaults.iam_role_path, null) + iam_role_description = try(each.value.iam_role_description, var.self_managed_node_group_defaults.iam_role_description, "Self managed node group IAM role") + iam_role_permissions_boundary = try(each.value.iam_role_permissions_boundary, var.self_managed_node_group_defaults.iam_role_permissions_boundary, null) + iam_role_tags = try(each.value.iam_role_tags, var.self_managed_node_group_defaults.iam_role_tags, {}) + iam_role_attach_cni_policy = try(each.value.iam_role_attach_cni_policy, var.self_managed_node_group_defaults.iam_role_attach_cni_policy, true) + # To better understand why this `lookup()` logic is required, see: + # https://github.com/hashicorp/terraform/issues/31646#issuecomment-1217279031 + iam_role_additional_policies = lookup(each.value, "iam_role_additional_policies", lookup(var.self_managed_node_group_defaults, "iam_role_additional_policies", {})) + create_iam_role_policy = try(each.value.create_iam_role_policy, var.self_managed_node_group_defaults.create_iam_role_policy, true) + iam_role_policy_statements = try(each.value.iam_role_policy_statements, var.self_managed_node_group_defaults.iam_role_policy_statements, []) + + # Access entry + create_access_entry = try(each.value.create_access_entry, var.self_managed_node_group_defaults.create_access_entry, true) + iam_role_arn = try(each.value.iam_role_arn, var.self_managed_node_group_defaults.iam_role_arn, null) + + # Autoscaling group schedule + create_schedule = try(each.value.create_schedule, var.self_managed_node_group_defaults.create_schedule, true) + schedules = try(each.value.schedules, var.self_managed_node_group_defaults.schedules, {}) + + # Security group + vpc_security_group_ids = compact(concat([local.node_security_group_id], try(each.value.vpc_security_group_ids, var.self_managed_node_group_defaults.vpc_security_group_ids, []))) + cluster_primary_security_group_id = try(each.value.attach_cluster_primary_security_group, var.self_managed_node_group_defaults.attach_cluster_primary_security_group, false) ? aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id : null + + tags = merge(var.tags, try(each.value.tags, var.self_managed_node_group_defaults.tags, {})) +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/outputs.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/outputs.tf new file mode 100644 index 00000000..e9c95589 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/outputs.tf @@ -0,0 +1,279 @@ +locals { + dualstack_oidc_issuer_url = try(replace(replace(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "https://oidc.eks.", "https://oidc-eks."), ".amazonaws.com/", ".api.aws/"), null) +} + +################################################################################ +# Cluster +################################################################################ + +output "cluster_arn" { + description = "The Amazon Resource Name (ARN) of the cluster" + value = try(aws_eks_cluster.this[0].arn, null) + + depends_on = [ + aws_eks_access_entry.this, + aws_eks_access_policy_association.this, + ] +} + +output "cluster_certificate_authority_data" { + description = "Base64 encoded certificate data required to communicate with the cluster" + value = try(aws_eks_cluster.this[0].certificate_authority[0].data, null) + + depends_on = [ + aws_eks_access_entry.this, + aws_eks_access_policy_association.this, + ] +} + +output "cluster_endpoint" { + description = "Endpoint for your Kubernetes API server" + value = try(aws_eks_cluster.this[0].endpoint, null) + + depends_on = [ + aws_eks_access_entry.this, + aws_eks_access_policy_association.this, + ] +} + +output "cluster_id" { + description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" + value = try(aws_eks_cluster.this[0].id, "") +} + +output "cluster_name" { + description = "The name of the EKS cluster" + value = try(aws_eks_cluster.this[0].name, "") + + depends_on = [ + aws_eks_access_entry.this, + aws_eks_access_policy_association.this, + ] +} + +output "cluster_oidc_issuer_url" { + description = "The URL on the EKS cluster for the OpenID Connect identity provider" + value = try(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, null) +} + +output "cluster_dualstack_oidc_issuer_url" { + description = "Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider" + value = local.dualstack_oidc_issuer_url +} + +output "cluster_version" { + description = "The Kubernetes version for the cluster" + value = try(aws_eks_cluster.this[0].version, null) +} + +output "cluster_platform_version" { + description = "Platform version for the cluster" + value = try(aws_eks_cluster.this[0].platform_version, null) +} + +output "cluster_status" { + description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`" + value = try(aws_eks_cluster.this[0].status, null) +} + +output "cluster_primary_security_group_id" { + description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" + value = try(aws_eks_cluster.this[0].vpc_config[0].cluster_security_group_id, null) +} + +output "cluster_service_cidr" { + description = "The CIDR block where Kubernetes pod and service IP addresses are assigned from" + value = var.cluster_ip_family == "ipv6" ? try(aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv6_cidr, null) : try(aws_eks_cluster.this[0].kubernetes_network_config[0].service_ipv4_cidr, null) +} + +output "cluster_ip_family" { + description = "The IP family used by the cluster (e.g. `ipv4` or `ipv6`)" + value = try(aws_eks_cluster.this[0].kubernetes_network_config[0].ip_family, null) +} + +################################################################################ +# Access Entry +################################################################################ + +output "access_entries" { + description = "Map of access entries created and their attributes" + value = aws_eks_access_entry.this +} + +output "access_policy_associations" { + description = "Map of eks cluster access policy associations created and their attributes" + value = aws_eks_access_policy_association.this +} + +################################################################################ +# KMS Key +################################################################################ + +output "kms_key_arn" { + description = "The Amazon Resource Name (ARN) of the key" + value = module.kms.key_arn +} + +output "kms_key_id" { + description = "The globally unique identifier for the key" + value = module.kms.key_id +} + +output "kms_key_policy" { + description = "The IAM resource policy set on the key" + value = module.kms.key_policy +} + +################################################################################ +# Cluster Security Group +################################################################################ + +output "cluster_security_group_arn" { + description = "Amazon Resource Name (ARN) of the cluster security group" + value = try(aws_security_group.cluster[0].arn, null) +} + +output "cluster_security_group_id" { + description = "ID of the cluster security group" + value = try(aws_security_group.cluster[0].id, null) +} + +################################################################################ +# Node Security Group +################################################################################ + +output "node_security_group_arn" { + description = "Amazon Resource Name (ARN) of the node shared security group" + value = try(aws_security_group.node[0].arn, null) +} + +output "node_security_group_id" { + description = "ID of the node shared security group" + value = try(aws_security_group.node[0].id, null) +} + +################################################################################ +# IRSA +################################################################################ + +output "oidc_provider" { + description = "The OpenID Connect identity provider (issuer URL without leading `https://`)" + value = try(replace(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, "https://", ""), null) +} + +output "oidc_provider_arn" { + description = "The ARN of the OIDC Provider if `enable_irsa = true`" + value = try(aws_iam_openid_connect_provider.oidc_provider[0].arn, null) +} + +output "cluster_tls_certificate_sha1_fingerprint" { + description = "The SHA1 fingerprint of the public key of the cluster's certificate" + value = try(data.tls_certificate.this[0].certificates[0].sha1_fingerprint, null) +} + +################################################################################ +# IAM Role +################################################################################ + +output "cluster_iam_role_name" { + description = "Cluster IAM role name" + value = try(aws_iam_role.this[0].name, null) +} + +output "cluster_iam_role_arn" { + description = "Cluster IAM role ARN" + value = try(aws_iam_role.this[0].arn, null) +} + +output "cluster_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.this[0].unique_id, null) +} + +################################################################################ +# EKS Auto Node IAM Role +################################################################################ + +output "node_iam_role_name" { + description = "EKS Auto node IAM role name" + value = try(aws_iam_role.eks_auto[0].name, null) +} + +output "node_iam_role_arn" { + description = "EKS Auto node IAM role ARN" + value = try(aws_iam_role.eks_auto[0].arn, null) +} + +output "node_iam_role_unique_id" { + description = "Stable and unique string identifying the IAM role" + value = try(aws_iam_role.eks_auto[0].unique_id, null) +} + +################################################################################ +# EKS Addons +################################################################################ + +output "cluster_addons" { + description = "Map of attribute maps for all EKS cluster addons enabled" + value = merge(aws_eks_addon.this, aws_eks_addon.before_compute) +} + +################################################################################ +# EKS Identity Provider +################################################################################ + +output "cluster_identity_providers" { + description = "Map of attribute maps for all EKS identity providers enabled" + value = aws_eks_identity_provider_config.this +} + +################################################################################ +# CloudWatch Log Group +################################################################################ + +output "cloudwatch_log_group_name" { + description = "Name of cloudwatch log group created" + value = try(aws_cloudwatch_log_group.this[0].name, null) +} + +output "cloudwatch_log_group_arn" { + description = "Arn of cloudwatch log group created" + value = try(aws_cloudwatch_log_group.this[0].arn, null) +} + +################################################################################ +# Fargate Profile +################################################################################ + +output "fargate_profiles" { + description = "Map of attribute maps for all EKS Fargate Profiles created" + value = module.fargate_profile +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +output "eks_managed_node_groups" { + description = "Map of attribute maps for all EKS managed node groups created" + value = module.eks_managed_node_group +} + +output "eks_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by EKS managed node groups" + value = compact(flatten([for group in module.eks_managed_node_group : group.node_group_autoscaling_group_names])) +} + +################################################################################ +# Self Managed Node Group +################################################################################ + +output "self_managed_node_groups" { + description = "Map of attribute maps for all self managed node groups created" + value = module.self_managed_node_group +} + +output "self_managed_node_groups_autoscaling_group_names" { + description = "List of the autoscaling group names created by self-managed node groups" + value = compact([for group in module.self_managed_node_group : group.autoscaling_group_name]) +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/templates/al2023_user_data.tpl b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/templates/al2023_user_data.tpl new file mode 100644 index 00000000..cc360e6d --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/templates/al2023_user_data.tpl @@ -0,0 +1,11 @@ +%{ if enable_bootstrap_user_data ~} +--- +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: ${cluster_name} + apiServerEndpoint: ${cluster_endpoint} + certificateAuthority: ${cluster_auth_base64} + cidr: ${cluster_service_cidr} +%{ endif ~} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/templates/bottlerocket_user_data.tpl b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/templates/bottlerocket_user_data.tpl new file mode 100644 index 00000000..666d6660 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/templates/bottlerocket_user_data.tpl @@ -0,0 +1,8 @@ +%{ if enable_bootstrap_user_data ~} +[settings.kubernetes] +"cluster-name" = "${cluster_name}" +"api-server" = "${cluster_endpoint}" +"cluster-certificate" = "${cluster_auth_base64}" +"cluster-dns-ip" = ${cluster_dns_ips} +%{ endif ~} +${bootstrap_extra_args ~} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/templates/linux_user_data.tpl b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/templates/linux_user_data.tpl new file mode 100644 index 00000000..d75d549c --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/templates/linux_user_data.tpl @@ -0,0 +1,12 @@ +%{ if enable_bootstrap_user_data ~} +#!/bin/bash +set -e +%{ endif ~} +${pre_bootstrap_user_data ~} +%{ if enable_bootstrap_user_data ~} +B64_CLUSTER_CA=${cluster_auth_base64} +API_SERVER_URL=${cluster_endpoint} +/etc/eks/bootstrap.sh ${cluster_name} ${bootstrap_extra_args} --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL \ + --ip-family ${cluster_ip_family} --service-${cluster_ip_family}-cidr ${cluster_service_cidr} +${post_bootstrap_user_data ~} +%{ endif ~} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/templates/windows_user_data.tpl b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/templates/windows_user_data.tpl new file mode 100644 index 00000000..9721d3cc --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/templates/windows_user_data.tpl @@ -0,0 +1,13 @@ +%{ if enable_bootstrap_user_data ~} + +%{ endif ~} +${pre_bootstrap_user_data ~} +%{ if enable_bootstrap_user_data ~} +[string]$EKSBinDir = "$env:ProgramFiles\Amazon\EKS" +[string]$EKSBootstrapScriptName = 'Start-EKSBootstrap.ps1' +[string]$EKSBootstrapScriptFile = "$EKSBinDir\$EKSBootstrapScriptName" +& $EKSBootstrapScriptFile -EKSClusterName ${cluster_name} -APIServerEndpoint ${cluster_endpoint} -Base64ClusterCA ${cluster_auth_base64} ${bootstrap_extra_args} 3>&1 4>&1 5>&1 6>&1 +$LastError = if ($?) { 0 } else { $Error[0].Exception.HResult } +${post_bootstrap_user_data ~} + +%{ endif ~} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/variables.tf b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/variables.tf new file mode 100644 index 00000000..b4881b29 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/aws-terraform-eks/variables.tf @@ -0,0 +1,687 @@ +variable "create" { + description = "Controls if resources should be created (affects nearly all resources)" + type = bool + default = true +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} + +variable "prefix_separator" { + description = "The separator to use between the prefix and the generated timestamp for resource names" + type = string + default = "-" +} + +################################################################################ +# Cluster +################################################################################ + +variable "cluster_name" { + description = "Name of the EKS cluster" + type = string + default = "" +} + +variable "cluster_version" { + description = "Kubernetes `.` version to use for the EKS cluster (i.e.: `1.27`)" + type = string + default = null +} + +variable "cluster_enabled_log_types" { + description = "A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html)" + type = list(string) + default = ["audit", "api", "authenticator"] +} + +variable "cluster_force_update_version" { + description = "Force version update by overriding upgrade-blocking readiness checks when updating a cluster" + type = bool + default = null +} + +variable "authentication_mode" { + description = "The authentication mode for the cluster. Valid values are `CONFIG_MAP`, `API` or `API_AND_CONFIG_MAP`" + type = string + default = "API_AND_CONFIG_MAP" +} + +variable "cluster_compute_config" { + description = "Configuration block for the cluster compute configuration" + type = any + default = {} +} + +variable "cluster_upgrade_policy" { + description = "Configuration block for the cluster upgrade policy" + type = any + default = {} +} + +variable "cluster_remote_network_config" { + description = "Configuration block for the cluster remote network configuration" + type = any + default = {} +} + +variable "cluster_zonal_shift_config" { + description = "Configuration block for the cluster zonal shift" + type = any + default = {} +} + +variable "cluster_additional_security_group_ids" { + description = "List of additional, externally created security group IDs to attach to the cluster control plane" + type = list(string) + default = [] +} + +variable "control_plane_subnet_ids" { + description = "A list of subnet IDs where the EKS cluster control plane (ENIs) will be provisioned. Used for expanding the pool of subnets used by nodes/node groups without replacing the EKS control plane" + type = list(string) + default = [] +} + +variable "subnet_ids" { + description = "A list of subnet IDs where the nodes/node groups will be provisioned. If `control_plane_subnet_ids` is not provided, the EKS cluster control plane (ENIs) will be provisioned in these subnets" + type = list(string) + default = [] +} + +variable "cluster_endpoint_private_access" { + description = "Indicates whether or not the Amazon EKS private API server endpoint is enabled" + type = bool + default = true +} + +variable "cluster_endpoint_public_access" { + description = "Indicates whether or not the Amazon EKS public API server endpoint is enabled" + type = bool + default = false +} + +variable "cluster_endpoint_public_access_cidrs" { + description = "List of CIDR blocks which can access the Amazon EKS public API server endpoint" + type = list(string) + default = ["0.0.0.0/0"] +} + +variable "cluster_ip_family" { + description = "The IP family used to assign Kubernetes pod and service addresses. Valid values are `ipv4` (default) and `ipv6`. You can only specify an IP family when you create a cluster, changing this value will force a new cluster to be created" + type = string + default = "ipv4" +} + +variable "cluster_service_ipv4_cidr" { + description = "The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks" + type = string + default = null +} + +variable "cluster_service_ipv6_cidr" { + description = "The CIDR block to assign Kubernetes pod and service IP addresses from if `ipv6` was specified when the cluster was created. Kubernetes assigns service addresses from the unique local address range (fc00::/7) because you can't specify a custom IPv6 CIDR block when you create the cluster" + type = string + default = null +} + +variable "outpost_config" { + description = "Configuration for the AWS Outpost to provision the cluster on" + type = any + default = {} +} + +variable "cluster_encryption_config" { + description = "Configuration block with encryption configuration for the cluster. To disable secret encryption, set this value to `{}`" + type = any + default = { + resources = ["secrets"] + } +} + +variable "attach_cluster_encryption_policy" { + description = "Indicates whether or not to attach an additional policy for the cluster IAM role to utilize the encryption key provided" + type = bool + default = true +} + +variable "cluster_tags" { + description = "A map of additional tags to add to the cluster" + type = map(string) + default = {} +} + +variable "create_cluster_primary_security_group_tags" { + description = "Indicates whether or not to tag the cluster's primary security group. This security group is created by the EKS service, not the module, and therefore tagging is handled after cluster creation" + type = bool + default = true +} + +variable "cluster_timeouts" { + description = "Create, update, and delete timeout configurations for the cluster" + type = map(string) + default = {} +} + +# TODO - hard code to false on next breaking change +variable "bootstrap_self_managed_addons" { + description = "Indicates whether or not to bootstrap self-managed addons after the cluster has been created" + type = bool + default = null +} + +################################################################################ +# Access Entry +################################################################################ + +variable "access_entries" { + description = "Map of access entries to add to the cluster" + type = any + default = {} +} + +variable "enable_cluster_creator_admin_permissions" { + description = "Indicates whether or not to add the cluster creator (the identity used by Terraform) as an administrator via access entry" + type = bool + default = false +} + +################################################################################ +# KMS Key +################################################################################ + +variable "create_kms_key" { + description = "Controls if a KMS key for cluster encryption should be created" + type = bool + default = true +} + +variable "kms_key_description" { + description = "The description of the key as viewed in AWS console" + type = string + default = null +} + +variable "kms_key_deletion_window_in_days" { + description = "The waiting period, specified in number of days. After the waiting period ends, AWS KMS deletes the KMS key. If you specify a value, it must be between `7` and `30`, inclusive. If you do not specify a value, it defaults to `30`" + type = number + default = null +} + +variable "enable_kms_key_rotation" { + description = "Specifies whether key rotation is enabled" + type = bool + default = true +} + +variable "kms_key_enable_default_policy" { + description = "Specifies whether to enable the default key policy" + type = bool + default = true +} + +variable "kms_key_owners" { + description = "A list of IAM ARNs for those who will have full key permissions (`kms:*`)" + type = list(string) + default = [] +} + +variable "kms_key_administrators" { + description = "A list of IAM ARNs for [key administrators](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-administrators). If no value is provided, the current caller identity is used to ensure at least one key admin is available" + type = list(string) + default = [] +} + +variable "kms_key_users" { + description = "A list of IAM ARNs for [key users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-default-allow-users)" + type = list(string) + default = [] +} + +variable "kms_key_service_users" { + description = "A list of IAM ARNs for [key service users](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html#key-policy-service-integration)" + type = list(string) + default = [] +} + +variable "kms_key_source_policy_documents" { + description = "List of IAM policy documents that are merged together into the exported document. Statements must have unique `sid`s" + type = list(string) + default = [] +} + +variable "kms_key_override_policy_documents" { + description = "List of IAM policy documents that are merged together into the exported document. In merging, statements with non-blank `sid`s will override statements with the same `sid`" + type = list(string) + default = [] +} + +variable "kms_key_aliases" { + description = "A list of aliases to create. Note - due to the use of `toset()`, values must be static strings and not computed values" + type = list(string) + default = [] +} + +################################################################################ +# CloudWatch Log Group +################################################################################ + +variable "create_cloudwatch_log_group" { + description = "Determines whether a log group is created by this module for the cluster logs. If not, AWS will automatically create one if logging is enabled" + type = bool + default = true +} + +variable "cloudwatch_log_group_retention_in_days" { + description = "Number of days to retain log events. Default retention - 90 days" + type = number + default = 90 +} + +variable "cloudwatch_log_group_kms_key_id" { + description = "If a KMS Key ARN is set, this key will be used to encrypt the corresponding log group. Please be sure that the KMS Key has an appropriate key policy (https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html)" + type = string + default = null +} + +variable "cloudwatch_log_group_class" { + description = "Specified the log class of the log group. Possible values are: `STANDARD` or `INFREQUENT_ACCESS`" + type = string + default = null +} + +variable "cloudwatch_log_group_tags" { + description = "A map of additional tags to add to the cloudwatch log group created" + type = map(string) + default = {} +} + +################################################################################ +# Cluster Security Group +################################################################################ + +variable "create_cluster_security_group" { + description = "Determines if a security group is created for the cluster. Note: the EKS service creates a primary security group for the cluster by default" + type = bool + default = true +} + +variable "cluster_security_group_id" { + description = "Existing security group ID to be attached to the cluster" + type = string + default = "" +} + +variable "vpc_id" { + description = "ID of the VPC where the cluster security group will be provisioned" + type = string + default = null +} + +variable "cluster_security_group_name" { + description = "Name to use on cluster security group created" + type = string + default = null +} + +variable "cluster_security_group_use_name_prefix" { + description = "Determines whether cluster security group name (`cluster_security_group_name`) is used as a prefix" + type = bool + default = true +} + +variable "cluster_security_group_description" { + description = "Description of the cluster security group created" + type = string + default = "EKS cluster security group" +} + +variable "cluster_security_group_additional_rules" { + description = "List of additional security group rules to add to the cluster security group created. Set `source_node_security_group = true` inside rules to set the `node_security_group` as source" + type = any + default = {} +} + +variable "cluster_security_group_tags" { + description = "A map of additional tags to add to the cluster security group created" + type = map(string) + default = {} +} + +################################################################################ +# EKS IPV6 CNI Policy +################################################################################ + +variable "create_cni_ipv6_iam_policy" { + description = "Determines whether to create an [`AmazonEKS_CNI_IPv6_Policy`](https://docs.aws.amazon.com/eks/latest/userguide/cni-iam-role.html#cni-iam-role-create-ipv6-policy)" + type = bool + default = false +} + +################################################################################ +# Node Security Group +################################################################################ + +variable "create_node_security_group" { + description = "Determines whether to create a security group for the node groups or use the existing `node_security_group_id`" + type = bool + default = true +} + +variable "node_security_group_id" { + description = "ID of an existing security group to attach to the node groups created" + type = string + default = "" +} + +variable "node_security_group_name" { + description = "Name to use on node security group created" + type = string + default = null +} + +variable "node_security_group_use_name_prefix" { + description = "Determines whether node security group name (`node_security_group_name`) is used as a prefix" + type = bool + default = true +} + +variable "node_security_group_description" { + description = "Description of the node security group created" + type = string + default = "EKS node shared security group" +} + +variable "node_security_group_additional_rules" { + description = "List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source" + type = any + default = {} +} + +variable "node_security_group_enable_recommended_rules" { + description = "Determines whether to enable recommended security group rules for the node security group created. This includes node-to-node TCP ingress on ephemeral ports and allows all egress traffic" + type = bool + default = true +} + +variable "node_security_group_tags" { + description = "A map of additional tags to add to the node security group created" + type = map(string) + default = {} +} + +variable "enable_efa_support" { + description = "Determines whether to enable Elastic Fabric Adapter (EFA) support" + type = bool + default = false +} + +################################################################################ +# IRSA +################################################################################ + +variable "enable_irsa" { + description = "Determines whether to create an OpenID Connect Provider for EKS to enable IRSA" + type = bool + default = true +} + +variable "openid_connect_audiences" { + description = "List of OpenID Connect audience client IDs to add to the IRSA provider" + type = list(string) + default = [] +} + +variable "include_oidc_root_ca_thumbprint" { + description = "Determines whether to include the root CA thumbprint in the OpenID Connect (OIDC) identity provider's server certificate(s)" + type = bool + default = true +} + +variable "custom_oidc_thumbprints" { + description = "Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s)" + type = list(string) + default = [] +} + +################################################################################ +# Cluster IAM Role +################################################################################ + +variable "create_iam_role" { + description = "Determines whether an IAM role is created for the cluster" + type = bool + default = true +} + +variable "iam_role_arn" { + description = "Existing IAM role ARN for the cluster. Required if `create_iam_role` is set to `false`" + type = string + default = null +} + +variable "iam_role_name" { + description = "Name to use on IAM role created" + type = string + default = null +} + +variable "iam_role_use_name_prefix" { + description = "Determines whether the IAM role name (`iam_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "iam_role_path" { + description = "The IAM role path" + type = string + default = null +} + +variable "iam_role_description" { + description = "Description of the role" + type = string + default = null +} + +variable "iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the IAM role" + type = string + default = null +} + +variable "iam_role_additional_policies" { + description = "Additional policies to be added to the IAM role" + type = map(string) + default = {} +} + +# TODO - will be removed in next breaking change; user can add the policy on their own when needed +variable "enable_security_groups_for_pods" { + description = "Determines whether to add the necessary IAM permission policy for security groups for pods" + type = bool + default = true +} + +variable "iam_role_tags" { + description = "A map of additional tags to add to the IAM role created" + type = map(string) + default = {} +} + +variable "cluster_encryption_policy_use_name_prefix" { + description = "Determines whether cluster encryption policy name (`cluster_encryption_policy_name`) is used as a prefix" + type = bool + default = true +} + +variable "cluster_encryption_policy_name" { + description = "Name to use on cluster encryption policy created" + type = string + default = null +} + +variable "cluster_encryption_policy_description" { + description = "Description of the cluster encryption policy created" + type = string + default = "Cluster encryption policy to allow cluster role to utilize CMK provided" +} + +variable "cluster_encryption_policy_path" { + description = "Cluster encryption policy path" + type = string + default = null +} + +variable "cluster_encryption_policy_tags" { + description = "A map of additional tags to add to the cluster encryption policy created" + type = map(string) + default = {} +} + +variable "dataplane_wait_duration" { + description = "Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed node group(s), self-managed node group(s), Fargate profile(s))" + type = string + default = "30s" +} + +variable "enable_auto_mode_custom_tags" { + description = "Determines whether to enable permissions for custom tags resources created by EKS Auto Mode" + type = bool + default = true +} + +################################################################################ +# EKS Addons +################################################################################ + +variable "cluster_addons" { + description = "Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name`" + type = any + default = {} +} + +variable "cluster_addons_timeouts" { + description = "Create, update, and delete timeout configurations for the cluster addons" + type = map(string) + default = {} +} + +################################################################################ +# EKS Identity Provider +################################################################################ + +variable "cluster_identity_providers" { + description = "Map of cluster identity provider configurations to enable for the cluster. Note - this is different/separate from IRSA" + type = any + default = {} +} + +################################################################################ +# EKS Auto Node IAM Role +################################################################################ + +variable "create_node_iam_role" { + description = "Determines whether an EKS Auto node IAM role is created" + type = bool + default = true +} + +variable "node_iam_role_name" { + description = "Name to use on the EKS Auto node IAM role created" + type = string + default = null +} + +variable "node_iam_role_use_name_prefix" { + description = "Determines whether the EKS Auto node IAM role name (`node_iam_role_name`) is used as a prefix" + type = bool + default = true +} + +variable "node_iam_role_path" { + description = "The EKS Auto node IAM role path" + type = string + default = null +} + +variable "node_iam_role_description" { + description = "Description of the EKS Auto node IAM role" + type = string + default = null +} + +variable "node_iam_role_permissions_boundary" { + description = "ARN of the policy that is used to set the permissions boundary for the EKS Auto node IAM role" + type = string + default = null +} + +variable "node_iam_role_additional_policies" { + description = "Additional policies to be added to the EKS Auto node IAM role" + type = map(string) + default = {} +} + +variable "node_iam_role_tags" { + description = "A map of additional tags to add to the EKS Auto node IAM role created" + type = map(string) + default = {} +} + +################################################################################ +# Fargate +################################################################################ + +variable "fargate_profiles" { + description = "Map of Fargate Profile definitions to create" + type = any + default = {} +} + +variable "fargate_profile_defaults" { + description = "Map of Fargate Profile default configurations" + type = any + default = {} +} + +################################################################################ +# Self Managed Node Group +################################################################################ + +variable "self_managed_node_groups" { + description = "Map of self-managed node group definitions to create" + type = any + default = {} +} + +variable "self_managed_node_group_defaults" { + description = "Map of self-managed node group default configurations" + type = any + default = {} +} + +################################################################################ +# EKS Managed Node Group +################################################################################ + +variable "eks_managed_node_groups" { + description = "Map of EKS managed node group definitions to create" + type = any + default = {} +} + +variable "eks_managed_node_group_defaults" { + description = "Map of EKS managed node group default configurations" + type = any + default = {} +} + +variable "putin_khuylo" { + description = "Do you agree that Putin doesn't respect Ukrainian sovereignty and territorial integrity? More info: https://en.wikipedia.org/wiki/Putin_khuylo!" + type = bool + default = true +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/facets.yaml b/modules/kubernetes_cluster/eks_standard/1.0/facets.yaml new file mode 100644 index 00000000..1a26e974 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/facets.yaml @@ -0,0 +1,342 @@ +clouds: + - aws +description: | + Creates an AWS EKS cluster with managed node groups using the official terraform-aws-eks module. + This flavor does NOT support EKS Auto Mode - for Auto Mode support, use the eks_auto flavor. +flavor: eks_standard +inputs: + cloud_account: + description: The AWS Cloud Account where the EKS cluster will be created + displayName: Cloud Account + optional: false + providers: + - aws + type: '@facets/aws_cloud_account' + network_details: + default: + resource_name: default + resource_type: network + displayName: Network + type: '@facets/aws-vpc-details' +intent: kubernetes_cluster +outputs: + default: + type: '@facets/kubernetes-details' + title: Kubernetes Cluster Output + providers: + kubernetes: + source: hashicorp/kubernetes + version: 2.38.0 + attributes: + host: attributes.cluster_endpoint + cluster_ca_certificate: attributes.cluster_ca_certificate + exec: + api_version: attributes.kubernetes_provider_exec.api_version + command: attributes.kubernetes_provider_exec.command + args: attributes.kubernetes_provider_exec.args + helm: + source: hashicorp/helm + version: 2.17.0 + attributes: + kubernetes: + host: attributes.cluster_endpoint + cluster_ca_certificate: attributes.cluster_ca_certificate + exec: + api_version: attributes.kubernetes_provider_exec.api_version + command: attributes.kubernetes_provider_exec.command + args: attributes.kubernetes_provider_exec.args + kubernetes-alpha: + source: hashicorp/kubernetes-alpha + version: 0.6.0 + attributes: + host: attributes.cluster_endpoint + cluster_ca_certificate: attributes.cluster_ca_certificate + exec: + api_version: attributes.kubernetes_provider_exec.api_version + command: attributes.kubernetes_provider_exec.command + args: attributes.kubernetes_provider_exec.args + attributes: + type: '@facets/eks' + title: EKS Cluster Attributes + description: Additional EKS cluster attributes without provider configuration +sample: + disabled: false + flavor: eks_standard + kind: kubernetes_cluster + spec: + cluster_addons: + coredns: + enabled: true + version: latest + kube_proxy: + enabled: true + version: latest + vpc_cni: + enabled: true + version: latest + ebs_csi: + enabled: true + version: latest + cluster_endpoint_private_access: true + cluster_endpoint_public_access: true + cluster_version: "1.33" + enable_cluster_encryption: false + version: "1.0" +spec: + description: Configure your Amazon EKS cluster with managed node groups + properties: + cluster_addons: + description: Managed EKS add-ons configuration + properties: + additional_addons: + description: Additional custom EKS add-ons (e.g., aws-ebs-csi-driver, aws-efs-csi-driver, snapshot-controller) + patternProperties: + ^[a-zA-Z0-9_-]+$: + properties: + configuration_values: + description: Optional JSON configuration values for the add-on + title: Configuration Values (JSON) + type: string + x-ui-yaml-editor: true + enabled: + default: true + description: Enable this add-on + title: Enabled + type: boolean + service_account_role_arn: + description: Optional IAM role ARN for the add-on's service account (IRSA) + title: Service Account Role ARN + type: string + x-ui-placeholder: arn:aws:iam::123456789012:role/addon-role + version: + description: Version of the add-on (use 'latest' for latest version) + title: Version + type: string + x-ui-placeholder: latest + type: object + title: Additional Add-ons + type: object + x-ui-yaml-editor: true + coredns: + properties: + enabled: + default: true + title: Enable CoreDNS + type: boolean + version: + default: latest + title: Version + type: string + x-ui-visible-if: + field: spec.cluster_addons.coredns.enabled + values: + - true + title: CoreDNS + type: object + kube_proxy: + properties: + enabled: + default: true + title: Enable Kube Proxy + type: boolean + version: + default: latest + title: Version + type: string + x-ui-visible-if: + field: spec.cluster_addons.kube_proxy.enabled + values: + - true + title: Kube Proxy + type: object + vpc_cni: + properties: + enabled: + default: true + title: Enable VPC CNI + type: boolean + version: + default: latest + title: Version + type: string + x-ui-visible-if: + field: spec.cluster_addons.vpc_cni.enabled + values: + - true + title: VPC CNI + type: object + ebs_csi: + properties: + enabled: + default: true + title: Enable EBS CSI Driver + type: boolean + version: + default: latest + title: Version + type: string + x-ui-visible-if: + field: spec.cluster_addons.ebs_csi.enabled + values: + - true + title: EBS CSI Driver + type: object + title: EKS Add-ons + type: object + x-ui-toggle: true + cluster_endpoint_private_access: + default: true + description: Whether the cluster API server endpoint is accessible from within the VPC + title: Enable Private Endpoint Access + type: boolean + cluster_endpoint_public_access: + default: true + description: Whether the cluster API server endpoint is accessible from the internet + title: Enable Public Endpoint Access + type: boolean + cluster_tags: + description: Additional tags for the EKS cluster + title: Cluster Tags + type: object + x-ui-yaml-editor: true + cluster_version: + default: "1.30" + description: EKS Kubernetes version + enum: + - "1.28" + - "1.29" + - "1.30" + - "1.31" + - "1.32" + - "1.33" + - "1.34" + - "1.35" + - "1.36" + - "1.37" + title: Kubernetes Version + type: string + default_node_pool: + description: Default system node pool configuration for running system workloads (CoreDNS, Karpenter, etc.) + properties: + capacity_type: + default: ON_DEMAND + description: Type of capacity for the default system node pool + enum: + - ON_DEMAND + - SPOT + title: Capacity Type + type: string + desired_size: + default: 2 + description: Desired number of nodes in the default system pool + minimum: 1 + title: Desired Size + type: integer + disk_size: + default: 50 + description: Disk size in GB for default system nodes + maximum: 1000 + minimum: 20 + title: Disk Size (GB) + type: integer + instance_types: + default: + - t3.medium + description: List of EC2 instance types for the default system node pool + items: + type: string + title: Instance Types + type: array + max_size: + default: 3 + description: Maximum number of nodes in the default system pool + minimum: 1 + title: Maximum Size + type: integer + min_size: + default: 1 + description: Minimum number of nodes in the default system pool + minimum: 1 + title: Minimum Size + type: integer + title: Default System Node Pool + type: object + x-ui-toggle: true + enable_cluster_encryption: + default: false + description: Enable encryption of Kubernetes secrets using AWS KMS + title: Enable Secrets Encryption + type: boolean + managed_node_groups: + description: Configure managed node groups for the EKS cluster + patternProperties: + ^[a-zA-Z0-9_-]+$: + properties: + capacity_type: + default: ON_DEMAND + description: Type of capacity for the node group + enum: + - ON_DEMAND + - SPOT + title: Capacity Type + type: string + desired_size: + default: 2 + description: Desired number of nodes + minimum: 1 + title: Desired Size + type: integer + disk_size: + default: 50 + description: Disk size in GB for worker nodes + maximum: 1000 + minimum: 20 + title: Disk Size (GB) + type: integer + instance_types: + default: + - t3.medium + description: List of EC2 instance types for the node group + items: + type: string + title: Instance Types + type: array + labels: + description: Kubernetes labels to apply to nodes + title: Node Labels + type: object + x-ui-yaml-editor: true + max_size: + default: 10 + description: Maximum number of nodes + minimum: 1 + title: Maximum Size + type: integer + min_size: + default: 1 + description: Minimum number of nodes + minimum: 0 + title: Minimum Size + type: integer + taints: + description: Kubernetes taints to apply to nodes + title: Node Taints + type: object + x-ui-yaml-editor: true + type: object + title: Managed Node Groups + type: object + required: + - cluster_version + title: EKS Cluster Configuration + type: object + x-ui-order: + - cluster_version + - cluster_endpoint_public_access + - cluster_endpoint_private_access + - enable_cluster_encryption + - cluster_addons + - default_node_pool + - managed_node_groups + - cluster_tags +version: "1.0" diff --git a/modules/kubernetes_cluster/eks_standard/1.0/main.tf b/modules/kubernetes_cluster/eks_standard/1.0/main.tf new file mode 100644 index 00000000..3cd5adf6 --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/main.tf @@ -0,0 +1,243 @@ +locals { + # Construct cluster name and ensure it doesn't exceed AWS limits + # IAM role name_prefix in terraform-aws-eks appends "-cluster-" (9 chars) to cluster_name + # Total limit is 38 chars, so cluster_name should be max 29 chars + full_cluster_name = "${var.instance_name}-${var.environment.unique_name}" + cluster_name = length(local.full_cluster_name) > 29 ? substr(local.full_cluster_name, 0, 29) : local.full_cluster_name + + # Merge environment cloud tags with cluster-specific tags + cluster_tags = merge( + var.environment.cloud_tags, + lookup(var.instance.spec, "cluster_tags", {}), + { + "facets:instance_name" = var.instance_name + "facets:environment" = var.environment.name + } + ) + + # Default system node pool - always created for system workloads (CoreDNS, Karpenter, etc.) + # User can configure these values via spec.default_node_pool + default_system_node_group = { + system = { + name = "sys" # Short name to avoid IAM role name length limits + + instance_types = lookup(lookup(var.instance.spec, "default_node_pool", {}), "instance_types", ["t3.medium"]) + capacity_type = lookup(lookup(var.instance.spec, "default_node_pool", {}), "capacity_type", "ON_DEMAND") + + min_size = lookup(lookup(var.instance.spec, "default_node_pool", {}), "min_size", 1) + max_size = lookup(lookup(var.instance.spec, "default_node_pool", {}), "max_size", 3) + desired_size = lookup(lookup(var.instance.spec, "default_node_pool", {}), "desired_size", 2) + + disk_size = lookup(lookup(var.instance.spec, "default_node_pool", {}), "disk_size", 50) + + labels = { + "workload-type" = "system" + "node-role" = "system" + } + + taints = [] + + # Use the private subnets from the network input + subnet_ids = var.inputs.network_details.attributes.private_subnet_ids + + tags = merge( + local.cluster_tags, + { + "Name" = "${local.cluster_name}-system-node" + } + ) + } + } + + # Build managed node groups configuration and merge with default system pool + user_node_groups = { + for ng_name, ng_config in lookup(var.instance.spec, "managed_node_groups", {}) : ng_name => { + # Truncate node group name to avoid IAM role name length limits + # Keep first 10 chars of node group name + name = substr(ng_name, 0, 10) + + instance_types = lookup(ng_config, "instance_types", ["t3.medium"]) + capacity_type = lookup(ng_config, "capacity_type", "ON_DEMAND") + + min_size = lookup(ng_config, "min_size", 1) + max_size = lookup(ng_config, "max_size", 10) + desired_size = lookup(ng_config, "desired_size", 2) + + disk_size = lookup(ng_config, "disk_size", 50) + + labels = lookup(ng_config, "labels", {}) + + # Convert taints from map to list format expected by terraform-aws-eks + taints = [ + for key, value in lookup(ng_config, "taints", {}) : { + key = key + value = value + effect = "NoSchedule" + } + ] + + # Use the private subnets from the network input + subnet_ids = var.inputs.network_details.attributes.private_subnet_ids + + tags = local.cluster_tags + } + } + + # Merge default system node group with user-defined node groups + eks_managed_node_groups = merge( + local.default_system_node_group, + local.user_node_groups + ) + + # Check if EBS CSI driver addon is enabled (default: true) + ebs_csi_enabled = lookup(lookup(var.instance.spec.cluster_addons, "ebs_csi", {}), "enabled", true) + + # Build cluster addons configuration - default addons + default_addons = { + vpc-cni = lookup(var.instance.spec.cluster_addons.vpc_cni, "enabled", true) ? { + addon_version = lookup(var.instance.spec.cluster_addons.vpc_cni, "version", "latest") == "latest" ? null : lookup(var.instance.spec.cluster_addons.vpc_cni, "version", null) + resolve_conflicts = "OVERWRITE" + service_account_role_arn = null + } : null + + kube-proxy = lookup(var.instance.spec.cluster_addons.kube_proxy, "enabled", true) ? { + addon_version = lookup(var.instance.spec.cluster_addons.kube_proxy, "version", "latest") == "latest" ? null : lookup(var.instance.spec.cluster_addons.kube_proxy, "version", null) + resolve_conflicts = "OVERWRITE" + service_account_role_arn = null + } : null + + coredns = lookup(var.instance.spec.cluster_addons.coredns, "enabled", true) ? { + addon_version = lookup(var.instance.spec.cluster_addons.coredns, "version", "latest") == "latest" ? null : lookup(var.instance.spec.cluster_addons.coredns, "version", null) + resolve_conflicts = "OVERWRITE" + service_account_role_arn = null + } : null + + aws-ebs-csi-driver = local.ebs_csi_enabled ? { + addon_version = lookup(lookup(var.instance.spec.cluster_addons, "ebs_csi", {}), "version", "latest") == "latest" ? null : lookup(lookup(var.instance.spec.cluster_addons, "ebs_csi", {}), "version", null) + resolve_conflicts = "OVERWRITE" + service_account_role_arn = aws_iam_role.ebs_csi_driver[0].arn + } : null + } + + # Build additional/custom addons configuration + additional_addons = { + for addon_name, addon_config in lookup(var.instance.spec.cluster_addons, "additional_addons", {}) : + addon_name => lookup(addon_config, "enabled", true) ? { + addon_version = lookup(addon_config, "version", "latest") == "latest" ? null : lookup(addon_config, "version", null) + resolve_conflicts = "OVERWRITE" + configuration_values = lookup(addon_config, "configuration_values", null) + service_account_role_arn = lookup(addon_config, "service_account_role_arn", null) + } : null + } + + # Merge default and additional addons + cluster_addons_config = merge( + local.default_addons, + local.additional_addons + ) + + # Filter out disabled addons + enabled_cluster_addons = { + for addon_name, addon_config in local.cluster_addons_config : + addon_name => addon_config if addon_config != null + } + + # KMS key for secrets encryption (only if enabled) + enable_kms_key = lookup(var.instance.spec, "enable_cluster_encryption", false) +} + +# KMS key for EKS secrets encryption (conditional) +resource "aws_kms_key" "eks" { + count = local.enable_kms_key ? 1 : 0 + + description = "EKS Secret Encryption Key for ${local.cluster_name}" + deletion_window_in_days = 7 + enable_key_rotation = true + + tags = local.cluster_tags +} + +resource "aws_kms_alias" "eks" { + count = local.enable_kms_key ? 1 : 0 + + name = "alias/eks-${local.cluster_name}" + target_key_id = aws_kms_key.eks[0].key_id +} + +# EKS Cluster using the official terraform-aws-eks module +module "eks" { + source = "./aws-terraform-eks" + + cluster_name = local.cluster_name + cluster_version = var.instance.spec.cluster_version + + # Network configuration + vpc_id = var.inputs.network_details.attributes.vpc_id + subnet_ids = concat( + var.inputs.network_details.attributes.private_subnet_ids, + lookup(var.inputs.network_details.attributes, "public_subnet_ids", []) + ) + + # Cluster endpoint access + cluster_endpoint_public_access = lookup(var.instance.spec, "cluster_endpoint_public_access", true) + cluster_endpoint_private_access = lookup(var.instance.spec, "cluster_endpoint_private_access", true) + + # Secrets encryption configuration + cluster_encryption_config = local.enable_kms_key ? { + provider_key_arn = aws_kms_key.eks[0].arn + resources = ["secrets"] + } : {} + + # Managed node groups + eks_managed_node_groups = local.eks_managed_node_groups + + # Cluster addons + cluster_addons = local.enabled_cluster_addons + + # IMPORTANT: Explicitly disable EKS Auto Mode since this is eks_standard flavor + # EKS Auto Mode is NOT supported in this module variant + # For Auto Mode support, use the eks_auto flavor instead + enable_cluster_creator_admin_permissions = true + + tags = local.cluster_tags +} + +# Data source to get cluster authentication token +data "aws_eks_cluster_auth" "cluster" { + name = module.eks.cluster_name +} + +# IAM Role for EBS CSI Driver (IRSA) +resource "aws_iam_role" "ebs_csi_driver" { + count = local.ebs_csi_enabled ? 1 : 0 + + name = "${local.cluster_name}-ebs-csi-driver" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Federated = module.eks.oidc_provider_arn + } + Action = "sts:AssumeRoleWithWebIdentity" + Condition = { + StringEquals = { + "${module.eks.oidc_provider}:aud" = "sts.amazonaws.com" + "${module.eks.oidc_provider}:sub" = "system:serviceaccount:kube-system:ebs-csi-controller-sa" + } + } + } + ] + }) + + tags = local.cluster_tags +} + +resource "aws_iam_role_policy_attachment" "ebs_csi_driver" { + count = local.ebs_csi_enabled ? 1 : 0 + + role = aws_iam_role.ebs_csi_driver[0].name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy" +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/outputs.tf b/modules/kubernetes_cluster/eks_standard/1.0/outputs.tf new file mode 100644 index 00000000..7f7d0aef --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/outputs.tf @@ -0,0 +1,38 @@ +locals { + output_attributes = { + cluster_endpoint = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + cluster_name = module.eks.cluster_name + cluster_version = module.eks.cluster_version + cluster_arn = module.eks.cluster_arn + cluster_id = module.eks.cluster_id + oidc_issuer_url = module.eks.cluster_oidc_issuer_url + oidc_provider = module.eks.oidc_provider + oidc_provider_arn = module.eks.oidc_provider_arn + node_iam_role_arn = try(module.eks.eks_managed_node_groups["system"].iam_role_arn, "") + node_iam_role_name = try(module.eks.eks_managed_node_groups["system"].iam_role_name, "") + node_security_group_id = module.eks.node_security_group_id + cluster_iam_role_arn = module.eks.cluster_iam_role_arn + cluster_primary_security_group_id = module.eks.cluster_primary_security_group_id + cluster_security_group_id = module.eks.cluster_security_group_id + cloud_provider = "AWS" + kubernetes_provider_exec = { + api_version = "client.authentication.k8s.io/v1beta1" + command = "bash" + args = ["-c", "command -v aws-iam-authenticator >/dev/null 2>&1 || (curl -sLo /tmp/aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.7.8/aws-iam-authenticator_0.7.8_linux_amd64 && chmod +x /tmp/aws-iam-authenticator && mv /tmp/aws-iam-authenticator /usr/local/bin/aws-iam-authenticator); aws-iam-authenticator token -i ${module.eks.cluster_name} --role ${var.inputs.cloud_account.attributes.aws_iam_role} -s facets-k8s-${var.instance_name} -e ${var.inputs.cloud_account.attributes.external_id} --region ${var.inputs.cloud_account.attributes.aws_region}"] + } + secrets = ["cluster_ca_certificate", "kubernetes_provider_exec"] + } + output_interfaces = { + kubernetes = { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + kubernetes_provider_exec = { + api_version = "client.authentication.k8s.io/v1beta1" + command = "bash" + args = ["-c", "command -v aws-iam-authenticator >/dev/null 2>&1 || (curl -sLo /tmp/aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.7.8/aws-iam-authenticator_0.7.8_linux_amd64 && chmod +x /tmp/aws-iam-authenticator && mv /tmp/aws-iam-authenticator /usr/local/bin/aws-iam-authenticator); aws-iam-authenticator token -i ${module.eks.cluster_name} --role ${var.inputs.cloud_account.attributes.aws_iam_role} -s facets-k8s-${var.instance_name} -e ${var.inputs.cloud_account.attributes.external_id} --region ${var.inputs.cloud_account.attributes.aws_region}"] + } + secrets = ["cluster_ca_certificate", "kubernetes_provider_exec"] + } + } +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/variables.tf b/modules/kubernetes_cluster/eks_standard/1.0/variables.tf new file mode 100644 index 00000000..75ff0a3f --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/variables.tf @@ -0,0 +1,92 @@ +variable "instance" { + type = object({ + kind = string + flavor = string + version = string + spec = object({ + cluster_version = string + cluster_endpoint_public_access = optional(bool, true) + cluster_endpoint_private_access = optional(bool, true) + enable_cluster_encryption = optional(bool, false) + + cluster_addons = optional(object({ + vpc_cni = optional(object({ + enabled = optional(bool, true) + version = optional(string, "latest") + }), {}) + kube_proxy = optional(object({ + enabled = optional(bool, true) + version = optional(string, "latest") + }), {}) + coredns = optional(object({ + enabled = optional(bool, true) + version = optional(string, "latest") + }), {}) + ebs_csi = optional(object({ + enabled = optional(bool, true) + version = optional(string, "latest") + }), {}) + additional_addons = optional(map(object({ + enabled = optional(bool, true) + version = optional(string, "latest") + configuration_values = optional(string) + service_account_role_arn = optional(string) + })), {}) + }), {}) + + managed_node_groups = optional(map(object({ + instance_types = optional(list(string), ["t3.medium"]) + min_size = optional(number, 1) + max_size = optional(number, 10) + desired_size = optional(number, 2) + capacity_type = optional(string, "ON_DEMAND") + disk_size = optional(number, 50) + labels = optional(map(string), {}) + taints = optional(map(string), {}) + })), {}) + + cluster_tags = optional(map(string), {}) + }) + }) + + validation { + condition = contains(["1.28", "1.29", "1.30", "1.31", "1.32", "1.33", "1.34", "1.35", "1.36", "1.37"], var.instance.spec.cluster_version) + error_message = "Kubernetes version must be one of: 1.28, 1.29, 1.30, 1.31, 1.32, 1.33, 1.34, 1.35, 1.36, 1.37" + } +} + +variable "instance_name" { + type = string + description = "Unique architectural name from blueprint" +} + +variable "environment" { + type = object({ + name = string + unique_name = string + cloud_tags = optional(map(string), {}) + }) + description = "Environment context including name and cloud tags" +} + +variable "inputs" { + type = object({ + cloud_account = object({ + attributes = object({ + aws_region = string + aws_account_id = string + aws_iam_role = string + external_id = optional(string) + session_name = optional(string) + }) + }) + network_details = object({ + attributes = object({ + vpc_id = string + private_subnet_ids = list(string) + public_subnet_ids = optional(list(string), []) + }) + }) + }) + description = "Inputs from dependent modules" +} diff --git a/modules/kubernetes_cluster/eks_standard/1.0/versions.tf b/modules/kubernetes_cluster/eks_standard/1.0/versions.tf new file mode 100644 index 00000000..1793595b --- /dev/null +++ b/modules/kubernetes_cluster/eks_standard/1.0/versions.tf @@ -0,0 +1,6 @@ +terraform { + required_version = ">= 1.5.0, < 2.0.0" + + # NO required_providers block - providers come from inputs via Facets + # Provider versions are defined in the output type schemas +} diff --git a/modules/kubernetes_node_pool/karpenter/1.0/DEPLOYMENT.md b/modules/kubernetes_node_pool/karpenter/1.0/DEPLOYMENT.md new file mode 100644 index 00000000..1dd975a5 --- /dev/null +++ b/modules/kubernetes_node_pool/karpenter/1.0/DEPLOYMENT.md @@ -0,0 +1,234 @@ +# Karpenter Deployment Guide + +## Prerequisites Checklist + +Before deploying Karpenter, ensure: + +### 1. Tag Your Subnets +Private subnets must be tagged for Karpenter discovery: + +```bash +CLUSTER_NAME="your-cluster-name" +VPC_ID=$(aws eks describe-cluster --name $CLUSTER_NAME --query "cluster.resourcesVpcConfig.vpcId" --output text) + +# Get private subnet IDs +SUBNET_IDS=$(aws ec2 describe-subnets \ + --filters "Name=vpc-id,Values=$VPC_ID" "Name=tag:kubernetes.io/role/internal-elb,Values=1" \ + --query 'Subnets[*].SubnetId' --output text) + +# Tag subnets +for SUBNET_ID in $SUBNET_IDS; do + aws ec2 create-tags \ + --resources $SUBNET_ID \ + --tags Key=karpenter.sh/discovery,Value=$CLUSTER_NAME +done +``` + +### 2. Tag Security Groups +Node security groups must be tagged: + +```bash +# Get node security group ID from EKS +SG_ID=$(aws eks describe-cluster --name $CLUSTER_NAME \ + --query "cluster.resourcesVpcConfig.clusterSecurityGroupId" --output text) + +# Tag security group +aws ec2 create-tags \ + --resources $SG_ID \ + --tags Key=karpenter.sh/discovery,Value=$CLUSTER_NAME +``` + +### 3. Verify OIDC Provider +Check that your cluster has OIDC provider: + +```bash +aws eks describe-cluster --name $CLUSTER_NAME \ + --query "cluster.identity.oidc.issuer" --output text +``` + +## Deploy Karpenter Module + +### Via Facets + +```yaml +kind: autoscaler +flavor: karpenter +version: "1.0" +metadata: + name: karpenter-autoscaler +spec: + karpenter_version: "1.0.1" + enable_spot_instances: true + enable_consolidation: true + interruption_handling: true + + node_pools: + default: + cpu_limits: "1000" + memory_limits: "1000Gi" + instance_families: [t3, t3a, t2] + instance_sizes: [medium, large, xlarge, 2xlarge] + capacity_types: [spot, on-demand] + architecture: [amd64] +``` + +## Verification + +### 1. Check Karpenter Controller +```bash +kubectl get pods -n kube-system -l app.kubernetes.io/name=karpenter +kubectl logs -n kube-system -l app.kubernetes.io/name=karpenter -f +``` + +### 2. Check NodePools +```bash +kubectl get nodepools +kubectl get ec2nodeclasses +``` + +### 3. Test Autoscaling + +Deploy test workload: +```bash +kubectl apply -f example-deployment.yaml +kubectl scale deployment inflate --replicas=5 +``` + +Watch Karpenter provision nodes: +```bash +kubectl logs -n kube-system -l app.kubernetes.io/name=karpenter -f +kubectl get nodes -w +``` + +Expected log output: +``` +INFO controller.provisioner created nodeclaim {"commit": "abc123", "nodeclaim": "default-xyz"} +INFO controller.nodeclaim launched instance {"commit": "abc123", "instance": "i-1234567890abcdef0"} +``` + +## Post-Deployment Tasks + +### 1. Remove Cluster Autoscaler (if present) +```bash +helm uninstall cluster-autoscaler -n kube-system +``` + +### 2. Scale Down Existing Node Groups +If migrating from ASG-based nodes: + +```bash +# Gradually reduce ASG minimums +aws autoscaling update-auto-scaling-group \ + --auto-scaling-group-name \ + --min-size 0 \ + --desired-capacity 0 +``` + +### 3. Monitor Cost Savings +Track instance costs in AWS Cost Explorer with tag filter: +``` +facets:component = karpenter +``` + +## Troubleshooting + +### Nodes not being created + +Check: +1. **Subnet tags**: `karpenter.sh/discovery=` +2. **Security group tags**: `karpenter.sh/discovery=` +3. **NodePool limits**: Not exceeded +4. **IAM permissions**: Controller role has correct policies + +View detailed errors: +```bash +kubectl logs -n kube-system -l app.kubernetes.io/name=karpenter | grep ERROR +``` + +### Insufficient capacity errors + +If seeing "InsufficientInstanceCapacity": +1. Add more instance families to node pool +2. Add more instance sizes +3. Enable multiple availability zones +4. Mix Spot and On-Demand + +### Consolidation not working + +Check: +1. `enable_consolidation: true` in spec +2. Pods have appropriate PodDisruptionBudgets +3. No pods with local storage +4. No DaemonSets blocking + +## Performance Tuning + +### For Batch Workloads +```yaml +node_pools: + batch: + instance_families: [c5, c5a, c6i, c5n] # More options + capacity_types: [spot] # Spot only for batch + enable_consolidation: true +``` + +### For Stable Services +```yaml +node_pools: + services: + instance_families: [t3, t3a] + capacity_types: [on-demand] # On-demand for stability + enable_consolidation: false # Less disruption +``` + +### For ML/GPU Workloads +```yaml +node_pools: + gpu: + instance_families: [p3, p4, g4dn] + capacity_types: [on-demand] + labels: + accelerator: nvidia +``` + +## Monitoring + +### Key Metrics to Watch + +```promql +# Node provisioning rate +rate(karpenter_nodes_created[5m]) + +# Node termination rate +rate(karpenter_nodes_terminated[5m]) + +# Pod startup latency +histogram_quantile(0.95, karpenter_pods_startup_duration_seconds) + +# Consolidation actions +karpenter_consolidation_actions_performed +``` + +### CloudWatch Dashboard + +Create dashboard with: +- Active nodes by node pool +- Pod scheduling latency +- Interruption events +- Cost per node pool + +## Best Practices + +1. **Start Conservative**: Begin with On-Demand, add Spot gradually +2. **Multiple Instance Types**: More options = better availability +3. **Set Realistic Limits**: Prevent runaway costs +4. **Use Labels**: Organize workloads by node pool +5. **Monitor Costs**: Track spending by node pool +6. **Test Interruptions**: Verify graceful handling +7. **Regular Updates**: Keep Karpenter version current + +## Additional Resources + +- [Karpenter Official Docs](https://karpenter.sh/) +- [AWS EKS Best Practices](https://aws.github.io/aws-eks-best-practices/karpenter/) +- [Karpenter Workshop](https://www.eksworkshop.com/docs/autoscaling/compute/karpenter/) diff --git a/modules/kubernetes_node_pool/karpenter/1.0/README.md b/modules/kubernetes_node_pool/karpenter/1.0/README.md new file mode 100644 index 00000000..0ba4f0d3 --- /dev/null +++ b/modules/kubernetes_node_pool/karpenter/1.0/README.md @@ -0,0 +1,284 @@ +# Karpenter Autoscaler Module + +Deploys Karpenter autoscaler for Amazon EKS clusters with automatic node provisioning and cost optimization. + +## Overview + +Karpenter is a Kubernetes node autoscaler that directly provisions EC2 instances based on pod requirements, providing: + +- **Faster Scaling**: 3-5x faster than Cluster Autoscaler (30-60 seconds vs 3-5 minutes) +- **Cost Optimization**: 20-50% cost savings through intelligent instance selection +- **Flexibility**: Automatically selects from 600+ instance types +- **Spot Instance Support**: Advanced Spot/On-Demand mixing with automatic fallback +- **Consolidation**: Automatic bin-packing and node replacement + +## Prerequisites + +Before deploying this module, ensure: + +1. **EKS Cluster**: You have an existing EKS cluster deployed +2. **OIDC Provider**: The cluster has an OIDC provider configured +3. **Subnet Tags**: Private subnets must be tagged with: + ``` + karpenter.sh/discovery = + ``` +4. **Security Group Tags**: Node security groups must be tagged with: + ``` + karpenter.sh/discovery = + ``` + +## Quick Start + +### Basic Configuration + +```yaml +kind: autoscaler +flavor: karpenter +version: "1.0" +spec: + karpenter_version: "1.0.1" + enable_spot_instances: true + enable_consolidation: true + node_pools: + default: + instance_families: [t3, t3a] + instance_sizes: [medium, large, xlarge] + capacity_types: [on-demand, spot] +``` + +### Advanced Configuration with Multiple Node Pools + +```yaml +spec: + karpenter_version: "1.0.1" + enable_spot_instances: true + enable_consolidation: true + interruption_handling: true + + node_pools: + # General purpose workloads + general: + cpu_limits: "500" + memory_limits: "500Gi" + instance_families: [t3, t3a, t2] + instance_sizes: [medium, large, xlarge] + capacity_types: [spot, on-demand] + architecture: [amd64] + labels: + workload-type: general + + # Compute-intensive workloads + compute: + cpu_limits: "200" + memory_limits: "400Gi" + instance_families: [c5, c5a, c6i] + instance_sizes: [xlarge, 2xlarge, 4xlarge] + capacity_types: [spot, on-demand] + architecture: [amd64] + labels: + workload-type: compute + taints: + dedicated: compute + + # Memory-intensive workloads + memory: + cpu_limits: "100" + memory_limits: "800Gi" + instance_families: [r5, r5a, r6i] + instance_sizes: [large, xlarge, 2xlarge] + capacity_types: [on-demand] + architecture: [amd64] + labels: + workload-type: memory + taints: + dedicated: memory + + # ARM-based workloads (cost-optimized) + arm: + cpu_limits: "200" + memory_limits: "400Gi" + instance_families: [t4g, c6g, m6g] + instance_sizes: [medium, large, xlarge] + capacity_types: [spot, on-demand] + architecture: [arm64] + labels: + workload-type: arm + arch: arm64 + + tags: + Team: platform + CostCenter: engineering +``` + +## Configuration Reference + +### Core Settings + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `karpenter_version` | string | - | Karpenter version to deploy (required) | +| `enable_spot_instances` | bool | `true` | Allow Spot instance provisioning | +| `enable_consolidation` | bool | `true` | Enable automatic node consolidation | +| `interruption_handling` | bool | `true` | Enable Spot interruption handling | + +### Node Pool Configuration + +Each node pool supports: + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `cpu_limits` | string | `"1000"` | Max CPU cores for this pool | +| `memory_limits` | string | `"1000Gi"` | Max memory for this pool | +| `instance_families` | list(string) | `["t3", "t3a"]` | EC2 instance families | +| `instance_sizes` | list(string) | `["medium", "large", "xlarge"]` | Instance sizes | +| `capacity_types` | list(string) | `["on-demand", "spot"]` | Capacity types | +| `architecture` | list(string) | `["amd64"]` | CPU architecture | +| `labels` | map(string) | `{}` | Kubernetes node labels | +| `taints` | map(string) | `{}` | Kubernetes node taints | + +## How Karpenter Works + +### Provisioning Flow + +1. **Pod Pending**: Kubernetes scheduler cannot place a pod +2. **Analysis**: Karpenter analyzes pod requirements (CPU, memory, labels, taints) +3. **Instance Selection**: Karpenter selects optimal instance type from configured pools +4. **Provisioning**: Directly launches EC2 instance via AWS API (30-45 seconds) +5. **Scheduling**: Pod is scheduled on the new node + +### Consolidation + +When enabled, Karpenter continuously: +- Monitors node utilization +- Identifies under-utilized nodes +- Replaces multiple small nodes with fewer large nodes +- Deletes empty nodes after 30 seconds + +### Interruption Handling + +For Spot instances, Karpenter: +- Monitors AWS interruption events via SQS/EventBridge +- Gracefully drains nodes before termination +- Automatically provisions replacement capacity + +## Instance Selection Example + +For a pod requesting: +```yaml +resources: + requests: + cpu: "2" + memory: "4Gi" +nodeSelector: + workload-type: compute +``` + +Karpenter will: +1. Look at the `compute` node pool +2. Filter instance families: `[c5, c5a, c6i]` +3. Filter sizes: `[xlarge, 2xlarge, 4xlarge]` +4. Check Spot vs On-Demand pricing +5. Select cheapest option (likely `c5a.xlarge` on Spot) + +## Cost Optimization Tips + +1. **Enable Spot**: Save 70-90% vs On-Demand + ```yaml + capacity_types: [spot, on-demand] + ``` + +2. **Use Instance Families**: Allow multiple families for better Spot availability + ```yaml + instance_families: [c5, c5a, c6i, c5n] + ``` + +3. **Enable Consolidation**: Automatically optimize node usage + ```yaml + enable_consolidation: true + ``` + +4. **Consider ARM**: 20% cheaper than x86 + ```yaml + architecture: [arm64, amd64] + ``` + +## Monitoring + +### Key Metrics + +Karpenter exposes Prometheus metrics: + +``` +karpenter_nodes_created +karpenter_nodes_terminated +karpenter_pods_startup_duration_seconds +karpenter_consolidation_actions_performed +``` + +### Logs + +View Karpenter controller logs: +```bash +kubectl logs -n kube-system -l app.kubernetes.io/name=karpenter -f +``` + +## Troubleshooting + +### Pods not being scheduled + +Check: +1. Node pool limits not exceeded +2. Instance requirements can be satisfied +3. Subnet/Security group tags are correct + +### Nodes not terminating + +Check: +1. Consolidation is enabled +2. Pods have appropriate PodDisruptionBudgets +3. No local storage preventing eviction + +### Spot interruptions causing issues + +Enable interruption handling: +```yaml +interruption_handling: true +``` + +## Migration from Cluster Autoscaler + +1. Deploy Karpenter alongside Cluster Autoscaler +2. Create NodePools with similar capacity +3. Cordon existing ASG nodes +4. Scale down ASG minimums to 0 +5. Let Karpenter provision new nodes +6. Remove Cluster Autoscaler when ready + +## Outputs + +The module provides: + +- `karpenter_namespace`: Namespace where Karpenter is deployed +- `karpenter_service_account`: ServiceAccount name +- `karpenter_version`: Deployed version +- `controller_role_arn`: IAM role ARN for controller +- `node_role_arn`: IAM role ARN for nodes +- `node_instance_profile_name`: Instance profile name +- `interruption_queue_name`: SQS queue for interruptions + +## Resources Created + +- IAM Role: Karpenter Controller (with OIDC) +- IAM Role: Karpenter Nodes +- IAM Instance Profile: For EC2 nodes +- Helm Release: Karpenter controller +- SQS Queue: Interruption handling (if enabled) +- EventBridge Rules: Spot interruptions (if enabled) +- NodePools: Per configuration +- EC2NodeClasses: Per configuration + +## References + +- [Karpenter Documentation](https://karpenter.sh/) +- [AWS Workshop](https://www.eksworkshop.com/docs/autoscaling/compute/karpenter/) +- [Best Practices](https://aws.github.io/aws-eks-best-practices/karpenter/) diff --git a/modules/kubernetes_node_pool/karpenter/1.0/example-deployment.yaml b/modules/kubernetes_node_pool/karpenter/1.0/example-deployment.yaml new file mode 100644 index 00000000..05040854 --- /dev/null +++ b/modules/kubernetes_node_pool/karpenter/1.0/example-deployment.yaml @@ -0,0 +1,84 @@ +# Example: Deploy a workload that will trigger Karpenter to provision nodes +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate + namespace: default +spec: + replicas: 0 # Scale this up to test Karpenter + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: inflate + image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 + resources: + requests: + cpu: 1 + memory: 1.5Gi +--- +# Example: Deployment that targets specific node pool +apiVersion: apps/v1 +kind: Deployment +metadata: + name: compute-workload + namespace: default +spec: + replicas: 0 + selector: + matchLabels: + app: compute-workload + template: + metadata: + labels: + app: compute-workload + spec: + nodeSelector: + workload-type: compute # Targets 'compute' node pool + tolerations: + - key: dedicated + operator: Equal + value: compute + effect: NoSchedule + containers: + - name: compute + image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 + resources: + requests: + cpu: 2 + memory: 4Gi + +--- +# Example: ARM workload +apiVersion: apps/v1 +kind: Deployment +metadata: + name: arm-workload + namespace: default +spec: + replicas: 0 + selector: + matchLabels: + app: arm-workload + template: + metadata: + labels: + app: arm-workload + spec: + nodeSelector: + kubernetes.io/arch: arm64 + workload-type: arm + containers: + - name: arm + image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 + resources: + requests: + cpu: 1 + memory: 2Gi diff --git a/modules/kubernetes_node_pool/karpenter/1.0/facets.yaml b/modules/kubernetes_node_pool/karpenter/1.0/facets.yaml new file mode 100644 index 00000000..b49ff378 --- /dev/null +++ b/modules/kubernetes_node_pool/karpenter/1.0/facets.yaml @@ -0,0 +1,155 @@ +clouds: + - aws +description: | + Creates Karpenter NodePool and EC2NodeClass for EKS clusters. + This module requires the karpenter controller module to be deployed first. + Each instance creates a single NodePool. Create multiple instances for multiple node pools. +flavor: karpenter +inputs: + cloud_account: + description: The AWS Cloud Account for EC2 instance provisioning + displayName: Cloud Account + optional: false + providers: + - aws + type: '@facets/aws_cloud_account' + kubernetes_details: + description: The EKS cluster where NodePool will be created + displayName: Kubernetes Cluster + optional: false + providers: + - kubernetes + type: '@facets/kubernetes-details' + network_details: + default: + resource_name: default + resource_type: network + displayName: Network + type: '@facets/aws-vpc-details' + karpenter_details: + description: Karpenter controller details (from karpenter module) + displayName: Karpenter Controller + optional: false + type: '@facets/karpenter-details' +intent: kubernetes_node_pool +outputs: + default: + type: '@facets/kubernetes_nodepool' + title: Kubernetes Nodepool + attributes: + type: '@facets/aws_karpenter_nodepool' + title: AWS Karpenter NodePool +sample: + disabled: false + flavor: karpenter + kind: kubernetes_node_pool + spec: + instance_families: + - t3 + - t3a + instance_sizes: + - medium + - large + - xlarge + capacity_types: + - on-demand + - spot + architecture: + - amd64 + cpu_limits: "1000" + memory_limits: 1000Gi + enable_consolidation: true + version: "1.0" +spec: + description: Configure Karpenter NodePool for your EKS cluster + properties: + instance_families: + default: + - t3 + - t3a + description: EC2 instance families to use (e.g., t3, c5, r5, m5) + items: + type: string + title: Instance Families + type: array + x-ui-placeholder: t3, c5, m5 + instance_sizes: + default: + - medium + - large + - xlarge + description: Instance sizes to allow (e.g., medium, large, xlarge) + items: + type: string + title: Instance Sizes + type: array + x-ui-placeholder: medium, large + capacity_types: + default: + - on-demand + - spot + description: Capacity types (on-demand, spot, or both) + items: + enum: + - on-demand + - spot + type: string + title: Capacity Types + type: array + architecture: + default: + - amd64 + description: CPU architecture (amd64, arm64, or both) + items: + enum: + - amd64 + - arm64 + type: string + title: Architecture + type: array + cpu_limits: + default: "1000" + description: Maximum CPU cores across all nodes in this pool + title: CPU Limits + type: string + x-ui-placeholder: "1000" + memory_limits: + default: 1000Gi + description: Maximum memory across all nodes in this pool + title: Memory Limits + type: string + x-ui-placeholder: 1000Gi + enable_consolidation: + default: true + description: Enable automatic node consolidation to optimize costs + title: Enable Consolidation + type: boolean + labels: + description: Kubernetes labels to apply to provisioned nodes + title: Node Labels + type: object + x-ui-yaml-editor: true + taints: + description: Kubernetes taints to apply to provisioned nodes + title: Node Taints + type: object + x-ui-yaml-editor: true + tags: + description: Additional AWS tags to apply to provisioned instances + title: Instance Tags + type: object + x-ui-yaml-editor: true + title: Karpenter NodePool Configuration + type: object + x-ui-order: + - instance_families + - instance_sizes + - capacity_types + - architecture + - cpu_limits + - memory_limits + - enable_consolidation + - labels + - taints + - tags +version: "1.0" diff --git a/modules/kubernetes_node_pool/karpenter/1.0/main.tf b/modules/kubernetes_node_pool/karpenter/1.0/main.tf new file mode 100644 index 00000000..fd79aaaf --- /dev/null +++ b/modules/kubernetes_node_pool/karpenter/1.0/main.tf @@ -0,0 +1,32 @@ +locals { + cluster_name = var.inputs.kubernetes_details.attributes.cluster_name + + # Use values from karpenter_details input (from karpenter controller module) + node_instance_profile_name = var.inputs.karpenter_details.attributes.node_instance_profile_name + node_role_arn = var.inputs.karpenter_details.attributes.node_role_arn + karpenter_namespace = var.inputs.karpenter_details.attributes.karpenter_namespace + karpenter_service_account = var.inputs.karpenter_details.attributes.karpenter_service_account + + # Merge environment tags with instance tags + instance_tags = merge( + var.environment.cloud_tags, + lookup(var.instance.spec, "tags", {}), + { + "facets:instance_name" = var.instance_name + "facets:environment" = var.environment.name + "facets:component" = "karpenter-nodepool" + } + ) + + # Output values + node_class_name = "${var.instance_name}-nodeclass" + node_pool_name = "${var.instance_name}-nodepool" + taints = [ + for taint_key, taint_config in lookup(var.instance.spec, "taints", {}) : { + key = taint_key + value = taint_config.value + effect = taint_config.effect + } + ] + labels = lookup(var.instance.spec, "labels", {}) +} diff --git a/modules/kubernetes_node_pool/karpenter/1.0/nodepools.tf b/modules/kubernetes_node_pool/karpenter/1.0/nodepools.tf new file mode 100644 index 00000000..475ec521 --- /dev/null +++ b/modules/kubernetes_node_pool/karpenter/1.0/nodepools.tf @@ -0,0 +1,145 @@ +# Create EC2NodeClass for this instance +resource "kubernetes_manifest" "ec2_node_class" { + manifest = { + apiVersion = "karpenter.k8s.aws/v1" + kind = "EC2NodeClass" + metadata = { + name = "${var.instance_name}-nodeclass" + } + spec = { + # Use the instance profile from karpenter_details input + instanceProfile = local.node_instance_profile_name + + # Specify AMI family so Karpenter can generate correct UserData + # EKS 1.33 requires AL2023 + amiFamily = "AL2023" + + amiSelectorTerms = [ + { + # Use Amazon Linux 2023 EKS optimized AMI + alias = "al2023@latest" + } + ] + subnetSelectorTerms = [ + { + tags = { + "karpenter.sh/discovery" = local.cluster_name + } + } + ] + securityGroupSelectorTerms = [ + { + tags = { + "karpenter.sh/discovery" = local.cluster_name + } + } + ] + } + } + + # Ignore fields that Karpenter controller manages + computed_fields = [ + "metadata.finalizers", + "metadata.annotations", + "status" + ] + + # Allow Terraform to override field manager conflicts with Karpenter controller + field_manager { + force_conflicts = true + } +} + +# Create NodePool for this instance +resource "kubernetes_manifest" "node_pool" { + manifest = { + apiVersion = "karpenter.sh/v1" + kind = "NodePool" + metadata = { + name = "${var.instance_name}-nodepool" + } + spec = { + template = { + metadata = { + labels = lookup(var.instance.spec, "labels", {}) + } + spec = merge( + { + requirements = concat( + [ + { + key = "kubernetes.io/arch" + operator = "In" + values = lookup(var.instance.spec, "architecture", ["amd64"]) + }, + { + key = "kubernetes.io/os" + operator = "In" + values = ["linux"] + }, + { + key = "karpenter.sh/capacity-type" + operator = "In" + values = lookup(var.instance.spec, "capacity_types", ["on-demand", "spot"]) + }, + # Generate list of instance types from families and sizes + { + key = "node.kubernetes.io/instance-type" + operator = "In" + values = flatten([ + for family in lookup(var.instance.spec, "instance_families", ["t3", "t3a"]) : [ + for size in lookup(var.instance.spec, "instance_sizes", ["medium", "large", "xlarge"]) : + "${family}.${size}" + ] + ]) + } + ], + [] + ) + nodeClassRef = { + group = "karpenter.k8s.aws" + kind = "EC2NodeClass" + name = "${var.instance_name}-nodeclass" + } + expireAfter = "720h" + }, + # Add taints if configured + length(lookup(var.instance.spec, "taints", {})) > 0 ? { + taints = [ + for taint_key, taint_config in lookup(var.instance.spec, "taints", {}) : { + key = taint_key + value = taint_config.value + effect = taint_config.effect + } + ] + } : {} + ) + } + limits = { + cpu = lookup(var.instance.spec, "cpu_limits", "1000") + memory = lookup(var.instance.spec, "memory_limits", "1000Gi") + } + disruption = { + consolidationPolicy = lookup(var.instance.spec, "enable_consolidation", true) ? "WhenEmptyOrUnderutilized" : "WhenEmpty" + consolidateAfter = "1m" + } + } + } + + # Ignore fields that Karpenter controller manages + computed_fields = [ + "metadata.finalizers", + "metadata.annotations", + "metadata.labels", + "status" + ] + + # Allow Terraform to override field manager conflicts with Karpenter controller + field_manager { + force_conflicts = true + } + + depends_on = [ + kubernetes_manifest.ec2_node_class + ] +} diff --git a/modules/kubernetes_node_pool/karpenter/1.0/outputs.tf b/modules/kubernetes_node_pool/karpenter/1.0/outputs.tf new file mode 100644 index 00000000..30b7c13c --- /dev/null +++ b/modules/kubernetes_node_pool/karpenter/1.0/outputs.tf @@ -0,0 +1,15 @@ +locals { + output_attributes = { + node_pool_name = "${var.instance_name}-nodepool" + node_class_name = "${var.instance_name}-nodeclass" + taints = [ + for taint_key, taint_config in lookup(var.instance.spec, "taints", {}) : { + key = taint_key + value = taint_config.value + effect = taint_config.effect + } + ] + node_selector = lookup(var.instance.spec, "labels", {}) + } + output_interfaces = {} +} diff --git a/modules/kubernetes_node_pool/karpenter/1.0/variables.tf b/modules/kubernetes_node_pool/karpenter/1.0/variables.tf new file mode 100644 index 00000000..85808eb2 --- /dev/null +++ b/modules/kubernetes_node_pool/karpenter/1.0/variables.tf @@ -0,0 +1,98 @@ +variable "instance" { + type = object({ + kind = string + flavor = string + version = string + spec = object({ + # Node pool instance configuration + instance_families = optional(list(string), ["t3", "t3a"]) + instance_sizes = optional(list(string), ["medium", "large", "xlarge"]) + capacity_types = optional(list(string), ["on-demand", "spot"]) + architecture = optional(list(string), ["amd64"]) + + # Node pool limits + cpu_limits = optional(string, "1000") + memory_limits = optional(string, "1000Gi") + + # Node pool behavior + enable_consolidation = optional(bool, true) + + # Node scheduling + labels = optional(map(string), {}) + taints = optional(map(object({ + value = string + effect = string + })), {}) + + # Tags + tags = optional(map(string), {}) + }) + }) +} + +variable "instance_name" { + type = string + description = "Unique architectural name from blueprint" +} + +variable "environment" { + type = object({ + name = string + unique_name = string + namespace = optional(string) + cloud_tags = optional(map(string), {}) + }) + description = "Environment context including name and cloud tags" +} + +variable "inputs" { + type = object({ + cloud_account = object({ + attributes = object({ + aws_region = string + aws_account_id = string + aws_iam_role = string + external_id = optional(string) + session_name = optional(string) + }) + }) + kubernetes_details = object({ + attributes = object({ + cluster_endpoint = string + cluster_ca_certificate = string + cluster_name = string + cluster_version = string + cluster_arn = string + cluster_id = string + oidc_issuer_url = string + oidc_provider = string + oidc_provider_arn = string + node_security_group_id = string + kubernetes_provider_exec = object({ + api_version = string + command = string + args = list(string) + }) + }) + }) + network_details = object({ + attributes = object({ + vpc_id = string + private_subnet_ids = list(string) + public_subnet_ids = list(string) + database_subnet_ids = optional(list(string), []) + }) + }) + # Required input - from karpenter controller module + karpenter_details = object({ + attributes = object({ + node_instance_profile_name = string + node_role_arn = string + controller_role_arn = optional(string) + karpenter_namespace = optional(string, "kube-system") + karpenter_service_account = optional(string, "karpenter") + }) + }) + }) + description = "Inputs from dependent modules" +} diff --git a/modules/kubernetes_node_pool/karpenter/1.0/versions.tf b/modules/kubernetes_node_pool/karpenter/1.0/versions.tf new file mode 100644 index 00000000..7117131f --- /dev/null +++ b/modules/kubernetes_node_pool/karpenter/1.0/versions.tf @@ -0,0 +1,3 @@ +terraform { + required_version = ">= 1.0" +} diff --git a/project-type/aws/base/kubernetes_cluster/instances/cluster.json b/project-type/aws/base/kubernetes_cluster/instances/cluster.json index 2b2fb4ee..2d2ea08f 100644 --- a/project-type/aws/base/kubernetes_cluster/instances/cluster.json +++ b/project-type/aws/base/kubernetes_cluster/instances/cluster.json @@ -1,5 +1,5 @@ { - "flavor": "eks", + "flavor": "eks_standard", "metadata": { "name": "eks-cluster" }, @@ -14,32 +14,39 @@ "resource_type": "network" } }, - "alias-flavors": [ - "default" - ], "disabled": true, "version": "1.0", "spec": { - "cluster": { - "cluster_endpoint_public_access": true, - "cluster_endpoint_public_access_cidrs": [ - "0.0.0.0/0" - ], - "cluster_addons": { - "eks-cluster-addons1": { - "name": "snapshot-controller", - "enabled": true, - "preserve": false - }, - "eks-cluster-addons2": { - "name": "metrics-server", - "enabled": true - } + "cluster_endpoint_private_access": true, + "cluster_endpoint_public_access": true, + "cluster_version": "1.33", + "enable_cluster_encryption": false, + "cluster_addons": { + "additional_addons": {}, + "coredns": { + "enabled": false, + "version": "latest" + }, + "kube_proxy": { + "enabled": false, + "version": "latest" }, - "cloudwatch": { - "log_group_retention_in_days": 90 + "vpc_cni": { + "enabled": false, + "version": "latest" } - } + }, + "default_node_pool": { + "capacity_type": "SPOT", + "desired_size": 1, + "disk_size": 50, + "instance_types": [ + "t3.medium" + ], + "max_size": 1, + "min_size": 1 + }, + "cluster_tags": {} }, "uiProps": {} } \ No newline at end of file diff --git a/project-type/aws/base/kubernetes_node_pool/instances/nodepool.json b/project-type/aws/base/kubernetes_node_pool/instances/nodepool.json index 23efaac7..e1a7c4eb 100644 --- a/project-type/aws/base/kubernetes_node_pool/instances/nodepool.json +++ b/project-type/aws/base/kubernetes_node_pool/instances/nodepool.json @@ -1,54 +1,48 @@ { + "flavor": "karpenter", "kind": "kubernetes_node_pool", - "flavor": "aws", - "version": "1.0", - "metadata": { - "name": "production-nodepool" - }, - "spec": { - "instance_requirements": { - "instance_classification": "Instance Types", - "instance_types": "m6i.large", - "cpu_range": "4,8,16", - "architectures": "amd64,arm64", - "capacity_types": "spot,on-demand" - }, - "scaling": { - "max_cpu": "500", - "max_memory": "2000Gi", - "consolidation_policy": "WhenEmptyOrUnderutilized", - "consolidation_delay": "30s" - }, - "networking": { - "subnet_type": "private" - }, - "storage": { - "disk_size": "100Gi", - "disk_iops": 3000, - "disk_throughput": 125 - }, - "scheduling": { - "node_labels": { - "environment": "production", - "workload-type": "general" - } - }, - "tags": { - "Environment": "production", - "Team": "platform", - "CostCenter": "engineering" - } - }, - "disabled": true, "inputs": { "kubernetes_details": { - "resource_name": "cluster", - "resource_type": "kubernetes_cluster", - "output_name": "attributes" + "resource_name": "eks-standard", + "resource_type": "kubernetes_cluster" + }, + "cloud_account": { + "resource_name": "eks-standard", + "resource_type": "cloud_account" }, "network_details": { - "resource_name": "network", + "resource_name": "eks-standard", "resource_type": "network" } + }, + "disabled": true, + "version": "1.0", + "metadata": { + "name": "production-nodepool" + }, + "spec": { + "install_karpenter": true, + "karpenter_version": "1.0.1", + "karpenter_replicas": 1, + "instance_families": [ + "t3", + "t3a" + ], + "instance_sizes": [ + "medium", + "large", + "xlarge" + ], + "capacity_types": [ + "on-demand", + "spot" + ], + "architecture": [ + "amd64" + ], + "cpu_limits": "1000", + "memory_limits": "1000Gi", + "enable_consolidation": true, + "interruption_handling": false } } \ No newline at end of file diff --git a/project-type/aws/project-type.yml b/project-type/aws/project-type.yml index 535b1313..3af809ef 100644 --- a/project-type/aws/project-type.yml +++ b/project-type/aws/project-type.yml @@ -12,7 +12,7 @@ modules: - intent: cloud_account flavor: aws_provider - intent: kubernetes_cluster - flavor: eks + flavor: eks-standard - intent: kubernetes_node_pool flavor: aws - intent: network