Handle potential conflicts during Kafka configuration updates

aws/config/README.md

@@ -8,6 +8,7 @@
 | <a name="requirement_helm"></a> [helm](#requirement\_helm) | ~> 2.11.0 |
 | <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | ~> 1.19.0 |
 | <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | ~> 2.24.0 |
+| <a name="requirement_random"></a> [random](#requirement\_random) | ~> 3.7.0 |
 
 ## Providers
 
@@ -18,6 +19,7 @@
 | <a name="provider_helm"></a> [helm](#provider\_helm) | 2.11.0 |
 | <a name="provider_kubectl"></a> [kubectl](#provider\_kubectl) | 1.19.0 |
 | <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | 2.24.0 |
+| <a name="provider_random"></a> [random](#provider\_random) | 3.7.2 |
 
 ## Modules
 
@@ -92,6 +94,7 @@
 | [kubernetes_namespace.kubernetes_dashboard](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource |
 | [kubernetes_secret_v1.dashboard_user](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource |
 | [kubernetes_service_account_v1.dashboard_user](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service_account_v1) | resource |
+| [random_id.msk_config](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource |
 | [archive_file.secret_rotation_lambda_artifact](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source |
 | [aws_eks_cluster.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source |
 | [aws_eks_cluster_auth.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
@@ -130,7 +133,7 @@
 | <a name="input_enable_s3"></a> [enable\_s3](#input\_enable\_s3) | Do you need S3? [true, false] | `bool` | n/a | yes |
 | <a name="input_enable_ses"></a> [enable\_ses](#input\_enable\_ses) | Do you need SES? [true, false] | `bool` | n/a | yes |
 | <a name="input_environment"></a> [environment](#input\_environment) | Environment name | `string` | `"dev"` | no |
-| <a name="input_kafka_version"></a> [kafka\_version](#input\_kafka\_version) | Version of the Kafka to be used for MSK | `string` | `"3.2.0"` | no |
+| <a name="input_kafka_version"></a> [kafka\_version](#input\_kafka\_version) | Version of the Kafka to be used for MSK | `string` | `"3.9.x"` | no |
 | <a name="input_karpenter_ami_version_alias"></a> [karpenter\_ami\_version\_alias](#input\_karpenter\_ami\_version\_alias) | Selector alias for the AMI version used by Karpenter EC2 node class | `string` | `"al2023@v20250519"` | no |
 | <a name="input_karpenter_node_pools"></a> [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for the Karpenter node pool(s) with each key being the node pool name | <pre>map(object({<br/>    architecture           = list(string)<br/>    os                     = list(string)<br/>    instance_capacity_type = list(string)<br/>    instance_category      = list(string)<br/>    instance_cpu           = list(string)<br/>  }))</pre> | `{}` | no |
 | <a name="input_karpenter_version"></a> [karpenter\_version](#input\_karpenter\_version) | n/a | `string` | `"1.3.3"` | no |

aws/config/msk.tf

@@ -26,6 +26,11 @@ resource "aws_iam_role_policy_attachment" "msk_policy_attachment" {
   role       = aws_iam_role.msk_role[0].name
 }
 
+resource "random_id" "msk_config" {
+  count       = var.enable_msk ? 1 : 0
+  byte_length = 4
+}
+
 resource "aws_security_group" "msk_cluster_access" {
   count = var.enable_msk ? 1 : 0
 
@@ -55,7 +60,7 @@ resource "aws_msk_configuration" "msk_configuration" {
   count = var.enable_msk ? 1 : 0
 
   kafka_versions = [var.kafka_version]
-  name           = "${var.eks_cluster_name}-msk-configuration"
+  name           = "${var.eks_cluster_name}-msk-configuration${random_id.msk_config[0].hex}"
 
   server_properties = <<PROPERTIES
 auto.create.topics.enable=false
@@ -72,6 +77,10 @@ socket.send.buffer.bytes=102400
 unclean.leader.election.enable=true
 zookeeper.session.timeout.ms=18000
 PROPERTIES
+
+  lifecycle {
+    create_before_destroy = true
+  }
 }
 
 resource "aws_cloudwatch_log_group" "msk_broker" {
@@ -147,6 +156,8 @@ resource "aws_msk_cluster" "msk_cluster" {
   lifecycle {
     ignore_changes = [broker_node_group_info[0].storage_info[0].ebs_storage_info[0].volume_size]
   }
+
+  depends_on = [aws_msk_configuration.msk_configuration]
 }
 
 output "radar_base_msk_bootstrap_brokers" {

aws/config/variables.tf

@@ -139,7 +139,7 @@ variable "kubernetes_dashboard_version" {
 variable "kafka_version" {
   type        = string
   description = "Version of the Kafka to be used for MSK"
-  default     = "3.2.0"
+  default     = "3.9.x"
 }
 
 variable "postgres_version" {

aws/config/versions.tf

@@ -20,6 +20,10 @@ terraform {
       source  = "hashicorp/archive"
       version = "~> 2.7.0"
     }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.7.0"
+    }
   }
   required_version = ">= 1.9.0"
 }