From 4f3c99a456a19d913000d41e349311a713a988bb Mon Sep 17 00:00:00 2001 From: patrick Date: Thu, 27 Jun 2024 13:31:08 +0100 Subject: [PATCH 1/3] add account_management_events topic --- dev-aws/kafka-shared-msk/iam/iam.tf | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/dev-aws/kafka-shared-msk/iam/iam.tf b/dev-aws/kafka-shared-msk/iam/iam.tf index 542f249b..98faa8d5 100644 --- a/dev-aws/kafka-shared-msk/iam/iam.tf +++ b/dev-aws/kafka-shared-msk/iam/iam.tf @@ -292,3 +292,37 @@ module "castle_processor" { consume_topics = [(kafka_topic.iam_credentials_v1.name)] consume_groups = ["iam.castle-processor"] } + + +resource "kafka_topic" "iam_account_management_events" { + name = "auth-customer.iam-account-management-events" + replication_factor = "3" + partitions = "15" + config = { + + # Use tiered storage + "remote.storage.enable" = "true" + # retain 100MB on each partition + "retention.bytes" = "104857600" + # keep data for 7 days + "retention.ms" = "604800000" + # keep data in hot storage for 2 days + "local.retention.ms" = "172800000" + # allow max 1 MB for a message + "max.message.bytes" = "1048576" + "compression.type" = "zstd" + "cleanup.policy" = "delete" + } +} + +module "iam_accounts_management_service" { + source = "../../../modules/tls-app" + cert_common_name = "clubhouse/account-management-service" + produce_topics = [[kafka_topic.iam_account_management_events.name]] +} + +module "iam_accounts_management_projector" { + source = "../../../modules/tls-app" + cert_common_name = "clubhouse/account_management-projector" + produce_topics = [(kafka_topic.iam_account_management_events.name)] +} From 43b844a56c3aa6651f3898657d18e474cd0bc3cd Mon Sep 17 00:00:00 2001 From: patrick Date: Thu, 27 Jun 2024 13:37:47 +0100 Subject: [PATCH 2/3] add topic to prod --- prod-aws/kafka-shared-msk/iam/iam.tf | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/prod-aws/kafka-shared-msk/iam/iam.tf b/prod-aws/kafka-shared-msk/iam/iam.tf index 67fa0fcc..336affdc 100644 --- a/prod-aws/kafka-shared-msk/iam/iam.tf +++ b/prod-aws/kafka-shared-msk/iam/iam.tf @@ -291,3 +291,38 @@ module "castle_processor" { consume_topics = [(kafka_topic.iam_credentials_v1.name)] consume_groups = ["iam.castle-processor"] } + + + +resource "kafka_topic" "iam_account_management_events" { + name = "auth-customer.iam-account-management-events" + replication_factor = "3" + partitions = "15" + config = { + + # Use tiered storage + "remote.storage.enable" = "true" + # retain 100MB on each partition + "retention.bytes" = "104857600" + # keep data for 7 days + "retention.ms" = "604800000" + # keep data in hot storage for 2 days + "local.retention.ms" = "172800000" + # allow max 1 MB for a message + "max.message.bytes" = "1048576" + "compression.type" = "zstd" + "cleanup.policy" = "delete" + } +} + +module "iam_accounts_management_service" { + source = "../../../modules/tls-app" + cert_common_name = "clubhouse/account-management-service" + produce_topics = [[kafka_topic.iam_account_management_events.name]] +} + +module "iam_accounts_management_projector" { + source = "../../../modules/tls-app" + cert_common_name = "clubhouse/account_management-projector" + produce_topics = [(kafka_topic.iam_account_management_events.name)] +} From 40bbd19de5f1384dd556ab0cf08478815e1f1582 Mon Sep 17 00:00:00 2001 From: patrick Date: Thu, 27 Jun 2024 14:51:41 +0100 Subject: [PATCH 3/3] update consumer groups and topics for acc-management projector --- dev-aws/kafka-shared-msk/iam/iam.tf | 3 ++- prod-aws/kafka-shared-msk/iam/iam.tf | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dev-aws/kafka-shared-msk/iam/iam.tf b/dev-aws/kafka-shared-msk/iam/iam.tf index 98faa8d5..872e3f39 100644 --- a/dev-aws/kafka-shared-msk/iam/iam.tf +++ b/dev-aws/kafka-shared-msk/iam/iam.tf @@ -324,5 +324,6 @@ module "iam_accounts_management_service" { module "iam_accounts_management_projector" { source = "../../../modules/tls-app" cert_common_name = "clubhouse/account_management-projector" - produce_topics = [(kafka_topic.iam_account_management_events.name)] + consume_topics = [(kafka_topic.iam_account_management_events.name)] + consume_groups = ["dex-account-management-project"] } diff --git a/prod-aws/kafka-shared-msk/iam/iam.tf b/prod-aws/kafka-shared-msk/iam/iam.tf index 336affdc..eea42b21 100644 --- a/prod-aws/kafka-shared-msk/iam/iam.tf +++ b/prod-aws/kafka-shared-msk/iam/iam.tf @@ -324,5 +324,6 @@ module "iam_accounts_management_service" { module "iam_accounts_management_projector" { source = "../../../modules/tls-app" cert_common_name = "clubhouse/account_management-projector" - produce_topics = [(kafka_topic.iam_account_management_events.name)] + consume_topics = [(kafka_topic.iam_account_management_events.name)] + consume_groups = ["dex-account-management-project"] }