move sqs queues to terraform

Author: devksingh4

Makefile

@@ -93,7 +93,7 @@ deploy_prod: check_account_prod
 	sam deploy $(common_params) --parameter-overrides $(run_env)=prod $(set_application_prefix)=$(application_key) $(set_application_name)="$(application_name)" S3BucketPrefix="$(s3_bucket_prefix)"
 	@echo "Deploying Terraform..."
 	$(eval MAIN_DISTRIBUTION_ID := $(shell aws cloudformation describe-stacks --stack-name $(application_key) --query "Stacks[0].Outputs[?OutputKey=='CloudfrontDistributionId'].OutputValue" --output text))
-	terraform -chdir=terraform/envs/prod init
+	terraform -chdir=terraform/envs/prod init -lockfile=readonly
 	terraform -chdir=terraform/envs/prod apply -auto-approve -var main_cloudfront_distribution_id="$(MAIN_DISTRIBUTION_ID)"
 	make postdeploy
 
@@ -102,7 +102,7 @@ deploy_dev: check_account_dev
 	sam deploy $(common_params) --parameter-overrides $(run_env)=dev $(set_application_prefix)=$(application_key) $(set_application_name)="$(application_name)" S3BucketPrefix="$(s3_bucket_prefix)"
 	@echo "Deploying Terraform..."
 	$(eval MAIN_DISTRIBUTION_ID := $(shell aws cloudformation describe-stacks --stack-name $(application_key) --query "Stacks[0].Outputs[?OutputKey=='CloudfrontDistributionId'].OutputValue" --output text))
-	terraform -chdir=terraform/envs/qa init
+	terraform -chdir=terraform/envs/qa init -lockfile=readonly
 	terraform -chdir=terraform/envs/qa apply -auto-approve -var main_cloudfront_distribution_id="$(MAIN_DISTRIBUTION_ID)"
 	make postdeploy
 
@@ -122,8 +122,8 @@ invalidate_cloudfront:
 install:
 	yarn -D
 	pip install cfn-lint
-	terraform -chdir=terraform/envs/qa init
-	terraform -chdir=terraform/envs/prod init
+	terraform -chdir=terraform/envs/qa init -lockfile=readonly
+	terraform -chdir=terraform/envs/prod init -lockfile=readonly
 
 test_live_integration: install
 	yarn test:live

cloudformation/main.yml

@@ -22,10 +22,6 @@ Parameters:
     Description: How long the SQS lambda is permitted to run (in seconds)
     Default: 180
     Type: Number
-  SqsMessageTimeout:
-    Description: MessageVisibilityTimeout for the SQS Lambda queue (should be at least (numMaxRetry + 1)*SqsLambdaTimeout)
-    Default: 720
-    Type: Number
   S3BucketPrefix:
     Description: S3 bucket prefix which will ensure global uniqueness
     Type: String
@@ -86,22 +82,14 @@ Resources:
         RunEnvironment: !Ref RunEnvironment
         LambdaFunctionName: !Sub ${ApplicationPrefix}-lambda
         SesEmailDomain: !FindInMap [General, !Ref RunEnvironment, SesDomain]
-        SqsQueueArn: !GetAtt AppSQSQueues.Outputs.MainQueueArn
+        SqsQueueArn: !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:infra-core-api-sqs-sales"
         LinkryKvArn: !GetAtt LinkryRecordsCloudfrontStore.Arn
 
   AppLogGroups:
     Type: AWS::Serverless::Application
     Properties:
       Location: ./logs.yml
 
-  AppSQSQueues:
-    Type: AWS::Serverless::Application
-    Properties:
-      Location: ./sqs.yml
-      Parameters:
-        QueueName: !Sub ${ApplicationPrefix}-sqs
-        MessageTimeout: !Ref SqsMessageTimeout
-
   LinkryRecordSetv4:
     Condition: IsDev
     Type: AWS::Route53::RecordSet
@@ -319,7 +307,7 @@ Resources:
       - AppSqsLambdaFunction
     Properties:
       BatchSize: 5
-      EventSourceArn: !GetAtt AppSQSQueues.Outputs.MainQueueArn
+      EventSourceArn: !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:infra-core-api-sqs"
       FunctionName: !Sub ${ApplicationPrefix}-sqs-lambda
       FunctionResponseTypes:
         - ReportBatchItemFailures
@@ -330,7 +318,7 @@ Resources:
       - AppSqsLambdaFunction
     Properties:
       BatchSize: 5
-      EventSourceArn: !GetAtt AppSQSQueues.Outputs.SalesEmailQueueArn
+      EventSourceArn: !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:infra-core-api-sqs-sales"
       FunctionName: !Sub ${ApplicationPrefix}-sqs-lambda
       FunctionResponseTypes:
         - ReportBatchItemFailures
@@ -1011,4 +999,4 @@ Outputs:
 
   SalesEmailQueueArn:
     Description: Sales Email Queue Arn
-    Value: !GetAtt AppSQSQueues.Outputs.SalesEmailQueueArn
+    Value: !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:infra-core-api-sqs-sales"

cspell.config.yaml

@@ -6,6 +6,7 @@ ignorePaths:
 dictionaryDefinitions: []
 dictionaries: []
 words:
+  - redrive
   - UIUC
 ignoreWords: []
 import: []

terraform/envs/prod/main.tf

@@ -18,10 +18,6 @@ provider "aws" {
   }
 }
 
-import {
-  to = aws_cloudwatch_log_group.main_app_logs
-  id = "/aws/lambda/${var.ProjectId}-lambda"
-}
 resource "aws_cloudwatch_log_group" "main_app_logs" {
   name              = "/aws/lambda/${var.ProjectId}-lambda"
   retention_in_days = var.LogRetentionDays
@@ -34,3 +30,8 @@ module "app_alarms" {
   priority_sns_arn                = var.GeneralSNSAlertArn
   standard_sns_arn                = var.PrioritySNSAlertArn
 }
+
+module "sqs_queues" {
+  source          = "../../modules/sqs"
+  resource_prefix = var.ProjectId
+}

terraform/envs/qa/main.tf

@@ -21,3 +21,7 @@ resource "aws_cloudwatch_log_group" "main_app_logs" {
   name              = "/aws/lambda/${var.ProjectId}-lambda"
   retention_in_days = var.LogRetentionDays
 }
+module "sqs_queues" {
+  source          = "../../modules/sqs"
+  resource_prefix = var.ProjectId
+}

terraform/modules/sqs/main.tf

@@ -0,0 +1,46 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+  }
+}
+
+resource "aws_sqs_queue" "app_dlq" {
+  name                       = "${var.resource_prefix}-sqs-dlq"
+  visibility_timeout_seconds = var.sqs_message_timeout
+  message_retention_seconds  = 1209600
+}
+
+resource "aws_sqs_queue" "app_queue" {
+  name                       = "${var.resource_prefix}-sqs"
+  visibility_timeout_seconds = var.sqs_message_timeout
+  redrive_policy = {
+    deadLetterTargetArn = aws_sqs_queue.app_dlq.arn
+    maxReceiveCount     = 3
+  }
+}
+
+resource "aws_sqs_queue" "sales_email_queue" {
+  name                       = "${var.resource_prefix}-sqs-sales"
+  visibility_timeout_seconds = var.sqs_message_timeout
+  redrive_policy = {
+    deadLetterTargetArn = aws_sqs_queue.app_dlq.arn
+    maxReceiveCount     = 3
+  }
+}
+
+output "main_queue_arn" {
+  description = "Main Queue Arn"
+  value       = aws_sqs_queue.app_queue.arn
+}
+
+output "dlq_arn" {
+  description = "Dead-letter Queue Arn"
+  value       = aws_sqs_queue.app_dlq.arn
+}
+
+output "sales_email_queue_arn" {
+  description = "Sales Email Queue Arn"
+  value       = aws_sqs_queue.sales_email_queue.arn
+}

terraform/modules/sqs/variables.tf

@@ -0,0 +1,16 @@
+variable "resource_prefix" {
+  type        = string
+  description = "Prefix before each resource"
+}
+
+variable "sqs_message_timeout" {
+  type        = number
+  description = "SQS Message timeout in seconds"
+  default     = 720
+}
+
+variable "dlq_message_retention" {
+  type        = number
+  description = "DLQ Message retention in seconds"
+  default     = 1209600
+}