diff --git a/Makefile b/Makefile index 8850b949..ed31d23a 100644 --- a/Makefile +++ b/Makefile @@ -93,7 +93,7 @@ deploy_prod: check_account_prod sam deploy $(common_params) --parameter-overrides $(run_env)=prod $(set_application_prefix)=$(application_key) $(set_application_name)="$(application_name)" S3BucketPrefix="$(s3_bucket_prefix)" @echo "Deploying Terraform..." $(eval MAIN_DISTRIBUTION_ID := $(shell aws cloudformation describe-stacks --stack-name $(application_key) --query "Stacks[0].Outputs[?OutputKey=='CloudfrontDistributionId'].OutputValue" --output text)) - terraform -chdir=terraform/envs/prod init + terraform -chdir=terraform/envs/prod init -lockfile=readonly terraform -chdir=terraform/envs/prod apply -auto-approve -var main_cloudfront_distribution_id="$(MAIN_DISTRIBUTION_ID)" make postdeploy @@ -102,7 +102,7 @@ deploy_dev: check_account_dev sam deploy $(common_params) --parameter-overrides $(run_env)=dev $(set_application_prefix)=$(application_key) $(set_application_name)="$(application_name)" S3BucketPrefix="$(s3_bucket_prefix)" @echo "Deploying Terraform..." $(eval MAIN_DISTRIBUTION_ID := $(shell aws cloudformation describe-stacks --stack-name $(application_key) --query "Stacks[0].Outputs[?OutputKey=='CloudfrontDistributionId'].OutputValue" --output text)) - terraform -chdir=terraform/envs/qa init + terraform -chdir=terraform/envs/qa init -lockfile=readonly terraform -chdir=terraform/envs/qa apply -auto-approve -var main_cloudfront_distribution_id="$(MAIN_DISTRIBUTION_ID)" make postdeploy @@ -119,11 +119,13 @@ invalidate_cloudfront: aws cloudfront wait invalidation-completed --distribution-id $(DISTRIBUTION_ID_2) --id $(INVALIDATION_ID_2) @echo "CloudFront invalidation completed!" +init_terraform: + terraform -chdir=terraform/envs/qa init + terraform -chdir=terraform/envs/prod init + install: yarn -D pip install cfn-lint - terraform -chdir=terraform/envs/qa init - terraform -chdir=terraform/envs/prod init test_live_integration: install yarn test:live @@ -131,9 +133,11 @@ test_live_integration: install test_unit: install yarn lint cfn-lint cloudformation/**/* + terraform -chdir=terraform/envs/qa init -reconfigure -backend=false -upgrade terraform -chdir=terraform/envs/qa fmt -check - terraform -chdir=terraform/envs/prod fmt -check terraform -chdir=terraform/envs/qa validate + terraform -chdir=terraform/envs/prod init -reconfigure -backend=false + terraform -chdir=terraform/envs/prod fmt -check terraform -chdir=terraform/envs/prod validate yarn prettier yarn test:unit @@ -149,3 +153,7 @@ dev_health_check: prod_health_check: curl -f https://core.acm.illinois.edu/api/v1/healthz && curl -f https://core.acm.illinois.edu + +lock_terraform: init_terraform + terraform -chdir=terraform/envs/qa providers lock -platform=windows_amd64 -platform=darwin_amd64 -platform=darwin_arm64 -platform=linux_amd64 -platform=linux_arm64 + terraform -chdir=terraform/envs/prod providers lock -platform=windows_amd64 -platform=darwin_amd64 -platform=darwin_arm64 -platform=linux_amd64 -platform=linux_arm64 diff --git a/cloudformation/main.yml b/cloudformation/main.yml index 8c424209..5ecaa07e 100644 --- a/cloudformation/main.yml +++ b/cloudformation/main.yml @@ -22,10 +22,6 @@ Parameters: Description: How long the SQS lambda is permitted to run (in seconds) Default: 180 Type: Number - SqsMessageTimeout: - Description: MessageVisibilityTimeout for the SQS Lambda queue (should be at least (numMaxRetry + 1)*SqsLambdaTimeout) - Default: 720 - Type: Number S3BucketPrefix: Description: S3 bucket prefix which will ensure global uniqueness Type: String @@ -86,7 +82,7 @@ Resources: RunEnvironment: !Ref RunEnvironment LambdaFunctionName: !Sub ${ApplicationPrefix}-lambda SesEmailDomain: !FindInMap [General, !Ref RunEnvironment, SesDomain] - SqsQueueArn: !GetAtt AppSQSQueues.Outputs.MainQueueArn + SqsQueueArn: !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:infra-core-api-sqs" LinkryKvArn: !GetAtt LinkryRecordsCloudfrontStore.Arn AppLogGroups: @@ -94,14 +90,6 @@ Resources: Properties: Location: ./logs.yml - AppSQSQueues: - Type: AWS::Serverless::Application - Properties: - Location: ./sqs.yml - Parameters: - QueueName: !Sub ${ApplicationPrefix}-sqs - MessageTimeout: !Ref SqsMessageTimeout - LinkryRecordSetv4: Condition: IsDev Type: AWS::Route53::RecordSet @@ -319,7 +307,7 @@ Resources: - AppSqsLambdaFunction Properties: BatchSize: 5 - EventSourceArn: !GetAtt AppSQSQueues.Outputs.MainQueueArn + EventSourceArn: !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:infra-core-api-sqs" FunctionName: !Sub ${ApplicationPrefix}-sqs-lambda FunctionResponseTypes: - ReportBatchItemFailures @@ -330,7 +318,7 @@ Resources: - AppSqsLambdaFunction Properties: BatchSize: 5 - EventSourceArn: !GetAtt AppSQSQueues.Outputs.SalesEmailQueueArn + EventSourceArn: !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:infra-core-api-sqs-sales" FunctionName: !Sub ${ApplicationPrefix}-sqs-lambda FunctionResponseTypes: - ReportBatchItemFailures @@ -1011,4 +999,4 @@ Outputs: SalesEmailQueueArn: Description: Sales Email Queue Arn - Value: !GetAtt AppSQSQueues.Outputs.SalesEmailQueueArn + Value: !Sub "arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:infra-core-api-sqs-sales" diff --git a/cloudformation/sqs.yml b/cloudformation/sqs.yml deleted file mode 100644 index c40e7d38..00000000 --- a/cloudformation/sqs.yml +++ /dev/null @@ -1,59 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Description: Stack SQS Queues -Transform: AWS::Serverless-2016-10-31 -Parameters: - QueueName: - Type: String - AllowedPattern: ^[a-zA-Z0-9]+[a-zA-Z0-9-]+[a-zA-Z0-9]+$ - MessageTimeout: - Type: Number -Resources: - AppDLQ: - Type: AWS::SQS::Queue - Properties: - QueueName: !Sub ${QueueName}-dlq - VisibilityTimeout: !Ref MessageTimeout - MessageRetentionPeriod: 1209600 - - AppQueue: - Type: AWS::SQS::Queue - Properties: - QueueName: !Ref QueueName - VisibilityTimeout: !Ref MessageTimeout - RedrivePolicy: - deadLetterTargetArn: - Fn::GetAtt: - - "AppDLQ" - - "Arn" - maxReceiveCount: 3 - SalesEmailQueue: - Type: AWS::SQS::Queue - Properties: - QueueName: !Sub ${QueueName}-sales - VisibilityTimeout: !Ref MessageTimeout - RedrivePolicy: - deadLetterTargetArn: - Fn::GetAtt: - - "AppDLQ" - - "Arn" - maxReceiveCount: 3 - -Outputs: - MainQueueArn: - Description: Main Queue Arn - Value: - Fn::GetAtt: - - AppQueue - - Arn - DLQArn: - Description: Dead-letter Queue Arn - Value: - Fn::GetAtt: - - AppDLQ - - Arn - SalesEmailQueueArn: - Description: Sales Email Queue Arn - Value: - Fn::GetAtt: - - SalesEmailQueue - - Arn diff --git a/cspell.config.yaml b/cspell.config.yaml index 57ffc007..d44fc9b4 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -6,6 +6,7 @@ ignorePaths: dictionaryDefinitions: [] dictionaries: [] words: + - redrive - UIUC ignoreWords: [] import: [] diff --git a/terraform/envs/prod/.terraform.lock.hcl b/terraform/envs/prod/.terraform.lock.hcl index 00f630cd..3e2c3752 100644 --- a/terraform/envs/prod/.terraform.lock.hcl +++ b/terraform/envs/prod/.terraform.lock.hcl @@ -5,7 +5,11 @@ provider "registry.terraform.io/hashicorp/aws" { version = "5.100.0" constraints = "~> 5.92" hashes = [ + "h1:H3mU/7URhP0uCRGK8jeQRKxx2XFzEqLiOq/L2Bbiaxs=", "h1:Ijt7pOlB7Tr7maGQIqtsLFbl7pSMIj06TVdkoSBcYOw=", + "h1:edXOJWE4ORX8Fm+dpVpICzMZJat4AX0VRCAy/xkcOc0=", + "h1:hd45qFU5cFuJMpFGdUniU9mVIr5LYVWP1uMeunBpYYs=", + "h1:wOhTPz6apLBuF7/FYZuCoXRK/MLgrNprZ3vXmq83g5k=", "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644", "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2", "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274", diff --git a/terraform/envs/prod/main.tf b/terraform/envs/prod/main.tf index 1c632218..86ace28f 100644 --- a/terraform/envs/prod/main.tf +++ b/terraform/envs/prod/main.tf @@ -1,3 +1,8 @@ +data "aws_caller_identity" "current" {} +locals { + account_id = data.aws_caller_identity.current.account_id +} + terraform { required_providers { aws = { @@ -7,6 +12,12 @@ terraform { } required_version = ">= 1.2" + backend "s3" { + bucket = "298118738376-terraform" + key = "infra-core-api" + region = "us-east-1" + use_lockfile = true + } } provider "aws" { @@ -17,7 +28,6 @@ provider "aws" { } } } - import { to = aws_cloudwatch_log_group.main_app_logs id = "/aws/lambda/${var.ProjectId}-lambda" @@ -34,3 +44,8 @@ module "app_alarms" { priority_sns_arn = var.GeneralSNSAlertArn standard_sns_arn = var.PrioritySNSAlertArn } + +module "sqs_queues" { + source = "../../modules/sqs" + resource_prefix = var.ProjectId +} diff --git a/terraform/envs/qa/.terraform.lock.hcl b/terraform/envs/qa/.terraform.lock.hcl index 00f630cd..3e2c3752 100644 --- a/terraform/envs/qa/.terraform.lock.hcl +++ b/terraform/envs/qa/.terraform.lock.hcl @@ -5,7 +5,11 @@ provider "registry.terraform.io/hashicorp/aws" { version = "5.100.0" constraints = "~> 5.92" hashes = [ + "h1:H3mU/7URhP0uCRGK8jeQRKxx2XFzEqLiOq/L2Bbiaxs=", "h1:Ijt7pOlB7Tr7maGQIqtsLFbl7pSMIj06TVdkoSBcYOw=", + "h1:edXOJWE4ORX8Fm+dpVpICzMZJat4AX0VRCAy/xkcOc0=", + "h1:hd45qFU5cFuJMpFGdUniU9mVIr5LYVWP1uMeunBpYYs=", + "h1:wOhTPz6apLBuF7/FYZuCoXRK/MLgrNprZ3vXmq83g5k=", "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644", "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2", "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274", diff --git a/terraform/envs/qa/main.tf b/terraform/envs/qa/main.tf index a8a6ed8f..2db6396c 100644 --- a/terraform/envs/qa/main.tf +++ b/terraform/envs/qa/main.tf @@ -7,6 +7,13 @@ terraform { } required_version = ">= 1.2" + + backend "s3" { + bucket = "427040638965-terraform" + key = "infra-core-api" + region = "us-east-1" + use_lockfile = true + } } provider "aws" { @@ -17,7 +24,15 @@ provider "aws" { } } } +import { + to = aws_cloudwatch_log_group.main_app_logs + id = "/aws/lambda/${var.ProjectId}-lambda" +} resource "aws_cloudwatch_log_group" "main_app_logs" { name = "/aws/lambda/${var.ProjectId}-lambda" retention_in_days = var.LogRetentionDays } +module "sqs_queues" { + source = "../../modules/sqs" + resource_prefix = var.ProjectId +} diff --git a/terraform/modules/sqs/main.tf b/terraform/modules/sqs/main.tf new file mode 100644 index 00000000..31215657 --- /dev/null +++ b/terraform/modules/sqs/main.tf @@ -0,0 +1,46 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + } + } +} + +resource "aws_sqs_queue" "app_dlq" { + name = "${var.resource_prefix}-sqs-dlq" + visibility_timeout_seconds = var.sqs_message_timeout + message_retention_seconds = 1209600 +} + +resource "aws_sqs_queue" "app_queue" { + name = "${var.resource_prefix}-sqs" + visibility_timeout_seconds = var.sqs_message_timeout + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.app_dlq.arn + maxReceiveCount = 3 + }) +} + +resource "aws_sqs_queue" "sales_email_queue" { + name = "${var.resource_prefix}-sqs-sales" + visibility_timeout_seconds = var.sqs_message_timeout + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.app_dlq.arn + maxReceiveCount = 3 + }) +} + +output "main_queue_arn" { + description = "Main Queue Arn" + value = aws_sqs_queue.app_queue.arn +} + +output "dlq_arn" { + description = "Dead-letter Queue Arn" + value = aws_sqs_queue.app_dlq.arn +} + +output "sales_email_queue_arn" { + description = "Sales Email Queue Arn" + value = aws_sqs_queue.sales_email_queue.arn +} diff --git a/terraform/modules/sqs/variables.tf b/terraform/modules/sqs/variables.tf new file mode 100644 index 00000000..3977281c --- /dev/null +++ b/terraform/modules/sqs/variables.tf @@ -0,0 +1,16 @@ +variable "resource_prefix" { + type = string + description = "Prefix before each resource" +} + +variable "sqs_message_timeout" { + type = number + description = "SQS Message timeout in seconds" + default = 720 +} + +variable "dlq_message_retention" { + type = number + description = "DLQ Message retention in seconds" + default = 1209600 +}