diff --git a/infrastructure/README.md b/infrastructure/README.md index 7bba80f951..20f0ce6545 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -21,7 +21,8 @@ Follow these steps to set up the infrastructure: ```bash cd infrastructure/backend/ ``` -*Note:* Optionally change the region: set `aws_region` in a `.tfvars` file. + +**Note:** Optionally change the region: set `aws_region` in a `.tfvars` file. - Initialize Terraform if needed: ```bash @@ -33,6 +34,10 @@ Follow these steps to set up the infrastructure: terraform apply ``` +**Note:** Copy the state bucket name from the output. + +**Note:** It is recommended to not destroy the backend resources unless absolutely necessary. + 2. **Setup Main Infrastructure (staging)**: - Navigate to the main infrastructure directory. If you are in `infrastructure/backend`, you can use: @@ -50,13 +55,23 @@ Follow these steps to set up the infrastructure: cat terraform.tfvars.example > terraform.tfvars ``` -- *Note:* Optionally change the region: - - set `aws_region` in a `.tfvars` file. - - set `region` in a `.tfbackend` file and provide it using `terraform init -backend-config=`. +- Create a local backend configuration file: + ```bash + touch terraform.tfbackend + ``` + +- Copy the contents from the example file: + ```bash + cat terraform.tfbackend.example > terraform.tfbackend + ``` + +*Note:* Update the state bucket name in `terraform.tfbackend` with the name of the state bucket created in the previous step. + +*Note:* Update defaults (e.g. `region`) as needed. - Initialize Terraform with the backend configuration: ```bash - terraform init + terraform init -backend-config=terraform.tfbackend ``` - Apply the changes to create the main infrastructure using the command: @@ -114,13 +129,15 @@ The Django backend deployment is managed by Zappa. This includes the API Gateway 5. **Deploy**: - - *Note*: Make sure to populate all `DJANGO_*` secrets that are set as `to-be-set-in-aws-console` + - **Note**: Make sure to populate all `DJANGO_*` secrets that are set as `to-be-set-in-aws-console` in the Parameter Store. The deployment might fail with no logs if secrets such as `DJANGO_SLACK_BOT_TOKEN` are invalid. ```bash zappa deploy staging ``` + - **Note**: If the deployment is successful but returns a `5xx` error, resolve the issues + and use `zappa undeploy staging` & `zappa deploy staging`. The command `zappa update staging` may not work. Once deployed, use the URL provided by Zappa to test the API. @@ -163,7 +180,7 @@ Migrate and load data into the new database. - Upload the fixture present in `backend/data` to `nest-fixtures` bucket using the following command: ```bash - aws s3 cp data/nest.json.gz s3://nest-fixtures/ + aws s3 cp data/nest.json.gz s3://owasp-nest-fixtures-/ ``` 3. **Run ECS Tasks**: @@ -188,6 +205,9 @@ Migrate and load data into the new database. zappa undeploy staging ``` +- Ensure all buckets and ECR repositories are empty. + +**Note:** Some resources have `prevent_destroy` set to `true`. Please set it to `false` before destruction. - To destroy Terraform infrastructure: ```bash diff --git a/infrastructure/backend/.terraform.lock.hcl b/infrastructure/backend/.terraform.lock.hcl index 55057fd2e7..22df00b9db 100644 --- a/infrastructure/backend/.terraform.lock.hcl +++ b/infrastructure/backend/.terraform.lock.hcl @@ -23,3 +23,23 @@ provider "registry.terraform.io/hashicorp/aws" { "zh:d479bad0a004e4893bf0ba6c6cd867fefd14000051bbe3de5b44a925e3d46cd5", ] } + +provider "registry.terraform.io/hashicorp/random" { + version = "3.7.2" + constraints = "3.7.2" + hashes = [ + "h1:356j/3XnXEKr9nyicLUufzoF4Yr6hRy481KIxRVpK0c=", + "zh:14829603a32e4bc4d05062f059e545a91e27ff033756b48afbae6b3c835f508f", + "zh:1527fb07d9fea400d70e9e6eb4a2b918d5060d604749b6f1c361518e7da546dc", + "zh:1e86bcd7ebec85ba336b423ba1db046aeaa3c0e5f921039b3f1a6fc2f978feab", + "zh:24536dec8bde66753f4b4030b8f3ef43c196d69cccbea1c382d01b222478c7a3", + "zh:29f1786486759fad9b0ce4fdfbbfece9343ad47cd50119045075e05afe49d212", + "zh:4d701e978c2dd8604ba1ce962b047607701e65c078cb22e97171513e9e57491f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b8434212eef0f8c83f5a90c6d76feaf850f6502b61b53c329e85b3b281cba34", + "zh:ac8a23c212258b7976e1621275e3af7099e7e4a3d4478cf8d5d2a27f3bc3e967", + "zh:b516ca74431f3df4c6cf90ddcdb4042c626e026317a33c53f0b445a3d93b720d", + "zh:dc76e4326aec2490c1600d6871a95e78f9050f9ce427c71707ea412a2f2f1a62", + "zh:eac7b63e86c749c7d48f527671c7aee5b4e26c10be6ad7232d6860167f99dbb0", + ] +} diff --git a/infrastructure/backend/main.tf b/infrastructure/backend/main.tf index 425f4b2bb6..b748cf0b64 100644 --- a/infrastructure/backend/main.tf +++ b/infrastructure/backend/main.tf @@ -5,6 +5,10 @@ terraform { source = "hashicorp/aws" version = "6.22.0" } + random = { + source = "hashicorp/random" + version = "3.7.2" + } } } @@ -46,6 +50,10 @@ data "aws_iam_policy_document" "state_https_only" { } } +resource "random_id" "suffix" { + byte_length = 4 +} + resource "aws_dynamodb_table" "state_lock" { name = "${var.project_name}-terraform-state-lock" billing_mode = "PAY_PER_REQUEST" @@ -58,20 +66,32 @@ resource "aws_dynamodb_table" "state_lock" { name = "LockID" type = "S" } + lifecycle { + prevent_destroy = true + } point_in_time_recovery { enabled = true } } resource "aws_s3_bucket" "logs" { # NOSONAR - bucket = "${var.project_name}-terraform-state-logs" + bucket = "${var.project_name}-terraform-state-logs-${random_id.suffix.hex}" + + lifecycle { + prevent_destroy = true + } tags = { Name = "${var.project_name}-terraform-state-logs" } } resource "aws_s3_bucket" "state" { # NOSONAR - bucket = "${var.project_name}-terraform-state" + bucket = "${var.project_name}-terraform-state-${random_id.suffix.hex}" + object_lock_enabled = true + + lifecycle { + prevent_destroy = true + } tags = { Name = "${var.project_name}-terraform-state" } @@ -115,6 +135,17 @@ resource "aws_s3_bucket_logging" "state" { target_prefix = "s3/" } +resource "aws_s3_bucket_object_lock_configuration" "state" { + bucket = aws_s3_bucket.state.id + + rule { + default_retention { + days = 30 + mode = "GOVERNANCE" + } + } +} + resource "aws_s3_bucket_policy" "logs" { bucket = aws_s3_bucket.logs.id policy = data.aws_iam_policy_document.logs.json diff --git a/infrastructure/modules/ecs/main.tf b/infrastructure/modules/ecs/main.tf index 0b91717efc..80fc8893c3 100644 --- a/infrastructure/modules/ecs/main.tf +++ b/infrastructure/modules/ecs/main.tf @@ -217,7 +217,7 @@ module "load_data_task" { set -e pip install --target=/tmp/awscli-packages awscli export PYTHONPATH="/tmp/awscli-packages:$PYTHONPATH" - python /tmp/awscli-packages/bin/aws s3 cp s3://${var.fixtures_s3_bucket}/nest.json.gz /tmp/nest.json.gz + python /tmp/awscli-packages/bin/aws s3 cp s3://${var.fixtures_bucket_name}/nest.json.gz /tmp/nest.json.gz python manage.py load_data --fixture-path /tmp/nest.json.gz EOT ] diff --git a/infrastructure/modules/ecs/variables.tf b/infrastructure/modules/ecs/variables.tf index 97de617acf..bfa7d41c27 100644 --- a/infrastructure/modules/ecs/variables.tf +++ b/infrastructure/modules/ecs/variables.tf @@ -30,7 +30,7 @@ variable "fixtures_read_only_policy_arn" { type = string } -variable "fixtures_s3_bucket" { +variable "fixtures_bucket_name" { description = "The name of the S3 bucket for fixtures" type = string } diff --git a/infrastructure/modules/storage/main.tf b/infrastructure/modules/storage/main.tf index 99fcca7ccf..a32b648a20 100644 --- a/infrastructure/modules/storage/main.tf +++ b/infrastructure/modules/storage/main.tf @@ -5,6 +5,10 @@ terraform { source = "hashicorp/aws" version = "6.22.0" } + random = { + source = "hashicorp/random" + version = "3.7.2" + } } } @@ -15,16 +19,19 @@ data "aws_iam_policy_document" "fixtures_read_only" { ] effect = "Allow" resources = [ - "arn:aws:s3:::${var.fixtures_s3_bucket}/*" + "arn:aws:s3:::${var.fixtures_bucket_name}-${random_id.suffix.hex}/*" ] } } +resource "random_id" "suffix" { + byte_length = 4 +} + module "fixtures_bucket" { source = "./modules/s3-bucket" - bucket_name = var.fixtures_s3_bucket - force_destroy = var.force_destroy_bucket + bucket_name = "${var.fixtures_bucket_name}-${random_id.suffix.hex}" tags = merge(var.common_tags, { Name = "${var.project_name}-${var.environment}-fixtures" }) @@ -33,8 +40,7 @@ module "fixtures_bucket" { module "zappa_bucket" { source = "./modules/s3-bucket" - bucket_name = var.zappa_s3_bucket - force_destroy = var.force_destroy_bucket + bucket_name = "${var.zappa_bucket_name}-${random_id.suffix.hex}" tags = merge(var.common_tags, { Name = "${var.project_name}-${var.environment}-zappa-deployments" }) diff --git a/infrastructure/modules/storage/modules/s3-bucket/main.tf b/infrastructure/modules/storage/modules/s3-bucket/main.tf index e4fd0d1c96..5f59a8bcbe 100644 --- a/infrastructure/modules/storage/modules/s3-bucket/main.tf +++ b/infrastructure/modules/storage/modules/s3-bucket/main.tf @@ -35,10 +35,13 @@ data "aws_iam_policy_document" "this" { } } -resource "aws_s3_bucket" "this" { #NOSONAR - bucket = var.bucket_name - force_destroy = var.force_destroy - tags = var.tags +resource "aws_s3_bucket" "this" { # NOSONAR + bucket = var.bucket_name + tags = var.tags + + lifecycle { + prevent_destroy = true + } } resource "aws_s3_bucket_policy" "this" { diff --git a/infrastructure/modules/storage/modules/s3-bucket/variables.tf b/infrastructure/modules/storage/modules/s3-bucket/variables.tf index a8e76c81bd..919b8c38e1 100644 --- a/infrastructure/modules/storage/modules/s3-bucket/variables.tf +++ b/infrastructure/modules/storage/modules/s3-bucket/variables.tf @@ -1,5 +1,5 @@ variable "abort_incomplete_multipart_upload_days" { - description = "Specifies the number of days after which an incomplete multipart upload is aborted." + description = "The number of days after which an incomplete multipart upload is aborted." type = number default = 7 } @@ -9,14 +9,8 @@ variable "bucket_name" { type = string } -variable "force_destroy" { - description = "If true, deletes all objects from the bucket when the bucket is destroyed." - type = bool - default = false -} - variable "noncurrent_version_expiration_days" { - description = "Specifies the number of days an object is noncurrent before it is expired." + description = "The number of days an object is noncurrent before it is expired." type = number default = 30 } diff --git a/infrastructure/modules/storage/variables.tf b/infrastructure/modules/storage/variables.tf index 6a197f5cce..936ac060e2 100644 --- a/infrastructure/modules/storage/variables.tf +++ b/infrastructure/modules/storage/variables.tf @@ -9,23 +9,17 @@ variable "environment" { type = string } -variable "fixtures_s3_bucket" { +variable "fixtures_bucket_name" { description = "The name of the S3 bucket for fixtures" type = string } -variable "force_destroy_bucket" { - description = "If true, deletes all objects from the bucket when the bucket is destroyed." - type = bool - default = false -} - variable "project_name" { description = "The name of the project" type = string } -variable "zappa_s3_bucket" { +variable "zappa_bucket_name" { description = "The name of the S3 bucket for Zappa deployments" type = string } diff --git a/infrastructure/staging/backend.tf b/infrastructure/staging/backend.tf index 138b77e5fe..8434ec82ac 100644 --- a/infrastructure/staging/backend.tf +++ b/infrastructure/staging/backend.tf @@ -1,9 +1,6 @@ terraform { backend "s3" { - bucket = "owasp-nest-terraform-state" - dynamodb_table = "owasp-nest-terraform-state-lock" - encrypt = true - key = "staging/terraform.tfstate" - region = "us-east-2" + encrypt = true + key = "staging/terraform.tfstate" } } diff --git a/infrastructure/staging/main.tf b/infrastructure/staging/main.tf index b68c5c4a10..a8cc0d8898 100644 --- a/infrastructure/staging/main.tf +++ b/infrastructure/staging/main.tf @@ -60,7 +60,7 @@ module "ecs" { ecs_sg_id = module.security.ecs_sg_id environment = var.environment fixtures_read_only_policy_arn = module.storage.fixtures_read_only_policy_arn - fixtures_s3_bucket = var.fixtures_s3_bucket + fixtures_bucket_name = var.fixtures_bucket_name private_subnet_ids = module.networking.private_subnet_ids project_name = var.project_name } @@ -109,8 +109,7 @@ module "storage" { common_tags = local.common_tags environment = var.environment - fixtures_s3_bucket = var.fixtures_s3_bucket - force_destroy_bucket = var.force_destroy_bucket + fixtures_bucket_name = var.fixtures_bucket_name project_name = var.project_name - zappa_s3_bucket = var.zappa_s3_bucket + zappa_bucket_name = var.zappa_bucket_name } diff --git a/infrastructure/staging/terraform.tfbackend.example b/infrastructure/staging/terraform.tfbackend.example new file mode 100644 index 0000000000..dc17c83323 --- /dev/null +++ b/infrastructure/staging/terraform.tfbackend.example @@ -0,0 +1,3 @@ +bucket = "${STATE_BUCKET_NAME}" +dynamodb_table = "owasp-nest-terraform-state-lock" +region = "us-east-2" diff --git a/infrastructure/staging/terraform.tfvars.example b/infrastructure/staging/terraform.tfvars.example index 118d8805e3..13077dbf8c 100644 --- a/infrastructure/staging/terraform.tfvars.example +++ b/infrastructure/staging/terraform.tfvars.example @@ -6,5 +6,4 @@ db_name = "owasp_nest" db_user = "owasp_nest_db_user" db_port = 5432 environment = "staging" -force_destroy_bucket = true project_name = "owasp-nest" diff --git a/infrastructure/staging/variables.tf b/infrastructure/staging/variables.tf index 8eb8bdcc93..eade7e669f 100644 --- a/infrastructure/staging/variables.tf +++ b/infrastructure/staging/variables.tf @@ -81,16 +81,10 @@ variable "environment" { } } -variable "force_destroy_bucket" { - description = "If true, deletes all objects from the bucket when the bucket is destroyed." - type = bool - default = false -} - -variable "fixtures_s3_bucket" { +variable "fixtures_bucket_name" { description = "The name of the S3 bucket for fixtures" type = string - default = "nest-fixtures" + default = "owasp-nest-fixtures" } variable "private_subnet_cidrs" { @@ -141,7 +135,7 @@ variable "vpc_cidr" { default = "10.0.0.0/16" } -variable "zappa_s3_bucket" { +variable "zappa_bucket_name" { description = "The name of the S3 bucket for Zappa deployments" type = string default = "owasp-nest-zappa-deployments"