Add Remote State Management (#2699)

* add remote state management

* refactor staging into its own directory

* change steps to account for staging

* bot suggestions

* sort alphabetically

* Update code

* Update documentation

* update formatting

* update code

Author: rudransh-shrivastava

.gitignore

@@ -25,8 +25,11 @@ __pycache__
 *.log
 *.pdf
 *.pem
+*.tfbackend
 *.tfstate
 *.tfstate.*
+*.tfvars
+!*.tfvars.example
 **/.terraform/
 backend/*nest-backend-dev*.tar.gz
 backend/*nest-backend-dev*.zip
@@ -46,7 +49,6 @@ frontend/pnpm-debug.log*
 frontend/test-results/
 frontend/yarn-debug.log*
 frontend/yarn-error.log*
-infrastructure/terraform.tfvars
 logs
 node_modules/
 TODO

cspell/custom-dict.txt

@@ -127,6 +127,7 @@ slackbot
 slideshare
 speakerdeck
 superfences
+tfbackend
 tiktok
 tsc
 turbopack

infrastructure/README.md

@@ -15,45 +15,56 @@ Ensure you have the following setup/installed:
 
 Follow these steps to set up the infrastructure:
 
-1. **Change the Directory**:
+1. **Setup Backend (one-time setup)**:
 
-   - Change the directory using the following command:
-
-     ```bash
-     cd infrastructure/
-     ```
-
-    *Note*: The following steps assume the current working directory is `infrastructure/`
-
-2. **Create Variables File**:
+- Navigate to the backend directory:
+  ```bash
+  cd infrastructure/backend/
+  ```
+*Note:* Optionally change the region: set `aws_region` in a `.tfvars` file.
 
-   - Create a local variables file in the `infrastructure` directory:
+- Initialize Terraform if needed:
+  ```bash
+  terraform init
+  ```
 
-     ```bash
-     touch terraform.tfvars
-     ```
+- Apply the changes to create the backend resources:
+  ```bash
+  terraform apply
+  ```
 
-   - Copy the contents from the template file into your new local environment file:
+2. **Setup Main Infrastructure (staging)**:
 
-     ```bash
-     cat terraform.tfvars.example > terraform.tfvars
-     ```
+- Navigate to the main infrastructure directory. If you are in `infrastructure/backend`, you can use:
+  ```bash
+  cd ../staging/
+  ```
 
-3. **Apply Changes**:
+- Create a local variables file:
+  ```bash
+  touch terraform.tfvars
+  ```
 
-   - Init terraform if needed:
+- Copy the contents from the example file:
+  ```bash
+  cat terraform.tfvars.example > terraform.tfvars
+  ```
 
-     ```bash
-     terraform init
-     ```
+- *Note:* Optionally change the region:
+  - set `aws_region` in a `.tfvars` file.
+  - set `region` in a `.tfbackend` file and provide it using `terraform init -backend-config=<file>`.
 
-   - Apply the changes and create the infrastructure using the following command:
+- Initialize Terraform with the backend configuration:
+  ```bash
+  terraform init
+  ```
 
-     ```bash
-     terraform apply
-     ```
+- Apply the changes to create the main infrastructure using the command:
+  ```bash
+  terraform apply
+  ```
 
-4. **Populate Secrets**:
+3. **Populate Secrets**
 
    - Visit the AWS Console > Systems Manager > Parameter Store.
    - Populate all `DJANGO_*` secrets that have `to-be-set-in-aws-console` value.
@@ -68,7 +79,7 @@ The Django backend deployment is managed by Zappa. This includes the API Gateway
    - Change the directory to `backend/` using the following command:
 
      ```bash
-     cd ../backend/
+     cd ../../backend/
      ```
 
     *Note*: The following steps assume the current working directory is `backend/`

infrastructure/backend/.terraform.lock.hcl

@@ -0,0 +1,174 @@
+terraform {
+  required_version = "1.14.0"
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "6.22.0"
+    }
+  }
+}
+
+data "aws_iam_policy_document" "logs" {
+  statement {
+    actions   = ["s3:PutObject"]
+    effect    = "Allow"
+    resources = ["${aws_s3_bucket.logs.arn}/*"]
+    sid       = "s3-log-delivery"
+
+    principals {
+      type        = "Service"
+      identifiers = ["logging.s3.amazonaws.com"]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "state_https_only" {
+  policy_id = "ForceHTTPS"
+
+  statement {
+    actions = ["s3:*"]
+    sid     = "HTTPSOnly"
+    effect  = "Deny"
+
+    condition {
+      test     = "Bool"
+      values   = ["false"]
+      variable = "aws:SecureTransport"
+    }
+    principals {
+      identifiers = ["*"]
+      type        = "AWS"
+    }
+    resources = [
+      aws_s3_bucket.state.arn,
+      "${aws_s3_bucket.state.arn}/*",
+    ]
+  }
+}
+
+resource "aws_dynamodb_table" "state_lock" {
+  name         = "${var.project_name}-terraform-state-lock"
+  billing_mode = "PAY_PER_REQUEST"
+  hash_key     = "LockID"
+  tags = {
+    Name = "${var.project_name}-terraform-state-lock"
+  }
+
+  attribute {
+    name = "LockID"
+    type = "S"
+  }
+  point_in_time_recovery {
+    enabled = true
+  }
+}
+
+resource "aws_s3_bucket" "logs" { # NOSONAR
+  bucket = "${var.project_name}-terraform-state-logs"
+  tags = {
+    Name = "${var.project_name}-terraform-state-logs"
+  }
+}
+
+resource "aws_s3_bucket" "state" { # NOSONAR
+  bucket = "${var.project_name}-terraform-state"
+  tags = {
+    Name = "${var.project_name}-terraform-state"
+  }
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "state" {
+  bucket = aws_s3_bucket.state.id
+
+  rule {
+    id     = "delete-old-versions"
+    status = "Enabled"
+
+    abort_incomplete_multipart_upload {
+      days_after_initiation = var.abort_incomplete_multipart_upload_days
+    }
+    noncurrent_version_expiration {
+      noncurrent_days = var.noncurrent_version_expiration_days
+    }
+  }
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "logs" {
+  bucket = aws_s3_bucket.logs.id
+
+  rule {
+    id     = "expire-logs"
+    status = "Enabled"
+
+    abort_incomplete_multipart_upload {
+      days_after_initiation = var.abort_incomplete_multipart_upload_days
+    }
+    expiration {
+      days = var.expire_log_days
+    }
+  }
+}
+
+resource "aws_s3_bucket_logging" "state" {
+  bucket        = aws_s3_bucket.state.id
+  target_bucket = aws_s3_bucket.logs.id
+  target_prefix = "s3/"
+}
+
+resource "aws_s3_bucket_policy" "logs" {
+  bucket = aws_s3_bucket.logs.id
+  policy = data.aws_iam_policy_document.logs.json
+}
+
+resource "aws_s3_bucket_policy" "state" {
+  bucket = aws_s3_bucket.state.id
+  policy = data.aws_iam_policy_document.state_https_only.json
+}
+
+resource "aws_s3_bucket_public_access_block" "logs" {
+  block_public_acls       = true
+  block_public_policy     = true
+  bucket                  = aws_s3_bucket.logs.id
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_public_access_block" "state" {
+  block_public_acls       = true
+  block_public_policy     = true
+  bucket                  = aws_s3_bucket.state.id
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "logs" {
+  bucket = aws_s3_bucket.logs.id
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "state" {
+  bucket = aws_s3_bucket.state.id
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
+resource "aws_s3_bucket_versioning" "state" {
+  bucket = aws_s3_bucket.state.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+resource "aws_s3_bucket_versioning" "logs" {
+  bucket = aws_s3_bucket.logs.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}

infrastructure/backend/main.tf

@@ -0,0 +1,9 @@
+output "dynamodb_table_name" {
+  description = "The name of the DynamoDB table for Terraform state locking"
+  value       = aws_dynamodb_table.state_lock.name
+}
+
+output "s3_bucket_name" {
+  description = "The name of the S3 bucket for Terraform state"
+  value       = aws_s3_bucket.state.bucket
+}

infrastructure/backend/outputs.tf

@@ -0,0 +1,29 @@
+variable "abort_incomplete_multipart_upload_days" {
+  description = "Specifies the number of days after which an incomplete multipart upload is aborted."
+  type        = number
+  default     = 7
+}
+
+variable "aws_region" {
+  description = "The AWS region to deploy resources in."
+  type        = string
+  default     = "us-east-2"
+}
+
+variable "expire_log_days" {
+  description = "The number of days to expire logs after."
+  type        = number
+  default     = 90
+}
+
+variable "noncurrent_version_expiration_days" {
+  description = "The number of days an object is noncurrent before it is expired."
+  type        = number
+  default     = 30
+}
+
+variable "project_name" {
+  description = "The name of the project."
+  type        = string
+  default     = "owasp-nest"
+}