Feat/allow pubic access to requirements.txt

ecs.tf

@@ -6,14 +6,21 @@ resource "aws_cloudwatch_log_group" "airflow" {
 }
 
 resource "aws_ecs_cluster" "airflow" {
-  name               = "${var.resource_prefix}-airflow-${var.resource_suffix}"
+  name = "${var.resource_prefix}-airflow-${var.resource_suffix}"
+
+  tags = local.common_tags
+}
+
+resource "aws_ecs_cluster_capacity_providers" "aws_ecs_cluster_capacity_providers" {
+  cluster_name = aws_ecs_cluster.airflow.name
+
   capacity_providers = ["FARGATE_SPOT", "FARGATE"]
 
   default_capacity_provider_strategy {
+    base              = 1
+    weight            = 100
     capacity_provider = "FARGATE_SPOT"
   }
-
-  tags = local.common_tags
 }
 
 resource "aws_ecs_task_definition" "airflow" {
@@ -35,7 +42,7 @@ resource "aws_ecs_task_definition" "airflow" {
         "image": "mikesir87/aws-cli",
         "name": "${local.airflow_sidecar_container_name}",
         "command": [
-            "/bin/bash -c \"aws s3 cp s3://${local.s3_bucket_name}/${local.s3_key} ${var.airflow_container_home} --recursive && chmod +x ${var.airflow_container_home}/${aws_s3_bucket_object.airflow_scheduler_entrypoint.key} && chmod +x ${var.airflow_container_home}/${aws_s3_bucket_object.airflow_webserver_entrypoint.key} && chmod -R 777 ${var.airflow_container_home}\""
+            "/bin/bash -c \"aws s3 cp s3://${local.s3_bucket_name}/${local.s3_key} ${var.airflow_container_home} --recursive && chmod +x ${var.airflow_container_home}/${aws_s3_object.airflow_scheduler_entrypoint.key} && chmod +x ${var.airflow_container_home}/${aws_s3_object.airflow_webserver_entrypoint.key} && chmod -R 777 ${var.airflow_container_home}\""
         ],
         "entryPoint": [
             "sh",
@@ -67,7 +74,7 @@ resource "aws_ecs_task_definition" "airflow" {
             }
         ],
         "command": [
-            "/bin/bash -c \"${var.airflow_container_home}/${aws_s3_bucket_object.airflow_init_entrypoint.key}\""
+            "/bin/bash -c \"${var.airflow_container_home}/${aws_s3_object.airflow_init_entrypoint.key}\""
         ],
         "entryPoint": [
             "sh",
@@ -106,7 +113,7 @@ resource "aws_ecs_task_definition" "airflow" {
             }
         ],
         "command": [
-            "/bin/bash -c \"${var.airflow_container_home}/${aws_s3_bucket_object.airflow_scheduler_entrypoint.key}\""
+            "/bin/bash -c \"${var.airflow_container_home}/${aws_s3_object.airflow_scheduler_entrypoint.key}\""
         ],
         "entryPoint": [
             "sh",
@@ -145,7 +152,7 @@ resource "aws_ecs_task_definition" "airflow" {
             }
         ],
         "command": [
-            "/bin/bash -c \"${var.airflow_container_home}/${aws_s3_bucket_object.airflow_webserver_entrypoint.key}\""
+            "/bin/bash -c \"${var.airflow_container_home}/${aws_s3_object.airflow_webserver_entrypoint.key}\""
         ],
         "entryPoint": [
             "sh",

locals.tf

@@ -17,7 +17,7 @@ locals {
   day                 = formatdate("D", local.timestamp)
 
   rds_name     = "${var.resource_prefix}-airflow-${var.resource_suffix}"
-  postgres_uri = var.postgres_uri != "" ? "postgresql+psycopg2://${var.rds_username}:${var.rds_password}@${var.postgres_uri}" : (var.airflow_executor == "Sequential" ? "" : "postgresql+psycopg2://${var.rds_username}:${var.rds_password}@${aws_db_instance.airflow[0].address}:${aws_db_instance.airflow[0].port}/${aws_db_instance.airflow[0].name}")
+  postgres_uri = var.postgres_uri != "" ? "postgresql+psycopg2://${var.rds_username}:${var.rds_password}@${var.postgres_uri}" : (var.airflow_executor == "Sequential" ? "" : "postgresql+psycopg2://${var.rds_username}:${var.rds_password}@${aws_db_instance.airflow[0].address}:${aws_db_instance.airflow[0].port}/${aws_db_instance.airflow[0].db_name}")
   db_uri       = var.airflow_executor == "Local" ? local.postgres_uri : "sqlite:////opt/airflow/airflow.db"
 
   s3_bucket_name = var.s3_bucket_name != "" ? var.s3_bucket_name : aws_s3_bucket.airflow[0].id

main.tf

@@ -1,9 +1,9 @@
 terraform {
-  required_version = "~> 0.15"
+  required_version = "~> 1.6.2"
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 3.12.0"
+      version = "~> 5.0"
     }
   }
 }

rds.tf

@@ -1,6 +1,6 @@
 resource "aws_db_instance" "airflow" {
   count                      = var.postgres_uri != "" || var.airflow_executor == "Sequential" ? 0 : 1
-  name                       = replace(title(local.rds_name), "-", "")
+  db_name                    = replace(title(local.rds_name), "-", "")
   allocated_storage          = var.rds_allocated_storage
   storage_type               = var.rds_storage_type
   engine                     = var.rds_engine

s3.tf

@@ -1,34 +1,61 @@
 resource "aws_s3_bucket" "airflow" {
   count  = var.s3_bucket_name == "" ? 1 : 0
   bucket = "${var.resource_prefix}-airflow-${var.resource_suffix}"
-  acl    = "private"
+  tags   = local.common_tags
+}
 
-  versioning {
-    enabled = true
-  }
+resource "aws_s3_bucket_public_access_block" "airflow" {
+  count  = var.s3_bucket_name == "" ? 1 : 0
+  bucket = aws_s3_bucket.airflow[0].id
 
-  server_side_encryption_configuration {
-    rule {
-      apply_server_side_encryption_by_default {
-        sse_algorithm = "aws:kms"
-      }
+  block_public_acls       = false
+  block_public_policy     = false
+  ignore_public_acls      = false
+  restrict_public_buckets = false
+}
+
+data "aws_iam_policy_document" "airflow" {
+  statement {
+    effect = "Allow"
+    principals {
+      type        = "*"
+      identifiers = ["*"]
     }
+    actions = [
+      "s3:GetObject"
+    ]
+    resources = [
+      "${aws_s3_bucket.airflow[0].arn}/startup/requirements.txt",
+    ]
   }
+}
+resource "aws_s3_bucket_policy" "airflow" {
+  bucket = aws_s3_bucket.airflow[0].id
+  policy = data.aws_iam_policy_document.airflow.json
+}
 
-  tags = local.common_tags
+resource "aws_s3_bucket_ownership_controls" "airflow" {
+  bucket = aws_s3_bucket.airflow[0].id
+  rule {
+    object_ownership = "ObjectWriter"
+  }
 }
 
-resource "aws_s3_bucket_public_access_block" "airflow" {
-  count  = var.s3_bucket_name == "" ? 1 : 0
+resource "aws_s3_bucket_acl" "airflow" {
+  depends_on = [aws_s3_bucket_ownership_controls.airflow]
+
   bucket = aws_s3_bucket.airflow[0].id
+  acl    = "private"
+}
 
-  block_public_acls       = true
-  block_public_policy     = true
-  ignore_public_acls      = true
-  restrict_public_buckets = true
+resource "aws_s3_bucket_versioning" "airflow" {
+  bucket = aws_s3_bucket.airflow[0].id
+  versioning_configuration {
+    status = "Enabled"
+  }
 }
 
-resource "aws_s3_bucket_object" "airflow_seed_dag" {
+resource "aws_s3_object" "airflow_seed_dag" {
   bucket = local.s3_bucket_name
   key    = "dags/airflow_seed_dag.py"
   content = templatefile("${path.module}/templates/dags/airflow_seed_dag.py", {
@@ -41,26 +68,32 @@ resource "aws_s3_bucket_object" "airflow_seed_dag" {
   })
 }
 
-resource "aws_s3_bucket_object" "airflow_example_dag" {
+resource "aws_s3_object" "airflow_example_dag" {
   count   = var.airflow_example_dag ? 1 : 0
   bucket  = local.s3_bucket_name
   key     = "dags/example_dag.py"
   content = templatefile("${path.module}/templates/dags/example_dag.py", {})
 }
 
-resource "aws_s3_bucket_object" "airflow_scheduler_entrypoint" {
-  bucket  = local.s3_bucket_name
-  key     = "startup/entrypoint_scheduler.sh"
-  content = templatefile("${path.module}/templates/startup/entrypoint_scheduler.sh", { AIRFLOW_HOME = var.airflow_container_home })
+resource "aws_s3_object" "airflow_scheduler_entrypoint" {
+  bucket = local.s3_bucket_name
+  key    = "startup/entrypoint_scheduler.sh"
+  content = templatefile("${path.module}/templates/startup/entrypoint_scheduler.sh", {
+    AIRFLOW_HOME             = var.airflow_container_home,
+    S3_URL_REQUIREMENTS_FILE = "https://${local.s3_bucket_name}.s3.${var.region}.amazonaws.com/startup/requirements.txt",
+  })
 }
 
-resource "aws_s3_bucket_object" "airflow_webserver_entrypoint" {
-  bucket  = local.s3_bucket_name
-  key     = "startup/entrypoint_webserver.sh"
-  content = templatefile("${path.module}/templates/startup/entrypoint_webserver.sh", { AIRFLOW_HOME = var.airflow_container_home })
+resource "aws_s3_object" "airflow_webserver_entrypoint" {
+  bucket = local.s3_bucket_name
+  key    = "startup/entrypoint_webserver.sh"
+  content = templatefile("${path.module}/templates/startup/entrypoint_webserver.sh", {
+    AIRFLOW_HOME             = var.airflow_container_home,
+    S3_URL_REQUIREMENTS_FILE = "https://${local.s3_bucket_name}.s3.${var.region}.amazonaws.com/startup/requirements.txt",
+  })
 }
 
-resource "aws_s3_bucket_object" "airflow_init_entrypoint" {
+resource "aws_s3_object" "airflow_init_entrypoint" {
   bucket = local.s3_bucket_name
   key    = "startup/entrypoint_init.sh"
   content = templatefile("${path.module}/templates/startup/entrypoint_init.sh", {
@@ -74,7 +107,7 @@ resource "aws_s3_bucket_object" "airflow_init_entrypoint" {
   })
 }
 
-resource "aws_s3_bucket_object" "airflow_requirements" {
+resource "aws_s3_object" "airflow_requirements" {
   count   = var.airflow_py_requirements_path == "" ? 0 : 1
   bucket  = local.s3_bucket_name
   key     = "startup/requirements.txt"

templates/startup/entrypoint_init.sh

@@ -8,6 +8,7 @@ if [[ "$airflow_major_version" == "1" ]]; then
     airflow initdb
 else
     airflow db init
+    airflow db migrate
 fi
 
 # add admin user if rbac enabled and not exists
@@ -17,15 +18,15 @@ if [[ "${RBAC_AUTH}" == "true" ]]; then
     if [[ "$airflow_major_version" == "1" ]]; then
         amount_of_users=$(python -c 'import sys;print((sys.argv.count("│") // 7) - 1)' $(airflow list_users))
     else
-        amount_of_users=$(python -c 'import sys;cmd_in = " ".join(sys.argv);print((cmd_in.count("|") // 5) - 1 if "No data found" not in cmd_in else 0)' $(airflow users list))  
+        amount_of_users=$(python -c 'import sys;cmd_in = " ".join(sys.argv);print((cmd_in.count("|") // 5) - 1 if "No data found" not in cmd_in else 0)' $(airflow users list))
     fi
 
     if [[ "$amount_of_users" == "0" ]]; then
         echo "Adding admin users, users list is empty!"
         if [[ "$airflow_major_version" == "1" ]]; then
             airflow create_user -r Admin -u ${RBAC_USERNAME} -e ${RBAC_EMAIL} -f ${RBAC_FIRSTNAME} -l ${RBAC_LASTNAME} -p ${RBAC_PASSWORD}
         else
-            airflow users create -r Admin -u ${RBAC_USERNAME} -e ${RBAC_EMAIL} -f ${RBAC_FIRSTNAME} -l ${RBAC_LASTNAME} -p ${RBAC_PASSWORD}  
+            airflow users create -r Admin -u ${RBAC_USERNAME} -e ${RBAC_EMAIL} -f ${RBAC_FIRSTNAME} -l ${RBAC_LASTNAME} -p ${RBAC_PASSWORD}
         fi
     else
         echo "No admin user added, users already exists!"

templates/startup/entrypoint_scheduler.sh

@@ -5,10 +5,17 @@ ls /opt/airflow/dags
 # Install boto and awscli for the seed dag
 python -m pip install awscli --user
 
-# Intall python packages through req.txt and pip (if exists)
-if [[ -f "${AIRFLOW_HOME}/startup/requirements.txt" ]]; then
+# Check if the requirements file exists and fetch it if it does
+if curl --head --fail -s ${S3_URL_REQUIREMENTS_FILE} > /dev/null; then
     echo "requirements.txt provided, installing it with pip"
-    python -m pip install -r ${AIRFLOW_HOME}/startup/requirements.txt --user
+    # Download the requirements.txt file to the AIRFLOW_HOME directory
+    curl -s -o "${AIRFLOW_HOME}/requirements.txt" ${S3_URL_REQUIREMENTS_FILE}
+
+    # Install the requirements using pip
+    pip install -r "${AIRFLOW_HOME}/requirements.txt" --user
+else
+    echo "requirements.txt provided, but not found in S3 bucket"
 fi
+
 # Run the airflow webserver
 airflow scheduler

templates/startup/entrypoint_webserver.sh

@@ -5,10 +5,16 @@ ls /opt/airflow/dags
 # Install boto and awscli for the seed dag
 python -m pip install awscli --user
 
-# Intall python packages through req.txt and pip (if exists)
-if [[ -f "${AIRFLOW_HOME}/startup/requirements.txt" ]]; then
+# Check if the requirements file exists and fetch it if it does
+if curl --head --fail -s ${S3_URL_REQUIREMENTS_FILE} > /dev/null; then
     echo "requirements.txt provided, installing it with pip"
-    python -m pip install -r ${AIRFLOW_HOME}/startup/requirements.txt --user
+    # Download the requirements.txt file to the AIRFLOW_HOME directory
+    curl -s -o "${AIRFLOW_HOME}/requirements.txt" ${S3_URL_REQUIREMENTS_FILE}
+
+    # Install the requirements using pip
+    pip install -r "${AIRFLOW_HOME}/requirements.txt" --user
+else
+    echo "requirements.txt provided, but not found in S3 bucket"
 fi
 
 export AIRFLOW__WEBSERVER__SECRET_KEY=$(openssl rand -hex 30)

templates/startup/requirements.txt

@@ -0,0 +1,2 @@
+# example
+pysftp==0.2.9

variables.tf

@@ -30,7 +30,7 @@ variable "airflow_image_name" {
 variable "airflow_image_tag" {
   type        = string
   description = "The tag of the airflow image"
-  default     = "2.0.1"
+  default     = "2.7.0"
 }
 
 variable "airflow_executor" {