fix: isolate dev, ci, prod environemnts; consolidate secrets (#1215)

* fix: isolate dev, ci, prod environemnts; consolidate secrets

* fix(settings): always define settings environment

* fix: pylint

Author: Jay-Lalwani

.env.example

@@ -1,49 +1,39 @@
-# Django
-DJANGO_SETTINGS_MODULE=?
-DEBUG=?
-SECRET_KEY=?
-
-ENVIRONMENT=?
-
-# review drive account
-REVIEW_DRIVE_ID=?
-REVIEW_DRIVE_EMAIL=?
-REVIEW_DRIVE_PASSWORD=?
-
-# local database
-DB_NAME=?
-DB_USER=?
-DB_PASSWORD=?
-DB_HOST=?
-DB_PORT=?
-
-# prod database
-PROD_DB_NAME=?
-PROD_DB_USER=?
-PROD_DB_PASSWORD=?
-PROD_DB_HOST=?
-PROD_DB_PORT=?
-
-# feedback
-EMAIL_HOST_USER=?
-EMAIL_HOST_PASSWORD=?
-
-# aws
-AWS_ACCESS_KEY_ID=?
-AWS_SECRET_ACCESS_KEY=?
-
-# s3
-AWS_STORAGE_BUCKET_NAME=?
-AWS_S3_REGION_NAME=?
-
-# cognito
-COGNITO_USER_POOL_ID=?
-COGNITO_APP_CLIENT_ID=?
-COGNITO_APP_CLIENT_SECRET=?
-COGNITO_DOMAIN=?
-COGNITO_REGION_NAME=?
-
-# ec2
-EC2_USER=?
-EC2_HOST=?
-PEM_KEY=?
+# =============================================================================
+# REQUIRED FOR ALL ENVIRONMENTS
+# =============================================================================
+SECRET_KEY=any-random-string
+
+# =============================================================================
+# LOCAL DEVELOPMENT (docker-compose, local_dump.sh, reset-db.sh)
+# =============================================================================
+DB_NAME=tcf_db
+DB_USER=tcf_django
+DB_PASSWORD=s3kr1t
+DB_HOST=tcf_db
+DB_PORT=5432
+
+# =============================================================================
+# OPTIONAL - For auth features (login/logout/profile)
+# =============================================================================
+# COGNITO_USER_POOL_ID=
+# COGNITO_APP_CLIENT_ID=
+# COGNITO_APP_CLIENT_SECRET=
+# COGNITO_DOMAIN=
+# COGNITO_REGION_NAME=us-east-1
+
+# =============================================================================
+# OPTIONAL - For prod_dump.sh script (must activate EC2 instances & get pem key)
+# =============================================================================
+# EC2_HOST=
+# EC2_USER=
+# PEM_KEY=path/to/your/key.pem
+# PROD_DB_HOST=
+# PROD_DB_USER=
+# PROD_DB_PASSWORD=
+
+# =============================================================================
+# OPTIONAL - For load_review_drive management command
+# =============================================================================
+# REVIEW_DRIVE_ID=
+# REVIEW_DRIVE_EMAIL=
+# REVIEW_DRIVE_PASSWORD=

.github/workflows/aws.yml

@@ -1,76 +1,30 @@
-# Based on https://github.com/actions/starter-workflows/blob/main/ci/django.yml
+# AWS ECS Deployment Workflow
 name: AWS Deployment
 
 on:
-  # CI must pass on the master branch
   workflow_run:
     workflows: ["Continuous Integration"]
     branches: [master]
     types:
       - completed
 
 env:
-  PYTHON_TARGET: 3.11
-  # Django
-  DJANGO_SETTINGS_MODULE: tcf_core.settings.prod
-  SECRET_KEY: ${{ secrets.SECRET_KEY }}
-  DEBUG: 0
-  ENVIRONMENT: production
-  # database
-  AWS_RDS_NAME: ${{ secrets.AWS_RDS_NAME }}
-  AWS_RDS_USER: ${{ secrets.AWS_RDS_USER }}
-  AWS_RDS_PASSWORD: ${{ secrets.AWS_DS_PASSWORD }}
-  AWS_RDS_HOST: ${{ secrets.AWS_RDS_HOST }}
-  AWS_RDS_PORT: ${{ secrets.AWS_RDS_PORT }}
-  # AWS Cognito
-  COGNITO_USER_POOL_ID: ${{ secrets.COGNITO_USER_POOL_ID }}
-  COGNITO_APP_CLIENT_ID: ${{ secrets.COGNITO_APP_CLIENT_ID }}
-  COGNITO_APP_CLIENT_SECRET: ${{ secrets.COGNITO_APP_CLIENT_SECRET }}
-  COGNITO_DOMAIN: ${{ secrets.COGNITO_DOMAIN }}
-  COGNITO_REGION_NAME: ${{ secrets.COGNITO_REGION_NAME }}
-  # email for account verification
-  EMAIL_HOST_USER: ${{ secrets.EMAIL_HOST_USER }}
-  EMAIL_HOST_PASSWORD: ${{ secrets.EMAIL_HOST_PASSWORD }}
-  # review drive account information
-  REVIEW_DRIVE_ID: ${{ secrets.REVIEW_DRIVE_ID }}
-  REVIEW_DRIVE_EMAIL: ${{ secrets.REVIEW_DRIVE_EMAIL }}
-  REVIEW_DRIVE_PASSWORD: ${{ secrets.REVIEW_DRIVE_PASSWORD }}
-  # aws ecs image
   AWS_REGION: us-east-1
   ECS_SERVICE: barrett-fogle-love-v1
   ECS_CLUSTER: tcf-fargate-cluster
   ECR_REPO: tcf/thecourseforum2
   ECR_CONTAINER_NAME: tcf-container
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
-  AWS_STORAGE_BUCKET_NAME: ${{ secrets.AWS_STORAGE_BUCKET_NAME }}
-  AWS_S3_CUSTOM_DOMAIN: ${{ secrets.AWS_S3_CUSTOM_DOMAIN }}
 
 jobs:
   deploy:
-    # if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout `master`
+      - name: Checkout master
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.workflow_run.head_sha }}
 
-      - name: Set up Python ${{ env.PYTHON_TARGET }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_TARGET }}
-          cache: 'pip'
-
-      - name: Install Python packages
-        run: |
-          python -m pip install --upgrade pip
-          pip install django django-environ django-storages boto3 django-cachalot djangorestframework django-filter psycopg2-binary
-
-      - name: Collect static files
-        run: python manage.py collectstatic --no-input
-
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -79,38 +33,37 @@ jobs:
           aws-region: ${{ env.AWS_REGION }}
 
       - name: Login to Amazon ECR
-        id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build and push
+      - name: Build and push Docker image
         uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile
           push: true
-          tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPO }}:${{github.sha}}
+          tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPO }}:${{ github.sha }}
           cache-from: type=registry,ref=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPO }}:buildcache
           cache-to: type=registry,ref=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPO }}:buildcache,mode=max
           provenance: false
 
-      - name: Get latest task definition
+      - name: Get current task definition
         run: |
           aws ecs describe-task-definition \
             --task-definition tcf-prod-task \
             --query taskDefinition > task-definition.json
 
-      - name: Add image in ECS task definition
+      - name: Update task definition with new image
         id: task-def
         uses: aws-actions/amazon-ecs-render-task-definition@v1
         with:
           task-definition: task-definition.json
           container-name: ${{ env.ECR_CONTAINER_NAME }}
-          image: ${{ env.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPO }}:${{github.sha}}
+          image: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPO }}:${{ github.sha }}
 
-      - name: Deploy Amazon ECS task definition
+      - name: Deploy to Amazon ECS
         uses: aws-actions/amazon-ecs-deploy-task-definition@v2
         with:
           task-definition: ${{ steps.task-def.outputs.task-definition }}

.github/workflows/ci.yml

@@ -5,40 +5,19 @@ on:
   pull_request:
     branches:
       - "*"
-  # The following is needed to run tests upon direct push to dev or master
   push:
     branches: [dev, master]
+
 env:
   PYTHON_TARGET: 3.11
-  # Django
   DJANGO_SETTINGS_MODULE: tcf_core.settings.ci
-  SECRET_KEY: ${{ secrets.SECRET_KEY }}
-  DEBUG: 1
-  ENVIRONMENT: dev
-  # database
-  DB_NAME: tcf_db # arbitrary string
-  DB_USER: postgres # default user
-  DB_PASSWORD: postgres # default password
-  DB_HOST: localhost # required for GitHub Actions
-  DB_PORT: 5432 # default port
-  # AWS Cognito
-  COGNITO_USER_POOL_ID: ${{ secrets.COGNITO_USER_POOL_ID }}
-  COGNITO_APP_CLIENT_ID: ${{ secrets.COGNITO_APP_CLIENT_ID }}
-  COGNITO_APP_CLIENT_SECRET: ${{ secrets.COGNITO_APP_CLIENT_SECRET }}
-  COGNITO_DOMAIN: ${{ secrets.COGNITO_DOMAIN }}
-  COGNITO_REGION_NAME: ${{ secrets.COGNITO_REGION_NAME }}
-  # email for account verification
-  EMAIL_HOST_USER: ${{ secrets.EMAIL_HOST_USER }}
-  EMAIL_HOST_PASSWORD: ${{ secrets.EMAIL_HOST_PASSWORD }}
-  # review drive account information
-  REVIEW_DRIVE_ID: ${{ secrets.REVIEW_DRIVE_ID }}
-  REVIEW_DRIVE_EMAIL: ${{ secrets.REVIEW_DRIVE_EMAIL }}
-  REVIEW_DRIVE_PASSWORD: ${{ secrets.REVIEW_DRIVE_PASSWORD }}
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
-  AWS_STORAGE_BUCKET_NAME: ${{ secrets.AWS_STORAGE_BUCKET_NAME }}
-  AWS_S3_CUSTOM_DOMAIN: ${{ secrets.AWS_S3_CUSTOM_DOMAIN }}
+  SECRET_KEY: ci-secret-key-not-for-production
+  # CI database (GitHub Actions postgres service)
+  DB_NAME: tcf_db
+  DB_USER: postgres
+  DB_PASSWORD: postgres
+  DB_HOST: localhost
+  DB_PORT: 5432
 
 jobs:
   pylint:
@@ -51,12 +30,12 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_TARGET }}
           cache: 'pip'
+          cache-dependency-path: 'requirements/*.txt'
 
-      - name: Install Python packages, excluding the unnecessary ones
+      - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          sed -i '/\(coverage\|types\-tqdm\|black\|isort\|gunicorn\|django\-heroku\|uwsgi\)/d' requirements.txt
-          pip install -r requirements.txt
+          pip install -r requirements/ci.txt
 
       - name: Run pylint
         run: |
@@ -71,12 +50,11 @@ jobs:
       postgres:
         image: postgres:15.4
         env:
-          POSTGRES_USER: ${{ env.DB_USER}}
+          POSTGRES_USER: ${{ env.DB_USER }}
           POSTGRES_PASSWORD: ${{ env.DB_PASSWORD }}
           POSTGRES_DB: ${{ env.DB_NAME }}
         ports:
           - 5432:5432
-        # needed because the postgres container does not provide a healthcheck
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
       - uses: actions/checkout@v4
@@ -86,16 +64,15 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_TARGET }}
           cache: 'pip'
+          cache-dependency-path: 'requirements/*.txt'
 
-      - name: Install Python packages, excluding the unnecessary ones
+      - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          sed -i '/\(lint\|mypy\|types\-tqdm\|black\|isort\|gunicorn\|django\-stubs\|uwsgi\)/d' requirements.txt
-          pip install -r requirements.txt
+          pip install -r requirements/ci.txt
 
       - name: Migrations & Tests
         run: |
-          envsubst < .env.example > .env
           python manage.py migrate
           coverage run manage.py test
 

Dockerfile

@@ -14,9 +14,10 @@ RUN apt-get update && \
 
 WORKDIR /app
 
-COPY requirements.txt /app
-
-RUN pip3 install -r requirements.txt --disable-pip-version-check --no-cache-dir
+# Copy requirements and install (use dev.txt for local, base.txt for prod)
+ARG REQUIREMENTS=requirements/base.txt
+COPY requirements/ /app/requirements/
+RUN pip3 install -r /app/${REQUIREMENTS} --disable-pip-version-check --no-cache-dir
 
 COPY . /app/
 

docker-compose.yml

@@ -2,14 +2,19 @@ version: "3"
 
 services:
   web:
-    build: .
+    build:
+      context: .
+      args:
+        REQUIREMENTS: requirements/dev.txt
     command:
-      bash -c "if [ \"${ENVIRONMENT}\" = 'dev' ]; then /wait-for-it.sh tcf_db:${DB_PORT} || exit 1; fi && \
+      bash -c "/wait-for-it.sh tcf_db:${DB_PORT} -- \
       python manage.py migrate && \
       python manage.py collectstatic --noinput && \
       python manage.py invalidate_cachalot tcf_website && \
       echo 'Starting Django Server...' && \
       python manage.py runserver 0.0.0.0:8000"
+    environment:
+      - DJANGO_SETTINGS_MODULE=tcf_core.settings.dev
     volumes:
       - .:/app
       - /app/db/ # exclude the subfolder to prevent potential interference

requirements.txt requirements/base.txtrequirements.txt renamed to requirements/base.txt

@@ -1,27 +1,17 @@
+# Core dependencies for all environments
 Django~=4.2.8
 asgiref~=3.6.0
 backoff~=2.2.1
-black~=24.1.1
 boto3~=1.37.4
-coverage~=7.3.3
 django-cachalot~=2.6.1
 django-environ~=0.11.2
 django-filter~=23.5
 django-storages~=1.14.5
-django-stubs~=4.2.7
 djangorestframework~=3.14.0
 gunicorn~=21.2.0
-html-linter~=0.4.0
-isort~=5.13.2
-mypy-extensions~=1.0.0
-mypy~=1.7.1
 numpy~=1.26.2
 pandas~=2.1.4
 psycopg[binary]>=3.2.12
-pylint-django~=2.5.5
-pylint~=3.0.3
+python-jose~=3.4.0
 requests~=2.31.0
 tqdm~=4.66.1
-types-tqdm~=4.66.0
-uWSGI~=2.0.28
-python-jose~=3.4.0

requirements/ci.txt

@@ -0,0 +1,10 @@
+# CI dependencies (includes base + testing/linting tools)
+-r base.txt
+
+# Linting (for pylint job)
+pylint~=3.0.3
+pylint-django~=2.5.5
+django-stubs~=4.2.7
+
+# Testing (for django job)
+coverage~=7.3.3