Fix running e2e backend

.github/workflows/run-ci-cd.yaml

@@ -251,7 +251,7 @@ jobs:
 
       - name: Run frontend end-to-end tests
         run: |
-          docker run --env-file frontend/.env.example owasp/nest:test-frontend-e2e-latest pnpm run test:e2e
+          docker run --env-file frontend/.env.e2e.example owasp/nest:test-frontend-e2e-latest pnpm run test:e2e
 
   set-release-version:
     name: Set release version

.github/workflows/setup-e2e-environment/action.yaml

@@ -7,10 +7,12 @@ runs:
     steps:
       - name: Wait for database to be ready
         run: |
-          until docker exec ${{ job.services.db.id }} pg_isready -U nest_user_e2e -d nest_db_e2e; do
-            echo "Waiting for database..."
-            sleep 5
-          done
+          timeout 1m bash -c '
+            until docker exec ${{ job.services.db.id }} pg_isready -U nest_user_e2e -d nest_db_e2e; do
+              echo "Waiting for database..."
+              sleep 5
+            done
+          '
         shell: bash
 
       - name: Install PostgreSQL client
@@ -21,7 +23,7 @@ runs:
         env:
           PGPASSWORD: nest_user_e2e_password
         run: |
-          gunzip -c backend/data/nest-e2e.sql.gz | psql -h localhost -U nest_user_e2e -d nest_db_e2e
+          gunzip -c backend/data/nest.sql.gz | psql -h localhost -U nest_user_e2e -d nest_db_e2e
         shell: bash
 
       - name: Build backend e2e image
@@ -43,6 +45,7 @@ runs:
             --env-file backend/.env.e2e.example \
             --network host \
             -p 9000:9000 \
+            -e DJANGO_DB_HOST=localhost \
             owasp/nest:test-backend-e2e-latest \
             sh -c '
               gunicorn wsgi:application --bind 0.0.0.0:9000
@@ -51,9 +54,11 @@ runs:
 
       - name: Waiting for the backend to be ready
         run: |
-          until wget --spider http://localhost:9000/a; do
-            echo "Waiting for backend..."
-            sleep 5
-          done
+          timeout 1m bash -c '
+            until wget --spider http://localhost:9000/a; do
+              echo "Waiting for backend..."
+              sleep 5
+            done
+          '
           echo "Backend is up!"
         shell: bash

CONTRIBUTING.md

@@ -286,6 +286,8 @@ Ensure that all `.env` files are saved in **UTF-8 format without BOM (Byte Order
 
 1. **Load Initial Data**:
 
+   - Make sure you have `gzip` installed on your machine.
+
    - Open a new terminal session and run the following command to populate the database with initial data from fixtures:
 
    ```bash

backend/.env.e2e.example

@@ -6,7 +6,7 @@ DJANGO_AWS_ACCESS_KEY_ID=None
 DJANGO_AWS_SECRET_ACCESS_KEY=None
 DJANGO_SETTINGS_MODULE=settings.e2e
 DJANGO_CONFIGURATION=E2E
-DJANGO_DB_HOST=None
+DJANGO_DB_HOST=db
 DJANGO_DB_NAME=nest_db_e2e
 DJANGO_DB_USER=nest_user_e2e
 DJANGO_DB_PASSWORD=nest_user_e2e_password

backend/Makefile

@@ -49,22 +49,7 @@ django-shell:
 
 dump-data:
 	@echo "Dumping Nest data"
-	@CMD="python manage.py dumpdata \
-		github \
-		owasp \
-		slack.Conversation \
-		slack.Member \
-		slack.Message \
-		slack.Workspace \
-		--indent=4 \
-		--natural-foreign \
-		--natural-primary -o data/nest.json" $(MAKE) exec-backend-command
-	@CMD="sed -E -i 's/(\"[^\"]*email\"): *\"([^\"]|\\\")*\"/\1: \"\"/g' data/nest.json" $(MAKE) exec-backend-command
-	@CMD="gzip -f data/nest.json" $(MAKE) exec-backend-command
-
-dump-data-e2e:
-	@echo "Dumping Nest e2e data"
-	@CMD="pg_dumpall -U nest_user_e2e --clean | gzip -9 > backend/data/nest-e2e.sql.gz" $(MAKE) exec-db-command-e2e
+	@CMD="./data/dump.sh" $(MAKE) exec-backend-command-it
 
 enrich-data: \
 	github-enrich-issues \
@@ -83,12 +68,18 @@ index-data:
 	@CMD="python manage.py algolia_update_synonyms" $(MAKE) exec-backend-command
 
 load-data:
+	@echo "Recreating Nest schema"
+	@CMD="psql -U nest_user_dev -d nest_db_dev -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public; GRANT ALL ON SCHEMA public TO nest_user_dev'" \
+	$(MAKE) exec-db-command-it 2>/dev/null
 	@echo "Loading Nest data"
-	@CMD="python manage.py load_data" $(MAKE) exec-backend-command
+	@gunzip -c backend/data/nest.sql.gz | docker exec -i nest-db psql -U nest_user_dev -d nest_db_dev
 
 load-data-e2e:
+	@echo "Recreating Nest e2e schema"
+	@CMD="psql -U nest_user_e2e -d nest_db_e2e -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public; GRANT ALL ON SCHEMA public TO nest_user_e2e'" \
+	$(MAKE) exec-db-command-e2e 2>/dev/null
 	@echo "Loading Nest e2e data"
-	@gunzip -c backend/data/nest-e2e.sql.gz | docker exec -i e2e-nest-db psql -U nest_user_e2e -d nest_db_e2e
+	@gunzip -c backend/data/nest.sql.gz | docker exec -i e2e-nest-db psql -U nest_user_e2e -d nest_db_e2e
 
 merge-migrations:
 	@CMD="python manage.py makemigrations --merge" $(MAKE) exec-backend-command

backend/apps/api/decorators/cache.py

@@ -36,6 +36,8 @@ def cache_response(
     def decorator(view_func):
         @wraps(view_func)
         def _wrapper(request, *args, **kwargs):
+            if settings.IS_E2E_ENVIRONMENT:
+                return view_func(request, *args, **kwargs)
             if request.method not in ("GET", "HEAD"):
                 return view_func(request, *args, **kwargs)
 

backend/apps/api/rest/v0/__init__.py

@@ -57,6 +57,19 @@
         ],
         "throttle": [],
     }
+
+elif settings.IS_E2E_ENVIRONMENT:
+    api_settings_customization = {
+        "auth": None,
+        "servers": [
+            {
+                "description": "E2E",
+                "url": settings.SITE_URL,
+            }
+        ],
+        "throttle": [],
+    }
+
 elif settings.IS_STAGING_ENVIRONMENT:
     api_settings_customization = {
         "servers": [

backend/data/dump.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+
+set -e
+
+# Cleanup function to ensure temp DB is dropped even on error
+cleanup() {
+    if [ -n "$TEMP_DB" ]; then
+        echo "Cleaning up temporary database $TEMP_DB..."
+        psql -h "$DJANGO_DB_HOST" -U "$DJANGO_DB_USER" -d postgres -c "DROP DATABASE IF EXISTS $TEMP_DB;" 2>/dev/null || true
+    fi
+}
+trap cleanup EXIT
+
+export PGPASSWORD="$DJANGO_DB_PASSWORD"
+export TEMP_DB="temp_$DJANGO_DB_NAME"
+
+# 1. Create a temporary copy of the database
+echo "Creating temporary database $TEMP_DB…"
+
+psql -h "$DJANGO_DB_HOST" -U "$DJANGO_DB_USER" -d postgres -c \
+    "CREATE DATABASE $TEMP_DB TEMPLATE $DJANGO_DB_NAME;"
+
+# 2. Generate all UPDATE statements dynamically
+UPDATES=$(psql -h "$DJANGO_DB_HOST" -U "$DJANGO_DB_USER" -d "$TEMP_DB" -Atqc "
+    SELECT 'UPDATE '
+        || quote_ident(n.nspname) || '.' || quote_ident(c.relname)
+        || ' SET ' || quote_ident(a.attname)
+        || ' = '''';'
+    FROM pg_attribute a
+    JOIN pg_class c ON c.oid = a.attrelid
+    JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE a.attname = 'email'
+      AND a.attnum > 0
+      AND NOT a.attisdropped
+      AND n.nspname NOT IN ('pg_catalog','information_schema');
+")
+
+if [ -z "$UPDATES" ]; then
+    echo "No email fields found to hide."
+else
+    echo "Hiding email addresses…"
+    echo "$UPDATES" | psql -h "$DJANGO_DB_HOST" -U "$DJANGO_DB_USER" -d "$TEMP_DB"
+fi
+
+# 3. Dump the DB
+echo "Creating dump…"
+pg_dump -h "$DJANGO_DB_HOST" -U "$DJANGO_DB_USER" -d "$TEMP_DB" | gzip -9 > ./data/nest.sql.gz
+
+# 4. Drop the temporary database
+echo "Dropping temporary database $TEMP_DB…"
+psql -h "$DJANGO_DB_HOST" -U "$DJANGO_DB_USER" -d postgres -c "DROP DATABASE $TEMP_DB;"
+
+echo "Dump created: data/nest.sql.gz"

backend/settings/e2e.py

@@ -9,6 +9,7 @@ class E2E(Base):
     """End-to-end testing configuration."""
 
     APP_NAME = "OWASP Nest E2E Testing"
+    SITE_URL = "http://localhost:9000"
 
     ALLOWED_ORIGINS = (
         "http://frontend:3000",  # NOSONAR
@@ -17,7 +18,6 @@ class E2E(Base):
     CORS_ALLOWED_ORIGINS = ALLOWED_ORIGINS
     CSRF_TRUSTED_ORIGINS = ALLOWED_ORIGINS
 
-    DEBUG = False
     IS_E2E_ENVIRONMENT = True
     LOGGING = {}
     PUBLIC_IP_ADDRESS = values.Value()

cspell/custom-dict.txt

@@ -2,6 +2,7 @@ Agentic
 Agsoc
 Aichi
 Aissue
+Atqc
 Aupdated
 BOTTOMPADDING
 CCSP
@@ -19,6 +20,7 @@ NOASSERTION
 NOSONAR
 Nadu
 Nominatim
+PGPASSWORD
 PLR
 PYTHONUNBUFFERED
 RUF
@@ -44,6 +46,7 @@ apk
 arithmatex
 arkid15r
 askowasp
+attisdropped
 bangbang
 bsky
 certbot
@@ -96,6 +99,7 @@ navlink
 nestbot
 noinput
 nosniff
+nspname
 openstreetmap
 owasppcitoolkit
 owtf