Fix running e2e backend

.github/workflows/run-ci-cd.yaml

@@ -251,7 +251,7 @@ jobs:
 
       - name: Run frontend end-to-end tests
         run: |
-          docker run --env-file frontend/.env.example owasp/nest:test-frontend-e2e-latest pnpm run test:e2e
+          docker run --env-file frontend/.env.e2e.example owasp/nest:test-frontend-e2e-latest pnpm run test:e2e
 
   set-release-version:
     name: Set release version

.github/workflows/setup-e2e-environment/action.yaml

@@ -7,23 +7,18 @@ runs:
     steps:
       - name: Wait for database to be ready
         run: |
-          until docker exec ${{ job.services.db.id }} pg_isready -U nest_user_e2e -d nest_db_e2e; do
-            echo "Waiting for database..."
-            sleep 5
-          done
+          timeout 5m bash -c '
+            until docker exec ${{ job.services.db.id }} pg_isready -U nest_user_e2e -d nest_db_e2e; do
+              echo "Waiting for database..."
+              sleep 5
+            done
+          '
         shell: bash
 
       - name: Install PostgreSQL client
         run: sudo apt-get install -y postgresql-client
         shell: bash
 
-      - name: Load Postgres data
-        env:
-          PGPASSWORD: nest_user_e2e_password
-        run: |
-          gunzip -c backend/data/nest-e2e.sql.gz | psql -h localhost -U nest_user_e2e -d nest_db_e2e
-        shell: bash
-
       - name: Build backend e2e image
         uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
         with:
@@ -43,17 +38,28 @@ runs:
             --env-file backend/.env.e2e.example \
             --network host \
             -p 9000:9000 \
+            -e DJANGO_DB_HOST=localhost \
             owasp/nest:test-backend-e2e-latest \
             sh -c '
+              python manage.py migrate &&
               gunicorn wsgi:application --bind 0.0.0.0:9000
           '
         shell: bash
 
       - name: Waiting for the backend to be ready
         run: |
-          until wget --spider http://localhost:9000/a; do
-            echo "Waiting for backend..."
-            sleep 5
-          done
+          timeout 5m bash -c '
+            until wget --spider http://localhost:9000/a; do
+              echo "Waiting for backend..."
+              sleep 5
+            done
+          '
           echo "Backend is up!"
         shell: bash
+
+      - name: Load Postgres data
+        env:
+          PGPASSWORD: nest_user_e2e_password
+        run: |
+          gunzip -c backend/data/nest.sql.gz | psql -h localhost -U nest_user_e2e -d nest_db_e2e
+        shell: bash

CONTRIBUTING.md

@@ -286,6 +286,8 @@ Ensure that all `.env` files are saved in **UTF-8 format without BOM (Byte Order
 
 1. **Load Initial Data**:
 
+   - Make sure you have `gzip` installed on your machine.
+
    - Open a new terminal session and run the following command to populate the database with initial data from fixtures:
 
    ```bash

backend/.env.e2e.example

@@ -6,7 +6,7 @@ DJANGO_AWS_ACCESS_KEY_ID=None
 DJANGO_AWS_SECRET_ACCESS_KEY=None
 DJANGO_SETTINGS_MODULE=settings.e2e
 DJANGO_CONFIGURATION=E2E
-DJANGO_DB_HOST=None
+DJANGO_DB_HOST=db
 DJANGO_DB_NAME=nest_db_e2e
 DJANGO_DB_USER=nest_user_e2e
 DJANGO_DB_PASSWORD=nest_user_e2e_password

backend/Makefile

@@ -49,22 +49,7 @@ django-shell:
 
 dump-data:
 	@echo "Dumping Nest data"
-	@CMD="python manage.py dumpdata \
-		github \
-		owasp \
-		slack.Conversation \
-		slack.Member \
-		slack.Message \
-		slack.Workspace \
-		--indent=4 \
-		--natural-foreign \
-		--natural-primary -o data/nest.json" $(MAKE) exec-backend-command
-	@CMD="sed -E -i 's/(\"[^\"]*email\"): *\"([^\"]|\\\")*\"/\1: \"\"/g' data/nest.json" $(MAKE) exec-backend-command
-	@CMD="gzip -f data/nest.json" $(MAKE) exec-backend-command
-
-dump-data-e2e:
-	@echo "Dumping Nest e2e data"
-	@CMD="pg_dumpall -U nest_user_e2e --clean | gzip -9 > backend/data/nest-e2e.sql.gz" $(MAKE) exec-db-command-e2e
+	@CMD="python manage.py dump_data" $(MAKE) exec-backend-command-it
 
 enrich-data: \
 	github-enrich-issues \
@@ -84,11 +69,11 @@ index-data:
 
 load-data:
 	@echo "Loading Nest data"
-	@CMD="python manage.py load_data" $(MAKE) exec-backend-command
+	@gunzip -c backend/data/nest.sql.gz | docker exec -i nest-db psql -U nest_user_dev -d nest_db_dev
 
 load-data-e2e:
 	@echo "Loading Nest e2e data"
-	@gunzip -c backend/data/nest-e2e.sql.gz | docker exec -i e2e-nest-db psql -U nest_user_e2e -d nest_db_e2e
+	@gunzip -c backend/data/nest.sql.gz | docker exec -i e2e-nest-db psql -U nest_user_e2e -d nest_db_e2e
 
 merge-migrations:
 	@CMD="python manage.py makemigrations --merge" $(MAKE) exec-backend-command

backend/apps/api/rest/v0/__init__.py

@@ -57,6 +57,17 @@
         ],
         "throttle": [],
     }
+elif settings.IS_E2E_ENVIRONMENT:
+    api_settings_customization = {
+        "auth": None,
+        "servers": [
+            {
+                "description": "E2E",
+                "url": settings.SITE_URL,
+            }
+        ],
+        "throttle": [],
+    }
 elif settings.IS_STAGING_ENVIRONMENT:
     api_settings_customization = {
         "servers": [

backend/apps/common/management/commands/dump_data.py

@@ -0,0 +1,158 @@
+"""Dump masked data from the database into a compressed file."""
+
+import os
+from pathlib import Path
+from subprocess import CalledProcessError, run
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+
+class Command(BaseCommand):
+    help = "Create a dump of selected db tables."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--output",
+            default=str(Path(settings.BASE_DIR) / "data" / "nest.sql.gz"),
+            help="Output dump path (default: data/nest.sql.gz)",
+        )
+        parser.add_argument(
+            "-t",
+            "--table",
+            action="append",
+            dest="tables",
+            default=["public.owasp_*", "public.github_*", "public.slack_*"],
+            help=(
+                "Table pattern to include. "
+                "Defaults: public.owasp_*, public.github_*, public.slack_*"
+            ),
+        )
+
+    def handle(self, *args, **options):
+        db = settings.DATABASES["default"]
+        name = db.get("NAME", "")
+        user = db.get("USER", "")
+        password = db.get("PASSWORD", "")
+        host = db.get("HOST", "localhost")
+        port = str(db.get("PORT", "5432"))
+        output_path = Path(options["output"]).resolve()
+        tables = options["tables"] or []
+        # Ensure output directory exists
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        temp_db = f"temp_{name}"
+        env = os.environ.copy()
+        if password:
+            env["PGPASSWORD"] = password
+
+        self.stdout.write(self.style.NOTICE(f"Creating temporary database: {temp_db}"))
+        try:
+            # 1) Create temp DB from template
+            self._psql(
+                host,
+                port,
+                user,
+                "postgres",
+                f"CREATE DATABASE {temp_db} TEMPLATE {name};",
+                env,
+            )
+
+            # 2) Hide emails
+            self.stdout.write(self.style.NOTICE("Hiding email fields in temp DB…"))
+            self._psql(host, port, user, temp_db, self._hide_emails(), env, via_stdin=True)
+
+            # 3) Dump selected tables
+            self.stdout.write(self.style.NOTICE(f"Creating dump at: {output_path}"))
+            dump_cmd = [
+                "pg_dump",
+                "-h",
+                host,
+                "-p",
+                port,
+                "-U",
+                user,
+                "-d",
+                temp_db,
+                "--compress=9",
+                "--clean",
+            ]
+            dump_cmd += [f"--table={table}" for table in tables]
+            dump_cmd += ["-f", str(output_path)]
+
+            run(dump_cmd, check=True, env=env)
+            self.stdout.write(self.style.SUCCESS(f"Dump created: {output_path}"))
+        except CalledProcessError as e:
+            message = f"Command failed: {e.cmd}"
+            raise CommandError(message) from e
+        finally:
+            # 4) Drop temp DB
+            self.stdout.write(self.style.NOTICE(f"Dropping temporary database: {temp_db}"))
+            try:
+                self._psql(
+                    host,
+                    port,
+                    user,
+                    "postgres",
+                    f"DROP DATABASE IF EXISTS {temp_db};",
+                    env,
+                )
+            except CalledProcessError:
+                # Best-effort cleanup
+                self.stderr.write(
+                    self.style.WARNING(f"Failed to drop temp DB {temp_db} (ignored).")
+                )
+
+    def _hide_emails(self) -> str:
+        # Uses a DO block to UPDATE every column named 'email' in non-system schemas
+        return """
+DO $$
+DECLARE
+    record RECORD;
+    statement TEXT;
+BEGIN
+    FOR record IN
+        SELECT quote_ident(n.nspname) AS schemaname,
+               quote_ident(c.relname) AS tablename,
+               quote_ident(a.attname) AS colname
+        FROM pg_attribute a
+        JOIN pg_class c ON c.oid = a.attrelid
+        JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE a.attname = 'email'
+          AND a.attnum > 0
+          AND NOT a.attisdropped
+          AND n.nspname NOT IN ('pg_catalog','information_schema','pg_toast')
+    LOOP
+        statement := format(
+            'UPDATE %s.%s SET %s = %L;', record.schemaname, record.tablename, record.colname, ''
+        );
+        EXECUTE statement;
+    END LOOP;
+END$$;
+""".strip()
+
+    def _psql(
+        self,
+        host: str,
+        port: str,
+        user: str,
+        dbname: str,
+        sql: str,
+        env: dict,
+        *,
+        via_stdin: bool = False,
+    ):
+        # Inputs are trusted; safe subprocess usage.
+        if via_stdin:
+            run(
+                ["psql", "-h", host, "-p", port, "-U", user, "-d", dbname],
+                input=sql.encode(),
+                check=True,
+                env=env,
+            )
+            return
+        run(
+            ["psql", "-h", host, "-p", port, "-U", user, "-d", dbname, "-c", sql],
+            check=True,
+            env=env,
+        )

backend/pyproject.toml

@@ -94,6 +94,10 @@ lint.per-file-ignores."**/management/commands/*.py" = [
   "D102", # https://docs.astral.sh/ruff/rules/undocumented-public-method/
   "T201", # https://docs.astral.sh/ruff/rules/print/
 ]
+lint.per-file-ignores."**/management/commands/dump_data.py" = [
+  "S603", # https://docs.astral.sh/ruff/rules/subprocess-without-shell-equals-true/,
+  "S607", # https://docs.astral.sh/ruff/rules/start-process-with-partial-path/,
+]
 lint.per-file-ignores."**/migrations/*.py" = [
   "D100",   # https://docs.astral.sh/ruff/rules/undocumented-public-module/
   "D101",   # https://docs.astral.sh/ruff/rules/undocumented-public-class/

backend/settings/e2e.py

@@ -9,6 +9,7 @@ class E2E(Base):
     """End-to-end testing configuration."""
 
     APP_NAME = "OWASP Nest E2E Testing"
+    SITE_URL = "http://localhost:9000"
 
     ALLOWED_ORIGINS = (
         "http://frontend:3000",  # NOSONAR
@@ -17,7 +18,6 @@ class E2E(Base):
     CORS_ALLOWED_ORIGINS = ALLOWED_ORIGINS
     CSRF_TRUSTED_ORIGINS = ALLOWED_ORIGINS
 
-    DEBUG = False
     IS_E2E_ENVIRONMENT = True
     LOGGING = {}
     PUBLIC_IP_ADDRESS = values.Value()

cspell/custom-dict.txt

@@ -2,6 +2,7 @@ Agentic
 Agsoc
 Aichi
 Aissue
+Atqc
 Aupdated
 BOTTOMPADDING
 CCSP
@@ -19,6 +20,7 @@ NOASSERTION
 NOSONAR
 Nadu
 Nominatim
+PGPASSWORD
 PLR
 PYTHONUNBUFFERED
 RUF
@@ -44,6 +46,7 @@ apk
 arithmatex
 arkid15r
 askowasp
+attisdropped
 bangbang
 bsky
 certbot
@@ -96,6 +99,7 @@ navlink
 nestbot
 noinput
 nosniff
+nspname
 openstreetmap
 owasppcitoolkit
 owtf