diff --git a/.github/workflows/run-ci-cd.yaml b/.github/workflows/run-ci-cd.yaml index 5ad623cf00..2703e81e93 100644 --- a/.github/workflows/run-ci-cd.yaml +++ b/.github/workflows/run-ci-cd.yaml @@ -251,7 +251,7 @@ jobs: - name: Run frontend end-to-end tests run: | - docker run --env-file frontend/.env.example owasp/nest:test-frontend-e2e-latest pnpm run test:e2e + docker run --env-file frontend/.env.e2e.example owasp/nest:test-frontend-e2e-latest pnpm run test:e2e set-release-version: name: Set release version diff --git a/.github/workflows/setup-e2e-environment/action.yaml b/.github/workflows/setup-e2e-environment/action.yaml index 6892826c39..63397bdd80 100644 --- a/.github/workflows/setup-e2e-environment/action.yaml +++ b/.github/workflows/setup-e2e-environment/action.yaml @@ -7,23 +7,18 @@ runs: steps: - name: Wait for database to be ready run: | - until docker exec ${{ job.services.db.id }} pg_isready -U nest_user_e2e -d nest_db_e2e; do - echo "Waiting for database..." - sleep 5 - done + timeout 5m bash -c ' + until docker exec ${{ job.services.db.id }} pg_isready -U nest_user_e2e -d nest_db_e2e; do + echo "Waiting for database..." + sleep 5 + done + ' shell: bash - name: Install PostgreSQL client run: sudo apt-get install -y postgresql-client shell: bash - - name: Load Postgres data - env: - PGPASSWORD: nest_user_e2e_password - run: | - gunzip -c backend/data/nest-e2e.sql.gz | psql -h localhost -U nest_user_e2e -d nest_db_e2e - shell: bash - - name: Build backend e2e image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 with: @@ -43,17 +38,28 @@ runs: --env-file backend/.env.e2e.example \ --network host \ -p 9000:9000 \ + -e DJANGO_DB_HOST=localhost \ owasp/nest:test-backend-e2e-latest \ sh -c ' + python manage.py migrate && gunicorn wsgi:application --bind 0.0.0.0:9000 ' shell: bash - name: Waiting for the backend to be ready run: | - until wget --spider http://localhost:9000/a; do - echo "Waiting for backend..." - sleep 5 - done + timeout 5m bash -c ' + until wget --spider http://localhost:9000/a; do + echo "Waiting for backend..." + sleep 5 + done + ' echo "Backend is up!" shell: bash + + - name: Load Postgres data + env: + PGPASSWORD: nest_user_e2e_password + run: | + gunzip -c backend/data/nest.sql.gz | psql -h localhost -U nest_user_e2e -d nest_db_e2e + shell: bash diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db8922a13f..a1973698a6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -286,6 +286,8 @@ Ensure that all `.env` files are saved in **UTF-8 format without BOM (Byte Order 1. **Load Initial Data**: + - Make sure you have `gzip` installed on your machine. + - Open a new terminal session and run the following command to populate the database with initial data from fixtures: ```bash diff --git a/backend/.env.e2e.example b/backend/.env.e2e.example index cd926019dd..7043b2cd60 100644 --- a/backend/.env.e2e.example +++ b/backend/.env.e2e.example @@ -6,7 +6,7 @@ DJANGO_AWS_ACCESS_KEY_ID=None DJANGO_AWS_SECRET_ACCESS_KEY=None DJANGO_SETTINGS_MODULE=settings.e2e DJANGO_CONFIGURATION=E2E -DJANGO_DB_HOST=None +DJANGO_DB_HOST=db DJANGO_DB_NAME=nest_db_e2e DJANGO_DB_USER=nest_user_e2e DJANGO_DB_PASSWORD=nest_user_e2e_password diff --git a/backend/Makefile b/backend/Makefile index 3d7ffacf59..5d31fc4f7f 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -49,22 +49,7 @@ django-shell: dump-data: @echo "Dumping Nest data" - @CMD="python manage.py dumpdata \ - github \ - owasp \ - slack.Conversation \ - slack.Member \ - slack.Message \ - slack.Workspace \ - --indent=4 \ - --natural-foreign \ - --natural-primary -o data/nest.json" $(MAKE) exec-backend-command - @CMD="sed -E -i 's/(\"[^\"]*email\"): *\"([^\"]|\\\")*\"/\1: \"\"/g' data/nest.json" $(MAKE) exec-backend-command - @CMD="gzip -f data/nest.json" $(MAKE) exec-backend-command - -dump-data-e2e: - @echo "Dumping Nest e2e data" - @CMD="pg_dumpall -U nest_user_e2e --clean | gzip -9 > backend/data/nest-e2e.sql.gz" $(MAKE) exec-db-command-e2e + @CMD="python manage.py dump_data" $(MAKE) exec-backend-command-it enrich-data: \ github-enrich-issues \ @@ -84,11 +69,11 @@ index-data: load-data: @echo "Loading Nest data" - @CMD="python manage.py load_data" $(MAKE) exec-backend-command + @gunzip -c backend/data/nest.sql.gz | docker exec -i nest-db psql -U nest_user_dev -d nest_db_dev load-data-e2e: @echo "Loading Nest e2e data" - @gunzip -c backend/data/nest-e2e.sql.gz | docker exec -i e2e-nest-db psql -U nest_user_e2e -d nest_db_e2e + @gunzip -c backend/data/nest.sql.gz | docker exec -i e2e-nest-db psql -U nest_user_e2e -d nest_db_e2e merge-migrations: @CMD="python manage.py makemigrations --merge" $(MAKE) exec-backend-command diff --git a/backend/apps/api/rest/v0/__init__.py b/backend/apps/api/rest/v0/__init__.py index 63d8277835..46b4aeefa9 100644 --- a/backend/apps/api/rest/v0/__init__.py +++ b/backend/apps/api/rest/v0/__init__.py @@ -57,6 +57,17 @@ ], "throttle": [], } +elif settings.IS_E2E_ENVIRONMENT: + api_settings_customization = { + "auth": None, + "servers": [ + { + "description": "E2E", + "url": settings.SITE_URL, + } + ], + "throttle": [], + } elif settings.IS_STAGING_ENVIRONMENT: api_settings_customization = { "servers": [ diff --git a/backend/apps/common/management/commands/dump_data.py b/backend/apps/common/management/commands/dump_data.py new file mode 100644 index 0000000000..2ea1f9a20f --- /dev/null +++ b/backend/apps/common/management/commands/dump_data.py @@ -0,0 +1,145 @@ +"""Dump masked data from the database into a compressed file.""" + +import contextlib +import os +from pathlib import Path +from subprocess import CalledProcessError, run + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from psycopg2 import ProgrammingError, connect, sql + +DB = settings.DATABASES["default"] +HOST = DB.get("HOST", "localhost") +PORT = str(DB.get("PORT", "5432")) +USERNAME = DB.get("USER", "") +PASSWORD = DB.get("PASSWORD", "") +NAME = DB.get("NAME", "") + + +class Command(BaseCommand): + help = "Create a dump of selected db tables." + + def add_arguments(self, parser): + parser.add_argument( + "--output", + default=str(Path(settings.BASE_DIR) / "data" / "nest.sql.gz"), + help="Output dump path (default: data/nest.sql.gz)", + ) + parser.add_argument( + "-t", + "--table", + action="append", + dest="tables", + default=[ + "public.owasp_*", + "public.github_*", + "public.slack_members", + "public.slack_workspaces", + "public.slack_conversations", + "public.slack_messages", + ], + help=( + "Table pattern to include. " + "Defaults: public.owasp_*, public.github_*, public.slack_members, " + "public.slack_workspaces, public.slack_conversations, public.slack_messages." + ), + ) + + def handle(self, *args, **options): + output_path = Path(options["output"]).resolve() + tables = options["tables"] or [] + output_path.parent.mkdir(parents=True, exist_ok=True) + + temp_db = f"temp_{NAME}" + env = os.environ.copy() + env["PGPASSWORD"] = PASSWORD + self.stdout.write(self.style.NOTICE(f"Creating temporary database: {temp_db}")) + try: + # 1) Create temp DB from template + self._execute_sql( + "postgres", + [f"CREATE DATABASE {temp_db} TEMPLATE {NAME};"], + ) + # 2) Get tables with email field + self.stdout.write(self.style.NOTICE("Fetching tables with email fields…")) + + table_list = self._execute_sql( + temp_db, + [self._table_list_query()], + ) + + # 3) Hide email fields + self.stdout.write(self.style.NOTICE("Hiding email fields in temp DB…")) + self._execute_sql(temp_db, self._hide_emails_queries([row[0] for row in table_list])) + # 4) Dump selected tables + self.stdout.write(self.style.NOTICE(f"Creating dump at: {output_path}")) + dump_cmd = [ + "pg_dump", + "-h", + HOST, + "-p", + PORT, + "-U", + USERNAME, + "-d", + temp_db, + "--compress=9", + "--clean", + ] + dump_cmd += [f"--table={table}" for table in tables] + dump_cmd += ["-f", str(output_path)] + + run(dump_cmd, check=True, env=env) + self.stdout.write(self.style.SUCCESS(f"Dump created: {output_path}")) + except CalledProcessError as e: + message = f"Command failed: {e.cmd}" + raise CommandError(message) from e + finally: + # 4) Drop temp DB + self.stdout.write(self.style.NOTICE(f"Dropping temporary database: {temp_db}")) + try: + self._execute_sql( + "postgres", + [f"DROP DATABASE IF EXISTS {temp_db};"], + ) + except CalledProcessError: + # Best-effort cleanup + self.stderr.write( + self.style.WARNING(f"Failed to drop temp DB {temp_db} (ignored).") + ) + + def _table_list_query(self) -> str: + return """ + SELECT table_name + FROM information_schema.columns + WHERE table_schema = 'public' AND column_name = 'email'; + """ + + def _hide_emails_queries(self, tables: list[str]) -> list[str]: + return [ + sql.SQL("UPDATE {table} SET email = '';").format(table=sql.Identifier(table)) + for table in tables + ] + + def _execute_sql( + self, + dbname: str, + sql_queries: list[str], + ): + connection = connect( + dbname=dbname, + user=USERNAME, + password=PASSWORD, + host=HOST, + port=PORT, + ) + connection.autocommit = True + rows = [] + with connection.cursor() as cursor: + for sql in sql_queries: + cursor.execute(sql) + with contextlib.suppress(ProgrammingError): + rows.extend(cursor.fetchall()) + connection.close() + return rows diff --git a/backend/apps/common/management/commands/load_data.py b/backend/apps/common/management/commands/load_data.py deleted file mode 100644 index 4705ab873b..0000000000 --- a/backend/apps/common/management/commands/load_data.py +++ /dev/null @@ -1,17 +0,0 @@ -"""A command to load OWASP Nest data.""" - -from django.core.management import call_command -from django.core.management.base import BaseCommand -from django.db import transaction - -from apps.core.utils import index - - -class Command(BaseCommand): - help = "Load OWASP Nest data." - - def handle(self, *_args, **_options) -> None: - """Load data into the OWASP Nest application.""" - with index.disable_indexing(), transaction.atomic(): - # Run loaddata - call_command("loaddata", "data/nest.json.gz", "-v", "3") diff --git a/backend/data/nest-e2e.sql.gz b/backend/data/nest-e2e.sql.gz deleted file mode 100644 index 83bba95a67..0000000000 Binary files a/backend/data/nest-e2e.sql.gz and /dev/null differ diff --git a/backend/data/nest.json.gz b/backend/data/nest.sql.gz similarity index 75% rename from backend/data/nest.json.gz rename to backend/data/nest.sql.gz index 1b03123b4d..44650c02a4 100644 Binary files a/backend/data/nest.json.gz and b/backend/data/nest.sql.gz differ diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 11e2a1ae3f..ee9b2d87ec 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -94,6 +94,10 @@ lint.per-file-ignores."**/management/commands/*.py" = [ "D102", # https://docs.astral.sh/ruff/rules/undocumented-public-method/ "T201", # https://docs.astral.sh/ruff/rules/print/ ] +lint.per-file-ignores."**/management/commands/dump_data.py" = [ + "S603", # https://docs.astral.sh/ruff/rules/subprocess-without-shell-equals-true/, + "S607", # https://docs.astral.sh/ruff/rules/start-process-with-partial-path/, +] lint.per-file-ignores."**/migrations/*.py" = [ "D100", # https://docs.astral.sh/ruff/rules/undocumented-public-module/ "D101", # https://docs.astral.sh/ruff/rules/undocumented-public-class/ diff --git a/backend/settings/e2e.py b/backend/settings/e2e.py index f47cf7e43d..4e3895c957 100644 --- a/backend/settings/e2e.py +++ b/backend/settings/e2e.py @@ -9,15 +9,22 @@ class E2E(Base): """End-to-end testing configuration.""" APP_NAME = "OWASP Nest E2E Testing" + SITE_URL = "http://localhost:9000" ALLOWED_ORIGINS = ( "http://frontend:3000", # NOSONAR "http://localhost:3000", ) + + CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + } + } + CORS_ALLOWED_ORIGINS = ALLOWED_ORIGINS CSRF_TRUSTED_ORIGINS = ALLOWED_ORIGINS - DEBUG = False IS_E2E_ENVIRONMENT = True LOGGING = {} PUBLIC_IP_ADDRESS = values.Value() diff --git a/backend/tests/apps/common/management/commands/dump_data_test.py b/backend/tests/apps/common/management/commands/dump_data_test.py new file mode 100644 index 0000000000..1c025e4e70 --- /dev/null +++ b/backend/tests/apps/common/management/commands/dump_data_test.py @@ -0,0 +1,89 @@ +from unittest.mock import MagicMock, patch + +from django.core.management import call_command +from django.test import override_settings + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": "db-name", + "USER": "db-user", + "PASSWORD": "db-pass", # NOSONAR + "HOST": "db-host", + "PORT": "5432", + } +} + + +class TestDumpDataCommand: + @override_settings(DATABASES=DATABASES) + @patch("apps.common.management.commands.dump_data.run") + @patch("apps.common.management.commands.dump_data.connect") + @patch("apps.common.management.commands.dump_data.Path") + @patch("apps.common.management.commands.dump_data.sql") + def test_dump_data(self, mock_sql, mock_path, mock_connect, mock_run): + # Mock psycopg2 connection/cursor + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_connect.return_value = mock_conn + mock_conn.cursor.return_value.__enter__.return_value = mock_cursor + mock_cursor.fetchall.return_value = [("public.users",), ("public.members",)] + mock_resolve = MagicMock() + mock_path.return_value.resolve.return_value = mock_resolve + mock_sql.SQL.return_value.format.return_value = "UPDATE public.users SET email = '';" + call_command( + "dump_data", + "--output", + "data/dump.sql.gz", + ) + + # Verify temp DB created from template + expected_temp_db = "temp_db-name" + mock_connect.assert_any_call( + dbname="postgres", + user="db-user", + # ruff: noqa: S106 + password="db-pass", # NOSONAR + host="db-host", + port="5432", + ) + mock_cursor.execute.assert_any_call( + f"CREATE DATABASE {expected_temp_db} TEMPLATE db-name;" + ) + executed_sql = [str(c.args[0]) for c in mock_cursor.execute.call_args_list] + assert "UPDATE public.users SET email = '';" in executed_sql + assert any( + """ + SELECT table_name + FROM information_schema.columns + WHERE table_schema = 'public' AND column_name = 'email'; + """.strip() + in str(query).strip() + for query in executed_sql + ) + + assert [ + "pg_dump", + "-h", + "db-host", + "-p", + "5432", + "-U", + "db-user", + "-d", + expected_temp_db, + "--compress=9", + "--clean", + "--table=public.owasp_*", + "--table=public.github_*", + "--table=public.slack_members", + "--table=public.slack_workspaces", + "--table=public.slack_conversations", + "--table=public.slack_messages", + "-f", + str(mock_resolve), + ] == mock_run.call_args[0][0] + # Ensure DROP DATABASE executed at the end + mock_cursor.execute.assert_any_call(f"DROP DATABASE IF EXISTS {expected_temp_db};") + mock_path.return_value.resolve.assert_called_once() + mock_resolve.parent.mkdir.assert_called_once_with(parents=True, exist_ok=True) diff --git a/backend/tests/apps/common/management/commands/load_data_test.py b/backend/tests/apps/common/management/commands/load_data_test.py deleted file mode 100644 index 0b384171c1..0000000000 --- a/backend/tests/apps/common/management/commands/load_data_test.py +++ /dev/null @@ -1,64 +0,0 @@ -import contextlib -from unittest.mock import MagicMock, patch - -from apps.common.management.commands.load_data import Command - - -class TestLoadDataCommand: - @patch("apps.core.utils.index.DisableIndexing.unregister_indexes") - @patch("apps.core.utils.index.DisableIndexing.register_indexes") - @patch("apps.common.management.commands.load_data.call_command") - @patch("apps.common.management.commands.load_data.transaction.atomic") - def test_handle( - self, - mock_atomic, - mock_call_command, - mock_register, - mock_unregister, - ): - mock_model = MagicMock() - mock_app_config = MagicMock() - mock_app_config.get_models.return_value = [mock_model] - - mock_atomic.return_value.__enter__ = MagicMock() - mock_atomic.return_value.__exit__ = MagicMock() - - mock_unregister.return_value = None - mock_register.return_value = None - - command = Command() - command.handle() - - mock_unregister.assert_called_once() - mock_register.assert_called_once() - - mock_call_command.assert_called_once_with("loaddata", "data/nest.json.gz", "-v", "3") - - mock_atomic.assert_called_once() - - @patch("apps.core.utils.index.DisableIndexing.unregister_indexes") - @patch("apps.core.utils.index.DisableIndexing.register_indexes") - @patch("apps.common.management.commands.load_data.call_command") - @patch("apps.common.management.commands.load_data.transaction.atomic") - def test_handle_with_exception_during_call_command( - self, - mock_atomic, - mock_call_command, - mock_register, - mock_unregister, - ): - """Test that indexing is re-enabled even if call_command fails.""" - mock_call_command.side_effect = Exception("Call command failed") - - command = Command() - with patch("contextlib.suppress") as mock_suppress: - mock_suppress.return_value.__enter__ = MagicMock() - mock_suppress.return_value.__exit__ = MagicMock() - - with contextlib.suppress(Exception): - command.handle() - - mock_unregister.assert_called_once() - mock_register.assert_called_once() - - mock_atomic.assert_called_once() diff --git a/cspell/custom-dict.txt b/cspell/custom-dict.txt index e4e9e9b81d..ca53cafbab 100644 --- a/cspell/custom-dict.txt +++ b/cspell/custom-dict.txt @@ -2,6 +2,7 @@ Agentic Agsoc Aichi Aissue +Atqc Aupdated BOTTOMPADDING CCSP @@ -19,6 +20,7 @@ NOASSERTION NOSONAR Nadu Nominatim +PGPASSWORD PLR PYTHONUNBUFFERED RUF @@ -44,6 +46,7 @@ apk arithmatex arkid15r askowasp +attisdropped bangbang bsky certbot @@ -96,6 +99,7 @@ navlink nestbot noinput nosniff +nspname openstreetmap owasppcitoolkit owtf