Merge branch 'dev' into 1177-notifications-update-slack-notification-pipeline

Author: CarsonDavis

.github/workflows/run_full_test_suite.yml

@@ -0,0 +1,37 @@
+name: Django Test Suite on PR
+
+on:
+  pull_request:
+    branches:
+      - dev
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+
+    services:
+      docker:
+        image: docker:24.0.5
+        options: --privileged
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Check out merged code
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+
+      - name: Build the Docker environment
+        run: docker-compose -f local.yml build
+
+      - name: Run test suite
+        env:
+          DJANGO_ENV: test
+        run: docker-compose -f local.yml run --rm django bash ./init.sh
+
+      - name: Cleanup
+        run: docker-compose -f local.yml down --volumes

.pre-commit-config.yaml

@@ -8,6 +8,8 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
+      - id: check-merge-conflict
+      - id: debug-statements
 
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.17.0
@@ -37,14 +39,41 @@ repos:
     hooks:
       - id: mypy
         args: ["--strict"]
-        # ignoring everything for now
-        exclude: .
-        additional_dependencies: [django-stubs, celery,  django-environ, django-extensions, django-crispy-forms,
-        crispy-bootstrap5, django-allauth, django-celery-beat, djangorestframework, djangorestframework-datatables,
-        django-debug-toolbar, psycopg2-binary, python-slugify, xmltodict, PyGithub, boto3, scrapy, types-requests]
+        exclude: "."
+        additional_dependencies:
+          - django-stubs
+          - celery
+          - django-environ
+          - django-extensions
+          - django-crispy-forms
+          - crispy-bootstrap5
+          - django-allauth
+          - django-celery-beat
+          - djangorestframework
+          - djangorestframework-datatables
+          - django-debug-toolbar
+          - psycopg2-binary
+          - python-slugify
+          - xmltodict
+          - PyGithub
+          - boto3
+          - scrapy
+          - types-requests
+
+  - repo: https://github.com/PyCQA/bandit
+    rev: '1.7.0'
+    hooks:
+      - id: bandit
+        args: ['-r', '--configfile=bandit-config.yml']
+
+  - repo: https://github.com/zricethezav/gitleaks
+    rev: 'v8.0.4'
+    hooks:
+      - id: gitleaks
+        args: ['--config=gitleaks-config.toml']
+
 
 
-# sets up .pre-commit-ci.yaml to ensure pre-commit dependencies stay up to date
 ci:
   autoupdate_schedule: weekly
   skip: []

CODE_STANDARDS.md

@@ -0,0 +1,63 @@
+# Coding Standards and Conventions for COSMOS
+
+## Overview
+To maintain high-quality code and ensure consistency across the entire COSMOS project, we have established coding standards and conventions. This document outlines the key standards and practices that all contributors are expected to follow. Adhering to these guidelines helps us to achieve a codebase that appears as if it were written by a single entity, regardless of the number of contributors.
+
+## Coding Standards
+
+### Formatting Standards
+- **Line Length**: Maximum of 120 characters per line to ensure readability across various environments.
+- **Code Formatting**: Utilize tools like Black for Python code to ensure consistent formatting across the entire codebase.
+- **Import Ordering**: Follow a consistent import order:
+  - Standard library imports.
+  - Third-party imports.
+  - Application-specific imports.
+
+### Naming Conventions
+- **Variables and Functions**: Use `snake_case`.
+- **Classes and Exceptions**: Use `CamelCase`.
+- **Constants**: Use `UPPER_CASE`.
+
+### Commenting
+- Inline comments should be used sparingly and only when necessary to explain "why" something is done, not "what" is done.
+- All public methods, classes, and modules should include docstrings that follow the [Google style guide](https://google.github.io/styleguide/pyguide.html).
+
+### Error Handling
+- Explicit is better than implicit. Raise exceptions rather than returning None or any error codes.
+- Use custom exceptions over generic exceptions when possible to make error handling more predictive.
+
+## Tool Configurations and Pre-commit Hooks
+
+To automate and enforce these standards, the following tools are configured with pre-commit hooks in our development process:
+
+### Pre-commit Hooks Setup
+
+To ensure that these tools are run automatically on every commit, contributors must set up pre-commit hooks locally. Run the following commands to install and configure pre-commit hooks:
+
+```bash
+pip install pre-commit
+pre-commit install
+pre-commit run --all-files
+```
+
+The following pre-commit hooks are configured:
+
+- trailing-whitespace, end-of-file-fixer, check-yaml, check-merge-conflict, debug-statements: Checks for common formatting issues.
+- pyupgrade: Automatically upgrades syntax for newer versions of the language.
+- black: Formats Python code to ensure consistent styling.
+- isort: Sorts imports alphabetically and automatically separated into sections.
+- flake8: Lints code to catch styling errors and potential bugs.
+- mypy: Checks type annotations to catch potential bugs.
+- bandit: Scans code for common security issues.
+- gitleaks: Prevents secrets from being committed to the repository.
+- hadolint: Lints Dockerfiles to ensure best practices and common conventions are followed.
+
+## Continuous Integration (CI)
+When a commit is pushed to a branch that is part of a Pull Request, our Continuous Integration (CI) pipeline automatically runs specified tools to check code quality, style, security and other standards. If these checks fail, the PR cannot be merged until all issues are resolved.
+
+## Quality Standards Enforcement
+- PRs must pass all checks from the configured pre-commit hooks and CI pipeline to be eligible for merging.
+- Code reviews additionally focus on logical errors and code quality beyond what automated tools can detect.
+
+## Conclusion
+By adhering to these standards and utilizing the tools set up, we maintain the high quality and consistency of our codebase, making it easier for developers to collaborate effectively.

README.md

@@ -225,6 +225,7 @@ $ pip install pre-commit
 $ pre-commit install
 $ pre-commit run --all-files
 ```
+For detailed information on the coding standards and conventions we enforce, please see our [Coding Standards and Conventions](CODE_STANDARDS.md).
 
 ### Sentry Setup
 

bandit-config.yml

@@ -0,0 +1,26 @@
+# bandit-config.yml
+skips:
+  - B101  # Skip assert used (often used in tests)
+  - B403  # Skip import from the pickle module
+
+exclude:
+  - ./tests/         # Exclude test directories
+  - ./migrations/    # Exclude migration directories
+  - ./venv/          # Exclude virtual environment
+
+tests:
+  - B105             # Include test for hardcoded password strings
+  - B602             # Include test for subprocess call with shell equals true
+
+profiles:
+  default:
+    include:
+      - B403         # Include test for dangerous default argument
+    exclude:
+      - B401         # Exclude test for import telnetlib
+
+# Set the severity level to focus on higher-risk issues
+severity: 'HIGH'
+
+# Set the confidence level to ensure that reported issues are likely true positives
+confidence: 'HIGH'

functional_tests/check_collection.py renamed to functional_tests/test_check_collection.py

@@ -0,0 +1,31 @@
+#!/bin/bash
+echo "Running all test cases across the project..."
+
+# Initialize a failure counter
+failure_count=0
+
+# Exclude tests in `document_classifier` and `functional_tests` directories
+excluded_dirs="document_classifier functional_tests"
+
+# Find all test files except those in excluded directories
+test_files=$(find . -type f -name "test_*.py" | grep -Ev "$(echo $excluded_dirs | sed 's/ /|/g')")
+
+# Run each test file
+for test_file in $test_files; do
+    echo "Running $test_file..."
+    pytest "$test_file"
+
+    # Check the exit status of pytest
+    if [ $? -ne 0 ]; then
+        echo "Test failed: $test_file"
+        failure_count=$((failure_count + 1))
+    fi
+done
+
+# Report the results
+if [ $failure_count -ne 0 ]; then
+    echo "$failure_count test(s) failed."
+    exit 1
+else
+    echo "All tests passed successfully!"
+fi

init.sh

@@ -63,6 +63,20 @@ def test_quality_check_perfect_triggers_public_query(self, mock_add):
 
 class TestReindexingStatusTransitions(TestCase):
     def setUp(self):
+        # Mock the GitHubHandler to return valid XML content
+        self.mock_github_handler = patch("sde_collections.models.collection.GitHubHandler").start()
+
+        self.mock_github_handler.return_value._get_file_contents.return_value.decoded_content = (
+            b'<?xml version="1.0" encoding="UTF-8"?>\n'
+            b"<Sinequa>\n"
+            b"    <KeepHashFragmentInUrl>false</KeepHashFragmentInUrl>\n"
+            b"    <CollectionSelection>Sample Collection</CollectionSelection>\n"
+            b"</Sinequa>"
+        )
+
+        self.addCleanup(patch.stopall)
+
+        # Create the collection with the mock applied
         self.collection = CollectionFactory(
             workflow_status=WorkflowStatusChoices.QUALITY_CHECK_PERFECT,
             reindexing_status=ReindexingStatusChoices.REINDEXING_NOT_NEEDED,

sde_collections/tests/test_models_collections.py

@@ -10,7 +10,6 @@
 
 from sde_indexing_helper.users.forms import UserAdminChangeForm
 from sde_indexing_helper.users.models import User
-from sde_indexing_helper.users.tests.factories import UserFactory
 from sde_indexing_helper.users.views import (
     UserRedirectView,
     UserUpdateView,
@@ -22,39 +21,45 @@
 
 class TestUserUpdateView:
     """
-    TODO:
-        extracting view initialization code as class-scoped fixture
-        would be great if only pytest-django supported non-function-scoped
-        fixture db access -- this is a work-in-progress for now:
-        https://github.com/pytest-dev/pytest-django/pull/258
+    Tests for the UserUpdateView.
     """
 
-    def dummy_get_response(self, request: HttpRequest):
+    @staticmethod
+    def dummy_get_response(request: HttpRequest):
+        """Dummy get_response method for middleware testing."""
         return None
 
     def test_get_success_url(self, user: User, rf: RequestFactory):
+        """
+        Test that UserUpdateView redirects to the correct success URL.
+        """
         view = UserUpdateView()
         request = rf.get("/fake-url/")
         request.user = user
-
         view.request = request
 
-        assert view.get_success_url() == f"/users/{user.username}/"
+        expected_url = f"/users/{user.username}/"
+        assert view.get_success_url() == expected_url, f"Expected {expected_url}, got {view.get_success_url()}"
 
     def test_get_object(self, user: User, rf: RequestFactory):
+        """
+        Test that UserUpdateView retrieves the correct user object.
+        """
         view = UserUpdateView()
         request = rf.get("/fake-url/")
         request.user = user
-
         view.request = request
 
         assert view.get_object() == user
 
     def test_form_valid(self, user: User, rf: RequestFactory):
+        """
+        Test that form submission in UserUpdateView processes correctly.
+        """
         view = UserUpdateView()
         request = rf.get("/fake-url/")
 
-        # Add the session/message middleware to the request
+        # Add session and message middleware
         SessionMiddleware(self.dummy_get_response).process_request(request)
         MessageMiddleware(self.dummy_get_response).process_request(request)
         request.user = user
@@ -72,26 +77,43 @@ def test_form_valid(self, user: User, rf: RequestFactory):
 
 
 class TestUserRedirectView:
+    """
+    Tests for the UserRedirectView.
+    """
+
     def test_get_redirect_url(self, user: User, rf: RequestFactory):
+        """
+        Test that UserRedirectView redirects to the "sde_collections:list" URL.
+        """
         view = UserRedirectView()
-        request = rf.get("/fake-url")
+        request = rf.get("/fake-url/")
         request.user = user
-
         view.request = request
 
-        assert view.get_redirect_url() == f"/users/{user.username}/"
+        expected_url = reverse("sde_collections:list")
+        assert view.get_redirect_url() == expected_url, f"Expected {expected_url}, got {view.get_redirect_url()}"
 
 
 class TestUserDetailView:
+    """
+    Tests for the user_detail_view function.
+    """
+
     def test_authenticated(self, user: User, rf: RequestFactory):
+        """
+        Test that an authenticated user can access their detail view.
+        """
         request = rf.get("/fake-url/")
-        request.user = UserFactory()
+        request.user = user
 
         response = user_detail_view(request, username=user.username)
 
         assert response.status_code == 200
 
     def test_not_authenticated(self, user: User, rf: RequestFactory):
+        """
+        Test that an unauthenticated user is redirected to the login page.
+        """
         request = rf.get("/fake-url/")
         request.user = AnonymousUser()