Write Customization tests in python (#2918)

* port everything integration test related to python

Author: hubertdeng123

.github/workflows/test.yml

@@ -55,11 +55,11 @@ jobs:
   integration-test:
     if: github.repository_owner == 'getsentry'
     runs-on: ubuntu-20.04
-    name: "integration test"
+    name: integration test ${{ matrix.compose_version }} - customizations ${{ matrix.customizations }}
     strategy:
       fail-fast: false
       matrix:
-        test_type: ["initial-install", "customizations"]
+        customizations: ["disabled", "enabled"]
         compose_version: ["v2.0.1", "v2.7.0"]
         include:
           - compose_version: "v2.0.1"
@@ -68,18 +68,19 @@ jobs:
             compose_path: "/usr/local/lib/docker/cli-plugins"
     env:
       COMPOSE_PROJECT_NAME: self-hosted-${{ strategy.job-index }}
-      SENTRY_DSN: https://[email protected]/6627632
-      REPORT_SELF_HOSTED_ISSUES: 1
+      REPORT_SELF_HOSTED_ISSUES: 0
+      SELF_HOSTED_TESTING_DSN: ${{ vars.SELF_HOSTED_TESTING_DSN }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Setup dev environment
         run: |
           pip install -r requirements-dev.txt
+          echo "PY_COLORS=1" >> "$GITHUB_ENV"
           ### pytest-sentry configuration ###
           if [ "$GITHUB_REPOSITORY" = "getsentry/self-hosted" ]; then
-            echo "PYTEST_SENTRY_DSN=${{ env.SELF_HOSTED_TESTING_DSN }}" >> $GITHUB_ENV
+            echo "PYTEST_SENTRY_DSN=$SELF_HOSTED_TESTING_DSN" >> $GITHUB_ENV
             echo "PYTEST_SENTRY_TRACES_SAMPLE_RATE=0" >> $GITHUB_ENV
 
             # This records failures on master to sentry in order to detect flakey tests, as it's
@@ -102,13 +103,29 @@ jobs:
           sudo chmod +x "${{ matrix.compose_path }}/docker-compose"
 
       - name: Install self-hosted
-        run: ./install.sh
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 10
+          max_attempts: 3
+          command: ./install.sh
 
       - name: Integration Test
-        run: ./integration-test.sh --${{ matrix.test_type }}
+        run: pytest --cov --junitxml=junit.xml --reruns 3 _integration-test/ --customizations=${{ matrix.customizations }}
 
       - name: Inspect failure
         if: failure()
         run: |
           docker compose ps
           docker compose logs
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: getsentry/self-hosted
+
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}

_integration-test/conftest.py

@@ -10,27 +10,49 @@
 TEST_PASS = "test123TEST"
 TIMEOUT_SECONDS = 60
 
+def pytest_addoption(parser):
+    parser.addoption("--customizations",  default="disabled")
 
 @pytest.fixture(scope="session", autouse=True)
-def configure_self_hosted_environment():
+def configure_self_hosted_environment(request):
     subprocess.run(["docker", "compose", "--ansi", "never", "up", "-d"], check=True)
     for i in range(TIMEOUT_SECONDS):
         try:
             response = httpx.get(SENTRY_TEST_HOST, follow_redirects=True)
-        except httpx.ConnectionError:
+        except httpx.NetworkError:
             time.sleep(1)
         else:
             if response.status_code == 200:
                 break
     else:
         raise AssertionError("timeout waiting for self-hosted to come up")
 
+    if request.config.getoption("--customizations") == "enabled":
+        os.environ['TEST_CUSTOMIZATIONS'] = "enabled"
+        script_content = '''\
+#!/bin/bash
+touch /created-by-enhance-image
+apt-get update
+apt-get install -y gcc libsasl2-dev python-dev libldap2-dev libssl-dev
+'''
+
+        with open('sentry/enhance-image.sh', 'w') as script_file:
+            script_file.write(script_content)
+        # Set executable permissions for the shell script
+        os.chmod('sentry/enhance-image.sh', 0o755)
+
+        # Write content to the requirements.txt file
+        with open('sentry/requirements.txt', 'w') as req_file:
+            req_file.write('python-ldap\n')
+        os.environ['MINIMIZE_DOWNTIME'] = "1"
+        subprocess.run(["./install.sh"], check=True)
     # Create test user
     subprocess.run(
         [
             "docker",
             "compose",
             "exec",
+            "-T",
             "web",
             "sentry",
             "createuser",
@@ -45,3 +67,8 @@ def configure_self_hosted_environment():
         check=True,
         text=True,
     )
+
+@pytest.fixture()
+def setup_backup_restore_env_variables():
+    os.environ['SENTRY_DOCKER_IO_DIR'] = os.path.join(os.getcwd(), 'sentry')
+    os.environ['SKIP_USER_CREATION'] = "1"

_integration-test/ensure-customizations-not-present.sh

@@ -1,13 +1,8 @@
 import subprocess
 import os
-import pytest
 
-@pytest.fixture()
-def setup_env():
-    os.environ['SENTRY_DOCKER_IO_DIR'] = os.path.join(os.getcwd(), 'sentry')
-    os.environ['SKIP_USER_CREATION'] = "1"
 
-def test_backup(setup_env):
+def test_backup(setup_backup_restore_env_variables):
     # Docker was giving me permissioning issues when trying to create this file and write to it even after giving read + write access
     # to group and owner. Instead, try creating the empty file and then give everyone write access to the backup file
     file_path = os.path.join(os.getcwd(), 'sentry', 'backup.json')
@@ -18,7 +13,7 @@ def test_backup(setup_env):
     subprocess.run([sentry_admin_sh, "export", "global", "/sentry-admin/backup.json", "--no-prompt"], check=True)
     assert os.path.getsize(file_path) > 0
 
-def test_import(setup_env):
+def test_import(setup_backup_restore_env_variables):
     # Bring postgres down and recreate the docker volume
     subprocess.run(["docker", "compose", "--ansi", "never", "stop", "postgres"], check=True)
     subprocess.run(["docker", "compose", "--ansi", "never", "rm", "-f", "-v", "postgres"], check=True)

_integration-test/ensure-customizations-work.sh

@@ -141,7 +141,18 @@ def test_custom_cas():
     try:
         subprocess.run(["./_integration-test/custom-ca-roots/setup.sh"], check=True)
         subprocess.run(
-            ["docker", "compose", "--ansi", "never", "run", "--no-deps", "web", "python3", "/etc/sentry/test-custom-ca-roots.py"], check=True
+            [
+                "docker",
+                "compose",
+                "--ansi",
+                "never",
+                "run",
+                "--no-deps",
+                "web",
+                "python3",
+                "/etc/sentry/test-custom-ca-roots.py",
+            ],
+            check=True,
         )
     finally:
         subprocess.run(["./_integration-test/custom-ca-roots/teardown.sh"], check=True)
@@ -171,3 +182,64 @@ def placeholder_fn():
         client,
         lambda x: len(json.loads(x)["data"]) > 0,
     )
+
+
+def test_customizations():
+    commands = [
+        [
+            "docker",
+            "compose",
+            "--ansi",
+            "never",
+            "run",
+            "--no-deps",
+            "web",
+            "bash",
+            "-c",
+            "if [ ! -e /created-by-enhance-image ]; then exit 1; fi",
+        ],
+        [
+            "docker",
+            "compose",
+            "--ansi",
+            "never",
+            "run",
+            "--no-deps",
+            "--entrypoint=/etc/sentry/entrypoint.sh",
+            "sentry-cleanup",
+            "bash",
+            "-c",
+            "if [ ! -e /created-by-enhance-image ]; then exit 1; fi",
+        ],
+        [
+            "docker",
+            "compose",
+            "--ansi",
+            "never",
+            "run",
+            "--no-deps",
+            "web",
+            "python",
+            "-c",
+            "import ldap",
+        ],
+        [
+            "docker",
+            "compose",
+            "--ansi",
+            "never",
+            "run",
+            "--no-deps",
+            "--entrypoint=/etc/sentry/entrypoint.sh",
+            "sentry-cleanup",
+            "python",
+            "-c",
+            "import ldap",
+        ]
+    ]
+    for command in commands:
+        result = subprocess.run(command, check=False)
+        if os.getenv("TEST_CUSTOMIZATIONS", "disabled") == "enabled":
+            assert result.returncode == 0
+        else:
+            assert result.returncode != 0

_integration-test/backup.py renamed to _integration-test/test_backup.py

@@ -1,5 +1,7 @@
+codecov-cli>=0.4.8
 sentry-sdk>=1.39.2
 pytest>=8.0.0
+pytest-cov>=4.1.0
 pytest-rerunfailures>=11.0
 pytest-sentry>=0.1.11
 httpx>=0.25.2