feat: create test_analytics app and database (#474)

* feat: create test_analytics app and database

- creates a new test_analytics django app
- modifies the db_settings to declare a new test analytics database
- modifies the db router to route the models from the new django app
  to the new database

* test: fix tests

* remove logs

* fix dummy_settings

* create migration for test_analytics app

Author: joseph-sentry

shared/django_apps/db_routers/__init__.py

@@ -12,46 +12,52 @@ class MultiDatabaseRouter:
     """
 
     def db_for_read(self, model, **hints):
-        if model._meta.app_label == "timeseries":
-            if settings.TIMESERIES_DATABASE_READ_REPLICA_ENABLED:
-                return "timeseries_read"
-            else:
-                return "timeseries"
-        else:
-            if settings.DATABASE_READ_REPLICA_ENABLED:
-                return "default_read"
-            else:
-                return "default"
+        match model._meta.app_label:
+            case "timeseries":
+                if settings.TIMESERIES_DATABASE_READ_REPLICA_ENABLED:
+                    return "timeseries_read"
+                else:
+                    return "timeseries"
+            case "test_analytics":
+                return "test_analytics"
+            case _:
+                if settings.DATABASE_READ_REPLICA_ENABLED:
+                    return "default_read"
+                else:
+                    return "default"
 
     def db_for_write(self, model, **hints):
-        if model._meta.app_label == "timeseries":
-            return "timeseries"
-        else:
-            return "default"
+        match model._meta.app_label:
+            case "timeseries":
+                return "timeseries"
+            case "test_analytics":
+                return "test_analytics"
+            case _:
+                return "default"
 
     def allow_migrate(self, db, app_label, model_name=None, **hints):
-        if (
-            db == "timeseries" or db == "timeseries_read"
-        ) and not settings.TIMESERIES_ENABLED:
-            log.warning("Skipping timeseries migration")
-            return False
-        if db == "default_read" or db == "timeseries_read":
-            log.warning("Skipping migration of read-only database")
-            return False
-        if app_label == "timeseries":
-            return db == "timeseries"
-        else:
-            return db == "default"
+        match db:
+            case "timeseries_read" | "test_analytics_read" | "default_read":
+                return False
+            case "timeseries":
+                if not settings.TIMESERIES_ENABLED:
+                    return False
+                return app_label == "timeseries"
+            case "test_analytics":
+                if not settings.TEST_ANALYTICS_DATABASE_ENABLED:
+                    return False
+                return app_label == "test_analytics"
+            case _:
+                return app_label not in {"timeseries", "test_analytics"}
 
     def allow_relation(self, obj1, obj2, **hints):
         obj1_app = obj1._meta.app_label
         obj2_app = obj2._meta.app_label
 
-        # cannot form relationship across default <-> timeseries dbs
-        if obj1_app == "timeseries" and obj2_app != "timeseries":
-            return False
-        if obj1_app != "timeseries" and obj2_app == "timeseries":
-            return False
-
-        # otherwise we allow it
-        return True
+        if obj1_app in {"timeseries", "test_analytics"} or obj2_app in {
+            "timeseries",
+            "test_analytics",
+        }:
+            return obj1_app == obj2_app
+        else:
+            return True

shared/django_apps/db_settings.py

@@ -158,6 +158,48 @@
             "CONN_MAX_AGE": CONN_MAX_AGE,
         }
 
+TEST_ANALYTICS_DATABASE_ENABLED = get_config(
+    "setup", "test_analytics_database", "enabled", default=True
+)
+
+test_analytics_database_url = get_config("services", "test_analytics_database_url")
+if test_analytics_database_url:
+    test_analytics_database_conf = urlparse(test_analytics_database_url)
+    TEST_ANALYTICS_DATABASE_NAME = test_analytics_database_conf.path.replace("/", "")
+    TEST_ANALYTICS_DATABASE_USER = test_analytics_database_conf.username
+    TEST_ANALYTICS_DATABASE_PASSWORD = test_analytics_database_conf.password
+    TEST_ANALYTICS_DATABASE_HOST = test_analytics_database_conf.hostname
+    TEST_ANALYTICS_DATABASE_PORT = test_analytics_database_conf.port
+else:
+    TEST_ANALYTICS_DATABASE_NAME = get_config(
+        "services", "test_analytics_database", "name", default="test_analytics"
+    )
+    TEST_ANALYTICS_DATABASE_USER = get_config(
+        "services", "test_analytics_database", "username", default="postgres"
+    )
+    TEST_ANALYTICS_DATABASE_PASSWORD = get_config(
+        "services", "test_analytics_database", "password", default="postgres"
+    )
+    TEST_ANALYTICS_DATABASE_HOST = get_config(
+        "services", "test_analytics_database", "host", default="postgres"
+    )
+    TEST_ANALYTICS_DATABASE_PORT = get_config(
+        "services", "test_analytics_database", "port", default=5432
+    )
+
+
+if TEST_ANALYTICS_DATABASE_ENABLED:
+    DATABASES["test_analytics"] = {
+        "ENGINE": "psqlextra.backend",
+        "NAME": TEST_ANALYTICS_DATABASE_NAME,
+        "USER": TEST_ANALYTICS_DATABASE_USER,
+        "PASSWORD": TEST_ANALYTICS_DATABASE_PASSWORD,
+        "HOST": TEST_ANALYTICS_DATABASE_HOST,
+        "PORT": TEST_ANALYTICS_DATABASE_PORT,
+        "CONN_MAX_AGE": CONN_MAX_AGE,
+    }
+
+
 # See https://django-postgres-extra.readthedocs.io/en/main/settings.html
 POSTGRES_EXTRA_DB_BACKEND_BASE: "django_prometheus.db.backends.postgresql"  # type: ignore
 

shared/django_apps/dummy_settings.py

@@ -33,6 +33,7 @@
     "shared.django_apps.profiling",
     "shared.django_apps.reports",
     "shared.django_apps.staticanalysis",
+    "shared.django_apps.test_analytics",
 ]
 
 # Needed for makemigrations to work
@@ -90,6 +91,14 @@
         "HOST": "timescale",
         "PORT": 5432,
     },
+    "test_analytics": {
+        "ENGINE": "psqlextra.backend",
+        "NAME": "test_analytics",
+        "USER": "postgres",
+        "PASSWORD": "postgres",
+        "HOST": "postgres",
+        "PORT": 5432,
+    },
 }
 
 # Password validation

shared/django_apps/test_analytics/__init__.py

@@ -0,0 +1,44 @@
+# Generated by Django 4.2.16 on 2025-01-17 22:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="Flake",
+            fields=[
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                ("repoid", models.IntegerField()),
+                ("test_id", models.BinaryField()),
+                ("recent_passes_count", models.IntegerField()),
+                ("count", models.IntegerField()),
+                ("fail_count", models.IntegerField()),
+                ("start_date", models.DateTimeField()),
+                ("end_date", models.DateTimeField(null=True)),
+            ],
+            options={
+                "db_table": "test_analytics_flake",
+                "indexes": [
+                    models.Index(
+                        fields=["repoid"], name="test_analyt_repoid_fcd881_idx"
+                    ),
+                    models.Index(
+                        fields=["test_id"], name="test_analyt_test_id_f504a1_idx"
+                    ),
+                    models.Index(
+                        fields=["repoid", "test_id"],
+                        name="test_analyt_repoid_0690c3_idx",
+                    ),
+                    models.Index(
+                        fields=["repoid", "end_date"],
+                        name="test_analyt_repoid_9e2402_idx",
+                    ),
+                ],
+            },
+        ),
+    ]

shared/django_apps/test_analytics/migrations/0001_initial.py

@@ -0,0 +1,28 @@
+from django.db import models
+
+# Added to avoid 'doesn't declare an explicit app_label and isn't in an application in INSTALLED_APPS' error\
+# Needs to be called the same as the API app
+TEST_ANALYTICS_APP_LABEL = "test_analytics"
+
+
+class Flake(models.Model):
+    id = models.BigAutoField(primary_key=True)
+
+    repoid = models.IntegerField()
+    test_id = models.BinaryField()
+
+    recent_passes_count = models.IntegerField()
+    count = models.IntegerField()
+    fail_count = models.IntegerField()
+    start_date = models.DateTimeField()
+    end_date = models.DateTimeField(null=True)
+
+    class Meta:
+        app_label = TEST_ANALYTICS_APP_LABEL
+        db_table = "test_analytics_flake"
+        indexes = [
+            models.Index(fields=["repoid"]),
+            models.Index(fields=["test_id"]),
+            models.Index(fields=["repoid", "test_id"]),
+            models.Index(fields=["repoid", "end_date"]),
+        ]

shared/django_apps/test_analytics/migrations/__init__.py

@@ -65,6 +65,30 @@ def test_allow_migrate_timeseries_disabled(self):
         assert router.allow_migrate("default", "timeseries") == False
         assert router.allow_migrate("default_read", "timeseries") == False
 
+    @override_settings(TEST_ANALYTICS_DATABASE_ENABLED=False)
+    def test_allow_migrate_test_analytics_disabled(self):
+        router = MultiDatabaseRouter()
+        assert router.allow_migrate("test_analytics", "test_analytics") == False
+        assert router.allow_migrate("test_analytics_read", "test_analytics") == False
+        assert router.allow_migrate("test_analytics", "default") == False
+        assert router.allow_migrate("test_analytics_read", "default") == False
+        assert router.allow_migrate("default", "default") == True
+        assert router.allow_migrate("default_read", "default") == False
+        assert router.allow_migrate("default", "test_analytics") == False
+        assert router.allow_migrate("default_read", "test_analytics") == False
+
+    @override_settings(TEST_ANALYTICS_DATABASE_ENABLED=True)
+    def test_allow_migrate_test_analytics_enabled(self):
+        router = MultiDatabaseRouter()
+        assert router.allow_migrate("test_analytics", "test_analytics") == True
+        assert router.allow_migrate("test_analytics_read", "test_analytics") == False
+        assert router.allow_migrate("test_analytics", "default") == False
+        assert router.allow_migrate("test_analytics_read", "default") == False
+        assert router.allow_migrate("default", "default") == True
+        assert router.allow_migrate("default_read", "default") == False
+        assert router.allow_migrate("default", "test_analytics") == False
+        assert router.allow_migrate("default_read", "test_analytics") == False
+
     def test_allow_relation(self, mocker):
         # At time of writing, the Django timeseries models don't live in this
         # repo so we're pretending a different model is from the timeseries app