simplify test setup

default.nix

@@ -128,6 +128,11 @@ rec {
         # psql doesn't take DATABASE_URL
         PGDATABASE = "nix-security-tracker";
         PGUSER = "nix-security-tracker";
+        CREDENTIALS_DIRECTORY = toString ./.credentials;
+        DJANGO_SETTINGS = builtins.toJSON {
+          SYNC_GITHUB_STATE_AT_STARTUP = false;
+          DEBUG = true;
+        };
       };
 
       # `./src/website/tracker/settings.py` by default looks for LOCAL_NIXPKGS_CHECKOUT
@@ -149,15 +154,12 @@ rec {
 
         ln -sf ${sources.htmx}/dist/htmx.js src/website/webview/static/htmx.min.js
 
-        mkdir -p .credentials
-        export CREDENTIALS_DIRECTORY=${builtins.toString ./.credentials}
+        mkdir -p $CREDENTIALS_DIRECTORY
+        # TODO(@fricklerhandwerk): move all configuration over to pydantic-settings
         touch .settings.py
         export USER_SETTINGS_FILE=${builtins.toString ./.settings.py}
       '';
     };
 
-  tests = import ./nix/tests/vm-basic.nix {
-    inherit pkgs;
-    wstModule = module;
-  };
+  tests = pkgs.callPackage ./nix/tests.nix { };
 }

nix/overlay.nix

@@ -42,6 +42,7 @@ in
       httpretty
       ipython
       psycopg2
+      pydantic-settings
       pygithub
       requests
       tqdm

nix/tests.nix

@@ -0,0 +1,50 @@
+{ lib, pkgs }:
+let
+  # TODO: specify project/service name globally
+  application = "web-security-tracker";
+  defaults = {
+    documentation.enable = lib.mkDefault false;
+
+    virtualisation = {
+      memorySize = 2048;
+      cores = 2;
+    };
+
+    services.${application} = {
+      enable = true;
+      production = false;
+      restart = "no"; # fail fast
+      domain = "example.org";
+      env = {
+        SYNC_GITHUB_STATE_AT_STARTUP = false;
+        DEBUG = true;
+      };
+      secrets = {
+        SECRET_KEY = pkgs.writeText "SECRET_KEY" "secret";
+        GH_CLIENT_ID = pkgs.writeText "gh_client" "bonjour";
+        GH_SECRET = pkgs.writeText "gh_secret" "secret";
+        GH_WEBHOOK_SECRET = pkgs.writeText "gh_secret" "webhook-secret";
+      };
+    };
+  };
+in
+lib.mapAttrs (name: test: pkgs.testers.runNixOSTest (test // { inherit name defaults; })) {
+  basic = {
+    nodes.server = _: { imports = [ ./web-security-tracker.nix ]; };
+    testScript = ''
+      server.wait_for_unit("${application}-server.service")
+      server.wait_for_unit("${application}-worker.service")
+
+      with subtest("Django application tests"):
+        # https://docs.djangoproject.com/en/5.0/topics/testing/overview/
+        server.succeed("wst-manage test shared")
+        server.succeed("wst-manage test webview")
+
+      with subtest("Check that stylesheet is served"):
+        machine.succeed("curl --fail -H 'Host: example.org' http://localhost/static/style.css")
+
+      with subtest("Check that admin interface is served"):
+        server.succeed("curl --fail -L -H 'Host: example.org' http://localhost/admin")
+    '';
+  };
+}

nix/web-security-tracker.nix

@@ -42,6 +42,11 @@ let
     name = "wst-manage";
 
     runtimeInputs = [ pkgs.git ];
+    runtimeEnv = environment;
+    excludeShellChecks = [
+      "SC2089"
+      "SC2090"
+    ];
 
     text = ''
       sudo="exec"
@@ -54,6 +59,13 @@ let
   };
   credentials = mapAttrsToList (name: secretPath: "${name}:${secretPath}") cfg.secrets;
   databaseUrl = "postgres:///web-security-tracker";
+
+  environment = {
+    DATABASE_URL = databaseUrl;
+    USER_SETTINGS_FILE = "${configFile}";
+    DJANGO_SETTINGS = builtins.toJSON cfg.env;
+  };
+
   # This script has access to the credentials, no matter where it is.
   wstExternalManageScript = writeScriptBin "wst-manage" ''
     #!${stdenv.shell}
@@ -67,7 +79,9 @@ let
       --property "Group=web-security-tracker" \
       --property "WorkingDirectory=/var/lib/web-security-tracker" \
       ${concatStringsSep "\n" (map (cred: "--property 'LoadCredential=${cred}' \\") credentials)}
-      --property "Environment=DATABASE_URL=${databaseUrl} USER_SETTINGS_FILE=${settingsFile}" \
+      --property "Environment=${
+        toString (lib.mapAttrsToList (name: value: "${name}=${value}") environment)
+      }" \
       "${wstManageScript}/bin/wst-manage" "$@"
   '';
 in
@@ -94,6 +108,10 @@ in
       type = types.nullOr types.str;
       default = null;
     };
+    env = mkOption {
+      type = types.attrsOf types.anything;
+      default = { };
+    };
     settings = mkOption {
       type = types.attrsOf types.anything;
       default = { };
@@ -132,6 +150,10 @@ in
   config = mkIf cfg.enable {
     environment.systemPackages = [ wstExternalManageScript ];
     services = {
+      web-security-tracker.env = {
+        SYNC_GITHUB_STATE_AT_STARTUP = mkDefault true;
+      };
+      # TODO(@fricklerhandwerk): move all configuration over to pydantic-settings
       web-security-tracker.settings = {
         STATIC_ROOT = mkDefault "/var/lib/web-security-tracker/static";
         DEBUG = mkDefault false;
@@ -198,10 +220,7 @@ in
             LogsDirectory = "web-security-tracker";
             LoadCredential = credentials;
           };
-          environment = {
-            DATABASE_URL = databaseUrl;
-            USER_SETTINGS_FILE = "${configFile}";
-          };
+          inherit environment;
         };
       in
       mapAttrs (_: recursiveUpdate defaults) {
@@ -216,9 +235,6 @@ in
           serviceConfig = {
             Restart = cfg.restart;
             TimeoutStartSec = lib.mkDefault "10m";
-            Environment = [
-              "SYNC_GITHUB_STATE_AT_STARTUP=true"
-            ];
           };
           preStart = ''
             # Auto-migrate on first run or if the package has changed

src/website/shared/apps.py

@@ -1,7 +1,5 @@
-import os
-import sys
-
 from django.apps import AppConfig
+from django.conf import settings
 
 
 class SharedConfig(AppConfig):
@@ -11,13 +9,8 @@ class SharedConfig(AppConfig):
     def ready(self) -> None:
         import shared.listeners  # noqa
 
-        # This hook is called on any `manage` subcommand.
-        # Only connect to GitHub when the server is started.
         # TODO: run this as a separate service, as this is almost exclusively a deployment concern
-        if os.environ.get("RUN_MAIN", None) is None and (
-            "runserver" in sys.argv
-            or os.environ.get("SYNC_GITHUB_STATE_AT_STARTUP", False)
-        ):
+        if settings.SYNC_GITHUB_STATE_AT_STARTUP:
             from shared.auth.github_state import GithubState
 
             self.github_state = GithubState()

src/website/tracker/settings.py

@@ -17,17 +17,49 @@
 
 import dj_database_url
 import sentry_sdk
+from pydantic import BaseModel, DirectoryPath, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
 from sentry_sdk.integrations.django import DjangoIntegration
 
+
+class Secrets(BaseSettings):
+    CREDENTIALS_DIRECTORY: DirectoryPath
+
+
+class Settings(BaseSettings):
+    # https://docs.pydantic.dev/latest/concepts/pydantic_settings/
+
+    model_config = SettingsConfigDict(
+        # https://docs.pydantic.dev/latest/concepts/pydantic_settings/#secrets
+        # https://systemd.io/CREDENTIALS/
+        secrets_dir=Secrets().CREDENTIALS_DIRECTORY,  # type: ignore[reportCallIssue]
+    )
+
+    class DjangoSettings(BaseModel):
+        # SECURITY WARNING: don't run with debug turned on in production!
+        DEBUG: bool = False
+        SYNC_GITHUB_STATE_AT_STARTUP: bool = Field(
+            description="""
+            Connect to GitHub when the service is started and update
+            team membership (security team and committers team)
+            of Nixpkgs maintainers in the evaluation database.
+            """
+        )
+
+    DJANGO_SETTINGS: DjangoSettings
+
+
+for key, value in Settings().dict()["DJANGO_SETTINGS"].items():  # type: ignore[reportCallIssue]
+    setattr(sys.modules[__name__], key, value)
+
+# TODO(@fricklerhandwerk): move all configuration over to pydantic-settings
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
 
 def get_secret(name: str, encoding: str = "utf-8") -> str:
-    credentials_dir = env.get("CREDENTIALS_DIRECTORY")
-
-    if credentials_dir is None:
-        raise RuntimeError("No credentials directory available.")
+    credentials_dir = Secrets().CREDENTIALS_DIRECTORY  # type: ignore[reportCallIssue]
 
     try:
         with open(f"{credentials_dir}/{name}", encoding=encoding) as f:
@@ -59,8 +91,6 @@ def get_secret(name: str, encoding: str = "utf-8") -> str:
 }
 
 ## Logging settings
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
@@ -81,7 +111,7 @@ def get_secret(name: str, encoding: str = "utf-8") -> str:
     },
     "handlers": {
         "console": {
-            "level": "DEBUG" if DEBUG else "INFO",
+            "level": "DEBUG" if DEBUG else "INFO",  # type: ignore # noqa: F821
             "filters": ["require_debug_true"],
             "class": "logging.StreamHandler",
             "formatter": "verbose",
@@ -107,7 +137,7 @@ def get_secret(name: str, encoding: str = "utf-8") -> str:
         },
         "shared": {
             "handlers": ["console", "mail_admins"],
-            "level": "DEBUG" if DEBUG else "INFO",
+            "level": "DEBUG" if DEBUG else "INFO",  # type: ignore # noqa: F821
             "filters": [],
         },
     },