(WIP) feat: deploy fail2ban using system manager

ansible/playbook.yml

@@ -13,7 +13,7 @@
           dest: "00-schema.sql",
         }
       - { source: "stat_extension.sql", dest: "01-extension.sql" }
-    
+
   environment:
     PATH: /usr/lib/postgresql/bin:{{ ansible_env.PATH }}
 
@@ -46,7 +46,7 @@
         - install-gotrue
         - install-supabase-internal
       when: debpkg_mode or nixpkg_mode
-  
+
     - name: Install PostgREST
       import_tasks: tasks/setup-postgrest.yml
       tags:
@@ -96,7 +96,7 @@
         src: files/apt_periodic
         dest: /etc/apt/apt.conf.d/10periodic
       when: debpkg_mode or nixpkg_mode
-      
+
     - name: Transfer init SQL files
       copy:
         src: files/{{ item.source }}
@@ -131,17 +131,12 @@
       tags:
         - install-supabase-internal
       when: debpkg_mode or stage2_nix
-      
+
     - name: Finalize AMI
       import_tasks: tasks/finalize-ami.yml
       tags:
         - install-supabase-internal
       when: debpkg_mode or nixpkg_mode
-
-    - name: Enhance fail2ban
-      import_tasks: tasks/setup-fail2ban.yml
-      when: debpkg_mode or nixpkg_mode
-
 
     # Install EC2 instance connect
     # Only for AWS images
@@ -218,7 +213,7 @@
         systemctl stop postgresql.service
       when: stage2_nix
 
-    - name: Remove osquery    
+    - name: Remove osquery
       become: yes
       shell: |
         sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile remove osquery"

flake.nix

@@ -33,7 +33,9 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
     system-manager = {
-      url = "github:numtide/system-manager";
+      # FIXME: remove custom branch when this PR is merged:
+      # https://github.com/numtide/system-manager/pull/266
+      url = "github:numtide/system-manager/users";
       inputs.nixpkgs.follows = "nixpkgs";
     };
   };

nix/systemConfigs.nix

@@ -1,9 +1,11 @@
 { self, inputs, ... }:
 let
   mkModules = system: [
+    self.systemModules.fail2ban
     ({
       services.nginx.enable = true;
       nixpkgs.hostPlatform = system;
+      supabase.services.fail2ban.enable = true;
     })
   ];
 

nix/systemModules/default.nix

@@ -4,6 +4,8 @@
 {
   imports = [ ./tests ];
   flake = {
-    systemModules = { };
+    systemModules = {
+      fail2ban = ./fail2ban.nix;
+    };
   };
 }

nix/systemModules/dummy-sshd.nix

@@ -0,0 +1,26 @@
+{ lib, ... }:
+{
+  options = {
+    services.openssh.settings.logLevel = lib.mkOption {
+      type = lib.types.str;
+    };
+  };
+}
+
+# FIXME: nix run .#check-system-manager
+# warning: Git tree '/data/yvan/wip/postgres' is dirty
+# error:
+#        … while evaluating 'strict' to select 'drvPath' on it
+#          at /builtin/derivation.nix:1:552:
+#        … while calling the 'derivationStrict' builtin
+#          at /builtin/derivation.nix:1:208:
+#        (stack trace truncated; use '--show-trace' to show the full trace)
+
+#        error: The option `services.openssh.settings.LogLevel' does not exist. Definition values:
+#        - In `/nix/store/8cpqym71jjq5frp06ypjsj1iwi3l0fln-source/nixos/modules/services/security/fail2ban.nix':
+#            {
+#              _type = "if";
+#              condition = false;
+#              content = {
+#                _type = "override";
+#            ...

nix/systemModules/fail2ban.nix

@@ -0,0 +1,110 @@
+{
+  lib,
+  nixosModulesPath,
+  config,
+  ...
+}:
+let
+  cfg = config.supabase.services.fail2ban;
+in
+{
+  imports = [
+    # We use a dummmy sshd module to workaround this error when importing "/services/networking/ssh/sshd.nix":
+    #   error: The option `users' in module `/nix/store/...-source/nix/modules'
+    #   would be a parent of the following options, but its type `attribute set' does not support nested options.
+    #   - option(s) with prefix `users.users' in module `/nix/store/...-source/nixos/modules/services/networking/ssh/sshd.nix'
+    # FIXME: it would be better to rely on userborn in system-manager:
+    # https://github.com/numtide/system-manager/pull/266
+    ./dummy-sshd.nix
+  ]
+  ++ map (path: nixosModulesPath + path) [
+    # "/config/console.nix"
+    # "/config/shells-environment.nix"
+    # "/config/system-path.nix"
+    # "/programs/i3lock.nix"
+    # "/programs/ssh.nix"
+    # "/security/pam.nix"
+    # "/services/networking/firewall.nix"
+    # "/services/networking/ssh/sshd.nix"
+    "/services/security/fail2ban.nix"
+    # "/system/boot/kernel.nix" # ERROR: The option `boot' in module `/nix/store/...-source/nix/modules/upstream/nixpkgs'
+    # would be a parent of the following options, but its type `raw value' does not support nested options.
+  ];
+
+  options = {
+    supabase.services.fail2ban = {
+      enable = lib.mkEnableOption "Fail2Ban";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    # TODO: (last bit form Ansible task)
+    # - name: Configure journald
+    #   copy:
+    #     src: files/fail2ban_config/jail-ssh.conf
+    #     dest: /etc/fail2ban/jail.d/sshd.local
+    #   when: debpkg_mode or nixpkg_mode
+    supabase.services.fail2ban = {
+      # enable = true; # TODO: don't use nixpkgs fail2ban
+      bantime = "3600";
+      jails = {
+        postgresql = {
+          settings = {
+            enabled = true;
+            port = "5432";
+            protocol = "tcp";
+            filter = "postgresql";
+            logpath = "/var/log/postgresql/auth-failures.csv";
+            maxretry = 3;
+            ignoreip = "192.168.0.0/16 172.17.1.0/20";
+          };
+        };
+        pgbouncer = {
+          settings = {
+            enabled = true;
+            port = "6543";
+            protocol = "tcp";
+            filter = "pgbouncer";
+            backend = "systemd[journalflags=1]";
+            maxretry = 3;
+          };
+        };
+      };
+      # TODO: extraPackages = [ pkgs.nftables ];
+    };
+
+    environment.etc = {
+      "fail2ban/jail.local".text = ''
+        [DEFAULT]
+        banaction = nftables-multiport
+        banaction_allports = nftables-allports
+      '';
+
+      "fail2ban/filter.d/postgresql.conf".text = ''
+        [Definition]
+        failregex = ^.*,.*,.*,.*,"<HOST>:.*password authentication failed for user.*$
+        ignoreregex = ^.*,.*,.*,.*,"127\.0\.0\.1.*password authentication failed for user.*$
+                      ^.*,.*,.*,.*,"<HOST>:.*password authentication failed for user ""supabase_admin".*$
+                      ^.*,.*,.*,.*,"<HOST>:.*password authentication failed for user ""supabase_auth_admin".*$
+                      ^.*,.*,.*,.*,"<HOST>:.*password authentication failed for user ""supabase_storage_admin".*$
+                      ^.*,.*,.*,.*,"<HOST>:.*password authentication failed for user ""authenticator".*$
+                      ^.*,.*,.*,.*,"<HOST>:.*password authentication failed for user ""pgbouncer".*$
+      '';
+
+      "fail2ban/filter.d/pgbouncer.conf".text = ''
+        [Definition]
+        failregex = ^.+@<HOST>:.+password authentication failed$
+        journalmatch = _SYSTEMD_UNIT=pgbouncer.service
+      '';
+    };
+
+    systemd.services.fail2ban = {
+      wantedBy = lib.mkForce [
+        "system-manager.target"
+      ];
+      # TODO:
+      # after = [ "nftables.service" ];
+      # wants = [ "nftables.service" ];
+    };
+  };
+}

nix/systemModules/tests/test_fail2ban.py

@@ -0,0 +1,11 @@
+# from time import sleep
+
+
+def test_fail2ban_service(host):
+    # sleep(5000) # Handy for interactive debugging (with docker exec -it $CONTAINER_ID /bin/bash)
+    assert host.service("fail2ban.service").is_valid
+    assert host.service("fail2ban.service").is_running, (
+        "Fail2Ban service should be running but failed: {}".format(
+            host.run("systemctl status fail2ban.service").stdout
+        )
+    )