Backup central key management (#484)

ewdurbin · web-flow · commit ce94e240e8c6 · 2024-08-28T15:23:27.000-04:00
* upgrade backup/downloads hosts to match prod deployments

* implement a centralized backup_ssh key management pillar

* migrate to using centralized backup ssh keys

* include docs for review

* correct the target_user for new hg backups

* save a line!
diff --git a/Vagrantfile b/Vagrantfile
@@ -2,14 +2,14 @@
 
 
 SERVERS = [
-  "backup-server",
+  {:name => "backup-server", :codename => "noble"},
   {:name => "bugs", :codename => "jammy", :ports => [8080]},
   {:name => "buildbot", :codename => "noble"},
   "cdn-logs",
   {:name => "codespeed", :codename => "jammy"},
   {:name => "consul", :codename => "jammy"},
-  "docs",
-  "downloads",
+  {:name => "docs", :codename => "noble"},
+  {:name => "downloads", :codename => "noble"},
   {:name => "hg", :codename => "noble"},
   {:name => "loadbalancer", :ports => [20000, 20001, 20002, 20003, 20004, 20005, 20010, 20011]},
   "mail",
diff --git a/conf/vagrant/master.conf b/conf/vagrant/master.conf
@@ -26,3 +26,4 @@ ext_pillar:
   - consul:
       key_path: /var/lib/consul/encryption_keys/primary.key
       acl_path: /var/lib/consul/acl_tokens/
+  - backup_ssh: {}
diff --git a/pillar/dev/backup/docs.sls b/pillar/dev/backup/docs.sls
@@ -0,0 +1,12 @@
+backup:
+  directories:
+    python-docs:
+      source_directory: /srv/
+      exclude:
+        - /srv/docsbuild
+      target_host: backup-server.vagrant.psf.io
+      target_directory: /backup/python-docs
+      target_user: python-docs
+      frequency: daily
+      increment_retention: 7D
+      user: root
diff --git a/pillar/dev/backup/downloads.sls b/pillar/dev/backup/downloads.sls
@@ -0,0 +1,10 @@
+backup:
+  directories:
+    python-downloads:
+      source_directory: /srv/
+      target_host: backup-server.vagrant.psf.io
+      target_directory: /backup/python-downloads
+      target_user: downloads
+      frequency: daily
+      increment_retention: 365D
+      user: root
diff --git a/pillar/dev/backup/server.sls b/pillar/dev/backup/server.sls
@@ -1,2 +1,2 @@
-backup-server:
-  backups: {}
+backup:
+  server: true
diff --git a/pillar/dev/top.sls b/pillar/dev/top.sls
@@ -11,7 +11,7 @@ base:
 
   'backup-server':
     - match: nodegroup
-    - backup.server
+    - backup.*
 
   'bugs':
     - match: nodegroup
@@ -35,11 +35,13 @@ base:
     - firewall.rs-lb-backend
     - groups.docs
     - secrets.docs
+    - backup.docs
 
   'downloads':
     - match: nodegroup
     - firewall.rs-lb-backend
     - groups.downloads
+    - backup.downloads
 
   'gnumailman':
     - match: nodegroup
diff --git a/pillar/prod/backup/bugs.sls b/pillar/prod/backup/bugs.sls
@@ -6,4 +6,5 @@ backup:
       target_directory: /backup/python-bugs
       target_user: python-bugs
       frequency: hourly
+      increment_retention: 30D
       user: root
diff --git a/pillar/prod/backup/buildbot.sls b/pillar/prod/backup/buildbot.sls
@@ -6,4 +6,5 @@ backup:
       target_directory: /backup/buildbot
       target_user: buildbot
       frequency: hourly
+      increment_retention: 90D
       user: root
diff --git a/pillar/prod/backup/docs.sls b/pillar/prod/backup/docs.sls
@@ -8,4 +8,5 @@ backup:
       target_directory: /backup/python-docs
       target_user: python-docs
       frequency: daily
+      increment_retention: 7D
       user: root
diff --git a/pillar/prod/backup/downloads.sls b/pillar/prod/backup/downloads.sls
@@ -6,4 +6,5 @@ backup:
       target_directory: /backup/python-downloads
       target_user: downloads
       frequency: daily
+      increment_retention: 365D
       user: root
diff --git a/pillar/prod/backup/gnumailman.sls b/pillar/prod/backup/gnumailman.sls
@@ -6,4 +6,5 @@ backup:
       target_directory: /backup/gnumailman-data
       target_user: gnumailman
       frequency: hourly
+      increment_retention: 90D
       user: root
diff --git a/pillar/prod/backup/hg.sls b/pillar/prod/backup/hg.sls
@@ -6,18 +6,21 @@ backup:
       target_directory: /backup/python-hg
       target_user: hg
       frequency: daily
+      increment_retention: 90D
       user: root
     hg-mercurial-static:
       source_directory: /usr/share/mercurial/templates/static/
       target_host: backup.sfo1.psf.io
       target_directory: /backup/hg-mercurial-static
-      target_user: root
+      target_user: hg
       frequency: daily
+      increment_retention: 90D
       user: root
     hg-svn-config:
       source_directory: /etc/apache2/svn_config/
       target_host: backup.sfo1.psf.io
       target_directory: /backup/hg-svn-config
-      target_user: root
+      target_user: hg
       frequency: daily
+      increment_retention: 90D
       user: root
diff --git a/pillar/prod/backup/mail.sls b/pillar/prod/backup/mail.sls
@@ -15,4 +15,5 @@ backup:
       target_directory: /backup/mail-python-org
       target_user: mail-python-org
       frequency: daily
+      increment_retention: 15D
       user: root
diff --git a/pillar/prod/backup/moin.sls b/pillar/prod/backup/moin.sls
@@ -6,4 +6,5 @@ backup:
       target_directory: /backup/moin
       target_user: moin
       frequency: daily
+      increment_retention: 90D
       user: root
diff --git a/pillar/prod/backup/server.sls b/pillar/prod/backup/server.sls
@@ -1,44 +1,5 @@
+backup:
+  server: true
 backup-server:
   volumes:
     /dev/sda: /backup
-  backups:
-    docs:
-      directory: /backup/python-docs
-      user: python-docs
-      increment_retention: 7D
-      authorized_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDhlpt0GMToIVMYBg5IvxXEE+D5rQQQEQxqzd8GFjA7GivE7jmxJJFHzDB+lA9mlaWEseNhDakzOma6PxDNdJ9lrBHDb/PeA/++oMsoQ2nU5BAbESXCrkSz9I6wh01oKGF4TytQNek4mv41R97eQioLRYFXsG0CvYsccudyQVwpDkhk/pBW3pqGudtY8JM3bjJI85EwcarQdqPj6dLy8STx8lTuOcSAOhLY5EPG34ZciHf3uFlgg6TYAkh5m8nT6nKEYsswQJIGqfJnLuTQVBuUODJ/tLQzjiOAPTcIKPJArPf/lAxqhuu6kiTX4aRl/gN68GnOvrgDvWbjVBXw3hrN
-    downloads:
-      directory: /backup/python-downloads
-      user: downloads
-      increment_retention: 365D
-      authorized_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCezDzZ3nfM5NpU6qAGXkU5122LmZHfe2+KjAdtgZr06OC9ke1kRO6mb+49JwO22zBxhdVOFEwiQsmtaeT3qh3FEcRB94rzvmBxwKiPuySMve4X7S+M/ozXDcJcdKnZ9jPwle2rJ9wag/0/6uCZtlHJFh0DZ4UI5Ttw6Pwq+X0T5ropD7i78OAbsaUn+lXU6k+ehIsWWjYjS/k8WFXW4WgMXchXk5AZYG7ZAOyWLLbmzDXMEqMmWe83EAArSF2fWOs1LoGyYRx4S1BVOo9w9HVAcbIPiccX0AtWLKzByoZ8fUILxdLmMeDrqohZXtbU/ci6V+AEBwNLRZsmvpfMeEJd
-    mail-python-org:
-      directory: /backup/mail-python-org
-      user: mail-python-org
-      increment_retention: 15D
-      authorized_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDN9voJiSP7mTsY8so/S8qMizKpJvLxFMWAyrYiTM41APvVpIU62JXfnU4nZxtPaDnqfyuXQzgYh7NgiqU7/OomQ5oyLzoZ6BH8kk4p1RT+tM1s9lR88jxalwSQqt7Av+p7qn4HuJkYAL0k0+AjHI559bFKtyDZYDpZz/JSP++keRqPXMtOk4Nd4z6KR18mzF5NV7rXNjHDExrpVb7kex8UVqXbNj8+dgl37PdXN4cAxlQoOALFbHxGGdxLqvJyalr1GZaxNRul6JUHaRFUkt6rVl90lp6+SO21i6hg5H3fL7eynJto6R0jDFiVNe6JfJs4XdXGYKIZlzhhqMOgbm0t
-    python-bugs:
-      directory: /backup/python-bugs
-      user: python-bugs
-      increment_retention: 30D
-      authorized_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsMBXD2hOm536YI0GMSratv8cM1CZ1M3J1bsvj2NqD9PEp10o3FD5ofr81kB+BTyFKMnpwxuP/dcoCfiY4dCF1COIa82nUtvuklFYTVybW8dL7DevWxoX0F6PeK8Ox+kcuASjmgx2UJ/pisKEIhFQYTF4bmevSRXbLv94461dxOO6j2MOgtJRGDmr/2OhA30VAnjMw1U+4flZd6FLodfq1udX8NVTBg05BIAwLNYLFrvLO8yMlqZzb4TbA53w29yyNIoSlXBLtG+K19mAA3ki+rqZdhdS+k6u1/u0AVUcDvmX1MrOtcvucy74SIesBDJfdyR7OFpHmAx4/aDPVdmGV
-    gnumailman-data:
-      directory: /backup/gnumailman-data
-      user: gnumailman
-      increment_retention: 90D
-      authorized_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDcqPefaRsYhkulCx/TtNOX4BaVqfGePoQraS8ELmujkrX3KP56u2uKshIUob8uCf/daRhlWnKKIyy8redICHc/+CmUSZjSgETgdbnVIYW17sZ+hCvME8AEc/FAObFrU9BEmOHU0LwA6SY2YclL9knz/Pd94aojXGhx5FjStJg/AfOuhcSzn1dgRfBERm9JPplgubFeVDocBJ816glLydh5Qg0TG1sgJtS2dczvhc0wwfQqnAVQtnhmxjVUBedj4efNaJhXK1N1aa+i5ev3k9Es6yXV/2jyAGEXAn8eiciVHCUt0vNDz+nNnckCHCQitkaA3VnF03Di5BaiJRTqyiyzRdtzh832O3DQLO54ep/S+jgYoFfajPnFbChHNTNpVI7VJHpsE8W8XEwvcf/ZU3sW5XHxfvwYq9KjOw7LWF26IT8pka2/WPClWeG3xKs10ndV2sNUmWpRJMIRQ82JFr2/4L23qzoI++7CDjb63X6eZeK+QtLTaj6pLel9nZVIapU=
-    hg:
-      directory: /backup/python-hg
-      user: hg
-      increment_retention: 90D
-      authorized_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDS0sTCNKlCfJd/TyiKW3HRwTUouo3+PvPOK3ddyfpY17bJ4KdpaMZgc7fNg5VKzFvvuHBqjvVJsdewP3LesLOuaQCQoSu1DniLoodZGRdJ9gqgtbRZf4ekzsn7E7WZUnVI0fbofvFWjbPt3PSxVtm8hCqwmwia53Ehh9G3xRurDhNUqIjrGcTStM3kloQHjKing+EGdCqPvikuwN1eMZXyNnt4zuoU4e39JGCBqRBfXumvrYvYzuNbAN8OZtNAfByzLFJ6DIWq0ihK6WS/KRYKGKivaaK26whafutfv44bP0w3LvZZyTMGGqiS/zLNPx0tkYK3JEt4bpLlyHZHbIBh
-    buildbot:
-      directory: /backup/buildbot
-      user: buildbot
-      increment_retention: 90D
-      authorized_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6ViDGniFFi9MeshjdKfh/qw3oXsYCLryh3t/wY9V43l209khYhXAh5k14QcbTu8b6H1MGNhq2jjMKLv2C2xzXubSZfUKFEhJp9MRG0xg3mxR9kGRu5wEmNbRavFKA2d0oiQFfMTRNUGCzPL5mn98EuFUuOtM+dMiXJ5eJdcFb5i0R8o31JzeaA37ogyYbmFYd20dsMlHEV7WdTILp0GeHxyq4t9NXMBu7cBvsLr4dSUQxlehTbHy5q0ZKWML0q1GVo65bAsTmh9byrEN5iUhWRRTTj/Pp9V15cYRtMc8qMTBNnDCKXtctfj3SuEUp47TCRbkyg2dFb/mUWCbVrgT9
-    moin:
-      directory: /backup/moin
-      user: moin
-      increment_retention: 90D
-      authorized_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDCr9RaD7FYNH4LSiC1d7UEwLo4o78+d8VZebRqsXfHH/J6xH6iBj8bEhzv08tVIVUb//T8fBM/N9bf9FYxiQPo/TouSkolaSNNkpPveXP3CtOUaR8f+W6C5Q4UfV0JRw0VRv7qfo5jjDpL7wMlq2wvI8ZS/0lODhUQg7TFUHSbaRM4j9CjokhQZLXoMa/V7M6S1OWjt9y3VrGNlT1EBAO/aQPmCNU8jStkJXvW/L9/jRU2RzTCGeuOBveKBe3HoXTaQbgqngilqZN+xfHNHtTMlF/KfgtZgl5l3C/nzlzkvBMSWOMiBLVtapUMdB6nNBAxY6Gb2OC5OjnMIoZ3X7efzKpasCJZ1mRwvaFU8XC2166+PVdWpj+UJ1Nv18Nl7Qzf3a04LOiV3s+0hxfYxDg1rsT+XyW/r65Tupp7u4MGgw1U/AtiH/Rsjl6IcGGsguMqQO6QlFcVPTTaEZvxAV54yMRlvzmc1hBDVbzKuUeE/yShpf1b9OsZODvyEjRFtDc=
diff --git a/salt/_extensions/pillar/backup_ssh.py b/salt/_extensions/pillar/backup_ssh.py
@@ -0,0 +1,50 @@
+import pathlib
+import subprocess
+
+
+def ext_pillar(minion_id, pillar, base_path="/etc/backup_keys/"):
+    base_path = pathlib.Path(base_path)
+    base_path.mkdir(parents=True, exist_ok=True)
+
+    is_server = pillar.get("backup", {}).get("server", False)
+
+    user_list = set()
+    for directory, directory_config in (
+        pillar.get("backup", {}).get("directories", {}).items()
+    ):
+        user_list.add(directory_config.get("target_user"))
+
+    user_keys = {}
+
+    for user in user_list:
+        user_private_key_path = base_path / f"{user}"
+        user_public_key_path = base_path / f"{user}.pub"
+
+        if not user_private_key_path.exists():
+            subprocess.run(
+                [
+                    "ssh-keygen",
+                    "-t",
+                    "ed25519",
+                    "-C",
+                    f"{user}@backup",
+                    "-f",
+                    user_private_key_path,
+                ]
+            )
+        if not user_public_key_path.exists():
+            with open(user_public_key_path, "w") as out_file:
+                subprocess.run(
+                    ["ssh-keygen", "-y", "-f", user_private_key_path], stdout=out_file
+                )
+
+        key_data = {"public": user_public_key_path.read_text()}
+        if not is_server:
+            key_data["private"] = user_private_key_path.read_text()
+
+        user_keys[user] = key_data
+
+    if is_server:
+        pillar["backup_directories"] = pillar.get("backup", {}).pop("directories")
+
+    return {"backup_keys": user_keys}
diff --git a/salt/backup/client/README.md b/salt/backup/client/README.md
@@ -9,6 +9,8 @@ Format for pillar data:
         postgres-archives:
           # Frequency of backup, currently {hourly, daily} are supported
           frequency: hourly
+          # Duration that increments are retained, in days
+          increment_retention: 365D
           # User to run backup as
           user: devpypi
           # Source Directory to backup
@@ -19,16 +21,10 @@ Format for pillar data:
           target_directory: /backup/postgres/archives
           # Target user on backup server
           target_user: devpypi
-          # SSH Private Key for backup server access as target_user
-          ssh_key: |
-            -----BEGIN RSA PRIVATE KEY-----
-            MIIEowIBAAKCAQEAsagAYbuOiROc0+vLjcKQXZqiP3mH3qXwPT1PAMWxkferuaz3
-            ...
-            pqMI3F6ButHsMcwjZotivkf3baM1FmnjIJ4oeTFPyagaLZdRCuiT
-            -----END RSA PRIVATE KEY-----
         # Backup example with pre/post/cleanup scripts
         postgres-base:
           frequency: daily
+          increment_retention: 30D
           user: postgres
           source_directory: /var/lib/pgsql/9.3/backups/base
           target_host: 172.16.57.201
@@ -40,10 +36,4 @@ Format for pillar data:
           post_script: '/usr/local/backup/postgres-archives/scripts/backup.bash'
           # Cleanup script to remove old backups
           cleanup_script: 'find /var/lib/pgsql/9.3/backups/base -maxdepth 1 -type d -mtime +7 -execdir rm -rf {} \;'
-          ssh_key: |
-            -----BEGIN RSA PRIVATE KEY-----
-            MIIEowIBAAKCAQEAsagAYbuOiROc0+vLjcKQXZqiP3mH3qXwPT1PAMWxkferuaz3
-            ...
-            pqMI3F6ButHsMcwjZotivkf3baM1FmnjIJ4oeTFPyagaLZdRCuiT
-            -----END RSA PRIVATE KEY-----
 
diff --git a/salt/backup/client/init.sls b/salt/backup/client/init.sls
@@ -19,7 +19,7 @@ include:
 {{ backup }}-ssh-key:
   file.managed:
     - name: /etc/backup/.ssh/id_rsa_{{ backup }}
-    - contents_pillar: backup-secret:directories:{{ backup }}:ssh_key
+    - contents_pillar: backup_keys:{{ config['target_user'] }}:private
     - user: {{ config['user'] }}
     - mode: "0600"
     - show_diff: False
diff --git a/salt/backup/server/init.sls b/salt/backup/server/init.sls
@@ -11,34 +11,34 @@ include:
     - group: root
     - mode: "0644"
 
-{% for backup, config in salt['pillar.get']('backup-server:backups', {}).items() %}
+{% for backup, config in salt['pillar.get']('backup_directories', {}).items() %}
 
 {{ backup }}-user:
   user.present:
-    - name: {{ config['user'] }}
+    - name: {{ config['target_user'] }}
 
 {{ backup }}-ssh:
   ssh_auth:
     - present
-    - user: {{ config['user'] }}
+    - user: {{ config['target_user'] }}
     - names:
-      - {{ config['authorized_key'] }}
+      - {{ salt['pillar.get']("backup_keys", {}).get(config['target_user'], {}).get('public') }}
     - options:
       - command="rdiff-backup server"
       - no-pty
       - no-port-forwarding
       - no-agent-forwarding
       - no-X11-forwarding
     - require:
-      - user: {{ config['user'] }}
+      - user: {{ config['target_user'] }}
 
 {{ backup }}:
   file.directory:
-    - name: {{ config['directory'] }}
-    - user: {{ config['user'] }}
+    - name: {{ config['target_directory'] }}
+    - user: {{ config['target_user'] }}
     - makedirs: True
     - require:
-      - user: {{ config['user'] }}
+      - user: {{ config['target_user'] }}
 
 {{ backup }}-increment-cleanup:
   file.managed:
@@ -50,6 +50,6 @@ include:
     - context:
         cron: '0 3 * * *'
         job_user: root
-        job_command: 'rdiff-backup --terminal-verbosity 1 --force remove increments --older-than {{ config['increment_retention'] }} {{ config['directory'] }}'
+        job_command: 'rdiff-backup --terminal-verbosity 1 --force remove increments --older-than {{ config['increment_retention'] }} {{ config['target_directory'] }}'
 
 {% endfor %}
