Put the Zenith client and MITM in the same pod

Author: Matt Pryor

roles/zenith_proxy/defaults/main.yml

@@ -8,9 +8,11 @@ zenith_sshd_port: 22
 zenith_proxy_podman_user: "{{ ansible_user }}"
 
 zenith_proxy_service_name: "{{ undef(hint = 'zenith_proxy_service_name is required') }}"
+zenith_proxy_client_service_name: "{{ zenith_proxy_service_name }}-client"
 zenith_proxy_mitm_service_name: "{{ zenith_proxy_service_name }}-mitm"
 
-zenith_proxy_container_name: "{{ zenith_proxy_service_name }}"
+zenith_proxy_pod_name: "{{ zenith_proxy_service_name }}"
+zenith_proxy_client_container_name: "{{ zenith_proxy_client_service_name }}"
 zenith_proxy_mitm_container_name: "{{ zenith_proxy_mitm_service_name }}"
 
 zenith_proxy_client_image_repository: ghcr.io/stackhpc/zenith-client
@@ -31,7 +33,6 @@ zenith_proxy_client_auth_skip: false
 zenith_proxy_client_auth_params: {}
 
 zenith_proxy_mitm_enabled: no
-zenith_proxy_mitm_host: "{{ ansible_default_ipv4.address }}"
 zenith_proxy_mitm_listen_port: 8080
 zenith_proxy_mitm_auth_inject: none  # valid values are 'basic' and 'bearer'
 zenith_proxy_mitm_auth_basic_username: >-

roles/zenith_proxy/files/podman-pod-infra-attach.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+#####
+# Small script that can be used to attach to the infra container of a pod
+#
+# Useful in a systemd service that starts a pod in order to track the execution
+#
+# Accepts a single argument which is the name of the pod whose infra container we should attach to
+#####
+
+set -e
+
+echo "[INFO] Finding infra container for pod '$1'"
+INFRA_CONTAINER_ID="$(podman pod inspect --format '{{.InfraContainerID}}' "$1")"
+
+echo "[INFO] Attaching to infra container '${INFRA_CONTAINER_ID}'"
+exec podman container attach --no-stdin ${INFRA_CONTAINER_ID}

roles/zenith_proxy/tasks/main.yml

@@ -1,13 +1,26 @@
 ---
 
-- name: Collect usernamespace facts
-  user_namespace_facts:
+- name: Install script for attaching to pod infra containers
+  copy:
+    src: podman-pod-infra-attach.sh
+    dest: /usr/bin/
+    mode: +x
+  become: true
 
-- name: Set facts containing sub-ids
-  set_fact:
-    # podman user is 1000
-    zenith_proxy_host_user_id: "{{ ansible_facts.subuid[zenith_proxy_podman_user]['start'] + 1000 - 1 }}"
-    zenith_proxy_host_group_id: "{{ ansible_facts.subgid[zenith_proxy_podman_user]['start'] + 1000 - 1 }}"
+- name: Create systemd unit for Zenith pod
+  template:
+    src: pod.service.j2
+    dest: /etc/systemd/system/{{ zenith_proxy_service_name }}.service
+  become: true
+  register: zenith_proxy_pod_systemd_unit
+
+- name: Ensure Zenith pod is started and enabled
+  service:
+    name: "{{ zenith_proxy_service_name }}.service"
+    state: "{{ 'restarted' if zenith_proxy_pod_systemd_unit is changed else 'started' }}"
+    enabled: yes
+    daemon_reload: "{{ zenith_proxy_pod_systemd_unit is changed }}"
+  become: true
 
 - block:
     - name: Create systemd unit file for MITM proxy
@@ -65,13 +78,13 @@
 - name: Create systemd unit file for Zenith client
   template:
     src: client.service.j2
-    dest: /etc/systemd/system/{{ zenith_proxy_service_name }}.service
+    dest: /etc/systemd/system/{{ zenith_proxy_client_service_name }}.service
   become: true
   register: zenith_proxy_client_systemd_unit
 
 - name: Ensure Zenith client is started and enabled
   service:
-    name: "{{ zenith_proxy_service_name }}.service"
+    name: "{{ zenith_proxy_client_service_name }}.service"
     state: >-
       {{
         'restarted'

roles/zenith_proxy/templates/client.service.j2

@@ -1,37 +1,34 @@
-# container-filebeat.service
-# based off
-#   podman generate systemd filebeat --restart-policy always --new --name
-# with pid/cidfiles replaced with --sdnotify=conmon approach
-
 [Unit]
-Description=Podman container-filebeat.service
-Documentation=man:podman-generate-systemd(1)
+Description=Podman {{ zenith_proxy_client_service_name }}.service
 Wants=network.target
 After=network-online.target
+BindsTo={{ zenith_proxy_service_name }}.service
+PartOf={{ zenith_proxy_service_name }}.service
+After={{ zenith_proxy_service_name }}.service
+{% if zenith_proxy_mitm_enabled %}
+Wants={{ zenith_proxy_mitm_service_name }}.service
+After={{ zenith_proxy_mitm_service_name }}.service
+{% endif %}
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
+Type=simple
 Restart=always
+User={{ zenith_proxy_podman_user }}
+Group={{ zenith_proxy_podman_user }}
 ExecStart=/usr/bin/podman run \
   --network slirp4netns:cidr={{ podman_cidr }} \
-  --sdnotify=conmon \
   --cgroups=no-conmon \
   --replace \
-  --name {{ zenith_proxy_container_name }} \
-  --restart=always \
+  --restart=no \
+  --pod {{ zenith_proxy_pod_name }} \
+  --name {{ zenith_proxy_client_container_name }} \
   --security-opt label=disable \
-  --detach=True \
   --volume /etc/zenith/{{ zenith_proxy_service_name }}:/etc/zenith:ro \
   --volume {{ zenith_proxy_service_name }}-ssh:/home/zenith/.ssh \
   {{ zenith_proxy_client_image }}
-ExecStop=/usr/bin/podman stop --ignore {{ zenith_proxy_container_name }} -t 10
-ExecStopPost=/usr/bin/podman rm --ignore -f {{ zenith_proxy_container_name }}
-KillMode=none
-Type=notify
-NotifyAccess=all
-User={{ zenith_proxy_podman_user }}
-Group={{ zenith_proxy_podman_user }}
-TimeoutStartSec=180
+ExecStop=/usr/bin/podman stop --ignore -t 10 {{ zenith_proxy_client_container_name }}
+ExecStopPost=/usr/bin/podman rm --ignore -f {{ zenith_proxy_client_container_name }}
 
 [Install]
 WantedBy=multi-user.target default.target

roles/zenith_proxy/templates/mitm.service.j2

@@ -1,26 +1,27 @@
-# container-filebeat.service
-# based off
-#   podman generate systemd filebeat --restart-policy always --new --name
-# with pid/cidfiles replaced with --sdnotify=conmon approach
+
 
 [Unit]
-Description=Podman container-filebeat.service
-Documentation=man:podman-generate-systemd(1)
+Description=Podman {{ zenith_proxy_mitm_service_name }}.service
 Wants=network.target
 After=network-online.target
+BindsTo={{ zenith_proxy_service_name }}.service
+PartOf={{ zenith_proxy_service_name }}.service
+After={{ zenith_proxy_service_name }}.service
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
+Type=simple
 Restart=always
+User={{ zenith_proxy_podman_user }}
+Group={{ zenith_proxy_podman_user }}
 ExecStart=/usr/bin/podman run \
   --network slirp4netns:cidr={{ podman_cidr }} \
-  --sdnotify=conmon \
   --cgroups=no-conmon \
   --replace \
+  --restart=no \
+  --pod {{ zenith_proxy_pod_name }} \
   --name {{ zenith_proxy_mitm_container_name }} \
-  --restart=always \
   --security-opt label=disable \
-  --detach=True \
   --env ZENITH_PROXY_LISTEN_PORT={{ zenith_proxy_mitm_listen_port }} \
   --env ZENITH_PROXY_UPSTREAM_SCHEME={{ zenith_proxy_upstream_scheme }} \
   --env ZENITH_PROXY_UPSTREAM_HOST={{ zenith_proxy_upstream_host }} \
@@ -39,14 +40,8 @@ ExecStart=/usr/bin/podman run \
   --env ZENITH_PROXY_AUTH_BEARER_TOKEN={{ zenith_proxy_mitm_auth_bearer_token }} \
 {% endif %}
   {{ zenith_proxy_mitm_image }}
-ExecStop=/usr/bin/podman stop --ignore {{ zenith_proxy_mitm_container_name }} -t 10
+ExecStop=/usr/bin/podman stop --ignore -t 10 {{ zenith_proxy_mitm_container_name }}
 ExecStopPost=/usr/bin/podman rm --ignore -f {{ zenith_proxy_mitm_container_name }}
-KillMode=none
-Type=notify
-NotifyAccess=all
-User={{ zenith_proxy_podman_user }}
-Group={{ zenith_proxy_podman_user }}
-TimeoutStartSec=180
 
 [Install]
 WantedBy=multi-user.target default.target

roles/zenith_proxy/templates/pod.service.j2

@@ -0,0 +1,19 @@
+[Unit]
+Description=Podman {{ zenith_proxy_service_name }}.service
+Wants=network.target
+After=network-online.target
+
+[Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
+Type=simple
+Restart=always
+User={{ zenith_proxy_podman_user }}
+Group={{ zenith_proxy_podman_user }}
+ExecStartPre=/usr/bin/podman pod create --replace --name {{ zenith_proxy_pod_name }}
+ExecStartPre=/usr/bin/podman pod start {{ zenith_proxy_pod_name }}
+ExecStart=/usr/bin/podman-pod-infra-attach.sh {{ zenith_proxy_pod_name }}
+ExecStop=/usr/bin/podman pod stop --ignore -t 10 {{ zenith_proxy_pod_name }}
+ExecStopPost=/usr/bin/podman pod rm --ignore -f {{ zenith_proxy_pod_name }}
+
+[Install]
+WantedBy=multi-user.target default.target

roles/zenith_proxy/templates/zenith-client.yaml.j2

@@ -9,7 +9,7 @@ verify_ssl: {{ 'yes' if zenith_registrar_verify_ssl else 'no' }}
 server_address: {{ zenith_sshd_host }}
 server_port: {{ zenith_sshd_port }}
 {% if zenith_proxy_mitm_enabled %}
-forward_to_host: {{ zenith_proxy_mitm_host }}
+forward_to_host: 127.0.0.1
 forward_to_port: {{ zenith_proxy_mitm_listen_port }}
 {% else %}
 forward_to_host: {{ zenith_proxy_upstream_host }}

slurm-infra.yml

@@ -144,6 +144,7 @@
         zenith_proxy_upstream_port: "{{ grafana_port }}"
         zenith_proxy_client_token: "{{ zenith_token_monitoring }}"
         zenith_proxy_client_auth_params: {}
+        zenith_proxy_mitm_enabled: yes
       when: zenith_subdomain_monitoring is defined
 
 - import_playbook: vendor/stackhpc/ansible-slurm-appliance/ansible/adhoc/hpctests.yml