AAP-60695 Add redis

docker/Dockerfile.backend

@@ -72,8 +72,8 @@ RUN useradd awx -u 1000 -g 0 --home-dir /var/lib/awx -s /bin/bash
 RUN python3.12 -m venv /venv
 RUN /venv/bin/pip install --upgrade pip wheel
 RUN mkdir /code /code/src
-COPY requirements.txt /code
-RUN /venv/bin/pip install -r /code/requirements.txt
+COPY requirements-build.txt /code
+RUN /venv/bin/pip install -r /code/requirements-build.txt
 
 # source .env
 COPY src/backend /code/src/backend

requirements-build.txt

@@ -1258,6 +1258,9 @@ uvicorn==0.38.0 \
     --hash=sha256:48c0afd214ceb59340075b4a052ea1ee91c16fbc2a9b1469cca0e54566977b02 \
     --hash=sha256:fd97093bdd120a2609fc0d3afe931d4d4ad688b6e75f0f929fde1bc36fe0e91d
     # via -r requirements-pinned.txt
+uwsgi==2.0.31 \
+    --hash=sha256:e8f8b350ccc106ff93a65247b9136f529c14bf96b936ac5b264c6ff9d0c76257
+    # via -r requirements-pinned.txt
 weasyprint==66.0 \
     --hash=sha256:82b0783b726fcd318e2c977dcdddca76515b30044bc7a830cc4fbe717582a6d0 \
     --hash=sha256:da71dc87dc129ac9cffdc65e5477e90365ab9dbae45c744014ec1d06303dde40

requirements-pinned.txt

@@ -23,9 +23,12 @@ split-settings==1.0.0
 urllib3==2.5.0
 uvicorn==0.38.0
 
+# Deployment dependencies
+uwsgi==2.0.31
+
 # Development dependencies
 pytest-cov==7.0.0
 pytest-django==4.11.1
 pytest-mock==3.15.1
 pytest==9.0.0
-time-machine==2.19.0
+time-machine==2.19.0

setup/collections/ansible_collections/ansible/containerized_installer/playbooks/dashboard_install.yml

@@ -18,7 +18,7 @@
         name: preflight
 
 - name: Install common container components
-  hosts: automationcontroller:automationeda:automationhub:database:execution_nodes:automationdashboard
+  hosts: automationcontroller:automationeda:automationhub:database:execution_nodes:automationdashboard:redis
   any_errors_fatal: true
   gather_facts: false
   become: false
@@ -61,6 +61,45 @@
         tasks_from: conf.yml
       when: not dashboard_pg_containerized
 
+- name: Install redis cluster cache
+  hosts: redis
+  any_errors_fatal: true
+  gather_facts: false
+  become: false
+  tasks:
+    - name: Install and configure redis cluster tcp socket
+      ansible.builtin.include_role:
+        name: redis
+      vars:
+        redis_unix_socket: false
+        redis_cluster: true
+      when: redis_mode | default('cluster') == 'cluster'
+
+- name: Install the redis cache
+  hosts: automationcontroller:automationeda:automationgateway:automationhub:automationdashboard
+  any_errors_fatal: true
+  gather_facts: false
+  become: false
+  tasks:
+    - name: Install and configure redis unix socket
+      ansible.builtin.include_role:
+        name: redis
+      when: >
+        inventory_hostname in groups.get('automationcontroller', []) or
+        inventory_hostname in groups.get('automationhub', []) or
+        (inventory_hostname in groups.get('automationeda', []) and groups.get('automationeda', []) | length == 1) or
+        (inventory_hostname in groups.get('automationdashboard', []) and groups.get('automationdashboard', []) | length == 1)
+
+    # group automationgateway in not defined when installing only automationdashboard
+    # - name: Install and configure redis tcp socket
+    #   ansible.builtin.include_role:
+    #     name: redis
+    #   vars:
+    #     redis_unix_socket: false
+    #   when:
+    #     - redis_mode | default('cluster') == 'standalone'
+    #     - inventory_hostname == groups['automationgateway'] | first
+
 - name: Install the Automation Dashboard
   any_errors_fatal: true
   hosts: automationdashboard

setup/collections/ansible_collections/ansible/containerized_installer/playbooks/dashboard_uninstall.yml

@@ -1,6 +1,6 @@
 ---
 - name: Collect services facts
-  hosts: automationcontroller:automationeda:automationhub:database:execution_nodes:automationdashboard
+  hosts: automationcontroller:automationeda:automationhub:database:execution_nodes:automationdashboard:redis
   gather_facts: false
   become: false
   tasks:
@@ -24,6 +24,47 @@
         name: automationdashboard
         tasks_from: uninstall.yml
 
+- name: Uninstall redis cluster
+  hosts: redis
+  any_errors_fatal: true
+  gather_facts: false
+  become: false
+  tasks:
+    - name: Uninstall redis cluster tcp socket
+      ansible.builtin.include_role:
+        name: redis
+        tasks_from: uninstall.yml
+      vars:
+        redis_unix_socket: false
+        redis_cluster: true
+      when: redis_mode | default('cluster') == 'cluster'
+
+- name: Uninstall redis
+  hosts: automationcontroller:automationeda:automationgateway:automationhub:automationdashboard
+  any_errors_fatal: true
+  gather_facts: false
+  become: false
+  tasks:
+    - name: Uninstall redis unix socket
+      ansible.builtin.include_role:
+        name: redis
+        tasks_from: uninstall.yml
+      when: >
+        inventory_hostname in groups.get('automationcontroller', []) or
+        inventory_hostname in groups.get('automationhub', []) or
+        (inventory_hostname in groups.get('automationeda', []) and groups.get('automationeda', []) | length == 1) or
+        (inventory_hostname in groups.get('automationdashboard', []) and groups.get('automationdashboard', []) | length == 1)
+
+    # - name: Uninstall redis tcp socket
+    #   ansible.builtin.include_role:
+    #     name: redis
+    #     tasks_from: uninstall.yml
+    #   vars:
+    #     redis_unix_socket: false
+    #   when:
+    #     - redis_mode | default('cluster') == 'standalone'
+    #     - inventory_hostname == groups['automationgateway'] | first
+
 - name: Uninstall the database
   hosts: database
   gather_facts: false

setup/collections/ansible_collections/ansible/containerized_installer/roles/automationdashboard/tasks/facts.yml

@@ -54,7 +54,7 @@
       - '{{ dashboard_conf_dir }}/launch_dashboard_web.sh:/usr/bin/launch_dashboard_web.sh:ro,z'
       # - '{{ receptor_conf_dir }}/receptor.conf:/etc/receptor/receptor.conf:ro,z'
       # - 'receptor_run:/run/receptor:U'
-      # - 'redis_run:/run/redis:z'
+      - 'redis_run:/run/redis:z'
       # - '{{ rsyslog_run_dir }}:/run/awx-rsyslog:z'
       - '{{ supervisor_run_dir }}:/run/supervisor:z'
       - '{{ dispatcher_run_dir }}:/run/dispatcher:z'

setup/collections/ansible_collections/ansible/containerized_installer/roles/automationdashboard/templates/django_dashboard_conf.py.j2

@@ -1,3 +1,6 @@
+with open("/etc/dashboard/SECRET_KEY") as fin:
+    SECRET_KEY = fin.read()
+
 ALLOWED_HOSTS=["*"]
 CORS_ALLOWED_ORIGINS = [
     "http://localhost:8080",
@@ -35,6 +38,8 @@ INITIAL_SYNC_SINCE = "{{ initial_sync_since }}"
 
 DISPATCHERD_DEBUGGING_SOCKFILE = "/run/dispatcher/automation-dashboard-dispatcher.sock"
 
+BROKER_URL = 'unix:///run/redis/redis.sock'
+
 SHOW_URLLIB3_INSECURE_REQUEST_WARNING = {{ show_urllib3_insecure_request_warning | default(True) }}
 # django DEBUG=0
 DEBUG = {{ django_debug | default(False) }}

setup/collections/ansible_collections/ansible/containerized_installer/roles/redis/defaults/main.yml

@@ -0,0 +1,20 @@
+---
+### container
+container_log_driver: journald
+
+### common
+archive_compressed_extension: "tar.gz"
+archive_decompressed_extension: "tar"
+
+### backup/restore
+redis_use_archive_compression: "{{ use_archive_compression | default(true) }}"
+
+redis_conf_dir: '{{ ansible_user_dir }}/aap/redis'
+redis_disable_tls: false
+redis_firewall_zone: public
+redis_port: 6379
+redis_unix_socket: true
+redis_cluster: false
+redis_cluster_replicas: 1
+redis_cluster_port: 16379
+...

setup/collections/ansible_collections/ansible/containerized_installer/roles/redis/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+- name: Restart redis
+  ansible.builtin.systemd:
+    name: 'redis-{{ _redis_suffix }}.service'
+    scope: user
+    state: restarted
+...

setup/collections/ansible_collections/ansible/containerized_installer/roles/redis/tasks/backup.yml

@@ -0,0 +1,16 @@
+---
+- name: Archive the redis data
+  ansible.containerized_installer.archive:
+    path:
+      - '{{ ansible_user_dir }}/aap/redis'
+      - '{{ ansible_user_dir }}/aap/tls'
+    dest: '{{ ansible_user_dir }}/aap/backups/redis.{{ redis_archive_extension }}'
+    format: '{{ redis_use_archive_compression | bool | ternary(omit, archive_decompressed_extension) }}'
+    mode: '0640'
+
+- name: Download the redis tarball
+  ansible.builtin.fetch:
+    src: '{{ ansible_user_dir }}/aap/backups/redis.{{ redis_archive_extension }}'
+    dest: '{{ hostvars["localhost"]["_backup_dir"] }}/redis_{{ inventory_hostname }}.{{ redis_archive_extension }}'
+    flat: true
+...

setup/collections/ansible_collections/ansible/containerized_installer/roles/redis/tasks/cluster.yml

@@ -0,0 +1,59 @@
+---
+- name: Set options fact for tcp socket
+  ansible.builtin.set_fact:
+    _options: '-h {{ _redis_hostname }} -p {{ redis_port }}'
+
+- name: Add tls options for tcp socket
+  ansible.builtin.set_fact:
+    _options: '{{ _options }} --tls --cert /var/lib/redis/server.crt --key /var/lib/redis/server.key'
+  when: not redis_disable_tls | bool
+
+- name: Check if the cluster is already initialized
+  ansible.builtin.stat:
+    path: '{{ redis_conf_dir }}/cluster.init'
+  register: _cluster_init
+
+- name: Configure the redis cluster
+  when: not _cluster_init.stat.exists | bool
+  block:
+    - name: Create the redis cluster
+      containers.podman.podman_container:
+        name: redis-cluster-init
+        image: '{{ _redis_image }}'
+        command: 'redis-cli {{ _options }} --cluster create {{ _redis_cluster_ips }} --cluster-replicas {{ redis_cluster_replicas }} --cluster-yes'
+        detach: false
+        rm: true
+        log_driver: '{{ container_log_driver }}'
+        network: host
+        volume: '{{ _volumes }}'
+        uidmap:
+          - '{{ redis_uid }}:0:1'
+          - '0:1:{{ redis_uid }}'
+          - '{{ redis_uid + 1 }}:{{ redis_uid + 1 }}:{{ 65536 - redis_uid }}'
+        gidmap:
+          - '{{ redis_gid }}:0:1'
+          - '0:1:{{ redis_gid }}'
+          - '{{ redis_gid + 1 }}:{{ redis_gid + 1 }}:{{ 65536 - redis_gid }}'
+        timeout: 120
+      run_once: true
+      register: _redis_cluster
+      changed_when: "'[OK] All 16384 slots covered' in _redis_cluster.stdout"
+
+  rescue:
+    - name: Cleanup redis_nodes.conf content
+      ansible.builtin.copy:
+        content: ''
+        dest: '{{ redis_conf_dir }}/redis_nodes.conf'
+        mode: '0640'
+
+    - name: Redis cluster created failed
+      ansible.builtin.fail:
+        msg: 'Please check the network and firewall configuration ({{ redis_port }}/{{ redis_cluster_port }})'
+      run_once: true
+
+- name: Create the initilized cluster file
+  ansible.builtin.file:
+    path: '{{ redis_conf_dir }}/cluster.init'
+    state: touch
+    mode: '0644'
+...

setup/collections/ansible_collections/ansible/containerized_installer/roles/redis/tasks/config.yml

@@ -0,0 +1,29 @@
+---
+- name: Create the redis config directory
+  ansible.builtin.file:
+    path: '{{ redis_conf_dir }}'
+    state: directory
+    mode: '0770'
+
+- name: Create the redis configuration file
+  ansible.builtin.template:
+    src: redis.conf.j2
+    dest: '{{ redis_conf_dir }}/redis-{{ _redis_suffix }}.conf'
+    mode: '0640'
+  notify: Restart redis
+
+- name: Create the redis user ACL file
+  ansible.builtin.template:
+    src: redis-users.acl.j2
+    dest: '{{ redis_conf_dir }}/redis-users.acl'
+    mode: '0640'
+  notify: Restart redis
+  when: not redis_unix_socket | bool
+
+- name: Create redis nodes file
+  ansible.builtin.file:
+    path: '{{ redis_conf_dir }}/redis_nodes.conf'
+    state: touch
+    mode: '0640'
+  when: redis_cluster | bool
+...

setup/collections/ansible_collections/ansible/containerized_installer/roles/redis/tasks/containers.yml

@@ -0,0 +1,27 @@
+---
+- name: Create the redis container
+  containers.podman.podman_container:
+    name: 'redis-{{ _redis_suffix }}'
+    image: '{{ _redis_image }}'
+    generate_systemd:
+      container_prefix: ''
+      path: '{{ ansible_user_dir }}/.config/systemd/user'
+      separator: ''
+    log_driver: '{{ container_log_driver }}'
+    network: host
+    volume: '{{ _volumes }}'
+    env:
+      REDIS_CONF: /etc/redis.conf
+    uidmap:
+      - '{{ redis_uid }}:0:1'
+      - '0:1:{{ redis_uid }}'
+      - '{{ redis_uid + 1 }}:{{ redis_uid + 1 }}:{{ 65536 - redis_uid }}'
+    gidmap:
+      - '{{ redis_gid }}:0:1'
+      - '0:1:{{ redis_gid }}'
+      - '{{ redis_gid + 1 }}:{{ redis_gid + 1 }}:{{ 65536 - redis_gid }}'
+    label: '{{ _autoupdate_label }}'
+    state: created
+    recreate: '{{ __containers_recreate | default(false) }}'
+  notify: Restart redis
+...