[CI] Fix bootstrap testing

This patch adjusts the CI in such a way that we
can finally test the correctness of bootstrap-servers.

 - It correctly installs the virtualenv at the beginning
 - It sets ansible_python_interpreter to this virtualenv
 - It runs bootstrap-servers between upgrades
 - It moves package installation from the CI playbook
   to ansible-collections-kolla

To summarize: this patch aligns behavior and tests the code
within the boundaries it should be tested in - that is, inside
a virtualenv. This naturally also tests the correctness of
ansible-collection-kolla, which installs packages and other
dependencies (now not mixed with OS py interpreter).
This is important, especially due to a bug in older versions
of Docker and requests, which required running bootstrap-servers.
Proof of this is that the rocky9-octavia job is finally working,
with no more issues with package cryptography (which was read from OS
python environment by ansible).

TL;DR: From now on, it runs correctly inside a virtualenv,
and the OS interpreter no longer interferes.

Depends-On: https://review.opendev.org/c/openstack/ansible-collection-kolla/+/946983

Change-Id: Ifea75c2e14ffececbe271bee1549e92bded90d48

Author: keuko

tests/run.yml

@@ -26,6 +26,7 @@
         upper_constraints_file: "{{ ansible_env.HOME }}/src/opendev.org/openstack/requirements/upper-constraints.txt"
         docker_image_tag_suffix: "{{ '-aarch64' if ansible_architecture == 'aarch64' else '' }}"
         kolla_ansible_venv_path: "{{ ansible_env.HOME }}/kolla-ansible-venv"
+        kolla_runtime_venv_path: "{{ ansible_env.HOME }}/kolla-runtime-venv"
         kolla_internal_fqdn: "kolla.example.com"
 
     - name: Install dig for Designate testing
@@ -52,6 +53,30 @@
       when:
         - scenario == "lets-encrypt"
 
+    # NOTE(kevko): Rocky Linux has Python 3.9 as the default, but we want to use Python 3.12 instead.
+    - name: Install Python3.12 and dependencies on RHEL derivatives
+      dnf:
+        name:
+          - python3.12
+          - python3.12-devel
+          - python3.12-pip
+        state: latest
+      when: ansible_facts.os_family == 'RedHat'
+      become: true
+
+    # NOTE(kevko): While in Rocky, virtualenv is included as part of the packages above, in Debuntu, virtualenv is in a separate package.
+    - name: Install python3-virtualenv
+      package:
+        name: python3-virtualenv
+        state: present
+        update_cache: "{{ True if ansible_facts.os_family == 'Debian' else omit }}"
+      become: true
+      when: ansible_facts.os_family != 'RedHat'
+
+    - name: Create Kolla runtime venv
+      command:
+        cmd: "{{ 'python3.12' if ansible_facts.os_family == 'RedHat' else 'python3' }} -m venv {{ kolla_runtime_venv_path }}"
+        creates: "{{ kolla_runtime_venv_path }}"
 
 - hosts: primary
   any_errors_fatal: true
@@ -236,37 +261,11 @@
         state: directory
       become: true
 
-    - name: Install Python3.12 and dependencies needed for dbus-python on RHEL derivatives
-      dnf:
-        name:
-          - python3.12
-          - python3.12-devel
-          - python3.12-pip
-          - gcc
-          - dbus-devel
-          - glib2-devel
-          - dbus-x11
-        state: latest
-      when: ansible_facts.os_family == 'RedHat'
-      become: true
-
     - name: Create Kolla Ansible venv
       command:
         cmd: "{{ 'python3.12' if ansible_facts.os_family == 'RedHat' else 'python3' }} -m venv {{ kolla_ansible_venv_path }}"
         creates: "{{ kolla_ansible_venv_path }}"
 
-    - name: Ensure the latest tested pip
-      pip:
-        name: "pip==23.*"
-        state: latest
-        virtualenv: "{{ kolla_ansible_venv_path }}"
-
-    - name: Ensure the latest tested setuptools
-      pip:
-        name: "setuptools==67.2.0"
-        state: latest
-        virtualenv: "{{ kolla_ansible_venv_path }}"
-
     - name: Install kolla-ansible and dependencies
       pip:
         extra_args: "-c {{ upper_constraints_file }}"

tests/templates/globals-default.j2

@@ -1,5 +1,6 @@
 ---
-ansible_python_interpreter: /usr/bin/python3
+virtualenv: "{{ kolla_runtime_venv_path }}"
+ansible_python_interpreter: "{{ kolla_runtime_venv_path }}/bin/python"
 
 kolla_base_distro: "{{ base_distro }}"
 network_interface: "{{ api_interface_name }}"

tests/upgrade.sh

@@ -13,6 +13,11 @@ function upgrade {
     source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
 
     kolla-ansible certificates -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/certificates
+    # Previous versions had older docker, requests requirements for example
+    # Therefore we need to run bootstrap again to ensure libraries are in
+    # proper versions (ansible-collection-kolla is different for new version, potentionally
+    # also dependencies).
+    kolla-ansible bootstrap-servers -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade-bootstrap
     # Skip rabbitmq-ha-precheck before the queues are migrated.
     kolla-ansible prechecks -i ${RAW_INVENTORY} --skip-tags rabbitmq-ha-precheck -vvv &> /tmp/logs/ansible/upgrade-prechecks-pre-rabbitmq