Merge "Switch to Dynamic Python Version Detection"

Author: Zuul

docker/base/Dockerfile.j2

@@ -23,7 +23,6 @@ LABEL kolla_version="{{ kolla_version }}"
 {% block base_header %}{% endblock %}
 
 ENV KOLLA_BASE_DISTRO={{ base_distro }} \
-    KOLLA_DISTRO_PYTHON_VERSION={{ distro_python_version }} \
     KOLLA_BASE_ARCH={{ base_arch }}
 
 

docker/horizon/Dockerfile.j2

@@ -66,7 +66,7 @@ RUN ln -s horizon-source/* horizon \
     && if [ "$(ls /plugins)" ]; then \
            {{ macros.install_pip(horizon_plugins_pip_packages) }}; \
        fi \
-    && for locale in  /var/lib/kolla/venv/lib/python{{distro_python_version}}/site-packages/*/locale; do \
+    && for locale in  /var/lib/kolla/venv/lib/python3/site-packages/*/locale; do \
            (cd ${locale%/*} && /var/lib/kolla/venv/bin/django-admin compilemessages) \
        done \
     && chmod 644 /usr/local/bin/kolla_extend_start

docker/horizon/extend_start.sh

@@ -5,7 +5,7 @@ set -o errexit
 FORCE_GENERATE="${FORCE_GENERATE:-no}"
 HASH_PATH=/var/lib/kolla/.settings.md5sum.txt
 
-SITE_PACKAGES="/var/lib/kolla/venv/lib/python${KOLLA_DISTRO_PYTHON_VERSION}/site-packages"
+SITE_PACKAGES="/var/lib/kolla/venv/lib/python3/site-packages"
 
 MANAGE_PY="/var/lib/kolla/venv/bin/python /var/lib/kolla/venv/bin/manage.py"
 

docker/kolla-toolbox/Dockerfile.j2

@@ -6,10 +6,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {% block kolla_toolbox_header %}{% endblock %}
 
 {% set venv_path='/opt/ansible' %}
-{% if base_package_type == 'rpm' %}
-    {% set distro_python_version = '3.12' %}
-{% endif %}
-{% set os_client_config= venv_path + '/lib/python' + distro_python_version + '/site-packages/os_client_config/defaults.json' %}
+{% set os_client_config= venv_path + '/lib/python3/site-packages/os_client_config/defaults.json' %}
 
 {% import "macros.j2" as macros with context %}
 
@@ -63,11 +60,32 @@ COPY apt_preferences_rabbitmq.{{ base_distro }} /etc/apt/preferences.d/rabbitmq
 {% endif %}
 {{ macros.install_packages(kolla_toolbox_packages | customizable("packages")) }}
 
+{# NOTE(kevko):
+  In all distros, there is always a /usr/bin/python3 -> python3.X.
+  However, this is disrupted for RHEL-based systems because, unlike
+  other distros, we upgrade Python above [1], but we forget to set
+  the default python3 to the new python3.X. As a result also, everything
+  outside of the virtual environment ends up running on Python 3.9.
+
+  So, let's correctly set python3 -> python3.X since we had to install
+  it and now have default python3 points to the new version.
+
+  [1] https://review.opendev.org/c/openstack/kolla/+/924245
+#}
+{% if base_package_type == 'rpm' %}
+RUN cd /usr/bin && \
+    rm -f python3 && \
+    ln -s python3.12 python3
+{% endif %}
+
 {% block kolla_toolbox_upper_constraints %}
 RUN mkdir -p /requirements \
     && curl -o /requirements/upper-constraints.txt ${UPPER_CONSTRAINTS_FILE:-https://releases.openstack.org/constraints/upper/{{ openstack_release }}} \
     && {{ macros.upper_constraints_remove("openstacksdk") }} \
-    && python{{ distro_python_version }} -m venv --system-site-packages {{ venv_path }}
+    && python3 -m venv --system-site-packages {{ venv_path }} \
+    && KOLLA_DISTRO_PYTHON_VERSION=$(/usr/bin/python3 -c "import sys; print('{}.{}'.format(sys.version_info.major, sys.version_info.minor))") \
+    && cd {{ venv_path }}/lib \
+    && ln -s python${KOLLA_DISTRO_PYTHON_VERSION} {{ venv_path }}/lib/python3
 {% endblock %}
 
 ENV PATH {{ venv_path }}/bin:$PATH
@@ -86,8 +104,8 @@ ENV PATH {{ venv_path }}/bin:$PATH
         'pyudev',
     ] %}
 
-RUN {{ macros.install_pip(['pip', 'wheel', 'setuptools'], python_version=distro_python_version) }} \
-    && {{ macros.install_pip((kolla_toolbox_pip_packages | customizable("pip_packages")), python_version=distro_python_version) }} \
+RUN {{ macros.install_pip(['pip', 'wheel', 'setuptools']) }} \
+    && {{ macros.install_pip((kolla_toolbox_pip_packages | customizable("pip_packages"))) }} \
     && mkdir -p /etc/ansible /usr/share/ansible \
     && echo 'localhost ansible_connection=local ansible_python_interpreter={{ venv_path }}/bin/python' > /etc/ansible/hosts \
     && sed -i 's|  "identity_api_version": "2.0",|  "identity_api_version": "3",|' {{ os_client_config }}

docker/openstack-base/Dockerfile.j2

@@ -195,7 +195,9 @@ RUN ln -s openstack-base-source/* /requirements \
 #}
     && {{ macros.upper_constraints_remove("ovs") }} \
     && mkdir -p /var/lib/kolla \
-    && python3 -m venv --system-site-packages /var/lib/kolla/venv
+    && python3 -m venv --system-site-packages /var/lib/kolla/venv \
+    && KOLLA_DISTRO_PYTHON_VERSION=$(/usr/bin/python3 -c "import sys; print('{}.{}'.format(sys.version_info.major, sys.version_info.minor))") \
+    && ln -s python${KOLLA_DISTRO_PYTHON_VERSION} /var/lib/kolla/venv/lib/python3
 
 {# Block used for overriding global version constraints.
 

docker/skyline/skyline-apiserver/extend_start.sh

@@ -3,7 +3,7 @@
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
-    SITE_PACKAGES="/var/lib/kolla/venv/lib/python${KOLLA_DISTRO_PYTHON_VERSION}/site-packages"
+    SITE_PACKAGES="/var/lib/kolla/venv/lib/python3/site-packages"
     pushd ${SITE_PACKAGES}/skyline_apiserver/db/alembic
     alembic upgrade head
     popd

kolla/image/kolla_worker.py

@@ -114,20 +114,14 @@ def __init__(self, conf):
         self.rpm_setup = self.build_rpm_setup(rpm_setup_config)
 
         if self.base in ['centos', 'rocky']:
-            self.conf.distro_python_version = "3.9"
             self.distro_package_manager = 'dnf'
             self.base_package_type = 'rpm'
         elif self.base in ['debian']:
-            self.conf.distro_python_version = "3.11"
             self.distro_package_manager = 'apt'
             self.base_package_type = 'deb'
         elif self.base in ['ubuntu']:
-            self.conf.distro_python_version = "3.10"
             self.distro_package_manager = 'apt'
             self.base_package_type = 'deb'
-        else:
-            # Assume worst
-            self.conf.distro_python_version = "3.8"
 
         if self.conf.distro_package_manager is not None:
             self.distro_package_manager = self.conf.distro_package_manager
@@ -147,7 +141,6 @@ def __init__(self, conf):
         self.image_statuses_unbuildable = dict()
         self.image_statuses_allowed_to_fail = dict()
         self.maintainer = conf.maintainer
-        self.distro_python_version = conf.distro_python_version
 
         try:
             self.engine_client = engine.getEngineClient(self.conf)
@@ -326,7 +319,6 @@ def create_dockerfiles(self):
                       'kolla_version': kolla_version,
                       'image_name': image_name,
                       'users': self.get_users(),
-                      'distro_python_version': self.distro_python_version,
                       'distro_package_manager': self.distro_package_manager,
                       'rpm_setup': self.rpm_setup,
                       'build_date': build_date,

kolla/tests/test_build.py

@@ -661,24 +661,6 @@ def test_build_rpm_setup(self):
         kolla = build.KollaWorker(self.conf)
         self.assertEqual(2, len(kolla.rpm_setup))
 
-    def test_build_distro_python_version_debian(self):
-        """check distro_python_version for Debian"""
-        self.conf.set_override('base', 'debian')
-        kolla = build.KollaWorker(self.conf)
-        self.assertEqual('3.11', kolla.distro_python_version)
-
-    def test_build_distro_python_version_ubuntu(self):
-        """check distro_python_version for Ubuntu"""
-        self.conf.set_override('base', 'ubuntu')
-        kolla = build.KollaWorker(self.conf)
-        self.assertEqual('3.10', kolla.distro_python_version)
-
-    def test_build_distro_python_version_centos(self):
-        """check distro_python_version for CentOS Stream 9"""
-        self.conf.set_override('base', 'centos')
-        kolla = build.KollaWorker(self.conf)
-        self.assertEqual('3.9', kolla.distro_python_version)
-
     def test_build_distro_package_manager(self):
         """check distro_package_manager conf value is taken"""
         self.conf.set_override('distro_package_manager', 'foo')

releasenotes/notes/remove-kolla-base-distro-d0376ae3aafffcc3.yaml

@@ -0,0 +1,5 @@
+---
+upgrade:
+  - |
+    Environment variable ``KOLLA_DISTRO_PYTHON_VERSION`` has
+    been dropped.