CI: fix kolla-ansible installation after cryptography 3.4 release

Installing kolla-ansible system-wide on CentOS 8 fails with:

    ModuleNotFoundError: No module named 'setuptools_rust'

This error appeared following the release of cryptography 3.4, which now
includes Rust code. It can be installed without Rust using a Python
wheel, but only with more recent pip than version 9.0.3 available as RPM
on CentOS 8. The cryptography bug report [1] recommends pip>=19.1.1.

This change switches to using pip --user when installing kolla-ansible.

Also fixes an issue with ansible-lint which was failing on
etc/kolla/globals.yml due to a missing space before comments.

This Victoria backport also includes
I47473de6f71c422db2238d653c2d8f379c55e79b, which fixes a similar issue
with ceph-ansible.
(cherry picked from commit 5fc7707)

This Victoria backport also includes
I4c65a428facdf6d4ce28d97a868589aeae4c856e, which drops lower-constraints
testing.  Per our PTG resolution [2] and general OpenStack resolution
[3], lower-constraints are not worth the extra work and confusion they
introduce.  This patch drops them along with all mentions.
(cherry picked from commit 604d85b)

[1] pyca/cryptography#5753
[2] http://lists.openstack.org/pipermail/openstack-discuss/2020-October/018445.html
[3] http://lists.openstack.org/pipermail/openstack-discuss/2020-December/019521.html

Change-Id: Ifaf1948ed5d42eebaa62d7bad375bbfc12b134d5
(cherry picked from commit 3dd6834)
Closes-Bug: #1915141

Author: markgoddard

lower-constraints.txt

@@ -9,6 +9,9 @@ export PYTHONUNBUFFERED=1
 function setup_ceph_ansible {
     # Prepare virtualenv for ceph-ansible deployment
     python3 -m venv --system-site-packages ~/ceph-venv
+    # NOTE(mgoddard): We need a recent pip to install the latest cryptography
+    # library. See https://github.com/pyca/cryptography/issues/5753
+    ~/ceph-venv/bin/pip install -I 'pip>=19.1.1'
     ~/ceph-venv/bin/pip install -Ir requirements.txt
     ~/ceph-venv/bin/pip install -IU selinux
 }

tests/deploy-ceph-ansible.sh

@@ -50,6 +50,8 @@
           - "--quiet"
 
 - hosts: primary
+  environment:
+    PATH: "{{ ansible_env.HOME + '/.local/bin:' + ansible_env.PATH }}"
   tasks:
     - name: check for existence of ara sqlite
       stat:

tests/post.yml

@@ -25,6 +25,8 @@
         dashboard_enabled: "{{ openstack_core_enabled }}"
         upper_constraints_file: "{{ ansible_env.HOME }}/src/opendev.org/openstack/requirements/upper-constraints.txt"
         docker_image_tag_suffix: "{{ '-aarch64' if ansible_architecture == 'aarch64' else '' }}"
+        pip_user_path_env:
+          PATH: "{{ ansible_env.HOME + '/.local/bin:' + ansible_env.PATH }}"
 
     - name: Install dig for Designate testing
       become: true
@@ -65,6 +67,7 @@
         # NOTE(yoctozepto): to avoid issues with IPv6 not enabled in the docker daemon
         # and since we don't need isolated networks here, use host networking
         network_mode: host
+  environment: "{{ pip_user_path_env }}"
   tasks:
     - name: detect whether need build images
       set_fact:
@@ -200,6 +203,14 @@
         state: directory
       become: true
 
+    # NOTE(mgoddard): We need a recent pip to install the latest cryptography
+    # library. See https://github.com/pyca/cryptography/issues/5753
+    - name: install pip 19.1.1+
+      pip:
+        name: "pip>=19.1.1"
+        executable: "pip3"
+        extra_args: "--user"
+
     - name: install kolla-ansible and dependencies
       vars:
         # Test latest ansible version on Ubuntu, minimum supported on others.
@@ -210,8 +221,7 @@
           - "ansible{{ ansible_version_constraint }}"
           - "ara<1.0.0"
         executable: "pip3"
-        extra_args: "-c {{ upper_constraints_file }}"
-      become: true
+        extra_args: "-c {{ upper_constraints_file }} --user"
 
     - name: get ARA callback plugin path
       command: "python3 -m ara.setup.callback_plugins"
@@ -486,6 +496,7 @@
 
 - hosts: primary
   any_errors_fatal: true
+  environment: "{{ pip_user_path_env }}"
   tasks:
     # Upgrade: update config.
     - block:
@@ -529,8 +540,7 @@
           pip:
             name: "{{ kolla_ansible_src_dir }}"
             executable: pip3
-            extra_args: "-c {{ upper_constraints_file }}"
-          become: true
+            extra_args: "-c {{ upper_constraints_file }} --user"
 
         # Update passwords.yml to include any new passwords added in this
         # release.
@@ -627,6 +637,7 @@
 
 - hosts: primary
   any_errors_fatal: true
+  environment: "{{ pip_user_path_env }}"
   tasks:
     - name: Run reconfigure.sh script
       script:

tests/run.yml

@@ -70,12 +70,6 @@ commands =
   rm -rf releasenotes/build
   sphinx-build -a -E -W -d releasenotes/build/doctrees --keep-going -b html releasenotes/source releasenotes/build/html
 
-[testenv:lower-constraints]
-deps =
-  -c{toxinidir}/lower-constraints.txt
-  -r{toxinidir}/test-requirements.txt
-  -r{toxinidir}/requirements.txt
-
 [testenv:linters]
 # Env vars and deps need to be defined in top level tox env
 setenv =
@@ -131,7 +125,7 @@ setenv = {[testenv:linters]setenv}
 deps = {[testenv:linters]deps}
 commands =
   python {toxinidir}/tools/validate-all-file.py
-  ansible-lint -p --exclude {toxinidir}/tests --exclude {toxinidir}/roles
+  ansible-lint -p --exclude {toxinidir}/tests --exclude {toxinidir}/roles --exclude {toxinidir}/etc
 
 [testenv:yamllint]
 deps = {[testenv:linters]deps}

tox.ini

@@ -14,7 +14,6 @@
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*
-      - ^lower-constraints.txt$
       - ^releasenotes/.*$
       - ^deploy-guide/.*$
       - ^test-requirements.txt$

zuul.d/base.yaml

@@ -3,7 +3,6 @@
     templates:
       - check-requirements
       - openstack-cover-jobs
-      - openstack-lower-constraints-jobs
       - openstack-python3-victoria-jobs
       - ansible-role-jobs
       - publish-openstack-docs-pti