Merge pull request #14 from stackhpc/upstream/wallaby-2022-06-20

markgoddard · web-flow · commit 264f7cc1db2c · 2022-06-21T09:31:16.000+01:00
Synchronise wallaby with upstream
diff --git a/ansible/group_vars/all/kolla b/ansible/group_vars/all/kolla
@@ -152,6 +152,8 @@ overcloud_container_image_regex_map:
     enabled: "{{ kolla_enable_gnocchi | bool }}"
   - regex: ^grafana
     enabled: "{{ kolla_enable_grafana | bool }}"
+  - regex: ^hacluster
+    enabled: "{{ kolla_enable_hacluster | bool }}"
   - regex: ^haproxy$
     enabled: "{{ kolla_enable_haproxy | bool }}"
   - regex: ^heat
@@ -199,7 +201,7 @@ overcloud_container_image_regex_map:
   - regex: neutron-server
     enabled: "{{ kolla_enable_neutron | bool }}"
   # Neutron SFC agent not currently supported on CentOS binary builds.
-  - regex: "neutron-\\(dhcp\\|l3\\|metadata\\|openvswitch\\)-agent"
+  - regex: "neutron-\\(dhcp\\|l3\\|metadata\\|linuxbridge\\|openvswitch\\)-agent"
     enabled: "{{ kolla_enable_neutron | bool }}"
   - regex: neutron-mlnx-agent
     enabled: "{{ kolla_enable_neutron_mlnx | bool }}"
@@ -540,6 +542,7 @@ kolla_enable_fluentd: "yes"
 kolla_enable_freezer: "no"
 kolla_enable_gnocchi: "no"
 kolla_enable_grafana: "{{ kolla_enable_monasca | bool }}"
+kolla_enable_hacluster: "no"
 kolla_enable_heat: "{{ kolla_enable_openstack_core | bool }}"
 kolla_enable_horizon: "{{ kolla_enable_openstack_core | bool }}"
 kolla_enable_influxdb: "{{ kolla_enable_monasca | bool or kolla_enable_cloudkitty | bool }}"
diff --git a/ansible/host-package-update.yml b/ansible/host-package-update.yml
@@ -8,9 +8,9 @@
     host_package_update_security: false
   tasks:
     - name: Update host packages
-      dnf:
+      package:
         name: "{{ host_package_update_packages }}"
-        security: "{{ host_package_update_security | bool }}"
+        security: "{{ host_package_update_security | bool if ansible_facts.os_family == 'RedHat' else omit }}"
         state: latest
-      when: ansible_facts.os_family == 'RedHat'
+        update_cache: "{{ True if ansible_facts.os_family == 'Debian' else omit }}"
       become: true
diff --git a/ansible/kolla-bifrost-hostvars.yml b/ansible/kolla-bifrost-hostvars.yml
@@ -16,7 +16,7 @@
       addressing_mode: static
       ipv4_interface_mac: "{% raw %}{{ extra.pxe_interface_mac | default }}{% endraw %}"
       ipv4_address: "{{ admin_oc_net_name | net_ip }}"
-      ipv4_subnet_mask: "{{ admin_oc_net_name | net_cidr | ipaddr('netmask') }}"
+      ipv4_subnet_mask: "{{ admin_oc_net_name | net_mask }}"
       # If the admin network does not have a gateway defined, use the
       # seed as a gateway to allow external access until other networks have
       # been configured.
diff --git a/ansible/provision-net.yml b/ansible/provision-net.yml
@@ -18,6 +18,7 @@
     venv: "{{ virtualenv_path }}/openstacksdk"
     provision_net:
       name: "{{ kolla_ironic_provisioning_network }}"
+      mtu: "{{ provision_wl_net_name | net_mtu | default(omit, True) }}"
       provider_network_type: "{% if provision_wl_net_name | net_vlan %}vlan{% else %}flat{% endif %}"
       provider_physical_network: "{{ provision_wl_net_name | net_physical_network | default('physnet1', True) }}"
       provider_segmentation_id: "{{ provision_wl_net_name | net_vlan }}"
@@ -31,6 +32,7 @@
           allocation_pool_end: "{{ provision_wl_net_name | net_neutron_allocation_pool_end }}"
     cleaning_net:
       name: "{{ kolla_ironic_cleaning_network }}"
+      mtu: "{{ cleaning_net_name | net_mtu | default(omit, True) }}"
       provider_network_type: "{% if cleaning_net_name | net_vlan %}vlan{% else %}flat{% endif %}"
       provider_physical_network: "{{ cleaning_net_name | net_physical_network | default('physnet1', True) }}"
       provider_segmentation_id: "{{ cleaning_net_name | net_vlan }}"
@@ -64,7 +66,7 @@
     - role: stackhpc.os-networks
       os_openstacksdk_install_epel: "{{ dnf_install_epel }}"
       os_openstacksdk_state: latest
-      os_openstacksdk_upper_constraints_file: "{{ pip_upper_constraints_file }}"
+      os_networks_upper_constraints_file: "{{ pip_upper_constraints_file }}"
       os_networks_venv: "{{ venv }}"
       os_networks_auth_type: "{{ openstack_auth_type }}"
       os_networks_auth: "{{ openstack_auth }}"
diff --git a/ansible/roles/kolla-ansible/library/kolla_passwords.py b/ansible/roles/kolla-ansible/library/kolla_passwords.py
@@ -181,7 +181,7 @@ def main():
     module = AnsibleModule(
         argument_spec = dict(
             dest=dict(default='/etc/kolla/passwords.yml', type='str'),
-            overrides=dict(default={}, type='dict'),
+            overrides=dict(default={}, type='dict', no_log=True),
             sample=dict(default='/usr/share/kolla-ansible/etc_examples/kolla/passwords.yml', type='str'),
             src=dict(default='/etc/kolla/passwords.yml', type='str'),
             vault_password=dict(type='str', no_log=True),
diff --git a/ansible/roles/kolla-ansible/templates/overcloud-top-level.j2 b/ansible/roles/kolla-ansible/templates/overcloud-top-level.j2
@@ -36,7 +36,7 @@ ansible_python_interpreter={{ kolla_ansible_target_venv }}/bin/python
 # Mapping from kolla-ansible group {{ kolla_group }} to top level kayobe
 # groups.
 [{{ kolla_group }}:children]
-{% for group in kolla_group_config.groups %}
+{% for group in renamed_groups %}
 {{ group }}
 {% endfor %}
 
diff --git a/ansible/roles/kolla-openstack/templates/glance.conf.j2 b/ansible/roles/kolla-openstack/templates/glance.conf.j2
@@ -1,34 +1,5 @@
 # {{ ansible_managed }}
 
-{% if kolla_enable_swift | bool %}
-[glance_store]
-
-default_store=swift
-stores=swift
-{% raw %}
-swift_store_user=service:{{ glance_keystone_user }}
-swift_store_key={{ glance_keystone_password }}
-swift_store_auth_version=2
-swift_store_auth_address={{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }}/v2.0
-{% endraw %}
-swift_store_endpoint_type=internalURL
-swift_store_create_container_on_put=True
-
-#swift_store_config_file=/etc/glance/glance-api.conf
-#default_swift_reference=swift_store_ref
-
-#[swift_store_ref]
-#{% raw %}
-#user=service:{{ glance_keystone_user }}
-#key={{ glance_keystone_password }}
-#auth_version=2
-#auth_address={{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }}/v2.0
-#user_domain_id=default
-#project_domain_id=default
-#{% endraw %}
-
-{% endif %}
-
 {% if kolla_extra_glance %}
 #######################
 # Extra configuration
diff --git a/ansible/roles/network-redhat/tasks/main.yml b/ansible/roles/network-redhat/tasks/main.yml
@@ -46,3 +46,4 @@
     name: veth
   vars:
     veth_interfaces: "{{ network_interfaces | net_ovs_veths }}"
+  when: "'overcloud' in group_names"
diff --git a/doc/source/configuration/reference/hosts.rst b/doc/source/configuration/reference/hosts.rst
@@ -768,11 +768,12 @@ For example:
 
    compute_lvm_groups_extra:
      - vgname: other-vg
-       disks: /dev/sdb
+       disks:
+         - /dev/sdb
        create: true
        lvnames:
          - lvname: other-vol
-           size: 100%
+           size: 100%FREE
            create: true
            mount: false
 
diff --git a/doc/source/configuration/reference/physical-network.rst b/doc/source/configuration/reference/physical-network.rst
@@ -15,7 +15,7 @@ configure`` command.  See :ref:`physical-network` for details.
 The following switch operating systems are currently supported:
 
 * Cumulus Linux (via `Network Command Line Utility (NCLU)
-  <https://docs.cumulusnetworks.com/display/DOCS/Network+Command+Line+Utility+-+NCLU>`__)
+  <https://docs.nvidia.com/networking-ethernet-software/cumulus-linux-44/System-Configuration/Network-Command-Line-Utility-NCLU/>`__)
 * Dell OS 6
 * Dell OS 9
 * Dell OS 10
diff --git a/kayobe/plugins/filter/networks.py b/kayobe/plugins/filter/networks.py
@@ -578,7 +578,7 @@ def net_configdrive_network_device(context, name, inventory_hostname=None):
     bootproto = net_bootproto(context, name, inventory_hostname)
     mtu = net_mtu(context, name, inventory_hostname)
     vlan = net_vlan(context, name, inventory_hostname)
-    if vlan:
+    if vlan and '.' in device:
         backend = [device.split('.')[0]]
     else:
         backend = None
@@ -591,7 +591,7 @@ def net_configdrive_network_device(context, name, inventory_hostname=None):
         'mtu': mtu,
         'backend': backend,
     }
-    if vlan:
+    if backend:
         interface['type'] = 'vlan'
     interface = {k: v for k, v in interface.items() if v is not None}
     return interface
diff --git a/molecule-requirements.txt b/molecule-requirements.txt
@@ -0,0 +1,12 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+ansible-lint>=3.0.0,<6.0.0,!=4.3.0 # MIT
+docker # Apache-2.0
+molecule # MIT
+molecule-docker # MIT
+pytest-molecule # MIT
+pytest-testinfra
+rich<12.1.0
+yamllint # GPLv3
diff --git a/playbooks/kayobe-overcloud-base/run.yml b/playbooks/kayobe-overcloud-base/run.yml
@@ -11,11 +11,6 @@
         cmd: "{{ kayobe_src_dir }}/dev/overcloud-deploy.sh &> {{ logs_dir }}/ansible/overcloud-deploy"
         executable: /bin/bash
 
-    - name: Ensure qemu-img is available for Tenks
-      command: docker exec -u root nova_libvirt bash -c 'apt update && apt -y install qemu-utils'
-      become: true
-      when: ansible_os_family == 'Debian'
-
     - name: Ensure test Tenks cluster is deployed
       shell:
         # Pass absolute source directory, since otherwise the `chdir` will
diff --git a/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2
@@ -36,5 +36,6 @@ compute_libvirt_sasl_password: "{% raw %}{{ lookup('password', '/tmp/libvirt-sas
 # Enable ironic for testing baremetal compute.
 kolla_enable_ironic: true
 
-# Disable heat to save disk space.
+# Disable heat and horizon to save disk space.
 kolla_enable_heat: false
+kolla_enable_horizon: false
diff --git a/playbooks/kayobe-seed-base/bifrost-overrides.yml.j2 b/playbooks/kayobe-seed-base/bifrost-overrides.yml.j2
@@ -1,7 +1,9 @@
 ---
-# Don't build an IPA deployment image, instead download upstream images.
+# Don't build an IPA deployment image.
 create_ipa_image: false
-download_ipa: true
+download_ipa: false
 
 # Don't build a disk image. It takes time and can be unreliable.
-use_cirros: true
+use_cirros: {{ not build_images }}
+# Copy a local file to avoid downloading Cirros image.
+cirros_deploy_image_upstream_url: file:///etc/os-release
diff --git a/playbooks/kayobe-seed-base/overrides.yml.j2 b/playbooks/kayobe-seed-base/overrides.yml.j2
@@ -33,7 +33,7 @@ aio_bridge_ports:
   - dummy1
 
 # Build seed deployment images (IPA) with extra-hardware element
-ipa_build_images: true
+ipa_build_images: {{ build_images }}
 ipa_build_dib_elements_extra:
   - "extra-hardware"
 
diff --git a/playbooks/kayobe-seed-base/run.yml b/playbooks/kayobe-seed-base/run.yml
@@ -10,20 +10,22 @@
         cmd: "{{ kayobe_src_dir }}/dev/seed-deploy.sh &> {{ logs_dir }}/ansible/seed-deploy"
         executable: /bin/bash
 
-    # NOTE(mgoddard): The stack user cannot read the kolla source code, which
-    # lives in the zuul user's home directory. Copy it to a readable location.
-    - name: Create a readable copy of kolla source code
-      become: true
-      copy:
-        src: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla'].src_dir }}"
-        dest: /tmp
-        owner: stack
-        group: stack
-        remote_src: true
+    - block:
+        # NOTE(mgoddard): The stack user cannot read the kolla source code, which
+        # lives in the zuul user's home directory. Copy it to a readable location.
+        - name: Create a readable copy of kolla source code
+          become: true
+          copy:
+            src: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla'].src_dir }}"
+            dest: /tmp
+            owner: stack
+            group: stack
+            remote_src: true
 
-    - name: Ensure base container image is built
-      shell:
-        cmd: >
-          source {{ kayobe_src_dir }}/dev/environment-setup.sh &&
-          kayobe seed container image build ^base &> {{ logs_dir }}/ansible/container-image-build
-        executable: /bin/bash
+        - name: Ensure base container image is built
+          shell:
+            cmd: >
+              source {{ kayobe_src_dir }}/dev/environment-setup.sh &&
+              kayobe seed container image build ^base &> {{ logs_dir }}/ansible/container-image-build
+            executable: /bin/bash
+      when: build_images
diff --git a/playbooks/kayobe-seed-upgrade-base/bifrost-overrides.yml.j2 b/playbooks/kayobe-seed-upgrade-base/bifrost-overrides.yml.j2
@@ -1,7 +1,9 @@
 ---
-# Don't build an IPA deployment image, instead download upstream images.
+# Don't build an IPA deployment image.
 create_ipa_image: false
-download_ipa: true
+download_ipa: false
 
 # Don't build a disk image. It takes time and can be unreliable.
 use_cirros: true
+# Copy a local file to avoid downloading Cirros image.
+cirros_deploy_image_upstream_url: file:///etc/os-release
diff --git a/releasenotes/notes/config-idemoptence-37846db82ecd9f43.yaml b/releasenotes/notes/config-idemoptence-37846db82ecd9f43.yaml
@@ -0,0 +1,4 @@
+fixes:
+  - |
+    Fixes an issue with idempotence of local Kolla Ansible configuration
+    generation.
diff --git a/releasenotes/notes/fix-hacluster-variable-48d1370651a11cda.yaml b/releasenotes/notes/fix-hacluster-variable-48d1370651a11cda.yaml
@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Fixes an issue with undefined kolla_enable_hacluster variable.
diff --git a/releasenotes/notes/fix-kolla-ansible-group-mapping-8fcd6cbb1e744e18.yaml b/releasenotes/notes/fix-kolla-ansible-group-mapping-8fcd6cbb1e744e18.yaml
@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixes Ansible inventory generation with some custom group mappings using
+    the same group names for Kayobe and Kolla Ansible. See `story 2009927
+    <https://storyboard.openstack.org/#!/story/2009927>`__ for details.
diff --git a/releasenotes/notes/fixes-keyerror-vlan-link-c177cf719e070df6.yaml b/releasenotes/notes/fixes-keyerror-vlan-link-c177cf719e070df6.yaml
@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    Fixes an issue with config drive generation for infrastructure and seed VMs
+    when using untagged interfaces. The symptom of this issue is that kayobe
+    cannot login to the instance. If you check the libvirt console log, you
+    will see ``KeyError: 'vlan_link'``. See `story 2009910
+    <https://storyboard.openstack.org/#!/story/2009910>`__ for details.
diff --git a/releasenotes/notes/hacluster-build-issue-2a8023e0cd80235a.yaml b/releasenotes/notes/hacluster-build-issue-2a8023e0cd80235a.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixes an issue where hacluster images are not built when the service is
+    enabled.
diff --git a/releasenotes/notes/kolla-passwords-overrides-no-log-57054ce64fae8143.yaml b/releasenotes/notes/kolla-passwords-overrides-no-log-57054ce64fae8143.yaml
@@ -0,0 +1,11 @@
+---
+security:
+  - |
+    Fixes an issue where any passwords in ``kolla_ansible_custom_passwords``
+    were exposed in Ansible logs. When using verbosity level 3 (``-vvv``), they
+    were also exposed in Ansible output.
+fixes:
+  - |
+    Fixes an issue where any passwords in ``kolla_ansible_custom_passwords``
+    were exposed in Ansible logs. When using verbosity level 3 (``-vvv``), they
+    were also exposed in Ansible output.
diff --git a/releasenotes/notes/patch-links-on-overcloud-e24dbc858d3399cc.yaml b/releasenotes/notes/patch-links-on-overcloud-e24dbc858d3399cc.yaml
@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixes an issue where patch links could be erroneously created
+    on hosts not in the overcloud group. See `Story 2009911
+    <https://storyboard.openstack.org/#!/story/2009911>`__ for details.
diff --git a/releasenotes/notes/pin-jinja2-988297e06a2cf790.yaml b/releasenotes/notes/pin-jinja2-988297e06a2cf790.yaml
@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Pins Jinja2 to less than ``3.1.0`` to avoid breaking changes.
diff --git a/releasenotes/notes/provision-net-mtu-befdda04224f49a6.yaml b/releasenotes/notes/provision-net-mtu-befdda04224f49a6.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixes an issue where the MTU defined in Kayobe was not applied to Ironic
+    provisioning and cleaning networks in Neutron.
diff --git a/releasenotes/notes/stackhpc-drac-check-mode-8097215f8eca9991.yaml b/releasenotes/notes/stackhpc-drac-check-mode-8097215f8eca9991.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixes failures to run ``kayobe overcloud bios raid configure`` by upgrading
+    the ``stackhpc.drac`` role to version 1.1.6.
diff --git a/releasenotes/notes/ubuntu-package-update-0db09fc57249b9fc.yaml b/releasenotes/notes/ubuntu-package-update-0db09fc57249b9fc.yaml
@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    Adds support for running package updates on Ubuntu hosts via the following
+    existing commands:
+
+    * ``kayobe seed host package update --packages <packages>``
+    * ``kayobe seed hypervisor host package update --packages <packages>``
+    * ``kayobe infra vm host package update --packages <packages>``
+    * ``kayobe overcloud host package update --packages <packages>``
diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,6 @@
 pbr>=2.0 # Apache-2.0
 ansible>=2.9.0,<2.11.0,!=2.9.8,!=2.9.12 # GPLv3
+Jinja2<3.1.0 # BSD
 cliff>=3.1.0 # Apache
 netaddr!=0.7.16,>=0.7.13 # BSD
 PyYAML>=3.10.0 # MIT
diff --git a/test-requirements.txt b/test-requirements.txt
@@ -6,13 +6,7 @@ ansible-lint>=3.0.0,<6.0.0,!=4.3.0 # MIT
 bashate>=0.2 # Apache-2.0
 coverage>=4.0 # Apache-2.0
 doc8 # Apache-2.0
-docker # Apache-2.0
 hacking>=3.0.1,<3.1.0 # Apache-2.0
-molecule # MIT
-molecule-docker # MIT
 oslotest>=1.10.0 # Apache-2.0
-paramiko
-pytest-molecule # MIT
-pytest-testinfra
 stestr # Apache-2.0
 yamllint # GPLv3
diff --git a/tox.ini b/tox.ini
@@ -62,7 +62,7 @@ commands =
 deps =
     -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/wallaby}
     -r{toxinidir}/requirements.txt
-    -r{toxinidir}/test-requirements.txt
+    -r{toxinidir}/molecule-requirements.txt
 whitelist_externals =
     bash
 commands =
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
