libvirt: make it possible to run libvirt on the host

In some cases it may be desirable to run the libvirt daemon on the host.
For example, when mixing host and container OS distributions or
versions.

This change makes it possible to disable the nova_libvirt container, by
setting enable_nova_libvirt_container to false. The default values of
some Docker mounts and other paths have been updated to point to default
host directories rather than Docker volumes when using a host libvirt
daemon.

This change does not handle migration of existing systems from using
a nova_libvirt container to libvirt on the host.

Depends-On: https://review.opendev.org/c/openstack/ansible-collection-kolla/+/830504

Change-Id: Ia1239069ccee39416b20959cbabad962c56693cf
(cherry picked from commit 4e41acd)

Author: markgoddard

ansible/group_vars/all.yml

@@ -691,6 +691,7 @@ enable_neutron_trunk: "no"
 enable_neutron_metering: "no"
 enable_neutron_infoblox_ipam_agent: "no"
 enable_neutron_port_forwarding: "no"
+enable_nova_libvirt_container: "{{ nova_compute_virt_type in ['kvm', 'qemu'] }}"
 enable_nova_serialconsole_proxy: "no"
 enable_nova_ssh: "yes"
 enable_octavia: "no"
@@ -1026,7 +1027,7 @@ nova_backend_ceph: "no"
 nova_backend: "{{ 'rbd' if nova_backend_ceph | bool else 'default' }}"
 # Valid options are [ kvm, qemu, vmware ]
 nova_compute_virt_type: "kvm"
-nova_instance_datadir_volume: "nova_compute"
+nova_instance_datadir_volume: "{{ 'nova_compute' if enable_nova_libvirt_container | bool else '/var/lib/nova' }}"
 nova_safety_upgrade: "no"
 # Valid options are [ none, novnc, spice ]
 nova_console: "novnc"

ansible/roles/ceilometer/defaults/main.yml

@@ -78,7 +78,7 @@ ceilometer_compute_default_volumes:
   - "/run/:/run/:shared"
   - "ceilometer:/var/lib/ceilometer/"
   - "kolla_logs:/var/log/kolla/"
-  - "nova_libvirt:/var/lib/libvirt"
+  - "{{ ceilometer_libvirt_volume }}:/var/lib/libvirt"
   - "{{ kolla_dev_repos_directory ~ '/ceilometer/ceilometer:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/ceilometer' if ceilometer_dev_mode | bool else '' }}"
 ceilometer_ipmi_default_volumes:
   - "{{ node_config_directory }}/ceilometer-ipmi/:{{ container_config_directory }}/:ro"
@@ -94,6 +94,8 @@ ceilometer_central_extra_volumes: "{{ ceilometer_extra_volumes }}"
 ceilometer_compute_extra_volumes: "{{ ceilometer_extra_volumes }}"
 ceilometer_ipmi_extra_volumes: "{{ ceilometer_extra_volumes }}"
 
+ceilometer_libvirt_volume: "{{ 'nova_libvirt' if enable_nova_libvirt_container | bool else '/var/lib/libvirt' }}"
+
 ####################
 # OpenStack
 ####################

ansible/roles/common/tasks/config.yml

@@ -128,7 +128,7 @@
       - name: "conf/input/04-openstack-wsgi.conf.j2"
         enabled: true
       - name: "conf/input/05-libvirt.conf.j2"
-        enabled: true
+        enabled: "{{ enable_nova | bool and enable_nova_libvirt_container | bool }}"
       - name: "conf/input/06-zookeeper.conf.j2"
         enabled: true
       - name: "conf/input/07-kafka.conf.j2"
@@ -222,7 +222,7 @@
       - { name: "neutron", enabled: "{{ enable_neutron | bool }}" }
       - { name: "neutron-tls-proxy", enabled: "{{ neutron_enable_tls_backend | bool }}" }
       - { name: "nova", enabled: "{{ enable_nova | bool }}" }
-      - { name: "nova-libvirt", enabled: "{{ enable_nova | bool and nova_compute_virt_type in ['kvm', 'qemu'] }}" }
+      - { name: "nova-libvirt", enabled: "{{ enable_nova | bool and enable_nova_libvirt_container | bool }}" }
       - { name: "octavia", enabled: "{{ enable_octavia | bool }}" }
       - { name: "openvswitch", enabled: "{{ enable_openvswitch | bool }}" }
       - { name: "outward-rabbitmq", enabled: "{{ enable_outward_rabbitmq | bool }}" }

ansible/roles/nova-cell/defaults/main.yml

@@ -3,7 +3,7 @@ nova_cell_services:
   nova-libvirt:
     container_name: nova_libvirt
     group: "{{ nova_cell_compute_group }}"
-    enabled: "{{ nova_compute_virt_type in ['kvm', 'qemu'] }}"
+    enabled: "{{ enable_nova_libvirt_container }}"
     image: "{{ nova_libvirt_image_full }}"
     pid_mode: "host"
     cgroupns_mode: "host"
@@ -398,7 +398,7 @@ nova_compute_default_volumes:
   - "/dev:/dev"
   - "kolla_logs:/var/log/kolla/"
   - "{% if enable_iscsid | bool %}iscsi_info:/etc/iscsi{% endif %}"
-  - "libvirtd:/var/lib/libvirt"
+  - "{{ nova_libvirt_volume }}:/var/lib/libvirt"
   - "{{ nova_instance_datadir_volume }}:/var/lib/nova/"
   - "{% if enable_shared_var_lib_nova_mnt | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}"
   - "{{ kolla_dev_repos_directory ~ '/nova/nova:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/nova' if nova_dev_mode | bool else '' }}"
@@ -428,6 +428,8 @@ nova_compute_ironic_extra_volumes: "{{ nova_extra_volumes }}"
 # Used by bootstrapping containers.
 nova_cell_bootstrap_extra_volumes: "{{ nova_extra_volumes }}"
 
+nova_libvirt_volume: "{{ 'libvirtd' if enable_nova_libvirt_container | bool else '/var/lib/libvirt' }}"
+
 ####################
 # HAProxy
 ####################

ansible/roles/nova-cell/handlers/main.yml

@@ -128,6 +128,12 @@
   changed_when: true
   no_log: true
 
+- name: Reload libvirtd
+  become: true
+  service:
+    name: libvirtd
+    state: reloaded
+
 - name: Restart nova-compute container
   vars:
     service_name: "nova-compute"

ansible/roles/nova-cell/tasks/config-host.yml

@@ -39,6 +39,7 @@
     dest: "/etc/udev/rules.d/99-kolla-kvm.rules"
     mode: "0644"
   when:
+    - enable_nova_libvirt_container | bool
     - nova_compute_virt_type == 'kvm'
     - inventory_hostname in groups[nova_cell_compute_group]
 
@@ -51,6 +52,7 @@
     name: qemu-kvm.service
     masked: true
   when:
+    - enable_nova_libvirt_container | bool
     - nova_compute_virt_type == 'kvm'
     - ansible_facts.distribution == 'Ubuntu'
     - inventory_hostname in groups[nova_cell_compute_group]

ansible/roles/nova-cell/tasks/config.yml

@@ -94,6 +94,7 @@
 - name: Copying over libvirt TLS keys
   include_tasks: config-libvirt-tls.yml
   when:
+    - inventory_hostname in groups[nova_cell_compute_group]
     - libvirt_tls | bool
     - libvirt_tls_manage_certs | bool
 

ansible/roles/nova-cell/tasks/external_ceph.yml

@@ -1,14 +1,4 @@
 ---
-- name: Ensuring config directory exists
-  file:
-    path: "{{ node_config_directory }}/{{ item }}"
-    state: "directory"
-    mode: "0770"
-  become: true
-  with_items:
-    - "nova-libvirt/secrets"
-  when: inventory_hostname in groups[nova_cell_compute_group]
-
 - name: Check nova keyring file
   stat:
     path: "{{ node_custom_config }}/nova/{{ ceph_nova_keyring }}"
@@ -31,10 +21,32 @@
     - cinder_backend_ceph | bool
     - external_ceph_cephx_enabled | bool
 
+- name: Extract nova key from file
+  set_fact:
+    nova_cephx_raw_key:
+      "{{ lookup('template', nova_cephx_keyring_file.stat.path) | regex_search('key\\s*=.*$', multiline=True) | regex_replace('key\\s*=\\s*(.*)\\s*', '\\1') }}"
+  changed_when: false
+  run_once: True
+  when:
+    - nova_backend == "rbd"
+    - external_ceph_cephx_enabled | bool
+
+- name: Extract cinder key from file
+  set_fact:
+    cinder_cephx_raw_key:
+      "{{ lookup('file', cinder_cephx_keyring_file.stat.path) | regex_search('key\\s*=.*$', multiline=True) | regex_replace('key\\s*=\\s*(.*)\\s*', '\\1') }}"
+  changed_when: false
+  run_once: True
+  when:
+    - cinder_backend_ceph | bool
+    - external_ceph_cephx_enabled | bool
+
 - name: Copy over ceph nova keyring file
   template:
     src: "{{ nova_cephx_keyring_file.stat.path }}"
     dest: "{{ node_config_directory }}/{{ item }}/"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
     mode: "0660"
   become: true
   with_items:
@@ -50,6 +62,8 @@
   template:
     src: "{{ cinder_cephx_keyring_file.stat.path }}"
     dest: "{{ node_config_directory }}/{{ item }}/"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
     mode: "0660"
   become: true
   with_items:  # NOTE: nova-libvirt does not need it
@@ -62,90 +76,119 @@
     - Restart {{ item }} container
 
 - name: Copy over ceph.conf
+  vars:
+    service: "{{ nova_cell_services[item] }}"
   template:
     src: "{{ node_custom_config }}/nova/ceph.conf"
     dest: "{{ node_config_directory }}/{{ item }}/"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
     mode: "0660"
   become: true
   with_items:
     - nova-compute
     - nova-libvirt
   when:
-    - inventory_hostname in groups[nova_cell_compute_group]
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
     - nova_backend == "rbd"
   notify:
     - Restart {{ item }} container
 
-- name: Pushing nova secret xml for libvirt
-  template:
-    src: "secret.xml.j2"
-    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ item.uuid }}.xml"
-    mode: "0600"
-  become: true
-  when:
-    - inventory_hostname in groups[nova_cell_compute_group]
-    - item.enabled | bool
-  with_items:
-    - uuid: "{{ rbd_secret_uuid }}"
-      name: "client.nova secret"
-      enabled: "{{ nova_backend == 'rbd' }}"
-    - uuid: "{{ cinder_rbd_secret_uuid }}"
-      name: "client.cinder secret"
-      enabled: "{{ cinder_backend_ceph }}"
-  notify:
-    - Restart nova-libvirt container
+- block:
+    - name: Ensure /etc/ceph directory exists (host libvirt)
+      file:
+        path: "/etc/ceph/"
+        state: "directory"
+        owner: "root"
+        group: "root"
+        mode: "0755"
+      become: true
 
-- name: Extract nova key from file
-  set_fact:
-    nova_cephx_raw_key:
-      "{{ lookup('template', nova_cephx_keyring_file.stat.path) | regex_search('key\\s*=.*$', multiline=True) | regex_replace('key\\s*=\\s*(.*)\\s*', '\\1') }}"
-  changed_when: false
-  run_once: True
+    - name: Copy over ceph.conf (host libvirt)
+      template:
+        src: "{{ node_custom_config }}/nova/ceph.conf"
+        dest: "/etc/ceph/ceph.conf"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      become: true
   when:
+    - not enable_nova_libvirt_container | bool
+    - inventory_hostname in groups[nova_cell_compute_group]
     - nova_backend == "rbd"
-    - external_ceph_cephx_enabled | bool
 
-- name: Extract cinder key from file
-  set_fact:
-    cinder_cephx_raw_key:
-      "{{ lookup('template', cinder_cephx_keyring_file.stat.path) | regex_search('key\\s*=.*$', multiline=True) | regex_replace('key\\s*=\\s*(.*)\\s*', '\\1') }}"
-  changed_when: false
-  run_once: True
-  when:
-    - cinder_backend_ceph | bool
-    - external_ceph_cephx_enabled | bool
+- block:
+    - name: Ensuring libvirt secrets directory exists
+      vars:
+        service: "{{ nova_cell_services['nova-libvirt'] }}"
+      file:
+        path: "{{ libvirt_secrets_dir }}"
+        state: "directory"
+        owner: "{{ config_owner_user }}"
+        group: "{{ config_owner_group }}"
+        mode: "0770"
+      become: true
+      when:
+        - inventory_hostname in groups[service.group]
 
-- name: Pushing secrets key for libvirt
-  copy:
-    content: "{{ item.result }}"
-    dest: "{{ node_config_directory }}/nova-libvirt/secrets/{{ item.uuid }}.base64"
-    mode: "0600"
-  become: true
-  when:
-    - inventory_hostname in groups[nova_cell_compute_group]
-    - item.enabled | bool
-    - external_ceph_cephx_enabled | bool
-  with_items:
-    # NOTE(yoctozepto): 'default' filter required due to eager evaluation of item content
-    # which will be undefined if the applicable condition is False
-    - uuid: "{{ rbd_secret_uuid }}"
-      result: "{{ nova_cephx_raw_key | default }}"
-      enabled: "{{ nova_backend == 'rbd' }}"
-    - uuid: "{{ cinder_rbd_secret_uuid }}"
-      result: "{{ cinder_cephx_raw_key | default }}"
-      enabled: "{{ cinder_backend_ceph }}"
-  notify:
-    - Restart nova-libvirt container
-  no_log: True
+    - name: Pushing nova secret xml for libvirt
+      vars:
+        service: "{{ nova_cell_services['nova-libvirt'] }}"
+      template:
+        src: "secret.xml.j2"
+        dest: "{{ libvirt_secrets_dir }}/{{ item.uuid }}.xml"
+        owner: "{{ config_owner_user }}"
+        group: "{{ config_owner_group }}"
+        mode: "0600"
+      become: true
+      when:
+        - inventory_hostname in groups[service.group]
+        - item.enabled | bool
+      with_items:
+        - uuid: "{{ rbd_secret_uuid }}"
+          name: "client.nova secret"
+          enabled: "{{ nova_backend == 'rbd' }}"
+        - uuid: "{{ cinder_rbd_secret_uuid }}"
+          name: "client.cinder secret"
+          enabled: "{{ cinder_backend_ceph }}"
+      notify: "{{ libvirt_restart_handlers }}"
 
-- name: Ensuring config directory has correct owner and permission
-  become: true
-  file:
-    path: "{{ node_config_directory }}/{{ item }}"
-    recurse: yes
-    owner: "{{ config_owner_user }}"
-    group: "{{ config_owner_group }}"
-  with_items:
-    - "nova-compute"
-    - "nova-libvirt/secrets"
-  when: inventory_hostname in groups[nova_cell_compute_group]
+    - name: Pushing secrets key for libvirt
+      vars:
+        service: "{{ nova_cell_services['nova-libvirt'] }}"
+      copy:
+        content: "{{ item.result }}"
+        dest: "{{ libvirt_secrets_dir }}/{{ item.uuid }}.base64"
+        owner: "{{ config_owner_user }}"
+        group: "{{ config_owner_group }}"
+        mode: "0600"
+      become: true
+      when:
+        - inventory_hostname in groups[service.group]
+        - item.enabled | bool
+        - external_ceph_cephx_enabled | bool
+      with_items:
+        # NOTE(yoctozepto): 'default' filter required due to eager evaluation of item content
+        # which will be undefined if the applicable condition is False
+        - uuid: "{{ rbd_secret_uuid }}"
+          result: "{{ nova_cephx_raw_key | default }}"
+          enabled: "{{ nova_backend == 'rbd' }}"
+        - uuid: "{{ cinder_rbd_secret_uuid }}"
+          result: "{{ cinder_cephx_raw_key | default }}"
+          enabled: "{{ cinder_backend_ceph }}"
+      notify: "{{ libvirt_restart_handlers }}"
+      no_log: True
+  vars:
+    libvirt_secrets_dir: >-
+      {{ (node_config_directory ~ '/nova-libvirt/secrets')
+         if enable_nova_libvirt_container | bool
+         else '/etc/libvirt/secrets' }}
+    # NOTE(mgoddard): When running libvirt as a host daemon, on CentOS it
+    # appears to pick up secrets automatically, while on Ubuntu it requires a
+    # reload. This may be due to differences in tested versions of libvirt
+    # (8.0.0 vs 6.0.0). Reload should be low overhead, so do it always.
+    libvirt_restart_handlers: >-
+      {{ ['Restart nova-libvirt container']
+         if enable_nova_libvirt_container | bool else
+         ['Reload libvirtd'] }}

ansible/roles/nova-cell/tasks/precheck.yml

@@ -111,13 +111,26 @@
     - nova_libvirt.enabled | bool
     - inventory_hostname in groups[nova_libvirt.group]
 
-- name: Checking that libvirt is not running
+- name: Checking that host libvirt is not running
   vars:
     nova_libvirt: "{{ nova_cell_services['nova-libvirt'] }}"
   stat: path=/var/run/libvirt/libvirt-sock
   register: result
   failed_when: result.stat.exists
   when:
-    - nova_compute_virt_type in ['kvm', 'qemu']
     - container_facts['nova_libvirt'] is not defined
+    - nova_libvirt.enabled | bool
+    - inventory_hostname in groups[nova_libvirt.group]
+
+- name: Checking that nova_libvirt container is not running
+  vars:
+    nova_libvirt: "{{ nova_cell_services['nova-libvirt'] }}"
+  fail:
+    msg: >-
+      The nova_libvirt container is running, however it has been disabled via
+      the 'enable_nova_libvirt_container' flag. Stop and remove the container
+      manually, taking care to migrate any state to the host libvirt daemon.
+  when:
+    - container_facts['nova_libvirt'] is defined
+    - not nova_libvirt.enabled | bool
     - inventory_hostname in groups[nova_libvirt.group]