Merge pull request #689 from stackhpc/upstream/master-2025-03-03

Synchronise master with upstream

Author: priteau

ansible/gather-facts.yml

@@ -44,7 +44,7 @@
     batch_size: "{{ ansible_play_batch | length }}"
     # Use a python list slice to divide the group up.
     # Syntax: [<start index>:<end index>:<step size>]
-    delegate_hosts: "{{ groups['all'][batch_index | int::batch_size | int] }}"
+    delegate_hosts: "{{ kolla_ansible_delegate_facts_hosts[batch_index | int::batch_size | int] }}"
   tasks:
     - name: Gather facts
       setup:

ansible/group_vars/all.yml

@@ -36,6 +36,12 @@ kolla_ansible_setup_filter: "{{ omit }}"
 # By default, we do not provide a gather subset.
 kolla_ansible_setup_gather_subset: "{{ omit }}"
 
+# This variable determines which hosts require facts when using --limit. Facts
+# will be gathered using delegation for hosts in this list that are not
+# included in the limit.
+# By default, this list includes all hosts.
+kolla_ansible_delegate_facts_hosts: "{{ groups['all'] }}"
+
 ###################
 # Kolla options
 ###################

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -24,7 +24,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['prometheus'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_port }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ prometheus_port }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -36,7 +36,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['prometheus-node-exporter'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_node_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_node_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -56,7 +56,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['prometheus-mysqld-exporter'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_mysqld_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_mysqld_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -69,7 +69,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['loadbalancer'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_haproxy_exporter_port }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ prometheus_haproxy_exporter_port }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -82,7 +82,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['rabbitmq'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_rabbitmq_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_rabbitmq_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -95,7 +95,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['prometheus-memcached-exporter'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_memcached_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_memcached_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -108,7 +108,7 @@ scrape_configs:
     static_configs:
 {% for host in groups["prometheus-cadvisor"] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_cadvisor_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_cadvisor_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -121,7 +121,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['fluentd'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -159,7 +159,7 @@ scrape_configs:
     static_configs:
 {% for host in groups["prometheus-elasticsearch-exporter"] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_elasticsearch_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_elasticsearch_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -204,7 +204,7 @@ scrape_configs:
     static_configs:
 {% for host in groups["prometheus-libvirt-exporter"] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_libvirt_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_libvirt_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -220,7 +220,7 @@ scrape_configs:
     static_configs:
 {% for host in groups["etcd"] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_etcd_integration_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_etcd_integration_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -232,7 +232,8 @@ scrape_configs:
   - job_name: ironic_prometheus_exporter
     static_configs:
 {% for host in groups['ironic-conductor'] %}
-      - targets: ["{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['ironic_prometheus_exporter_port'] }}"]
+      - targets:
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['ironic_prometheus_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -245,7 +246,7 @@ scrape_configs:
     static_configs:
 {% for host in groups['prometheus-alertmanager'] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -257,7 +258,7 @@ scrape_configs:
     static_configs:
 {% for host in groups["loadbalancer"] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['proxysql_prometheus_exporter_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['proxysql_prometheus_exporter_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"
@@ -270,7 +271,7 @@ alerting:
     - static_configs:
 {% for host in groups["prometheus-alertmanager"] %}
       - targets:
-        - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
+        - '{{ 'api' | kolla_address(host, override_var='prometheus_target_address') | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}'
 {% if hostvars[host].prometheus_instance_label | default(false, true) %}
         labels:
           instance: "{{ hostvars[host].prometheus_instance_label }}"

doc/source/reference/logging-and-monitoring/prometheus-guide.rst

@@ -243,3 +243,16 @@ For example:
       labels:
         instance: host1
 
+Target address
+~~~~~~~~~~~~~~
+
+By default, Prometheus server uses the IP of the API interface of scrape
+targets when collecting metrics. This may be overridden by setting
+``prometheus_target_address`` as a host variable. The value of this host
+variable must be a valid IPv4 or IPv6 address.
+
+Prometheus server is one of the few instances where we need to know IP
+addresses of all other hosts in the cloud. Being able to specify these via
+``prometheus_target_address`` allows us to operate when facts are not available
+for all hosts. This could be due to some hosts being unreachable or having
+previously failed.

doc/source/reference/storage/cinder-guide-pure.rst

@@ -71,7 +71,12 @@ If you wish to use any of these parameters then refer to the
 `Service Configuration <https://docs.openstack.org/kolla-ansible/latest/admin/advanced-configuration.html#openstack-service-configuration-in-kolla>`_
 documentation for instructions using the INI update strategy.
 
-The use of this backend requires that the ``purestorage`` SDK package is
-installed in the ``cinder-volume`` container. To do this follow the steps
+The use of this backend requires that an additional Python SDK package is
+installed in the ``cinder-volume`` container.
+
+Prior to 2024.2 (Dalmatian) the ``purestorage`` SDK is required. From
+2024.2 (Dalmatian) the SDK to install is called ``py-pure-client``.
+
+To install the appropriate SDK follow the steps
 outlined in the `kolla image building guide <https://docs.openstack.org/kolla/latest/admin/image-building.html>`_
 particularly the ``Package Customisation`` and ``Custom Repos`` sections.

doc/source/user/ansible-tuning.rst

@@ -160,3 +160,44 @@ A max fail percentage may be set for specific services using
 
    kolla_max_fail_percentage: 50
    nova_max_fail_percentage: 25
+
+Delegated fact gathering
+------------------------
+
+When Kolla Ansible is executed with a ``--limit`` argument, the scope of an
+operation is limited to the hosts in the limit. For example:
+
+.. code-block:: console
+
+   kolla-ansible deploy --limit control
+
+Due to the nature of configuring clustered software services, there are cases
+where we need to know information about other hosts. Most often this is related
+to their hostname or network addresses. To make this work, Kolla Ansible
+gathers facts for hosts outside of the limit using `delegated fact gathering
+<https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_delegation.html#delegating-facts>`__.
+
+By default, Kolla Ansible gathers facts for all hosts. Because delegated facts
+are gathered serially in batches by the active hosts, this can take a long time
+when there are not many hosts in the limit. If you know that facts are not
+required for all hosts, it is possible to reduce the set of hosts eligible for
+delegated fact gathering by setting ``kolla_ansible_delegate_facts_hosts`` to a
+list of hosts. This may be done permanently in ``globals.yml`` or temporarily
+for the duration of a command using the ``-e`` argument.
+
+The exact requirements will depend upon configuration and inventory, but here
+are some rules of thumb:
+
+* Facts are typically required for all controllers, regardless of which hosts
+  are in the limit. This is due to references to RabbitMQ and Memcache
+  connection strings etc.
+* Prometheus server requires facts for all other hosts to generate scrape
+  configs for node exporter, cAdvisor, etc. Specifically it uses the IP address
+  of the API interface. This may be avoided by hard-coding
+  ``prometheus_target_address`` in the inventory for each host.
+* Configuration of ``/etc/hosts`` during the ``bootstrap-servers`` command
+  requires facts for all other hosts. Specifically it uses the IP address of
+  the API interface, and the ``hostname`` and ``nodename`` facts.
+* Noting the above exceptions, compute nodes are fairly independent. Other
+  hosts do not need to know their facts, and they do not need to know other
+  hosts' facts.

etc/kolla/globals.yml

@@ -5,6 +5,9 @@
 # commented parameters are shown here, To override the default value uncomment
 # the parameter and change its value.
 
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes
+
 ###################
 # Ansible options
 ###################
@@ -21,15 +24,18 @@
 # By default, we do not provide a gather subset.
 #kolla_ansible_setup_gather_subset: "{{ omit }}"
 
-# Dummy variable to allow Ansible to accept this file.
-workaround_ansible_issue_8743: yes
-
 # This variable may be used to set the maximum failure percentage for all
 # plays. More fine-grained control is possible via per-service variables, e.g.
 # nova_max_fail_percentage. The default behaviour is to set a max fail
 # percentage of 100, which is equivalent to not setting it.
 #kolla_max_fail_percentage:
 
+# This variable determines which hosts require facts when using --limit. Facts
+# will be gathered using delegation for hosts in this list that are not
+# included in the limit.
+# By default, this list includes all hosts.
+#kolla_ansible_delegate_facts_hosts: "{{ groups['all'] }}"
+
 ###############
 # Kolla options
 ###############

kolla_ansible/kolla_address.py

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import ipaddress
+
 from jinja2.filters import pass_context
 from jinja2.runtime import Undefined
 
@@ -22,7 +24,7 @@
 
 
 @pass_context
-def kolla_address(context, network_name, hostname=None):
+def kolla_address(context, network_name, hostname=None, override_var=None):
     """returns IP address on the requested network
 
     The output is affected by '<network_name>_*' variables:
@@ -33,6 +35,8 @@ def kolla_address(context, network_name, hostname=None):
     :param network_name: string denoting the name of the network to get IP
                          address for, e.g. 'api'
     :param hostname: to override host which address is retrieved for
+    :param override_var: optional name of a host variable that can be used
+                         to override the IP address
     :returns: string with IP address
     """
 
@@ -58,6 +62,25 @@ def kolla_address(context, network_name, hostname=None):
 
     del hostvars  # remove for clarity (no need for other hosts)
 
+    if override_var is not None:
+        if override_var in host:
+            try:
+                # Use ipaddress to test IPv4/6 validity of
+                # the string returned from override_var and
+                # return the string-formatted, valid IP address
+                ip = ipaddress.ip_address(host[override_var])
+                return format(ip)
+            except ValueError:
+                # Catch ValueError from ipaddress and make the
+                # output more useful for operators
+                raise FilterError("variable '{override_var}' for "
+                                  "host '{hostname}' is set to "
+                                  "'{value}', which is not a valid "
+                                  "IPv4 or IPv6 address"
+                                  .format(override_var=override_var,
+                                          hostname=hostname,
+                                          value=host[override_var]))
+
     # NOTE(yoctozepto): variable "host" will *not* return Undefined
     # same applies to all its children (act like plain dictionary)
 

kolla_ansible/tests/unit/test_address_filters.py

@@ -325,6 +325,71 @@ def test_valid_ipv6_config_do_not_ignore_any_vip_address(self):
         })
         self.assertEqual(addr, kolla_address(context, 'api'))
 
+    def test_override_var_valid_ipv4(self):
+        addr = '192.0.2.1'
+        override_var = 'my_ip_address'
+        context = self._make_context({
+            'inventory_hostname': 'primary',
+            'hostvars': {
+                'primary': {
+                    override_var: addr,
+                },
+            },
+        })
+        self.assertEqual(
+            addr, kolla_address(context, 'api', override_var=override_var))
+
+    def test_override_var_valid_ipv6(self):
+        addr = 'fd::'
+        override_var = 'my_ip_address'
+        context = self._make_context({
+            'inventory_hostname': 'primary',
+            'hostvars': {
+                'primary': {
+                    override_var: addr,
+                },
+            },
+        })
+        self.assertEqual(
+            addr, kolla_address(context, 'api', override_var=override_var))
+
+    def test_override_var_invalid(self):
+        addr = 'this-is-an-fqdn.example.com'
+        override_var = 'my_ip_address'
+        context = self._make_context({
+            'inventory_hostname': 'primary',
+            'hostvars': {
+                'primary': {
+                    override_var: addr,
+                },
+            },
+        })
+        self.assertRaises(
+            FilterError, kolla_address, context, 'api',
+            override_var=override_var)
+
+    def test_override_var_missing(self):
+        addr = '192.0.2.1'
+        override_var = 'my_ip_address'
+        context = self._make_context({
+            'inventory_hostname': 'primary',
+            'hostvars': {
+                'primary': {
+                    'api_address_family': 'ipv4',
+                    'api_interface': 'fake-interface',
+                    'ansible_facts': {
+                        'fake_interface': {
+                            'ipv4': {
+                                'address': addr,
+                            },
+                        },
+                    },
+                },
+            },
+        })
+        self.assertEqual(
+            addr, kolla_address(context, 'api', None, override_var))
+
 
 class TestKollaUrlFilter(unittest.TestCase):
 

releasenotes/notes/delegate-facts-hosts-a3c8bd588c805ffa.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds a new variable, ``kolla_ansible_delegate_facts_hosts``, that may be
+    used to control which hosts require facts when using ``--limit``. Its
+    default value is ``groups['all']``.