ovn: Add support for OVN SB Relay

Add support for deploying OVN SB relays [1].
This is enabled by default to improve large scale scenarios.

Default mechanism deploys one relay group per 50 ovn-controller hosts
and uses random but idempotent method to choose a relay for a given host.

neutron-server will connect directly to SB cluster, not via the relays.

[1]: https://docs.ovn.org/en/latest/tutorials/ovn-ovsdb-relay.html

Co-Authored-By: Krzysztof Tomaszewski <ktomaszewski@cloudferro.com>

Depends-On: https://review.opendev.org/c/openstack/kolla/+/928898

Change-Id: Id7d6973668d8b2b194913b533f8cb756c6708536
(cherry picked from commit 7b95939)

Author: mnasiadka

ansible/group_vars/all.yml

@@ -616,8 +616,16 @@ opensearch_dashboards_listen_port: "{{ opensearch_dashboards_port }}"
 
 ovn_nb_db_port: "6641"
 ovn_sb_db_port: "6642"
+# OVN SB Relay related variables
+ovn_sb_db_relay_count: "{{ ((groups['ovn-controller'] | length) / ovn_sb_db_relay_compute_per_relay | int) | round(0, 'ceil') | int }}"
+ovn_sb_db_relay_compute_per_relay: "50"
+ovn_sb_db_relay_port_prefix: "1664"
+ovn_sb_db_relay_port: "{{ ovn_sb_db_relay_port_prefix ~ ovn_sb_db_relay_client_group_id }}"
+ovn_sb_db_relay_client_group_id: "{{ range(1, ovn_sb_db_relay_count | int + 1) | random(seed=inventory_hostname) }}"
 ovn_nb_connection: "{% for host in groups['ovn-nb-db'] %}tcp:{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ ovn_nb_db_port }}{% if not loop.last %},{% endif %}{% endfor %}"
-ovn_sb_connection: "{% for host in groups['ovn-sb-db'] %}tcp:{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ ovn_sb_db_port }}{% if not loop.last %},{% endif %}{% endfor %}"
+ovn_sb_connection: "{{ ovn_sb_connection_relay if enable_ovn_sb_db_relay | bool else ovn_sb_connection_no_relay }}"
+ovn_sb_connection_no_relay: "{% for host in groups['ovn-sb-db'] %}tcp:{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ ovn_sb_db_port }}{% if not loop.last %},{% endif %}{% endfor %}"
+ovn_sb_connection_relay: "{% for host in groups['ovn-sb-db-relay'] %}tcp:{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ ovn_sb_db_relay_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 
 ovsdb_port: "6640"
 
@@ -924,6 +932,7 @@ enable_octavia_driver_agent: "{{ enable_octavia | bool and neutron_plugin_agent
 enable_octavia_jobboard: "{{ enable_octavia | bool and 'amphora' in octavia_provider_drivers }}"
 enable_openvswitch: "{{ enable_neutron | bool and neutron_plugin_agent != 'linuxbridge' }}"
 enable_ovn: "{{ enable_neutron | bool and neutron_plugin_agent == 'ovn' }}"
+enable_ovn_sb_db_relay: "no"
 enable_ovs_dpdk: "no"
 enable_osprofiler: "no"
 enable_placement: "{{ enable_nova | bool or enable_zun | bool }}"

ansible/inventory/all-in-one

@@ -592,6 +592,9 @@ ovn-database
 [ovn-sb-db:children]
 ovn-database
 
+[ovn-sb-db-relay:children]
+ovn-database
+
 [venus-api:children]
 venus
 

ansible/inventory/multinode

@@ -611,6 +611,9 @@ ovn-database
 [ovn-sb-db:children]
 ovn-database
 
+[ovn-sb-db-relay:children]
+ovn-database
+
 [venus-api:children]
 venus
 

ansible/roles/neutron/templates/ml2_conf.ini.j2

@@ -37,7 +37,7 @@ max_header_size = 38
 
 [ovn]
 ovn_nb_connection = {{ ovn_nb_connection }}
-ovn_sb_connection = {{ ovn_sb_connection }}
+ovn_sb_connection = {{ ovn_sb_connection_no_relay }}
 ovn_metadata_enabled = True
 enable_distributed_floating_ip = {{ neutron_ovn_distributed_fip | bool }}
 ovn_emit_need_to_frag = True

ansible/roles/octavia/templates/octavia.conf.j2

@@ -29,7 +29,7 @@ enabled_provider_agents = {{ octavia_provider_agents }}
 {% if neutron_plugin_agent == 'ovn' %}
 [ovn]
 ovn_nb_connection = {{ ovn_nb_connection }}
-ovn_sb_connection = {{ ovn_sb_connection }}
+ovn_sb_connection = {{ ovn_sb_connection_no_relay }}
 {% endif %}
 
 [haproxy_amphora]

ansible/roles/ovn-db/defaults/main.yml

@@ -21,6 +21,15 @@ ovn_db_services:
     image: "{{ ovn_sb_db_image_full }}"
     volumes: "{{ ovn_sb_db_default_volumes + ovn_sb_db_extra_volumes }}"
     dimensions: "{{ ovn_sb_db_dimensions }}"
+  ovn-sb-db-relay:
+    container_name: ovn_sb_db_relay
+    group: ovn-sb-db-relay
+    enabled: "{{ enable_ovn_sb_db_relay | bool }}"
+    image: "{{ ovn_sb_db_relay_image_full }}"
+    iterate: true
+    iterate_var: "{{ ovn_sb_db_relay_count | int }}"
+    volumes: "{{ ovn_sb_db_relay_default_volumes + ovn_sb_db_relay_extra_volumes }}"
+    dimensions: "{{ ovn_sb_db_relay_dimensions }}"
 
 
 ####################
@@ -40,9 +49,14 @@ ovn_sb_db_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docke
 ovn_sb_db_tag: "{{ ovn_tag }}"
 ovn_sb_db_image_full: "{{ ovn_sb_db_image }}:{{ ovn_sb_db_tag }}"
 
+ovn_sb_db_relay_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/ovn-sb-db-relay"
+ovn_sb_db_relay_tag: "{{ ovn_tag }}"
+ovn_sb_db_relay_image_full: "{{ ovn_sb_db_relay_image }}:{{ ovn_sb_db_relay_tag }}"
+
 ovn_northd_dimensions: "{{ default_container_dimensions }}"
 ovn_nb_db_dimensions: "{{ default_container_dimensions }}"
 ovn_sb_db_dimensions: "{{ default_container_dimensions }}"
+ovn_sb_db_relay_dimensions: "{{ default_container_dimensions }}"
 
 ovn_northd_default_volumes:
   - "{{ node_config_directory }}/ovn-northd/:{{ container_config_directory }}/:ro"
@@ -58,11 +72,16 @@ ovn_sb_db_default_volumes:
   - "/etc/localtime:/etc/localtime:ro"
   - "ovn_sb_db:/var/lib/openvswitch/ovn-sb/"
   - "kolla_logs:/var/log/kolla/"
+ovn_sb_db_relay_default_volumes:
+  - "{{ node_config_directory }}/ovn-sb-db-relay{{ ('-' + item | string) if item is defined }}/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "kolla_logs:/var/log/kolla/"
 
 ovn_db_extra_volumes: "{{ default_extra_volumes }}"
 ovn_northd_extra_volumes: "{{ ovn_db_extra_volumes }}"
 ovn_nb_db_extra_volumes: "{{ ovn_db_extra_volumes }}"
 ovn_sb_db_extra_volumes: "{{ ovn_db_extra_volumes }}"
+ovn_sb_db_relay_extra_volumes: "{{ ovn_db_extra_volumes }}"
 
 #####
 # OVN
@@ -75,6 +94,19 @@ ovn_openflow_probe_interval: "60"
 ovn_db_inactivity_probe: "60000"
 ovn_sb_db_inactivity_probe: "{{ ovn_db_inactivity_probe }}"
 ovn_nb_db_inactivity_probe: "{{ ovn_db_inactivity_probe }}"
+ovn_sb_db_relay_active_inactivity_probe: "{{ ovn_db_inactivity_probe | int * 2}}"
+ovn_sb_db_relay_passive_inactivity_probe: "{{ ovn_db_inactivity_probe }}"
+ovn_sb_db_relay_max_backoff: "{{ ovn_db_inactivity_probe }}"
+
+# TODO(mnnasiadka): remove that once Debian gets OVS 3.3+
+ovn_sb_db_relay_config_file_args: >-
+  {% if kolla_base_distro == 'debian' %}
+  --db-sb-relay-remote={{ ovn_sb_connection_no_relay }}
+  -- --remote=ptcp:{{ ovn_sb_db_relay_port_prefix ~ ovn_sb_db_relay_group_id }}:{{ api_interface_address | put_address_in_context('url') }}
+  {% else %}
+  --db-sb-relay-config-file=/etc/ovn/ovsdb-relay.json
+  {% endif %}
+
 # OVN startup commands
 ovn_nb_command: >-
   /usr/share/ovn/scripts/ovn-ctl run_nb_ovsdb
@@ -94,5 +126,13 @@ ovn_sb_command: >-
   --db-sb-pidfile=/run/ovn/ovnsb_db.pid
   --db-sb-file=/var/lib/openvswitch/ovn-sb/ovnsb.db
   --ovn-sb-logfile=/var/log/kolla/openvswitch/ovn-sb-db.log
+ovn_sb_relay_command: >-
+  /usr/share/ovn/scripts/ovn-ctl run_sb_relay_ovsdb
+  --db-sb-relay-ctrl-sock=/run/ovn/ovnsb_db_relay_{{ ovn_sb_db_relay_group_id }}.ctl
+  --db-sb-relay-pidfile=/run/ovn/ovnsb_db_relay_{{ ovn_sb_db_relay_group_id }}.pid
+  --db-sb-relay-sock=/run/ovn/ovnsb_db_{{ ovn_sb_db_relay_group_id }}.sock
+  --db-sb-relay-use-remote-in-db=no
+  --ovn-sb-relay-logfile=/var/log/kolla/openvswitch/ovn-sb-relay-{{ ovn_sb_db_relay_group_id }}.log
+  {{ ovn_sb_db_relay_config_file_args }}
 # Workaround: pause after restarting containers to allow for leader election.
 ovn_leader_election_pause: 5

ansible/roles/ovn-db/handlers/main.yml

@@ -25,6 +25,20 @@
     volumes: "{{ service.volumes | reject('equalto', '') | list }}"
     dimensions: "{{ service.dimensions }}"
 
+- name: Restart ovn-sb-db-relay container
+  vars:
+    service_name: "ovn-sb-db-relay"
+    service: "{{ ovn_db_services[service_name] }}"
+  become: true
+  kolla_container:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}_{{ item }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes | reject('equalto', '') | list }}"
+    dimensions: "{{ service.dimensions }}"
+  loop: "{{ range(1, (ovn_sb_db_relay_count | int) + 1) | list }}"
+
 - name: Restart ovn-northd container
   vars:
     service_name: "ovn-northd"

ansible/roles/ovn-db/tasks/bootstrap-db.yml

@@ -56,3 +56,20 @@
   until: check_ovn_sb_db_port is success
   retries: 10
   delay: 6
+
+- name: Wait for ovn-sb-db-relay
+  vars:
+    ovn_sb_db_relay_group_id: "{{ item }}"
+    ovn_sb_db_relay_port: "{{ ovn_sb_db_relay_port_prefix ~ ovn_sb_db_relay_group_id }}"
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ ovn_sb_db_relay_port }}"
+    connect_timeout: 1
+    timeout: 60
+  register: check_ovn_sb_db_relay_port
+  until: check_ovn_sb_db_relay_port is success
+  retries: 10
+  delay: 6
+  when:
+    - enable_ovn_sb_db_relay | bool
+  loop: "{{ range(1, (ovn_sb_db_relay_count | int) +1) }}"

ansible/roles/ovn-db/tasks/config-relay.yml

@@ -0,0 +1,35 @@
+---
+- name: Ensuring config directories exist for OVN relay containers
+  vars:
+    ovn_sb_db_relay_group_id: "{{ item }}"
+  file:
+    path: "{{ node_config_directory }}/ovn-sb-db-relay-{{ item }}"
+    state: "directory"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+    mode: "0770"
+  become: true
+
+- name: Copying over config.json files for OVN relay services
+  vars:
+    # NOTE(mnasiadka): Used in the ovn_sb_relay_command template in defaults/main.yml
+    ovn_sb_db_relay_group_id: "{{ item }}"
+  template:
+    src: "ovn-sb-db-relay.json.j2"
+    dest: "{{ node_config_directory }}/ovn-sb-db-relay-{{ item }}/config.json"
+    mode: "0660"
+  become: true
+  notify:
+    - Restart ovn-sb-db-relay container
+
+- name: Generate config files for OVN relay services
+  vars:
+    # NOTE(mnasiadka): Used in the ovn_sb_relay_command template in defaults/main.yml
+    ovn_sb_db_relay_group_id: "{{ item }}"
+  template:
+    src: "ovsdb-relay.json.j2"
+    dest: "{{ node_config_directory }}/ovn-sb-db-relay-{{ item }}/ovsdb-relay.json"
+    mode: "0660"
+  become: true
+  notify:
+    - Restart ovn-sb-db-relay container

ansible/roles/ovn-db/tasks/config.yml

@@ -8,6 +8,7 @@
     mode: "0770"
   become: true
   with_dict: "{{ ovn_db_services | select_services_enabled_and_mapped_to_host }}"
+  when: item.key != 'ovn-sb-db-relay'
 
 - name: Copying over config.json files for services
   template:
@@ -16,3 +17,11 @@
     mode: "0660"
   become: true
   with_dict: "{{ ovn_db_services | select_services_enabled_and_mapped_to_host }}"
+  notify:
+    - Restart {{ item.key }} container
+  when: item.key != 'ovn-sb-db-relay'
+
+- name: Ensure configuration for relays exists
+  include_tasks: config-relay.yml
+  loop: "{{ range(1, ovn_sb_db_relay_count | int + 1) | list }}"
+  when: enable_ovn_sb_db_relay | bool