Align Ceph deployment to OSP 17

This patch adds a new Ceph role which is responsible to execute a set of
tasks and deploy the cluster in the standalone scenario. This works
aligns dev-install to the new approach introduced in Wallaby/Zed, where
the Ceph bootstrap is not part of the overcloud deploy anymore.

In particular, as per [1], Ceph expects to see:
1. an already provisioned metal
2. an already provisioned storage_network
3. reserved VIPs (if needed)

There are still actions happening during the overcloud deployment (pool
creation, ganesha deployment) to finalize the cloud, but the initial
bootstrap process has been moved to a set of overcloud ceph related
commands, executed through the related tripleo.operator.

Note that a similar approach has been recently introduced in oooq [2].

[1] https://docs.openstack.org/project-deploy-guide/tripleo-docs/latest/features/deployed_ceph.html#deployed-ceph-workflow
[2] https://github.com/openstack/tripleo-quickstart-extras/blob/master/roles/standalone/tasks/ceph-install.yml

Signed-off-by: Francesco Pantano <[email protected]>
Co-Authored-By: Emilien Macchi <[email protected]>

Author: fmount

playbooks/install_stack.yaml

@@ -198,6 +198,11 @@
         src: "{{ net_config_template }}"
         dest: "{{ ansible_env.HOME }}/dev-install_net_config.yaml"
 
+    - name: Run os-net-config # noqa no-changed-when
+      ansible.builtin.command: "os-net-config --exit-on-validation-errors -d -c {{ ansible_env.HOME }}/dev-install_net_config.yaml"
+      become: true
+      become_user: root
+
     - name: Read dev-install_net_config.yaml
       ansible.builtin.slurp:
         src: "{{ ansible_env.HOME }}/dev-install_net_config.yaml"
@@ -375,14 +380,6 @@
         owner: stack
         mode: '644'
 
-    - name: Add ceph to enabled services
-      ansible.builtin.set_fact:
-        service_envs: "{{ service_envs | union(ceph_env) }}"
-      vars:
-        ceph_env:
-          - "/usr/share/openstack-tripleo-heat-templates/environments/{{ ceph_env_base }}/{{ ceph_env_name }}.yaml"
-      when: ceph_enabled
-
     - name: Install tuned cpu partionioning profile
       ansible.builtin.yum:
         name:
@@ -474,6 +471,12 @@
           when:
             - (tripleo_repos_branch is defined and tripleo_repos_branch not in ['train', 'ussuri', 'victoria']) or (tripleo_repos_branch is not defined)
 
+    - name: Deploy Ceph
+      ansible.builtin.include_role:
+        name: ceph
+      when:
+        - ceph_enabled
+
     - name: Run TripleO deploy
       ansible.builtin.import_role:
         name: tripleo.operator.tripleo_deploy

playbooks/roles/ceph/tasks/main.yaml

@@ -0,0 +1,121 @@
+- name: Tasks for ceph-adm
+  when: ceph_adm_enabled
+  block:
+    - name: Set facts for ceph-adm
+      ansible.builtin.set_fact:
+        stack: standalone
+        mon_ip: "{{ local_ip }}"
+        ceph_network: "{{ control_plane_cidr }}"
+        network_data: "{{ ansible_env.HOME }}/network_data.yaml"
+        deployed_network: "{{ ansible_env.HOME }}/deployed_network.yaml"
+        osd_spec: "{{ ansible_env.HOME }}/osd_spec.yaml"
+        initial_ceph_conf: "{{ ansible_env.HOME }}/initial_ceph.conf"
+        ceph_spec: "{{ ansible_env.HOME }}/ceph_spec.yaml"
+        containers_prep: "{{ ansible_env.HOME }}/containers-prepare-parameters.yaml"
+        deployed_ceph: "{{ ansible_env.HOME }}/deployed_ceph.yaml"
+
+    - name: Configure Storage Network
+      ansible.builtin.include_tasks: storage-network.yml
+      vars:
+        interface: br-ctlplane
+        dummy_ip_cidr: "{{ mon_ip ~ '/' ~ control_plane_prefix | string }}"
+        dummy_vip: "{{ control_plane_ip }}"
+        start: "{{ control_plane_cidr | nthhost(4) }}"
+        end: "{{ control_plane_cidr | nthhost(250) }}"
+
+    - name: Create OSD spec file
+      ansible.builtin.copy:
+        dest: "{{ osd_spec }}"
+        content: |
+          data_devices:
+            paths:
+          {% if ceph_devices is defined and ceph_devices | length > 0 %}
+              {% for d in ceph_devices %}
+              - {{ d }}
+              {% endfor %}
+          {% else %}
+              - /dev/vg_ceph/data
+          {% endif %}
+        mode: '644'
+
+    - name: Create initial ceph.conf file
+      ansible.builtin.copy:
+        dest: "{{ initial_ceph_conf }}"
+        content: |
+          [global]
+          osd_crush_chooseleaf_type = 0
+          osd pool default size = 1
+          [mon]
+          mon_warn_on_pool_no_redundancy = false
+          [mgr]
+          mgr/cephadm/log_to_cluster_level = debug
+        mode: '644'
+
+    - name: Create Ceph Spec file via tripleo-operator-ansible
+      collections:
+        - tripleo.operator
+      ansible.builtin.include_role:
+        name: tripleo_ceph_spec
+      vars:
+        tripleo_ceph_spec_standalone: true
+        tripleo_ceph_spec_overwrite: true
+        tripleo_ceph_spec_mon_ip: "{{ mon_ip }}"
+        tripleo_ceph_spec_stack: "{{ stack }}"
+        tripleo_ceph_spec_file: "{{ ceph_spec }}"
+        tripleo_ceph_spec_osd_spec: "{{ osd_spec }}"
+        tripleo_ceph_spec_debug: true
+        tripleo_ceph_spec_generate_scripts: true
+
+    - name: Create ceph-admin user via tripleo-operator-ansible
+      collections:
+        - tripleo.operator
+      ansible.builtin.include_role:
+        name: tripleo_ceph_user
+      vars:
+        tripleo_ceph_user_spec: "{{ ceph_spec }}"
+        tripleo_ceph_user_enable: true
+        tripleo_ceph_user_stack: "{{ stack }}"
+        tripleo_ceph_user_standalone: true
+        tripleo_ceph_user_debug: true
+        tripleo_ceph_user_generate_scripts: true
+
+    - name: Deploy Ceph via tripleo-operator-ansible
+      collections:
+        - tripleo.operator
+      ansible.builtin.include_role:
+        name: tripleo_ceph_deploy
+      vars:
+        tripleo_ceph_deploy_standalone: true
+        tripleo_ceph_deploy_single_host_defaults: true
+        tripleo_ceph_deploy_skip_user_create: true
+        tripleo_ceph_deploy_skip_hosts_config: true
+        tripleo_ceph_deploy_skip_container_registry_config: true
+        tripleo_ceph_deploy_mon_ip: "{{ mon_ip }}"
+        tripleo_ceph_deploy_spec: "{{ ceph_spec }}"
+        tripleo_ceph_deploy_stack: "{{ stack }}"
+        tripleo_ceph_deploy_config: "{{ initial_ceph_conf }}"
+        tripleo_ceph_deploy_output: "{{ deployed_ceph }}"
+        tripleo_ceph_deploy_container_image_prepare: "{{ containers_prep }}"
+        tripleo_ceph_deploy_cephadm_extra_args: "--log-to-file --skip-mon-network"
+        tripleo_ceph_deploy_force: true
+        tripleo_ceph_deploy_become: true
+        tripleo_ceph_deploy_overwrite: true
+        tripleo_ceph_deploy_debug: true
+        tripleo_ceph_deploy_generate_scripts: true
+        tripleo_ceph_deploy_network_data: "{{ network_data }}"
+        tripleo_ceph_deploy_cluster_network_name: storage
+
+- name: Add ceph to enabled services
+  ansible.builtin.set_fact:
+    service_envs: "{{ service_envs | union(ceph_env) }}"
+  vars:
+    ceph_env:
+      - "/usr/share/openstack-tripleo-heat-templates/environments/{{ ceph_env_base }}/{{ ceph_env_name }}.yaml"
+
+- name: Add ceph network to enabled services
+  when: ceph_adm_enabled
+  ansible.builtin.set_fact:
+    service_envs: "{{ service_envs | union(deployed_ceph_envs) }}"
+  vars:
+    deployed_ceph_envs:
+      - "{{ deployed_ceph }}"

playbooks/roles/ceph/tasks/storage-network.yml

@@ -0,0 +1,76 @@
+---
+- name: Create network_data file
+  ansible.builtin.copy:
+    dest: "{{ network_data }}"
+    mode: '644'
+    content: |
+      - name: Storage
+        mtu: 1350
+        vip: true
+        name_lower: storage
+        dns_domain: storage.mydomain.tld.
+        service_net_map_replace: storage
+        subnets:
+          storage_subnet:
+            ip_subnet: '{{ ceph_network }}'
+            allocation_pools: [{'start': '{{ start }}', 'end': '{{ end }}'}]
+
+- name: Create deployed_network environment file (with VIPs)
+  ansible.builtin.copy:
+    dest: "{{ deployed_network }}"
+    mode: '644'
+    content: |
+      resource_registry:
+        OS::TripleO::Network::Ports::ControlPlaneVipPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_vip_ctlplane.yaml
+        OS::TripleO::Network::Ports::StorageVipPort: network/ports/deployed_vip_storage.yaml
+        OS::TripleO::Network: /usr/share/openstack-tripleo-heat-templates/network/deployed_networks.yaml
+      parameter_defaults:
+        NodePortMap:
+          standalone:
+            ctlplane:
+              ip_address: {{ local_ip }}
+              ip_subnet: {{ local_ip }}/{{ control_plane_prefix }}
+              ip_address_uri: {{ local_ip }}
+            storage:
+              ip_address: {{ mon_ip }}
+              ip_subnet: {{ dummy_ip_cidr }}
+              ip_address_uri: {{ mon_ip }}
+        ControlPlaneVipData:
+          fixed_ips:
+            - ip_address: {{ control_plane_ip }}
+          name: control_virtual_ip
+          network:
+            tags:
+              - "{{ control_plane_cidr }}"
+          subnets:
+            - ip_version: 4
+        VipPortMap:
+          storage:
+            ip_address: {{ dummy_vip }}
+            ip_address_uri: {{ dummy_vip }}
+            ip_subnet: {{ dummy_vip ~ '/' ~ control_plane_prefix | string }}
+        DeployedNetworkEnvironment:
+          net_attributes_map:
+            storage:
+              network:
+                dns_domain: storage.mydomain.tld.
+                mtu: 1350
+                name: storage
+                tags:
+                  - tripleo_network_name=Storage
+                  - tripleo_net_idx=0
+                  - tripleo_service_net_map_replace=storage
+                  - tripleo_vip=true
+              subnets:
+                storage_subnet:
+                  cidr: {{ ceph_network }}
+                  dns_nameservers: []
+                  gateway_ip: null
+                  host_routes: []
+                  ip_version: 4
+                  name: storage_subnet
+          net_cidr_map:
+            storage:
+              - {{ ceph_network }}
+          net_ip_version_map:
+            storage: 4

playbooks/templates/standalone_parameters.ceph_ansible.yaml.j2

@@ -1,5 +1,9 @@
 #vi:syntax=yaml
 
+  CephPoolDefaultPgNum: 8
+  CephPoolDefaultSize: 1
+  CephSpecFqdn: true
+
 {% if ceph_devices is defined %}
   CephAnsibleDisksConfig:
     osd_scenario: lvm

playbooks/templates/standalone_parameters.cephadm.yaml.j2

@@ -116,15 +116,10 @@ parameter_defaults:
     tripleo_kernel_defer_reboot: true
 
 {% if ceph_enabled %}
-  {% if ceph_adm_enabled %}
-{% include "standalone_parameters.cephadm.yaml.j2" %}
-  {% else %}
+  {% if not ceph_adm_enabled %}
 {% include "standalone_parameters.ceph_ansible.yaml.j2" %}
   {% endif %}
 
-  CephPoolDefaultPgNum: 8
-  CephPoolDefaultSize: 1
-  CephSpecFqdn: true
   NovaComputeStartupDelay: 180
 
   # Don't use ceph for local ephemeral storage by default: its performance is