Skip to content

Commit ea44f97

Browse files
markgoddardmarkgoddard
committed
Fix cephadm bootstrap host delegation issues
Sometimes we see a bootstrap failure, where cephadm cannot read the previously generated SSH key. This seems to be because it generates the key on one host, then tries to bootstrap on another. This can happen when running under Kayobe, and the ansible_host variable is not respected with delegate_to. There is another issue, where the facts seen when using delegate_to are of the original host, but the task is executed on the delegate host. This results in cephadm bootstrap getting the wrong value for --mon-ip, and failing a port binding check. This change reworks the cephadm role to avoid delegate_to, and instead uses include_tasks and a condition, to only include for a single bootstrap host. Fixes: #62
1 parent e6dbe19 commit ea44f97

File tree

9 files changed

+168
-168
lines changed

9 files changed

+168
-168
lines changed

‎roles/cephadm/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ All Ceph hosts must be in the `ceph` group.
3333
* `cephadm_custom_repos`: If enabled - the role won't define yum/apt repositories (default: False)
3434
* `cephadm_package_update`: If enabled - cephadm package will be updated to latest version (default: False)
3535
* Bootstrap settings
36+
* `cephadm_bootstrap_host`: The host on which to bootstrap Ceph (default: `groups['mons'][0]`)
3637
* `cephadm_enable_dashboard`: If enabled - dashboard service on MGR will be enabled (default: False)
3738
* `cephadm_enable_firewalld`: If enabled - firewalld will be installed and rules will be applied (default: False)
3839
* `cephadm_enable_monitoring`: If enabled - cephadm monitoring stack will be deployed i.e. prometheus/node-exporters/grafana (default: False)

‎roles/cephadm/defaults/main.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ cephadm_registry_url: ""
1515
cephadm_registry_username: ""
1616
cephadm_registry_password: ""
1717
# Bootstrap settings
18+
cephadm_bootstrap_host: "{{ groups['mons'][0] }}"
1819
cephadm_enable_dashboard: False
1920
cephadm_enable_firewalld: False
2021
cephadm_enable_monitoring: False

‎roles/cephadm/tasks/bootstrap.yml‎

Lines changed: 70 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -1,83 +1,78 @@
11
---
22
- name: Bootstrap cephadm
3-
block:
4-
- name: Bootstrap cephadm
5-
vars:
6-
mon_ip: "{{ hostvars[inventory_hostname].ansible_facts[cephadm_public_interface].ipv4.address }}"
7-
monitoring_stack: "{{ '--skip-monitoring-stack' if not (cephadm_enable_monitoring | bool) else '' }}"
8-
dashboard: "{{ '--skip-dashboard' if not cephadm_enable_dashboard | bool else '' }}"
9-
firewalld: "{{ '--skip-firewalld' if not cephadm_enable_firewalld | bool else '' }}"
10-
command:
11-
cmd: >
12-
cephadm
13-
{% if cephadm_image | length > 0 %}
14-
--image={{ cephadm_image }}
15-
{% endif %}
16-
bootstrap
17-
{{ monitoring_stack }}
18-
{{ dashboard }}
19-
{{ firewalld }}
20-
--ssh-private-key={{ cephadm_ssh_private_key }}
21-
--ssh-public-key={{ cephadm_ssh_public_key }}
22-
{% if cephadm_ssh_user | length > 0 %}
23-
--ssh-user "{{ cephadm_ssh_user }}"
24-
{% endif %}
25-
{% if cephadm_registry_url | length > 0 %}
26-
--registry-url={{ cephadm_registry_url }}
27-
--registry-username={{ cephadm_registry_username }}
28-
--registry-password={{ cephadm_registry_password }}
29-
{% endif %}
30-
--skip-pull
31-
{% if cephadm_fsid | length > 0 %}
32-
--fsid={{ cephadm_fsid }}
33-
{% endif %}
34-
--mon-ip={{ mon_ip }}
35-
{{ cephadm_bootstrap_additional_parameters }}
36-
become: true
37-
when: not cephadm_check_ceph_conf.stat.exists
3+
vars:
4+
mon_ip: "{{ hostvars[inventory_hostname].ansible_facts[cephadm_public_interface].ipv4.address }}"
5+
monitoring_stack: "{{ '--skip-monitoring-stack' if not (cephadm_enable_monitoring | bool) else '' }}"
6+
dashboard: "{{ '--skip-dashboard' if not cephadm_enable_dashboard | bool else '' }}"
7+
firewalld: "{{ '--skip-firewalld' if not cephadm_enable_firewalld | bool else '' }}"
8+
command:
9+
cmd: >
10+
cephadm
11+
{% if cephadm_image | length > 0 %}
12+
--image={{ cephadm_image }}
13+
{% endif %}
14+
bootstrap
15+
{{ monitoring_stack }}
16+
{{ dashboard }}
17+
{{ firewalld }}
18+
--ssh-private-key={{ cephadm_ssh_private_key }}
19+
--ssh-public-key={{ cephadm_ssh_public_key }}
20+
{% if cephadm_ssh_user | length > 0 %}
21+
--ssh-user "{{ cephadm_ssh_user }}"
22+
{% endif %}
23+
{% if cephadm_registry_url | length > 0 %}
24+
--registry-url={{ cephadm_registry_url }}
25+
--registry-username={{ cephadm_registry_username }}
26+
--registry-password={{ cephadm_registry_password }}
27+
{% endif %}
28+
--skip-pull
29+
{% if cephadm_fsid | length > 0 %}
30+
--fsid={{ cephadm_fsid }}
31+
{% endif %}
32+
--mon-ip={{ mon_ip }}
33+
{{ cephadm_bootstrap_additional_parameters }}
34+
become: true
35+
when: not cephadm_check_ceph_conf.stat.exists
3836

39-
- name: Set public network
40-
command:
41-
cmd: "cephadm shell -- ceph config set global public_network {{ cephadm_public_network }}"
42-
become: true
37+
- name: Set public network
38+
command:
39+
cmd: "cephadm shell -- ceph config set global public_network {{ cephadm_public_network }}"
40+
become: true
4341

44-
- name: Set cluster network
45-
command:
46-
cmd: "cephadm shell -- ceph config set global cluster_network {{ cephadm_cluster_network }}"
47-
when: cephadm_cluster_network | length > 0
48-
become: true
42+
- name: Set cluster network
43+
command:
44+
cmd: "cephadm shell -- ceph config set global cluster_network {{ cephadm_cluster_network }}"
45+
when: cephadm_cluster_network | length > 0
46+
become: true
4947

50-
- name: Get cluster fsid
51-
command:
52-
cmd: "cephadm shell -- ceph fsid"
53-
when: cephadm_fsid | length == 0
54-
become: true
55-
register: cephadm_fsid_current
48+
- name: Get cluster fsid
49+
command:
50+
cmd: "cephadm shell -- ceph fsid"
51+
when: cephadm_fsid | length == 0
52+
become: true
53+
register: cephadm_fsid_current
5654

57-
- name: Template out cluster.yml
58-
vars:
59-
fsid: "{{ cephadm_fsid if cephadm_fsid | length > 0 else cephadm_fsid_current.stdout }}"
60-
template:
61-
src: "templates/cluster.yml.j2"
62-
dest: "/var/run/ceph/{{ fsid }}/cephadm_cluster.yml"
63-
owner: root
64-
group: root
65-
mode: 0644
66-
become: true
67-
run_once: True
68-
69-
- name: Apply spec
70-
command:
71-
cmd: >
72-
cephadm shell --
73-
ceph orch apply -i /var/run/ceph/cephadm_cluster.yml
74-
become: true
55+
- name: Template out cluster.yml
56+
vars:
57+
fsid: "{{ cephadm_fsid if cephadm_fsid | length > 0 else cephadm_fsid_current.stdout }}"
58+
template:
59+
src: "templates/cluster.yml.j2"
60+
dest: "/var/run/ceph/{{ fsid }}/cephadm_cluster.yml"
61+
owner: root
62+
group: root
63+
mode: 0644
64+
become: true
65+
run_once: True
7566

76-
- name: Install ceph cli on mon hosts
77-
command:
78-
cmd: "cephadm install ceph"
79-
become: true
80-
when: cephadm_install_ceph_cli
67+
- name: Apply spec
68+
command:
69+
cmd: >
70+
cephadm shell --
71+
ceph orch apply -i /var/run/ceph/cephadm_cluster.yml
72+
become: true
8173

82-
delegate_to: "{{ groups['mons'][0] }}"
83-
run_once: True
74+
- name: Install ceph cli on mon hosts
75+
command:
76+
cmd: "cephadm install ceph"
77+
become: true
78+
when: cephadm_install_ceph_cli

‎roles/cephadm/tasks/destroy.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,17 @@
44
cmd: "cephadm shell -- ceph fsid"
55
become: true
66
register: cephadm_destroy_fsid
7-
delegate_to: "{{ groups['mons'][0] }}"
8-
run_once: True
97
changed_when: false
108
failed_when: false
9+
when: inventory_hostname == cephadm_bootstrap_host
1110

1211
- name: Destroy cluster
12+
vars:
13+
fsid_result: "{{ hostvars[cephadm_bootstrap_host].cephadm_destroy_fsid }}"
1314
command:
14-
cmd: "cephadm rm-cluster --fsid {{ cephadm_destroy_fsid.stdout }} --force"
15+
cmd: "cephadm rm-cluster --fsid {{ fsid_result.stdout }} --force"
1516
become: true
16-
when: cephadm_destroy_fsid.rc != 1
17+
when: fsid_result.rc != 1
1718

1819
- name: Remove ssh keys
1920
file:

‎roles/cephadm/tasks/main.yml‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,27 @@
11
---
22
- include_tasks: "destroy.yml"
3-
when: cephadm_recreate | bool
3+
when:
4+
- cephadm_recreate | bool
45

56
- include_tasks: "prechecks.yml"
67
when: not cephadm_skip_prechecks | bool
78

89
- include_tasks: "pkg_{{ ansible_facts.os_family | lower }}.yml"
910

10-
- include_tasks: "prereqs.yml"
11+
- include_tasks: "prereqs_bootstrap.yml"
12+
when:
13+
- inventory_hostname == cephadm_bootstrap_host
14+
15+
- import_tasks: "prereqs.yml"
1116

1217
- include_tasks: "bootstrap.yml"
13-
when: cephadm_bootstrap | bool
18+
when:
19+
- cephadm_bootstrap | bool
20+
- inventory_hostname == cephadm_bootstrap_host
21+
22+
- import_tasks: "osds.yml"
1423

15-
- include_tasks: "osds.yml"
24+
- include_tasks: "osds_spec.yml"
25+
when:
26+
- cephadm_osd_spec | length > 0
27+
- inventory_hostname == cephadm_bootstrap_host

‎roles/cephadm/tasks/osds.yml‎

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -11,36 +11,3 @@
1111
delegate_to: "{{ omit if 'mons' in group_names else groups['mons'][0] }}"
1212
when: cephadm_osd_devices | length > 0
1313
with_items: "{{ cephadm_osd_devices }}"
14-
15-
- name: Add OSDs
16-
block:
17-
- name: Get cluster fsid
18-
command:
19-
cmd: "cephadm shell -- ceph fsid"
20-
when: cephadm_fsid | length == 0
21-
become: true
22-
register: cephadm_fsid_current
23-
changed_when: false
24-
25-
- name: Template out osd_spec.yml
26-
vars:
27-
fsid: "{{ cephadm_fsid if cephadm_fsid | length > 0 else cephadm_fsid_current.stdout }}"
28-
copy:
29-
content: "{{ cephadm_osd_spec | to_nice_yaml if cephadm_osd_spec is mapping else cephadm_osd_spec }}"
30-
dest: "/var/run/ceph/{{ fsid }}/osd_spec.yml"
31-
owner: root
32-
group: root
33-
mode: 0644
34-
when: cephadm_osd_spec | length > 0
35-
become: true
36-
37-
- name: Apply OSDs spec
38-
command:
39-
cmd: >
40-
cephadm shell --
41-
ceph orch apply -i /var/run/ceph/osd_spec.yml
42-
when: cephadm_osd_spec | length > 0
43-
become: true
44-
45-
delegate_to: "{{ groups['mons'][0] }}"
46-
run_once: True

‎roles/cephadm/tasks/osds_spec.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
---
2+
- name: Get cluster fsid
3+
command:
4+
cmd: "cephadm shell -- ceph fsid"
5+
when: cephadm_fsid | length == 0
6+
become: true
7+
register: cephadm_fsid_current
8+
changed_when: false
9+
10+
- name: Template out osd_spec.yml
11+
vars:
12+
fsid: "{{ cephadm_fsid if cephadm_fsid | length > 0 else cephadm_fsid_current.stdout }}"
13+
copy:
14+
content: "{{ cephadm_osd_spec | to_nice_yaml if cephadm_osd_spec is mapping else cephadm_osd_spec }}"
15+
dest: "/var/run/ceph/{{ fsid }}/osd_spec.yml"
16+
owner: root
17+
group: root
18+
mode: 0644
19+
become: true
20+
21+
- name: Apply OSDs spec
22+
command:
23+
cmd: >
24+
cephadm shell --
25+
ceph orch apply -i /var/run/ceph/osd_spec.yml
26+
become: true

‎roles/cephadm/tasks/prereqs.yml‎

Lines changed: 1 addition & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,9 @@
11
---
2-
- name: Prepare cephadm bootstrap dependencies
3-
block:
4-
- name: Ensure /etc/ceph directory exists
5-
file:
6-
path: /etc/ceph
7-
state: directory
8-
owner: root
9-
group: root
10-
mode: 0755
11-
become: true
12-
13-
- name: Check if /etc/ceph/ceph.conf exists
14-
stat:
15-
path: /etc/ceph/ceph.conf
16-
register: cephadm_check_ceph_conf
17-
18-
- name: Check if cephadm ssh key exists
19-
stat:
20-
path: "{{ cephadm_ssh_private_key }}"
21-
register: cephadm_check_ceph_id
22-
23-
- name: Generate ssh key for cephadm
24-
openssh_keypair:
25-
path: "{{ cephadm_ssh_private_key }}"
26-
size: 4096
27-
comment: "ceph-{{ cephadm_fsid }}"
28-
when: not cephadm_check_ceph_id.stat.exists
29-
register: cephadm_ssh_key
30-
become: true
31-
32-
- name: Save public key
33-
copy:
34-
content: "{{ cephadm_ssh_key.public_key | default }}"
35-
dest: "{{ cephadm_ssh_public_key }}"
36-
owner: root
37-
group: root
38-
mode: 0644
39-
become: true
40-
when: not cephadm_check_ceph_id.stat.exists
41-
42-
- name: Slurp public key if already exist
43-
slurp:
44-
src: "{{ cephadm_ssh_public_key }}"
45-
register: cephadm_ssh_public_key_slurp
46-
when: cephadm_check_ceph_id.stat.exists
47-
48-
delegate_to: "{{ groups['mons'][0] }}"
49-
run_once: True
50-
512
- name: Copy cephadm public key to all hosts
52-
vars:
53-
content: "{{ cephadm_ssh_public_key_slurp.content | b64decode if cephadm_check_ceph_id.stat.exists else cephadm_ssh_key.public_key }}"
543
authorized_key:
554
user: "{{ cephadm_ssh_user }}"
565
state: present
57-
key: "{{ content }}"
6+
key: "{{ hostvars[cephadm_bootstrap_host].cephadm_ssh_public_key_content }}"
587
when: "cephadm_ssh_user | length > 0"
598
become: true
609

‎roles/cephadm/tasks/prereqs_bootstrap.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
---
2+
- name: Ensure /etc/ceph directory exists
3+
file:
4+
path: /etc/ceph
5+
state: directory
6+
owner: root
7+
group: root
8+
mode: 0755
9+
become: true
10+
11+
- name: Check if /etc/ceph/ceph.conf exists
12+
stat:
13+
path: /etc/ceph/ceph.conf
14+
register: cephadm_check_ceph_conf
15+
16+
- name: Check if cephadm ssh key exists
17+
stat:
18+
path: "{{ cephadm_ssh_private_key }}"
19+
register: cephadm_check_ceph_id
20+
21+
- name: Generate ssh key for cephadm
22+
openssh_keypair:
23+
path: "{{ cephadm_ssh_private_key }}"
24+
size: 4096
25+
comment: "ceph-{{ cephadm_fsid }}"
26+
when: not cephadm_check_ceph_id.stat.exists
27+
register: cephadm_ssh_key
28+
become: true
29+
30+
- name: Save public key
31+
copy:
32+
content: "{{ cephadm_ssh_key.public_key | default }}"
33+
dest: "{{ cephadm_ssh_public_key }}"
34+
owner: root
35+
group: root
36+
mode: 0644
37+
become: true
38+
when: not cephadm_check_ceph_id.stat.exists
39+
40+
- name: Slurp public key
41+
slurp:
42+
src: "{{ cephadm_ssh_public_key }}"
43+
register: cephadm_ssh_public_key_slurp
44+
when: cephadm_check_ceph_id.stat.exists
45+
46+
- name: Set a fact about the SSH public key
47+
set_fact:
48+
cephadm_ssh_public_key_content: "{{ cephadm_ssh_public_key_slurp.content | b64decode if cephadm_check_ceph_id.stat.exists else cephadm_ssh_key.public_key }}"

0 commit comments

Comments
 (0)