Merge pull request #69 from lae/feature/ceph-updates

lae · web-flow · commit c34d262d2065 · 2019-10-07T10:17:31.000-07:00
A few Ceph-specific updates
diff --git a/.gitignore b/.gitignore
@@ -2,7 +2,7 @@
 *.*~*
 *.bak
 
-ansible.cfg
+/ansible.cfg
 *.retry
 /.project
 /.pydevproject
diff --git a/.travis.yml b/.travis.yml
@@ -15,7 +15,7 @@ env:
 install:
 - if [ "$ANSIBLE_GIT_VERSION" ]; then pip install "https://github.com/ansible/ansible/archive/${ANSIBLE_GIT_VERSION}.tar.gz";
   else pip install "ansible${ANSIBLE_VERSION}"; fi;
-  pip install --pre ansible-lint; pip install jmespath
+  pip install --pre ansible-lint; pip install jmespath netaddr
 - ansible --version
 - ansible-galaxy install lae.travis-lxc,v0.8.1
 - ansible-playbook tests/install.yml -i tests/inventory
@@ -25,11 +25,11 @@ before_script: cd tests/
 script:
 - ansible-lint ../ || true
 - ansible-playbook -i inventory deploy.yml --syntax-check
-- ansible-playbook -i inventory -v deploy.yml --skip skiponlxc
-- 'ANSIBLE_STDOUT_CALLBACK=debug unbuffer ansible-playbook --skip skiponlxc -vv
-  -i inventory deploy.yml > idempotency.log 2>&1 || (e=$?; cat idempotency.log; exit $e)'
+- 'ansible-playbook -i inventory -v deploy.yml --skip skiponlxc & pid=$!; { while true; do sleep 1; kill -0 $pid 2>/dev/null || break; printf "\0"; done }'
+- 'ANSIBLE_STDOUT_CALLBACK=debug ANSIBLE_DISPLAY_SKIPPED_HOSTS=no ANSIBLE_DISPLAY_OK_HOSTS=no
+  unbuffer ansible-playbook --skip skiponlxc -vv -i inventory deploy.yml &> idempotency.log'
 - 'grep -A1 "PLAY RECAP" idempotency.log | grep -qP "changed=0 .*failed=0 .*" &&
-  (echo "Idempotence: PASS"; exit 0) || (echo "Idempotence: FAIL"; exit 1)'
+  (echo "Idempotence: PASS"; exit 0) || (echo "Idempotence: FAIL"; cat idempotency.log; exit 1)'
 - ANSIBLE_STDOUT_CALLBACK=debug ansible-playbook -i inventory -v test.yml
 notifications:
   webhooks:
diff --git a/README.md b/README.md
@@ -340,9 +340,9 @@ For example:
 This will ask for a sudo password, then login to the `admin1` user (using public
 key auth - add `-k` for pw) and run the playbook.
 
-That's it! You should now have a fully deployed Proxmox cluster. You may want to
-create Ceph storage on it afterward, which this role does not (yet?) do, and
-other tasks possibly, but the hard part is mostly complete.
+That's it! You should now have a fully deployed Proxmox cluster. You may want
+to create Ceph storage on it afterwards (see Ceph for more info) and other
+tasks possibly, but the hard part is mostly complete.
 
 
 ## Example Playbook
@@ -553,6 +553,16 @@ documentation.
 
 ## Ceph configuration
 
+*This section could use a little more love. If you are actively using this role
+to manage your PVE Ceph cluster, please feel free to flesh this section more
+thoroughly and open a pull request! See issue #68.*
+
+**PVE Ceph management with this role is experimental.** While users have
+successfully used this role to deploy PVE Ceph, it is not fully tested in CI
+(due to a lack of usable block devices to use as OSDs in Travis CI). Please
+deploy a test environment with your configuration first prior to prod, and
+report any issues if you run into any.
+
 This role can configure the Ceph storage system on your Proxmox hosts.
 
 ```
@@ -592,6 +602,9 @@ pve_ceph_fs:
     mountpoint: /srv/proxmox/backup
 ```
 
+`pve_ceph_network` by default uses the `ipaddr` filter, which requires the
+`netaddr` library to be installed and usable by your Ansible controller.
+
 ## Contributors
 
 Musee Ullah ([@lae](https://github.com/lae), <lae@lae.is>) - Main developer  
diff --git a/defaults/main.yml b/defaults/main.yml
@@ -17,7 +17,7 @@ pve_zfs_enabled: no
 # pve_zfs_options: "parameters to pass to zfs module"
 # pve_zfs_zed_email: "email address for zfs events"
 pve_ceph_enabled: false
-pve_ceph_repository_line: "{{ pve_ceph_repo }}"
+pve_ceph_repository_line: "deb http://download.proxmox.com/debian/{% if ansible_distribution_release == 'stretch' %}ceph-luminous stretch{% else %}ceph-nautilus buster{% endif %} main"
 pve_ceph_network: "{{ (ansible_default_ipv4.network +'/'+ ansible_default_ipv4.netmask) | ipaddr('net') }}"
 pve_ceph_mon_group: "{{ pve_group }}"
 pve_ceph_mds_group: "{{ pve_group }}"
@@ -43,4 +43,4 @@ pve_groups: []
 pve_users: []
 pve_acls: []
 pve_storages: []
-pve_ssh_port: 22
+pve_ssh_port: 22
diff --git a/tasks/ceph.yml b/tasks/ceph.yml
@@ -1,20 +1,4 @@
 # This is an Ansible version of what "pveceph install" actually does
-- name: Configure Ceph package source
-  apt_repository:
-    repo: '{{ pve_ceph_repository_line }}'
-    filename: ceph.list
-    state: present
-
-- name: Install Ceph packages
-  apt:
-    name:
-      - ceph
-      - ceph-common
-      - ceph-mds
-      - ceph-fuse
-      - gdisk
-    install_recommends: false
-
 - name: Install custom Ceph systemd service
   copy:
     src: /usr/share/doc/pve-manager/examples/ceph.service
@@ -34,79 +18,77 @@
   when:
   - "ansible_distribution_release == 'stretch'"
 
-- name: Create initial Ceph config
-  command: 'pveceph init --network {{ pve_ceph_network }}'
-  args:
-    creates: /etc/ceph/ceph.conf
-  when: inventory_hostname == groups[pve_ceph_mon_group][0]
+- block:
+  - name: Create initial Ceph config
+    command: 'pveceph init --network {{ pve_ceph_network }}'
+    args:
+      creates: /etc/ceph/ceph.conf
 
-- name: Create initial Ceph monitor
-  command: 'pveceph mon create'
-  args:
-    creates: '/var/lib/ceph/mon/ceph-{{ ansible_hostname }}/'
-  register: _ceph_initial_mon
-  when: inventory_hostname == groups[pve_ceph_mon_group][0]
+  - name: Create initial Ceph monitor
+    command: 'pveceph mon create'
+    args:
+      creates: '/var/lib/ceph/mon/ceph-{{ ansible_hostname }}/'
+    register: _ceph_initial_mon
 
-- name: Fail if initial monitor creation failed
-  fail:
-    msg: 'Ceph intial monitor creation failed.'
-  when: hostvars[groups[pve_ceph_mon_group][0]]._ceph_initial_mon is failed
+  - name: Fail if initial monitor creation failed
+    fail:
+      msg: 'Ceph intial monitor creation failed.'
+    when: _ceph_initial_mon is failed
+
+  when: inventory_hostname == groups[pve_ceph_mon_group][0]
 
 - name: Create additional Ceph monitors
   command: 'pveceph mon create'
   args:
     creates: '/var/lib/ceph/mon/ceph-{{ ansible_hostname }}/'
-  when: not inventory_hostname == groups[pve_ceph_mon_group][0]
+  when: inventory_hostname != groups[pve_ceph_mon_group][0]
 
 - name: Create Ceph OSDs
   command: >-
     pveceph osd create {{ item.device }}
-    {{ ("block.db" in item) | ternary("--journal_dev", "") }} {{ item["block.db"] | default("") }}
+    {% if "block.db" in item %}--journal_dev {{ item["block.db"] }}{% endif %}
   args:
     creates: '{{ item.device }}1'
   with_items: '{{ pve_ceph_osds }}'
 
-- name: List Ceph CRUSH rules
-  command: 'ceph osd crush rule ls'
-  changed_when: false
-  register: ceph_crush
-  when: inventory_hostname == groups[pve_ceph_mon_group][0]
+- block:
+  - name: List Ceph CRUSH rules
+    command: 'ceph osd crush rule ls'
+    changed_when: false
+    register: _ceph_crush
+
+  - name: Create Ceph CRUSH rules
+    command: >-
+      ceph osd crush rule create-replicated
+      {{ item.name }} default host {{ item.class | default("") }}
+    when: item.name not in _ceph_crush.stdout_lines
+    with_items: '{{ pve_ceph_crush_rules }}'
+
+  - name: List Ceph Pools
+    command: ceph osd pool ls
+    changed_when: false
+    register: _ceph_pools
+
+  - name: Create Ceph Pools
+    command: >-
+      pveceph pool create {{ item.name }}
+      {% if 'storage' in item %}
+      --add_storages {{ item.storage }}
+      {% endif %}
+      {% if 'application' in item %}
+      --application {{ item.application }}
+      {% endif %}
+      {% if 'rule' in item %}
+      --crush_rule {{ item.rule }}
+      {% endif %}
+      {% if 'pgs' in item %}
+      --pg_num {{ item.pgs }}
+      {% endif %}
+    when: item.name not in _ceph_pools.stdout_lines
+    with_items: '{{ pve_ceph_pools }}'
 
-- name: Create Ceph CRUSH rules
-  command: >-
-    ceph osd crush rule create-replicated
-    {{ item.name }} default host {{ item.class | default("") }}
-  when:
-    - inventory_hostname == groups[pve_ceph_mon_group][0]
-    - item.name not in ceph_crush.stdout_lines
-  with_items: '{{ pve_ceph_crush_rules }}'
-
-- name: List Ceph Pools
-  command: ceph osd pool ls
-  changed_when: false
-  register: ceph_pools
   when: inventory_hostname == groups[pve_ceph_mon_group][0]
 
-- name: Create Ceph Pools
-  command: >-
-    pveceph pool create {{ item.name }}
-    {% if 'storage' in item %}
-    --add_storages {{ item.storage }}
-    {% endif %}
-    {% if 'application' in item %}
-    --application {{ item.application }}
-    {% endif %}
-    {% if 'rule' in item %}
-    --crush_rule {{ item.rule }}
-    {% endif %}
-    {% if 'pgs' in item %}
-    --pg_num {{ item.pgs }}
-    {% endif %}
-  when:
-    - inventory_hostname == groups[pve_ceph_mon_group][0]
-    - item.name not in ceph_pools.stdout_lines
-  with_items: '{{ pve_ceph_pools }}'
-
 - name: Create Ceph MDS servers
   command: pveceph mds create
   args:
@@ -122,60 +104,52 @@
   delay: 2
   when: _ceph_mds_create is changed
 
-- name: List Ceph Filesystems
-  command: ceph fs ls -f json
-  changed_when: false
-  when:
-    - inventory_hostname == groups[pve_ceph_mon_group][0]
-    - pve_ceph_fs | length > 0
-  register: ceph_fs
-
-- name: Create Ceph Filesystems
-  command: >-
-    pveceph fs create
-    --name {{ item.name }}
-    --add-storage {{ item.storage }}
-    --pg_num {{ item.pgs }}
-  register: ceph_fs_create
-  failed_when: ceph_fs_create.stderr
-  when:
-    - inventory_hostname == groups[pve_ceph_mon_group][0]
-    - item.name not in (ceph_fs.stdout | from_json | map(attribute="name"))
-  with_items: '{{ pve_ceph_fs }}'
-
-- name: Get Ceph Filesystem pool CRUSH rules
-  command: 'ceph -f json osd pool get {{ item.0.name }}_{{ item.1 }} crush_rule'
-  changed_when: false
-  when:
-    - inventory_hostname == groups[pve_ceph_mon_group][0]
-    - pve_ceph_fs | length > 0
-  register: ceph_fs_rule
-  loop: '{{ pve_ceph_fs | product(["data", "metadata"]) | list }}'
-
-- name: Set Ceph Filesystem pool CRUSH rules
-  command: >-
-    ceph osd pool set {{ item.item.0.name }}_{{ item.item.1 }} crush_rule {{ item.item.0.rule }}
-  when:
-    - inventory_hostname == groups[pve_ceph_mon_group][0]
-    - item.item.0.rule != (item.stdout | from_json).crush_rule
-  loop: '{{ ceph_fs_rule.results }}'
-  loop_control:
-    label: '{{ item.item.0.name }}_{{ item.item.1 }}'
-
-- name: Create Ceph filesystem key
-  command: 'ceph auth get-or-create client.{{ item.name }} osd "allow rw pool={{ item.name }}_data" mon "allow r" mds "allow rw"'
-  register: ceph_fs_auth
-  changed_when: '"added key" in ceph_fs_auth.stdout'
-  when:
-    - inventory_hostname == groups[pve_ceph_mon_group][0]
-    - item.mountpoint is defined
-  loop: '{{ pve_ceph_fs }}'
+- block:
+  - name: List Ceph Filesystems
+    command: ceph fs ls -f json
+    changed_when: false
+    when: pve_ceph_fs | length > 0
+    register: _ceph_fs
+
+  - name: Create Ceph Filesystems
+    command: >-
+      pveceph fs create
+      --name {{ item.name }}
+      --add-storage {{ item.storage }}
+      --pg_num {{ item.pgs }}
+    register: _ceph_fs_create
+    failed_when: _ceph_fs_create.stderr
+    when: item.name not in (_ceph_fs.stdout | from_json | map(attribute="name"))
+    with_items: '{{ pve_ceph_fs }}'
+
+  - name: Get Ceph Filesystem pool CRUSH rules
+    command: 'ceph -f json osd pool get {{ item.0.name }}_{{ item.1 }} crush_rule'
+    changed_when: false
+    when: pve_ceph_fs | length > 0
+    register: _ceph_fs_rule
+    loop: '{{ pve_ceph_fs | product(["data", "metadata"]) | list }}'
+
+  - name: Set Ceph Filesystem pool CRUSH rules
+    command: >-
+      ceph osd pool set {{ item.item.0.name }}_{{ item.item.1 }} crush_rule {{ item.item.0.rule }}
+    when: item.item.0.rule != (item.stdout | from_json).crush_rule
+    loop: '{{ _ceph_fs_rule.results }}'
+    loop_control:
+      label: '{{ item.item.0.name }}_{{ item.item.1 }}'
+
+  - name: Create Ceph filesystem key
+    command: 'ceph auth get-or-create client.{{ item.name }} osd "allow rw pool={{ item.name }}_data" mon "allow r" mds "allow rw"'
+    register: _ceph_fs_auth
+    changed_when: '"added key" in _ceph_fs_auth.stdout'
+    when: item.mountpoint is defined
+    loop: '{{ pve_ceph_fs }}'
+  when: inventory_hostname == groups[pve_ceph_mon_group][0]
 
 - name: Fetch Ceph filesystem key
   command: 'ceph auth print-key client.{{ item.name }}'
   args:
     creates: '/etc/ceph/{{ item.name }}.secret'
-  register: ceph_fs_key
+  register: _ceph_fs_key
   when: item.mountpoint is defined
   loop: '{{ pve_ceph_fs }}'
 
@@ -187,7 +161,7 @@
     mode: '0600'
     content: '{{ item.stdout }}'
   when: item is changed
-  loop: '{{ ceph_fs_key.results }}'
+  loop: '{{ _ceph_fs_key.results }}'
   loop_control:
     label: '{{ item.item }}'
 
diff --git a/tasks/identify_needed_packages.yml b/tasks/identify_needed_packages.yml
@@ -17,6 +17,11 @@
     _pve_install_packages: "{{ _pve_install_packages | union(['zfsutils-linux', 'zfs-initramfs', 'zfs-zed']) }}"
   when: "pve_zfs_enabled | bool"
 
+- name: Stage Ceph packages if Ceph is enabled
+  set_fact:
+    _pve_install_packages: "{{ _pve_install_packages | union(['ceph', 'ceph-common', 'ceph-mds', 'ceph-fuse', 'gdisk']) }}"
+  when: "pve_ceph_enabled | bool"
+
 - name: Stage any extra packages the user has specified
   set_fact:
     _pve_install_packages: "{{ _pve_install_packages | union(pve_extra_packages) }}"
diff --git a/tasks/main.yml b/tasks/main.yml
@@ -65,12 +65,20 @@
     state: present
   register: _pve_repo
 
+- name: Add Proxmox Ceph repository
+  apt_repository:
+    repo: '{{ pve_ceph_repository_line }}'
+    filename: ceph
+    state: present
+  register: _pve_ceph_repo
+  when: "pve_ceph_enabled | bool"
+
 - name: Run apt-get dist-upgrade on repository changes
   apt:
     update_cache: yes
     cache_valid_time: 3600
     upgrade: dist
-  when: _pve_repo is changed
+  when: _pve_repo is changed or _pve_ceph_repo is changed
   retries: 2
   register: _dist_upgrade
   until: _dist_upgrade is succeeded
diff --git a/tests/ansible.cfg b/tests/ansible.cfg
@@ -0,0 +1,2 @@
+[defaults]
+callback_whitelist = profile_tasks
diff --git a/tests/group_vars/all b/tests/group_vars/all
@@ -9,6 +9,7 @@ pve_run_system_upgrades: true
 pve_watchdog: ipmi
 pve_zfs_enabled: yes
 pve_zfs_zed_email: root@localhost
+pve_ceph_enabled: yes
 pve_ssl_private_key: "{{ lookup('file', ssl_host_key_path) }}"
 pve_ssl_certificate: "{{ lookup('file', ssl_host_cert_path) }}"
 pve_cluster_enabled: yes
diff --git a/tests/install.yml b/tests/install.yml
diff --git a/vars/debian-buster.yml b/vars/debian-buster.yml
diff --git a/vars/debian-stretch.yml b/vars/debian-stretch.yml

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+[defaults]`
	`2`	`+callback_whitelist = profile_tasks`