diff --git a/etc/kayobe/ansible/nova-compute-drain.yml b/etc/kayobe/ansible/nova-compute-drain.yml index a29ac3169..d06544f74 100644 --- a/etc/kayobe/ansible/nova-compute-drain.yml +++ b/etc/kayobe/ansible/nova-compute-drain.yml @@ -5,70 +5,8 @@ tags: - nova-compute-drain vars: - venv: "{{ virtualenv_path }}/openstack" - live_migration_fatal: true + nova_compute_migration_fatal: true tasks: - - name: Set up openstack cli virtualenv - pip: - virtualenv: "{{ venv }}" - name: - - python-openstackclient - state: latest - extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + - include_role: + name: nova-compute-drain run_once: true - delegate_to: "{{ groups['controllers'][0] }}" - vars: - ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" - - - block: - - name: Query instances - command: > - {{ venv }}/bin/openstack - server list --host {{ ansible_facts.nodename }} - --all-projects - --status ACTIVE - --format json - register: instances - - - name: Live migrate instances - command: > - {{ venv }}/bin/openstack - --os-compute-api-version 2.25 - server migrate - {{ instance_uuid }} - --live-migration - --wait - loop: "{{ instances.stdout | from_json }}" - loop_control: - label: "{{ instance_uuid }}" - vars: - instance_uuid: "{{ item.ID | default }}" - register: result - failed_when: - - live_migration_fatal | bool - - result is failed - - - name: Query instances - command: > - {{ venv }}/bin/openstack - server list --host {{ ansible_facts.nodename }} - --all-projects - --status ACTIVE - --format json - register: instances - - - name: Fail if there are instances still on the host - fail: - msg: > - Instances still on {{ inventory_hostname }}: {{ instances.stdout | from_json }} - when: - - live_migration_fatal | bool - - instances.stdout | from_json | length > 0 - - delegate_to: "{{ groups['controllers'][0] }}" - environment: "{{ openstack_auth_env }}" - when: - - "'compute' in group_names" - - groups['compute'] | length > 1 - vars: - ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/defaults/main.yml b/etc/kayobe/ansible/roles/nova-compute-drain/defaults/main.yml new file mode 100644 index 000000000..3722cff60 --- /dev/null +++ b/etc/kayobe/ansible/roles/nova-compute-drain/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +nova_compute_drain_venv: "{{ virtualenv_path }}/openstack" +nova_compute_drain_delegate_host: "{{ groups['controllers'][0] }}" diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml new file mode 100644 index 000000000..9f863d1cd --- /dev/null +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/cold-migrate.yml @@ -0,0 +1,38 @@ +--- + +- block: + - name: "Cold migrate instance: {{ instance_uuid }}" + command: > + {{ nova_compute_drain_venv }}/bin/openstack + --os-compute-api-version 2.25 + server migrate + {{ instance_uuid }} + --wait + register: result + + - name: "Wait for VERIFY_RESIZE: {{ instance_uuid }}" + command: > + {{ nova_compute_drain_venv }}/bin/openstack server show {{ instance_uuid }} -f value -c status + register: result + until: result.stdout == 'VERIFY_RESIZE' or result.stdout == 'SHUTOFF' + retries: 10 + delay: 30 + + - name: "Confirm resize: {{ instance_uuid }}" + command: > + {{ nova_compute_drain_venv }}/bin/openstack server migrate confirm {{ instance_uuid }} + when: result.stdout == 'VERIFY_RESIZE' + + - name: "Wait for SHUTOFF: {{ instance_uuid }}" + command: > + {{ nova_compute_drain_venv }}/bin/openstack server show {{ instance_uuid }} -f value -c status + register: result + until: result.stdout == 'SHUTOFF' + retries: 10 + delay: 30 + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ nova_compute_drain_delegate_host }}" + vars: + ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host }}" + rescue: + - meta: noop diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/instance-info.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/instance-info.yml new file mode 100644 index 000000000..390639469 --- /dev/null +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/instance-info.yml @@ -0,0 +1,16 @@ +--- +- name: Query instances + command: > + {{ nova_compute_drain_venv }}/bin/openstack + server list --host {{ ansible_facts.nodename }} + --all-projects + --format json + register: instances + delegate_to: "{{ nova_compute_drain_delegate_host }}" + environment: "{{ openstack_auth_env }}" + vars: + ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host }}" + +- name: Set fact containing list of instances + set_fact: + nova_compute_drain_instance_info: "{{ instances.stdout | from_json }}" diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml new file mode 100644 index 000000000..fa73f78eb --- /dev/null +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/live-migrate.yml @@ -0,0 +1,17 @@ +--- + +- name: "Live migrate instance: {{ instance_uuid }}" + command: > + {{ nova_compute_drain_venv }}/bin/openstack + --os-compute-api-version 2.25 + server migrate + {{ instance_uuid }} + --live-migration + --wait + delegate_to: "{{ nova_compute_drain_delegate_host }}" + environment: "{{ openstack_auth_env }}" + vars: + ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host }}" + register: result + failed_when: + - result is failed diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/main.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/main.yml new file mode 100644 index 000000000..fa78da2e4 --- /dev/null +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/main.yml @@ -0,0 +1,33 @@ +--- +- include_tasks: + name: setup.yml + +- include_tasks: + name: instance-info.yml + +- include_tasks: + name: live-migrate.yml + loop: "{{ nova_compute_drain_instance_info | selectattr('Status', 'equalto', 'ACTIVE') | list }}" + loop_control: + label: "{{ item.ID | default }}" + vars: + instance_uuid: "{{ item.ID | default }}" + +- include_tasks: + name: cold-migrate.yml + loop: "{{ nova_compute_drain_instance_info | selectattr('Status', 'equalto', 'SHUTOFF') | list }}" + loop_control: + label: "{{ item.ID | default }}" + vars: + instance_uuid: "{{ item.ID | default }}" + +- include_tasks: + name: instance-info.yml + +- name: Fail if there are instances still on the host + fail: + msg: > + Instances still on {{ inventory_hostname }}: {{ instances.stdout | from_json }} + when: + - nova_compute_migration_fatal | bool + - nova_compute_drain_instance_info | length > 0 \ No newline at end of file diff --git a/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml new file mode 100644 index 000000000..5c7023408 --- /dev/null +++ b/etc/kayobe/ansible/roles/nova-compute-drain/tasks/setup.yml @@ -0,0 +1,27 @@ +--- + +- name: Initiate openstack cli virtualenv + pip: + virtualenv: "{{ nova_compute_drain_venv }}" + name: + - pip + - setuptools + state: latest + virtualenv_command: /usr/bin/python3.6 -m venv + delegate_to: "{{ nova_compute_drain_delegate_host }}" + vars: + # NOTE: Without this, the delegate ansible_host variable will not + # be respected when using delegate_to. + ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host | default(nova_compute_drain_delegate_host) }}" + +- name: Install openstack CLI tools in virtualenv + pip: + virtualenv: "{{ nova_compute_drain_venv }}" + name: + - python-openstackclient + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + delegate_to: "{{ nova_compute_drain_delegate_host }}" + vars: + # NOTE: Without this, the delegate's ansible_host variable will not + # be respected when using delegate_to. + ansible_host: "{{ hostvars[nova_compute_drain_delegate_host].ansible_host | default(nova_compute_drain_delegate_host) }}"