Merge remote-tracking branch 'origin/stable/victoria' into stackhpc/victora-ironic-rebalance

Author: jovial

doc/source/admin/evacuate.rst

@@ -97,3 +97,17 @@ instances up and running.
    using a pattern you might want to use the ``--strict`` flag which got introduced
    in version 10.2.0 to make sure nova matches the ``FAILED_HOST``
    exactly.
+
+.. note::
+   .. code-block:: bash
+
+      +------+--------+--------------+
+      | Name | Status | Task State   |
+      +------+--------+--------------+
+      | vm_1 | ACTIVE | powering-off |
+      +------------------------------+
+
+   If the instance task state is not None, evacuation will be possible. However,
+   depending on the ongoing operation, there may be clean up required in other
+   services which the instance was using, such as neutron, cinder, glance, or
+   the storage backend.

nova/compute/api.py

@@ -5185,7 +5185,7 @@ def live_migrate_abort(self, context, instance, migration_id,
     @reject_vtpm_instances(instance_actions.EVACUATE)
     @block_accelerators(until_service=SUPPORT_ACCELERATOR_SERVICE_FOR_REBUILD)
     @check_instance_state(vm_state=[vm_states.ACTIVE, vm_states.STOPPED,
-                                    vm_states.ERROR])
+                                    vm_states.ERROR], task_state=None)
     def evacuate(self, context, instance, host, on_shared_storage,
                  admin_password=None, force=None):
         """Running evacuate to target host.
@@ -5212,7 +5212,7 @@ def evacuate(self, context, instance, host, on_shared_storage,
             context, instance.uuid)
 
         instance.task_state = task_states.REBUILDING
-        instance.save(expected_task_state=[None])
+        instance.save(expected_task_state=None)
         self._record_action_start(context, instance, instance_actions.EVACUATE)
 
         # NOTE(danms): Create this as a tombstone for the source compute

nova/tests/functional/integrated_helpers.py

@@ -94,6 +94,12 @@ def router(self):
             return rpc.ClientRouter(default_client)
 
 
+# placeholder used as a default parameter value to distinguish between the case
+# when the parameter is specified by the caller with None from the case when it
+# was not specified
+NOT_SPECIFIED = object()
+
+
 class InstanceHelperMixin:
 
     def _wait_for_server_parameter(
@@ -493,12 +499,42 @@ def _unshelve_server(self, server, expected_state='ACTIVE'):
         self.api.post_server_action(server['id'], {'unshelve': {}})
         return self._wait_for_state_change(server, expected_state)
 
-    def _evacuate_server(self, server, host, expected_state='ACTIVE'):
+    def _evacuate_server(
+            self, server, extra_post_args=None, expected_host=None,
+            expected_state='ACTIVE', expected_task_state=NOT_SPECIFIED,
+            expected_migration_status='done'):
         """Evacuate a server."""
-        self.api.post_server_action(server['id'], {'evacuate': {}})
-        self._wait_for_server_parameter(
-            self.server, {'OS-EXT-SRV-ATTR:host': host,
-                          'status': expected_state})
+        api = getattr(self, 'admin_api', self.api)
+
+        post = {'evacuate': {}}
+        if extra_post_args:
+            post['evacuate'].update(extra_post_args)
+
+        expected_result = {'status': expected_state}
+        if expected_host:
+            expected_result['OS-EXT-SRV-ATTR:host'] = expected_host
+        if expected_task_state is not NOT_SPECIFIED:
+            expected_result['OS-EXT-STS:task_state'] = expected_task_state
+
+        api.post_server_action(server['id'], post)
+
+        # NOTE(gibi): The order of waiting for the migration and returning
+        # a fresh server from _wait_for_server_parameter is important as
+        # the compute manager sets status of the instance before sets the
+        # host and finally sets the migration status. So waiting for the
+        # migration first makes the returned server object more consistent.
+        self._wait_for_migration_status(server, [expected_migration_status])
+        return self._wait_for_server_parameter(server, expected_result)
+
+    def _start_server(self, server):
+        self.api.post_server_action(server['id'], {'os-start': None})
+        return self._wait_for_state_change(server, 'ACTIVE')
+
+    def _stop_server(self, server, wait_for_stop=True):
+        self.api.post_server_action(server['id'], {'os-stop': None})
+        if wait_for_stop:
+            return self._wait_for_state_change(server, 'SHUTOFF')
+        return server
 
 
 class PlacementHelperMixin:

nova/tests/functional/regressions/test_bug_1978983.py

@@ -0,0 +1,71 @@
+# Copyright 2022 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from nova import test
+from nova.tests import fixtures as nova_fixtures
+from nova.tests.functional import fixtures as func_fixtures
+from nova.tests.functional import integrated_helpers
+
+
+class EvacuateServerWithTaskState(
+    test.TestCase, integrated_helpers.InstanceHelperMixin,
+):
+    """Regression test for bug 1978983
+    If instance task state is powering-off or not None
+    instance should be allowed to evacuate.
+    """
+
+    def setUp(self):
+        super().setUp()
+        # Stub out external dependencies.
+        self.useFixture(nova_fixtures.NeutronFixture(self))
+        self.useFixture(nova_fixtures.GlanceFixture(self))
+        self.useFixture(func_fixtures.PlacementFixture())
+        self.useFixture(nova_fixtures.HostNameWeigherFixture())
+
+        # Start nova controller services.
+        self.start_service('conductor')
+        self.start_service('scheduler')
+
+        api_fixture = self.useFixture(nova_fixtures.OSAPIFixture(
+            api_version='v2.1'))
+        self.api = api_fixture.admin_api
+        self.api.microversion = 'latest'
+
+        self.src = self._start_compute(host='host1')
+        self.dest = self._start_compute(host='host2')
+
+    def test_evacuate_instance(self):
+        """Evacuating a server
+        """
+        server = self._create_server(networks=[])
+
+        server = self._wait_for_state_change(server, 'ACTIVE')
+        self.assertEqual(self.src.host, server['OS-EXT-SRV-ATTR:host'])
+
+        # stop host1 compute service
+        self.src.stop()
+        self.api.put_service_force_down(self.src.service_ref.uuid, True)
+
+        # poweroff instance
+        self._stop_server(server, wait_for_stop=False)
+        server = self._wait_for_server_parameter(
+            server, {'OS-EXT-STS:task_state': 'powering-off'})
+
+        # evacuate instance
+        server = self._evacuate_server(
+            server, expected_host=self.dest.host
+        )
+        self.assertEqual(self.dest.host, server['OS-EXT-SRV-ATTR:host'])

nova/tests/functional/test_servers.py

@@ -8181,7 +8181,8 @@ def test_evacuate_ok(self):
         arqs = self.cyborg.fake_get_arqs_for_instance(self.server['id'])
         compute_to_stop, compute_to_evacuate = self._test_evacuate(
             self.server, self.NUM_HOSTS)
-        self._evacuate_server(self.server, compute_to_evacuate.host)
+        self._evacuate_server(self.server,
+            expected_host=compute_to_evacuate.host)
         compute_to_stop.start()
         self.server = self.api.get_server(self.server['id'])
         arqs_new = self.cyborg.fake_get_arqs_for_instance(self.server['id'])

releasenotes/notes/ignore-instance-task-state-for-evacuation-e000f141d0153638.yaml

@@ -0,0 +1,11 @@
+---
+fixes:
+  - |
+    If compute service is down in source node and user try to stop
+    instance, instance gets stuck at powering-off, hence evacuation fails with
+    msg: Cannot 'evacuate' instance <instance-id> while it is in
+    task_state powering-off.
+    It is now possible for evacuation to ignore the vm task state.
+    For more details see: `bug 1978983`_
+
+    .. _`bug 1978983`: https://bugs.launchpad.net/nova/+bug/1978983