Allow update cluster status with admin context

Trust token can be deleted outside of magnum,
But when trust token not found, the periodic update status job will
stay in inprogress unless another cluster action triggered.

Propose to use admin context when trust can not be found in periodic
update status job.

Story: 2010232
Task: 46031

Change-Id: I9cc9a0e654fb26ebec517e3413a592ac6613777c
(cherry picked from commit 1ed78a4)

Author: ricolin

magnum/drivers/common/driver.py

@@ -141,7 +141,7 @@ def get_driver_for_cluster(cls, context, cluster):
             context, cluster.cluster_template_id)
         return cls.get_driver(ct.server_type, ct.cluster_distro, ct.coe)
 
-    def update_cluster_status(self, context, cluster):
+    def update_cluster_status(self, context, cluster, use_admin_ctx=False):
         """Update the cluster status based on underlying orchestration
 
            This is an optional method if your implementation does not need

magnum/drivers/heat/driver.py

@@ -120,13 +120,16 @@ def delete_nodegroup(self, context, cluster, nodegroup):
             osc = clients.OpenStackClients(context)
             self._delete_stack(context, osc, nodegroup.stack_id)
 
-    def update_cluster_status(self, context, cluster):
+    def update_cluster_status(self, context, cluster, use_admin_ctx=False):
         if cluster.stack_id is None:
             # NOTE(mgoddard): During cluster creation it is possible to poll
             # the cluster before its heat stack has been created. See bug
             # 1682058.
             return
-        stack_ctx = mag_ctx.make_cluster_context(cluster)
+        if use_admin_ctx:
+            stack_ctx = context
+        else:
+            stack_ctx = mag_ctx.make_cluster_context(cluster)
         poller = HeatPoller(clients.OpenStackClients(stack_ctx), context,
                             cluster, self)
         poller.poll_and_check()

magnum/service/periodic.py

@@ -23,6 +23,7 @@
 from pycadf import cadftaxonomy as taxonomy
 
 from magnum.common import context
+from magnum.common import exception
 from magnum.common import profiler
 from magnum.common import rpc
 from magnum.conductor import monitors
@@ -68,7 +69,19 @@ def update_status(self):
         # get the driver for the cluster
         cdriver = driver.Driver.get_driver_for_cluster(self.ctx, self.cluster)
         # ask the driver to sync status
-        cdriver.update_cluster_status(self.ctx, self.cluster)
+        try:
+            cdriver.update_cluster_status(self.ctx, self.cluster)
+        except exception.AuthorizationFailure as e:
+            trust_ex = ("Could not find trust: %s" % self.cluster.trust_id)
+            # Try to use admin context if trust not found.
+            # This will make sure even with trust got deleted out side of
+            # Magnum, we still be able to check cluster status
+            if trust_ex in str(e):
+                cdriver.update_cluster_status(
+                    self.ctx, self.cluster, use_admin_ctx=True)
+            else:
+                raise
+
         LOG.debug("Status for cluster %s updated to %s (%s)",
                   self.cluster.id, self.cluster.status,
                   self.cluster.status_reason)

magnum/tests/unit/service/test_periodic.py

@@ -17,6 +17,7 @@
 from oslo_utils import uuidutils
 
 from magnum.common import context
+from magnum.common import exception
 from magnum.common.rpc_service import CONF
 from magnum.db.sqlalchemy import api as dbapi
 from magnum.drivers.common import driver
@@ -189,6 +190,32 @@ def _mock_update_status(context, cluster):
         self.mock_driver.update_cluster_status.side_effect = (
             _mock_update_status)
 
+    @mock.patch('magnum.drivers.common.driver.Driver.get_driver_for_cluster')
+    def test_update_status_non_trusts_error(self, mock_get_driver):
+        mock_get_driver.return_value = self.mock_driver
+        trust_ex = ("Unknown Keystone error")
+        self.mock_driver.update_cluster_status.side_effect = \
+            exception.AuthorizationFailure(client='keystone', message=trust_ex)
+        self.assertRaises(
+            exception.AuthorizationFailure,
+            periodic.ClusterUpdateJob(
+                self.context, self.cluster1).update_status
+        )
+        self.assertEqual(1, self.mock_driver.update_cluster_status.call_count)
+
+    @mock.patch('magnum.drivers.common.driver.Driver.get_driver_for_cluster')
+    def test_update_status_trusts_not_found(self, mock_get_driver):
+        mock_get_driver.return_value = self.mock_driver
+        trust_ex = ("Could not find trust: %s" % self.cluster1.trust_id)
+        self.mock_driver.update_cluster_status.side_effect = \
+            exception.AuthorizationFailure(client='keystone', message=trust_ex)
+        self.assertRaises(
+            exception.AuthorizationFailure,
+            periodic.ClusterUpdateJob(
+                self.context, self.cluster1).update_status
+        )
+        self.assertEqual(2, self.mock_driver.update_cluster_status.call_count)
+
     @mock.patch('oslo_service.loopingcall.FixedIntervalLoopingCall',
                 new=fakes.FakeLoopingCall)
     @mock.patch('magnum.drivers.common.driver.Driver.get_driver_for_cluster')