Merge "Warn when starting services with older than N-1 computes" into stable/victoria

Author: Zuul

doc/source/contributor/ptl-guide.rst

@@ -257,6 +257,9 @@ Immediately after RC
 
     * Example: https://review.opendev.org/543580
 
+  * Bump the oldest supported compute service version
+    * https://review.opendev.org/#/c/738482/
+
 * Create the launchpad series for the next cycle
 
 * Set the development focus of the project to the new cycle series

nova/api/openstack/wsgi_app.py

@@ -23,6 +23,7 @@
 from nova import exception
 from nova import objects
 from nova import service
+from nova import utils
 
 CONF = cfg.CONF
 
@@ -40,6 +41,11 @@ def _get_config_files(env=None):
 
 
 def _setup_service(host, name):
+    try:
+        utils.raise_if_old_compute()
+    except exception.TooOldComputeService as e:
+        logging.getLogger(__name__).warning(str(e))
+
     binary = name if name.startswith('nova-') else "nova-%s" % name
 
     ctxt = context.get_admin_context()

nova/exception.py

@@ -559,6 +559,14 @@ class ServiceTooOld(Invalid):
                 "Unable to continue.")
 
 
+class TooOldComputeService(Invalid):
+    msg_fmt = _("Current Nova version does not support computes older than "
+                "%(oldest_supported_version)s but the minimum compute service "
+                "level in your %(scope)s is %(min_service_level)d and the "
+                "oldest supported service level is "
+                "%(oldest_supported_service)d.")
+
+
 class DestinationDiskExists(Invalid):
     msg_fmt = _("The supplied disk path (%(path)s) already exists, "
                 "it is expected not to exist.")

nova/objects/service.py

@@ -192,6 +192,13 @@
     {'compute_rpc': '5.12'},
 )
 
+# This is used to raise an error at service startup if older than N-1 computes
+# are detected. Update this at the beginning of every release cycle
+OLDEST_SUPPORTED_SERVICE_VERSION = 'Ussuri'
+SERVICE_VERSION_ALIASES = {
+    'Ussuri': 41
+}
+
 
 # TODO(berrange): Remove NovaObjectDictCompat
 @base.NovaObjectRegistry.register

nova/service.py

@@ -255,6 +255,17 @@ def create(cls, host=None, binary=None, topic=None, manager=None,
                           periodic_fuzzy_delay=periodic_fuzzy_delay,
                           periodic_interval_max=periodic_interval_max)
 
+        # NOTE(gibi): This have to be after the service object creation as
+        # that is the point where we can safely use the RPC to the conductor.
+        # E.g. the Service.__init__ actually waits for the conductor to start
+        # up before it allows the service to be created. The
+        # raise_if_old_compute() depends on the RPC to be up and does not
+        # implement its own retry mechanism to connect to the conductor.
+        try:
+            utils.raise_if_old_compute()
+        except exception.TooOldComputeService as e:
+            LOG.warning(str(e))
+
         return service_obj
 
     def kill(self):

nova/tests/functional/test_service.py

@@ -10,7 +10,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from unittest import mock
+
 from nova import context as nova_context
+from nova.objects import service
 from nova import test
 from nova.tests import fixtures as nova_fixtures
 from nova.tests.functional import fixtures as func_fixtures
@@ -97,3 +100,40 @@ def test_service_start_resets_cell_cache(self):
         self.metadata.start()
         # Cell cache should be empty after the service reset.
         self.assertEqual({}, nova_context.CELL_CACHE)
+
+
+class TestOldComputeCheck(
+        test.TestCase, integrated_helpers.InstanceHelperMixin):
+
+    def test_conductor_warns_if_old_compute(self):
+        old_version = service.SERVICE_VERSION_ALIASES[
+            service.OLDEST_SUPPORTED_SERVICE_VERSION] - 1
+        with mock.patch(
+                "nova.objects.service.get_minimum_version_all_cells",
+                return_value=old_version):
+            self.start_service('conductor')
+            self.assertIn(
+                'Current Nova version does not support computes older than',
+                self.stdlog.logger.output)
+
+    def test_api_warns_if_old_compute(self):
+        old_version = service.SERVICE_VERSION_ALIASES[
+            service.OLDEST_SUPPORTED_SERVICE_VERSION] - 1
+        with mock.patch(
+                "nova.objects.service.get_minimum_version_all_cells",
+                return_value=old_version):
+            self.useFixture(nova_fixtures.OSAPIFixture(api_version='v2.1'))
+            self.assertIn(
+                'Current Nova version does not support computes older than',
+                self.stdlog.logger.output)
+
+    def test_compute_warns_if_old_compute(self):
+        old_version = service.SERVICE_VERSION_ALIASES[
+            service.OLDEST_SUPPORTED_SERVICE_VERSION] - 1
+        with mock.patch(
+                "nova.objects.service.get_minimum_version_all_cells",
+                return_value=old_version):
+            self._start_compute('host1')
+            self.assertIn(
+                'Current Nova version does not support computes older than',
+                self.stdlog.logger.output)

nova/tests/unit/api/openstack/test_requestlog.py

@@ -42,7 +42,8 @@ def setUp(self):
         # this is the minimal set of magic mocks needed to convince
         # the API service it can start on it's own without a database.
         mocks = ['nova.objects.Service.get_by_host_and_binary',
-                 'nova.objects.Service.create']
+                 'nova.objects.Service.create',
+                 'nova.utils.raise_if_old_compute']
         self.stdlog = fixtures.StandardLogging()
         self.useFixture(self.stdlog)
         for m in mocks:

nova/tests/unit/test_fixtures.py

@@ -90,6 +90,7 @@ def test_debug_logging(self):
 class TestOSAPIFixture(testtools.TestCase):
     @mock.patch('nova.objects.Service.get_by_host_and_binary')
     @mock.patch('nova.objects.Service.create')
+    @mock.patch('nova.utils.raise_if_old_compute', new=mock.Mock())
     def test_responds_to_version(self, mock_service_create, mock_get):
         """Ensure the OSAPI server responds to calls sensibly."""
         self.useFixture(output.CaptureOutput())

nova/tests/unit/test_service.py

@@ -268,6 +268,24 @@ def test_reset(self):
             serv.reset()
             mock_reset.assert_called_once_with()
 
+    @mock.patch('nova.conductor.api.API.wait_until_ready')
+    @mock.patch('nova.utils.raise_if_old_compute')
+    def test_old_compute_version_check_happens_after_wait_for_conductor(
+            self, mock_check_old, mock_wait):
+        obj_base.NovaObject.indirection_api = mock.MagicMock()
+
+        def fake_wait(*args, **kwargs):
+            mock_check_old.assert_not_called()
+
+        mock_wait.side_effect = fake_wait
+
+        service.Service.create(
+            self.host, self.binary, self.topic,
+            'nova.tests.unit.test_service.FakeManager')
+
+        mock_check_old.assert_called_once_with()
+        mock_wait.assert_called_once_with(mock.ANY)
+
 
 class TestWSGIService(test.NoDBTestCase):
 

nova/tests/unit/test_utils.py

@@ -37,6 +37,7 @@
 from nova import exception
 from nova.objects import base as obj_base
 from nova.objects import instance as instance_obj
+from nova.objects import service as service_obj
 from nova import test
 from nova.tests import fixtures as nova_fixtures
 from nova.tests.unit.objects import test_objects
@@ -1207,3 +1208,89 @@ def test_get_sdk_adapter_conf_group_fail(self):
         self.mock_get_confgrp.assert_called_once_with(self.service_type)
         self.mock_connection.assert_not_called()
         self.mock_get_auth_sess.assert_not_called()
+
+
+class TestOldComputeCheck(test.NoDBTestCase):
+
+    @mock.patch('nova.objects.service.get_minimum_version_all_cells')
+    def test_no_compute(self, mock_get_min_service):
+        mock_get_min_service.return_value = 0
+
+        utils.raise_if_old_compute()
+
+        mock_get_min_service.assert_called_once_with(
+            mock.ANY, ['nova-compute'])
+
+    @mock.patch('nova.objects.service.get_minimum_version_all_cells')
+    def test_old_but_supported_compute(self, mock_get_min_service):
+        oldest = service_obj.SERVICE_VERSION_ALIASES[
+            service_obj.OLDEST_SUPPORTED_SERVICE_VERSION]
+        mock_get_min_service.return_value = oldest
+
+        utils.raise_if_old_compute()
+
+        mock_get_min_service.assert_called_once_with(
+            mock.ANY, ['nova-compute'])
+
+    @mock.patch('nova.objects.service.get_minimum_version_all_cells')
+    def test_new_compute(self, mock_get_min_service):
+        mock_get_min_service.return_value = service_obj.SERVICE_VERSION
+
+        utils.raise_if_old_compute()
+
+        mock_get_min_service.assert_called_once_with(
+            mock.ANY, ['nova-compute'])
+
+    @mock.patch('nova.objects.service.Service.get_minimum_version')
+    def test_too_old_compute_cell(self, mock_get_min_service):
+        self.flags(group='api_database', connection=None)
+        oldest = service_obj.SERVICE_VERSION_ALIASES[
+            service_obj.OLDEST_SUPPORTED_SERVICE_VERSION]
+        mock_get_min_service.return_value = oldest - 1
+
+        ex = self.assertRaises(
+            exception.TooOldComputeService, utils.raise_if_old_compute)
+
+        self.assertIn('cell', str(ex))
+        mock_get_min_service.assert_called_once_with(mock.ANY, 'nova-compute')
+
+    @mock.patch('nova.objects.service.get_minimum_version_all_cells')
+    def test_too_old_compute_top_level(self, mock_get_min_service):
+        self.flags(group='api_database', connection='fake db connection')
+        oldest = service_obj.SERVICE_VERSION_ALIASES[
+            service_obj.OLDEST_SUPPORTED_SERVICE_VERSION]
+        mock_get_min_service.return_value = oldest - 1
+
+        ex = self.assertRaises(
+            exception.TooOldComputeService, utils.raise_if_old_compute)
+
+        self.assertIn('system', str(ex))
+        mock_get_min_service.assert_called_once_with(
+            mock.ANY, ['nova-compute'])
+
+    @mock.patch.object(utils.LOG, 'warning')
+    @mock.patch('nova.objects.service.Service.get_minimum_version')
+    @mock.patch('nova.objects.service.get_minimum_version_all_cells')
+    def test_api_db_is_configured_but_the_service_cannot_access_db(
+            self, mock_get_all, mock_get, mock_warn):
+        # This is the case when the nova-compute service is wrongly configured
+        # with db credentials but nova-compute is never allowed to access the
+        # db directly.
+        mock_get_all.side_effect = exception.DBNotAllowed(
+            binary='nova-compute')
+
+        oldest = service_obj.SERVICE_VERSION_ALIASES[
+            service_obj.OLDEST_SUPPORTED_SERVICE_VERSION]
+        mock_get.return_value = oldest - 1
+
+        ex = self.assertRaises(
+            exception.TooOldComputeService, utils.raise_if_old_compute)
+
+        self.assertIn('cell', str(ex))
+        mock_get_all.assert_called_once_with(mock.ANY, ['nova-compute'])
+        mock_get.assert_called_once_with(mock.ANY, 'nova-compute')
+        mock_warn.assert_called_once_with(
+            'This service is configured for access to the API database but is '
+            'not allowed to directly access the database. You should run this '
+            'service without the [api_database]/connection config option. The '
+            'service version check will only query the local cell.')