Merge "Limit the number of concurrent snapshots"

Author: Zuul

nova/compute/manager.py

@@ -542,6 +542,11 @@ def __init__(self, compute_driver=None, *args, **kwargs):
                 CONF.max_concurrent_builds)
         else:
             self._build_semaphore = compute_utils.UnlimitedSemaphore()
+        if CONF.max_concurrent_snapshots > 0:
+            self._snapshot_semaphore = eventlet.semaphore.Semaphore(
+                CONF.max_concurrent_snapshots)
+        else:
+            self._snapshot_semaphore = compute_utils.UnlimitedSemaphore()
         if CONF.max_concurrent_live_migrations > 0:
             self._live_migration_executor = futurist.GreenThreadPoolExecutor(
                 max_workers=CONF.max_concurrent_live_migrations)
@@ -3819,8 +3824,9 @@ def snapshot_instance(self, context, image_id, instance):
                       instance=instance)
             return
 
-        self._snapshot_instance(context, image_id, instance,
-                                task_states.IMAGE_SNAPSHOT)
+        with self._snapshot_semaphore:
+            self._snapshot_instance(context, image_id, instance,
+                                    task_states.IMAGE_SNAPSHOT)
 
     def _snapshot_instance(self, context, image_id, instance,
                            expected_task_state):

nova/conf/compute.py

@@ -661,6 +661,20 @@
 
 * 0 : treated as unlimited.
 * Any positive integer representing maximum concurrent builds.
+"""),
+    cfg.IntOpt('max_concurrent_snapshots',
+        default=5,
+        min=0,
+        help="""
+Maximum number of instance snapshot operations to run concurrently.
+This limit is enforced to prevent snapshots overwhelming the
+host/network/storage and causing failure. This value can be set per
+compute node.
+
+Possible Values:
+
+* 0 : treated as unlimited.
+* Any positive integer representing maximum concurrent snapshots.
 """),
     cfg.IntOpt('max_concurrent_live_migrations',
         default=1,

nova/tests/unit/compute/test_compute_mgr.py

@@ -748,6 +748,40 @@ def test_max_concurrent_builds_semaphore_unlimited(self):
         self.assertIsInstance(compute._build_semaphore,
                               compute_utils.UnlimitedSemaphore)
 
+    @mock.patch('nova.objects.Instance.save')
+    @mock.patch('nova.compute.manager.ComputeManager.'
+                '_snapshot_instance')
+    def _test_max_concurrent_snapshots(self, mock_si, mock_inst_save):
+
+        with mock.patch.object(self.compute,
+                               '_snapshot_semaphore') as mock_sem:
+            instance = objects.Instance(uuid=uuidutils.generate_uuid())
+            for i in (1, 2, 3):
+                self.compute.snapshot_instance(self.context,
+                                               mock.sentinel.image,
+                                               instance)
+            self.assertEqual(3, mock_sem.__enter__.call_count)
+
+    def test_max_concurrent_snapshots_limited(self):
+        self.flags(max_concurrent_snapshots=2)
+        self._test_max_concurrent_snapshots()
+
+    def test_max_concurrent_snapshots_unlimited(self):
+        self.flags(max_concurrent_snapshots=0)
+        self._test_max_concurrent_snapshots()
+
+    def test_max_concurrent_snapshots_semaphore_limited(self):
+        self.flags(max_concurrent_snapshots=123)
+        self.assertEqual(123,
+                         manager.ComputeManager()._snapshot_semaphore.balance)
+
+    def test_max_concurrent_snapshots_semaphore_unlimited(self):
+        self.flags(max_concurrent_snapshots=0)
+        compute = manager.ComputeManager()
+        self.assertEqual(0, compute._snapshot_semaphore.balance)
+        self.assertIsInstance(compute._snapshot_semaphore,
+                              compute_utils.UnlimitedSemaphore)
+
     def test_nil_out_inst_obj_host_and_node_sets_nil(self):
         instance = fake_instance.fake_instance_obj(self.context,
                                                    uuid=uuids.instance,

releasenotes/notes/max-concurrent-snapshots-21a0a437dbe1044a.yaml

@@ -0,0 +1,12 @@
+---
+features:
+  - |
+    A new configuration option, ``[DEFAULT]/max_concurrent_snapshots``,
+    has been added. This allow operator to configure maximum concurrent
+    snapshots on a compute host and prevent resource overuse related
+    to snapshot.
+upgrade:
+  - |
+    Previously, the number of concurrent snapshots was unlimited, now it is
+    limited via ``[DEFAULT]/max_concurrent_snapshots``, which currently
+    defaults to 5.