Merge "Add --sleep option for archive_deleted_rows --until-complete"

Author: Zuul

doc/source/cli/nova-manage.rst

@@ -227,6 +227,7 @@ db archive_deleted_rows
 
     nova-manage db archive_deleted_rows [--max_rows <rows>] [--verbose]
       [--until-complete] [--before <date>] [--purge] [--all-cells] [--task-log]
+      [--sleep]
 
 Move deleted rows from production tables to shadow tables. Note that the
 corresponding rows in the ``instance_mappings``, ``request_specs`` and
@@ -240,7 +241,7 @@ stopping at 0, or use the :option:`--until-complete` option.
 
 .. versionchanged:: 24.0.0 (Xena)
 
-    Added :option:`--task-log` option.
+    Added :option:`--task-log`, :option:`--sleep` options.
 
 .. rubric:: Options
 
@@ -295,7 +296,12 @@ stopping at 0, or use the :option:`--until-complete` option.
     record data via the `/os-instance_usage_audit_log`__ API (example:
     Telemetry).
 
-.. __: https://docs.openstack.org/api-ref/compute/#server-usage-audit-log-os-instance-usage-audit-log
+    .. __: https://docs.openstack.org/api-ref/compute/#server-usage-audit-log-os-instance-usage-audit-log
+
+.. option:: --sleep
+
+    The amount of time in seconds to sleep between batches when
+    :option:`--until-complete` is used. Defaults to 0.
 
 .. rubric:: Return codes
 

nova/cmd/manage.py

@@ -26,6 +26,7 @@
 import os
 import re
 import sys
+import time
 import traceback
 from urllib import parse as urlparse
 
@@ -249,9 +250,14 @@ def version(self):
                 '``--before`` option to avoid races for those consuming '
                 '``task_log`` record data via the '
                 '``/os-instance_usage_audit_log`` API (example: Telemetry).'))
-    def archive_deleted_rows(self, max_rows=1000, verbose=False,
-                             until_complete=False, purge=False,
-                             before=None, all_cells=False, task_log=False):
+    @args('--sleep', type=int, metavar='<seconds>', dest='sleep',
+          help='The amount of time in seconds to sleep between batches when '
+               '``--until-complete`` is used. Defaults to 0.')
+    def archive_deleted_rows(
+        self, max_rows=1000, verbose=False,
+        until_complete=False, purge=False,
+        before=None, all_cells=False, task_log=False, sleep=0,
+    ):
         """Move deleted rows from production tables to shadow tables.
 
         Returns 0 if nothing was archived, 1 if some number of rows were
@@ -344,7 +350,8 @@ def sort_func(item):
                         verbose,
                         before_date,
                         cell_name,
-                        task_log)
+                        task_log,
+                        sleep)
                 except KeyboardInterrupt:
                     interrupt = True
                     break
@@ -377,8 +384,10 @@ def sort_func(item):
         # NOTE(danms): Return nonzero if we archived something
         return int(bool(table_to_rows_archived))
 
-    def _do_archive(self, table_to_rows_archived, cctxt, max_rows,
-                    until_complete, verbose, before_date, cell_name, task_log):
+    def _do_archive(
+        self, table_to_rows_archived, cctxt, max_rows,
+        until_complete, verbose, before_date, cell_name, task_log, sleep,
+    ):
         """Helper function for archiving deleted rows for a cell.
 
         This will archive deleted rows for a cell database and remove the
@@ -398,6 +407,8 @@ def _do_archive(self, table_to_rows_archived, cctxt, max_rows,
         :param cell_name: Name of the cell or None if not archiving across all
             cells
         :param task_log: Whether to archive task_log table rows
+        :param sleep: The amount of time in seconds to sleep between batches
+            when ``until_complete`` is True.
         """
         ctxt = context.get_admin_context()
         while True:
@@ -437,6 +448,8 @@ def _do_archive(self, table_to_rows_archived, cctxt, max_rows,
                 break
             if verbose:
                 sys.stdout.write('.')
+            # Optionally sleep between batches to throttle the archiving.
+            time.sleep(sleep)
         return total_rows_archived
 
     @args('--before', metavar='<before>', dest='before',

nova/tests/unit/cmd/test_manage.py

@@ -315,17 +315,20 @@ def test_archive_deleted_rows_verbose(self):
         # Tests that we get table output.
         self._test_archive_deleted_rows(verbose=True)
 
+    @mock.patch('time.sleep')
     @mock.patch.object(db, 'archive_deleted_rows')
     @mock.patch.object(objects.CellMappingList, 'get_all')
     def test_archive_deleted_rows_until_complete(self, mock_get_all,
-                                                 mock_db_archive,
-                                                 verbose=False):
+                                                 mock_db_archive, mock_sleep,
+                                                 verbose=False,
+                                                 sleep=0):
         mock_db_archive.side_effect = [
             ({'instances': 10, 'instance_extra': 5}, list(), 15),
             ({'instances': 5, 'instance_faults': 1}, list(), 6),
             ({}, list(), 0)]
         result = self.commands.archive_deleted_rows(20, verbose=verbose,
-                                                    until_complete=True)
+                                                    until_complete=True,
+                                                    sleep=sleep)
         self.assertEqual(1, result)
         if verbose:
             expected = """\
@@ -353,10 +356,15 @@ def test_archive_deleted_rows_until_complete(self, mock_get_all,
                 test.MatchType(context.RequestContext), 20, before=None,
                 task_log=False),
         ])
+        self.assertEqual(2, mock_sleep.call_count)
+        mock_sleep.assert_has_calls([mock.call(sleep), mock.call(sleep)])
 
     def test_archive_deleted_rows_until_complete_quiet(self):
         self.test_archive_deleted_rows_until_complete(verbose=False)
 
+    def test_archive_deleted_rows_until_complete_sleep(self):
+        self.test_archive_deleted_rows_until_complete(sleep=30)
+
     @mock.patch('nova.db.main.api.purge_shadow_tables')
     @mock.patch.object(db, 'archive_deleted_rows')
     @mock.patch.object(objects.CellMappingList, 'get_all')

releasenotes/notes/archive-sleep-a0cc3d3e7784e5df.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    A ``--sleep`` option has been added to the ``nova-manage db
+    archive_deleted_rows`` CLI. When this command is run with the
+    ``--until-complete`` option, the process will archive rows in batches
+    in a tight loop, which can cause problems in busy environments where
+    the aggressive archiving interferes with other requests trying to write
+    to the database. The ``--sleep`` option can be used to specify a time to
+    sleep between batches of rows while archiving with ``--until-complete``,
+    allowing the process to be throttled.