Add --sleep option for archive_deleted_rows --until-complete

Currently, when 'nova-manage db archive_deleted_rows' is run with
the --until-complete option, the process will archive rows in batches
in a tight loop, which can cause problems in busy environments where
the aggressive archiving interferes with other requests trying to write
to the database.

This adds an option for users to specify an amount of time in seconds
to sleep between batches of rows while archiving with --until-complete,
allowing the process to be throttled.

Closes-Bug: #1912579

Change-Id: I638b2fa78b81919373e607458e6f68a7983a79aa

Author: melwitt

doc/source/cli/nova-manage.rst

@@ -227,6 +227,7 @@ db archive_deleted_rows
 
     nova-manage db archive_deleted_rows [--max_rows <rows>] [--verbose]
       [--until-complete] [--before <date>] [--purge] [--all-cells] [--task-log]
+      [--sleep]
 
 Move deleted rows from production tables to shadow tables. Note that the
 corresponding rows in the ``instance_mappings``, ``request_specs`` and
@@ -240,7 +241,7 @@ stopping at 0, or use the :option:`--until-complete` option.
 
 .. versionchanged:: 24.0.0 (Xena)
 
-    Added :option:`--task-log` option.
+    Added :option:`--task-log`, :option:`--sleep` options.
 
 .. rubric:: Options
 
@@ -295,7 +296,12 @@ stopping at 0, or use the :option:`--until-complete` option.
     record data via the `/os-instance_usage_audit_log`__ API (example:
     Telemetry).
 
-.. __: https://docs.openstack.org/api-ref/compute/#server-usage-audit-log-os-instance-usage-audit-log
+    .. __: https://docs.openstack.org/api-ref/compute/#server-usage-audit-log-os-instance-usage-audit-log
+
+.. option:: --sleep
+
+    The amount of time in seconds to sleep between batches when
+    :option:`--until-complete` is used. Defaults to 0.
 
 .. rubric:: Return codes
 

nova/cmd/manage.py

@@ -26,6 +26,7 @@
 import os
 import re
 import sys
+import time
 import traceback
 from urllib import parse as urlparse
 
@@ -249,9 +250,14 @@ def version(self):
                 '``--before`` option to avoid races for those consuming '
                 '``task_log`` record data via the '
                 '``/os-instance_usage_audit_log`` API (example: Telemetry).'))
-    def archive_deleted_rows(self, max_rows=1000, verbose=False,
-                             until_complete=False, purge=False,
-                             before=None, all_cells=False, task_log=False):
+    @args('--sleep', type=int, metavar='<seconds>', dest='sleep',
+          help='The amount of time in seconds to sleep between batches when '
+               '``--until-complete`` is used. Defaults to 0.')
+    def archive_deleted_rows(
+        self, max_rows=1000, verbose=False,
+        until_complete=False, purge=False,
+        before=None, all_cells=False, task_log=False, sleep=0,
+    ):
         """Move deleted rows from production tables to shadow tables.
 
         Returns 0 if nothing was archived, 1 if some number of rows were
@@ -344,7 +350,8 @@ def sort_func(item):
                         verbose,
                         before_date,
                         cell_name,
-                        task_log)
+                        task_log,
+                        sleep)
                 except KeyboardInterrupt:
                     interrupt = True
                     break
@@ -377,8 +384,10 @@ def sort_func(item):
         # NOTE(danms): Return nonzero if we archived something
         return int(bool(table_to_rows_archived))
 
-    def _do_archive(self, table_to_rows_archived, cctxt, max_rows,
-                    until_complete, verbose, before_date, cell_name, task_log):
+    def _do_archive(
+        self, table_to_rows_archived, cctxt, max_rows,
+        until_complete, verbose, before_date, cell_name, task_log, sleep,
+    ):
         """Helper function for archiving deleted rows for a cell.
 
         This will archive deleted rows for a cell database and remove the
@@ -398,6 +407,8 @@ def _do_archive(self, table_to_rows_archived, cctxt, max_rows,
         :param cell_name: Name of the cell or None if not archiving across all
             cells
         :param task_log: Whether to archive task_log table rows
+        :param sleep: The amount of time in seconds to sleep between batches
+            when ``until_complete`` is True.
         """
         ctxt = context.get_admin_context()
         while True:
@@ -437,6 +448,8 @@ def _do_archive(self, table_to_rows_archived, cctxt, max_rows,
                 break
             if verbose:
                 sys.stdout.write('.')
+            # Optionally sleep between batches to throttle the archiving.
+            time.sleep(sleep)
         return total_rows_archived
 
     @args('--before', metavar='<before>', dest='before',

nova/tests/unit/cmd/test_manage.py

@@ -315,17 +315,20 @@ def test_archive_deleted_rows_verbose(self):
         # Tests that we get table output.
         self._test_archive_deleted_rows(verbose=True)
 
+    @mock.patch('time.sleep')
     @mock.patch.object(db, 'archive_deleted_rows')
     @mock.patch.object(objects.CellMappingList, 'get_all')
     def test_archive_deleted_rows_until_complete(self, mock_get_all,
-                                                 mock_db_archive,
-                                                 verbose=False):
+                                                 mock_db_archive, mock_sleep,
+                                                 verbose=False,
+                                                 sleep=0):
         mock_db_archive.side_effect = [
             ({'instances': 10, 'instance_extra': 5}, list(), 15),
             ({'instances': 5, 'instance_faults': 1}, list(), 6),
             ({}, list(), 0)]
         result = self.commands.archive_deleted_rows(20, verbose=verbose,
-                                                    until_complete=True)
+                                                    until_complete=True,
+                                                    sleep=sleep)
         self.assertEqual(1, result)
         if verbose:
             expected = """\
@@ -353,10 +356,15 @@ def test_archive_deleted_rows_until_complete(self, mock_get_all,
                 test.MatchType(context.RequestContext), 20, before=None,
                 task_log=False),
         ])
+        self.assertEqual(2, mock_sleep.call_count)
+        mock_sleep.assert_has_calls([mock.call(sleep), mock.call(sleep)])
 
     def test_archive_deleted_rows_until_complete_quiet(self):
         self.test_archive_deleted_rows_until_complete(verbose=False)
 
+    def test_archive_deleted_rows_until_complete_sleep(self):
+        self.test_archive_deleted_rows_until_complete(sleep=30)
+
     @mock.patch('nova.db.main.api.purge_shadow_tables')
     @mock.patch.object(db, 'archive_deleted_rows')
     @mock.patch.object(objects.CellMappingList, 'get_all')

releasenotes/notes/archive-sleep-a0cc3d3e7784e5df.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    A ``--sleep`` option has been added to the ``nova-manage db
+    archive_deleted_rows`` CLI. When this command is run with the
+    ``--until-complete`` option, the process will archive rows in batches
+    in a tight loop, which can cause problems in busy environments where
+    the aggressive archiving interferes with other requests trying to write
+    to the database. The ``--sleep`` option can be used to specify a time to
+    sleep between batches of rows while archiving with ``--until-complete``,
+    allowing the process to be throttled.