optionally increasing check_num_items retry count

Change-Id: Id94f13db33735be535cadeb4d53ea1a6b433aa16
Reviewed-on: http://review.couchbase.org/c/perfrunner/+/152569
Tested-by: Build Bot <[email protected]>
Reviewed-by: Bo-Chun Wang <[email protected]>
Reviewed-by: Korrigan Clark <[email protected]>

Author: korry8911

perfrunner/helpers/monitor.py

@@ -973,10 +973,12 @@ def _wait_for_replica_count_match(self, host, bucket):
             if time.time() - start_time > self.TIMEOUT:
                 raise Exception('Replica items monitoring got stuck')
 
-    def monitor_num_items(self, host: str, bucket: str, num_items: int):
+    def monitor_num_items(self, host: str, bucket: str, num_items: int, max_retry: int=None):
         logger.info('Checking the number of items in {}'.format(bucket))
+        if not max_retry:
+            max_retry = self.MAX_RETRY
         retries = 0
-        while retries < self.MAX_RETRY:
+        while retries < max_retry:
             curr_items = self._get_num_items(host, bucket, total=True)
             if curr_items == num_items:
                 break

perfrunner/tests/__init__.py

@@ -250,7 +250,7 @@ def wait_for_indexing(self):
             for server in self.index_nodes:
                 self.monitor.monitor_indexing(server)
 
-    def check_num_items(self, bucket_items: dict = None):
+    def check_num_items(self, bucket_items: dict = None, max_retry: int=None):
         if bucket_items:
             for target in self.target_iterator:
                 num_items = bucket_items.get(target.bucket, None)
@@ -259,7 +259,8 @@ def check_num_items(self, bucket_items: dict = None):
                     self.monitor.monitor_num_items(
                         target.node,
                         target.bucket,
-                        num_items
+                        num_items,
+                        max_retry=max_retry
                     )
         elif getattr(self.test_config.load_settings, 'collections', None):
             for target in self.target_iterator:
@@ -274,14 +275,14 @@ def check_num_items(self, bucket_items: dict = None):
                     num_load_targets * \
                     (1 + self.test_config.bucket.replica_number)
                 self.monitor.monitor_num_items(target.node, target.bucket,
-                                               num_items)
+                                               num_items, max_retry=max_retry)
         else:
             num_items = self.test_config.load_settings.items * (
                 1 + self.test_config.bucket.replica_number
             )
             for target in self.target_iterator:
                 self.monitor.monitor_num_items(target.node, target.bucket,
-                                               num_items)
+                                               num_items, max_retry=max_retry)
 
     def reset_kv_stats(self):
         master_node = next(self.cluster_spec.masters)

perfrunner/tests/operator.py

@@ -71,7 +71,7 @@ class OperatorBackupRestoreTest(OperatorBackupTest):
     def restore(self):
         logger.info('Running restore')
         self.remote.create_restore()
-        self.check_num_items()
+        self.check_num_items(max_retry=3600)
 
     def flush_buckets(self):
         logger.info('Flushing bucket')