Bundle Analysis: fix processor retry (#1081)

Author: JerrySentry

tasks/bundle_analysis_processor.py

@@ -30,6 +30,8 @@
 class BundleAnalysisProcessorTask(
     BaseCodecovTask, name=bundle_analysis_processor_task_name
 ):
+    max_retries = 5
+
     def run_impl(
         self,
         db_session,
@@ -75,7 +77,7 @@ def run_impl(
                     previous_result,
                 )
         except LockRetry as retry:
-            self.retry(max_retries=5, countdown=retry.countdown)
+            self.retry(countdown=retry.countdown)
 
     def process_impl_within_lock(
         self,
@@ -139,9 +141,22 @@ def process_impl_within_lock(
             result: ProcessingResult = report_service.process_upload(
                 commit, upload, compare_sha
             )
-            if result.error and result.error.is_retryable and self.request.retries == 0:
-                # retryable error and no retry has already be scheduled
-                self.retry(max_retries=5, countdown=20)
+            if (
+                result.error
+                and result.error.is_retryable
+                and self.request.retries < self.max_retries
+            ):
+                log.warn(
+                    "Attempting to retry bundle analysis upload",
+                    extra=dict(
+                        repoid=repoid,
+                        commit=commitid,
+                        commit_yaml=commit_yaml,
+                        params=params,
+                        result=result.as_dict(),
+                    ),
+                )
+                self.retry(countdown=30 * (2**self.request.retries))
             result.update_upload(carriedforward=carriedforward)
 
             processing_results.append(result.as_dict())

tasks/tests/unit/test_bundle_analysis_processor_task.py

@@ -176,7 +176,7 @@ def test_bundle_analysis_processor_task_error(
 
     assert commit.state == "error"
     assert upload.state == "error"
-    retry.assert_called_once_with(countdown=20, max_retries=5)
+    retry.assert_called_once_with(countdown=30)
 
 
 def test_bundle_analysis_processor_task_general_error(
@@ -362,7 +362,7 @@ def test_bundle_analysis_processor_task_locked(
     assert result is None
 
     assert upload.state == "started"
-    retry.assert_called_once_with(countdown=ANY, max_retries=5)
+    retry.assert_called_once_with(countdown=ANY)
 
 
 def test_bundle_analysis_process_upload_rate_limit_error(
@@ -431,7 +431,7 @@ def test_bundle_analysis_process_upload_rate_limit_error(
 
     assert commit.state == "error"
     assert upload.state == "error"
-    retry.assert_called_once_with(countdown=20, max_retries=5)
+    retry.assert_called_once_with(countdown=30)
 
 
 def test_bundle_analysis_process_associate_no_parent_commit_id(