fix: retry observation reporting task (#17296)

* fix: retry observation reporting task

After seeing the occasional error raised from the upstream service,
retry a few times

Signed-off-by: Mike Fiedler <[email protected]>

* narrow exceptions caught to `requests`-related

Signed-off-by: Mike Fiedler <[email protected]>

---------

Signed-off-by: Mike Fiedler <[email protected]>

Author: miketheman

warehouse/observations/tasks.py

@@ -17,6 +17,8 @@
 from base64 import b64encode
 from textwrap import dedent
 
+from requests.exceptions import RequestException
+
 from warehouse import db, tasks
 from warehouse.helpdesk.interfaces import IHelpDeskService
 
@@ -51,7 +53,13 @@ def execute_observation_report(config: Configurator, session: SA_Session):
         config.task(report_observation_to_helpscout).delay(obj.id)
 
 
-@tasks.task(bind=True, ignore_result=True, acks_late=True)
+@tasks.task(
+    bind=True,
+    ignore_result=True,
+    acks_late=True,
+    autoretry_for=(RequestException,),
+    retry_backoff=True,
+)
 def report_observation_to_helpscout(task, request: Request, model_id: UUID) -> None:
     """
     Report an Observation to HelpScout for further tracking.