Fix webhook mock expectations for retry in tests

Author: tomach

crate/operator/webhooks.py

@@ -30,13 +30,18 @@
 from aiohttp import ClientError, ClientResponseError, TCPConnector
 
 from crate.operator import __version__
+from crate.operator.config import config
 from crate.operator.utils.crd import has_compute_changed
 
 RETRYABLE_STATUS_CODES = {502, 503, 504}
 RETRY_MAX_ATTEMPTS = 5
 RETRY_BASE_DELAY = 0.5
 
 
+def should_retry(retry: bool) -> bool:
+    return retry and not config.TESTING
+
+
 class WebhookEvent(str, enum.Enum):
     SCALE = "scale"
     UPGRADE = "upgrade"
@@ -306,6 +311,8 @@ async def _send(
             )
             return None
 
+        retry = should_retry(retry)
+
         payload = WebhookPayload(
             event=event,
             status=status,

tests/test_change_compute.py

@@ -274,6 +274,7 @@ async def test_change_compute_from_request_to_limit(
         name,
         compute_changed_data=mock.ANY,
         unsafe=mock.ANY,
+        retry=mock.ANY,
         logger=mock.ANY,
     )
     assert await was_notification_sent(

tests/test_cratedb.py

@@ -232,6 +232,7 @@ async def test_cratedb_health_ping(
             WebhookClusterHealthPayload(status="GREEN"),
             WebhookStatus.SUCCESS,
             mock.ANY,
+            retry=False,
         ),
         err_msg="Failed to ping cluster.",
         timeout=DEFAULT_TIMEOUT * 5,

tests/test_expand_volume.py

@@ -87,6 +87,7 @@ async def test_expand_cluster_storage(
         name,
         feedback_data=mock.ANY,
         unsafe=mock.ANY,
+        retry=mock.ANY,
         logger=mock.ANY,
     )
 

tests/test_restore_backup.py

@@ -221,6 +221,7 @@ async def test_restore_backup_aws(
                 "action": WebhookAction.RESTORE_SNAPSHOT.value,
             },
             unsafe=mock.ANY,
+            retry=mock.ANY,
             logger=mock.ANY,
         ),
         err_msg="In progress notification has not been sent.",
@@ -369,6 +370,7 @@ async def test_restore_backup_aws(
                 "action": WebhookAction.RESTORE_SNAPSHOT.value,
             },
             unsafe=mock.ANY,
+            retry=mock.ANY,
             logger=mock.ANY,
         ),
         err_msg="Success notification has not been sent.",
@@ -510,6 +512,7 @@ async def test_restore_backup_azure_blob(
                 "action": WebhookAction.RESTORE_SNAPSHOT.value,
             },
             unsafe=mock.ANY,
+            retry=mock.ANY,
             logger=mock.ANY,
         ),
         err_msg="In progress notification has not been sent.",
@@ -655,6 +658,7 @@ async def test_restore_backup_azure_blob(
                 "action": WebhookAction.RESTORE_SNAPSHOT.value,
             },
             unsafe=mock.ANY,
+            retry=mock.ANY,
             logger=mock.ANY,
         ),
         err_msg="Success notification has not been sent.",
@@ -770,6 +774,7 @@ async def test_restore_backup_create_repo_fails(
                 "action": WebhookAction.RESTORE_SNAPSHOT.value,
             },
             unsafe=mock.ANY,
+            retry=mock.ANY,
             logger=mock.ANY,
         ),
         err_msg="In progress notification has not been sent.",
@@ -792,6 +797,7 @@ async def test_restore_backup_create_repo_fails(
                 "action": WebhookAction.RESTORE_SNAPSHOT.value,
             },
             unsafe=mock.ANY,
+            retry=mock.ANY,
             logger=mock.ANY,
         ),
         err_msg="Exception notification has not been sent.",

tests/test_scale.py

@@ -277,6 +277,7 @@ async def test_scale_cluster(
         name,
         scale_data=mock.ANY,
         unsafe=mock.ANY,
+        retry=mock.ANY,
         logger=mock.ANY,
     )
     assert await was_notification_sent(

tests/test_update_backups_schedule.py

@@ -126,6 +126,7 @@ async def test_update_backups_schedule(
         name,
         backup_schedule_data=mock.ANY,
         unsafe=mock.ANY,
+        retry=mock.ANY,
         logger=mock.ANY,
     )
 

tests/test_upgrade.py

@@ -185,6 +185,7 @@ async def test_upgrade_cluster(
         name,
         upgrade_data=mock.ANY,
         unsafe=mock.ANY,
+        retry=mock.ANY,
         logger=mock.ANY,
     )
     assert await was_notification_sent(
@@ -426,6 +427,7 @@ async def test_upgrade_rollback_on_failure(
             "action": WebhookAction.UPGRADE.value,
         },
         unsafe=mock.ANY,
+        retry=mock.ANY,
         logger=mock.ANY,
     )
     assert await was_notification_sent(

tests/test_webhooks.py

@@ -21,6 +21,7 @@
 
 import json
 import logging
+from unittest import mock
 
 import pytest
 from aiohttp import web
@@ -430,16 +431,20 @@ async def test_retries_then_success(self):
             "secr3t password",
         )
 
-        response = await client.send_notification(
-            "my-namespace",
-            "my-cluster",
-            WebhookEvent.UPGRADE,
-            WebhookUpgradePayload(
-                old_registry="a", new_registry="b", old_version="c", new_version="d"
-            ),
-            WebhookStatus.SUCCESS,
-            logging.getLogger(__name__),
-        )
+        with (
+            mock.patch("crate.operator.config.config.TESTING", False),
+            mock.patch("crate.operator.webhooks.RETRY_BASE_DELAY", 0.001),
+        ):
+            response = await client.send_notification(
+                "my-namespace",
+                "my-cluster",
+                WebhookEvent.UPGRADE,
+                WebhookUpgradePayload(
+                    old_registry="a", new_registry="b", old_version="c", new_version="d"
+                ),
+                WebhookStatus.SUCCESS,
+                logging.getLogger(__name__),
+            )
 
         assert response.status == 200
         assert self.app["calls"] == 3
@@ -473,16 +478,20 @@ async def test_retry_exhausted(self):
             "secr3t password",
         )
 
-        response = await client.send_notification(
-            "my-namespace",
-            "my-cluster",
-            WebhookEvent.UPGRADE,
-            WebhookUpgradePayload(
-                old_registry="a", new_registry="b", old_version="c", new_version="d"
-            ),
-            WebhookStatus.SUCCESS,
-            logging.getLogger(__name__),
-        )
+        with (
+            mock.patch("crate.operator.config.config.TESTING", False),
+            mock.patch("crate.operator.webhooks.RETRY_BASE_DELAY", 0.001),
+        ):
+            response = await client.send_notification(
+                "my-namespace",
+                "my-cluster",
+                WebhookEvent.UPGRADE,
+                WebhookUpgradePayload(
+                    old_registry="a", new_registry="b", old_version="c", new_version="d"
+                ),
+                WebhookStatus.SUCCESS,
+                logging.getLogger(__name__),
+            )
 
         assert response.status == 502
         assert self.app["calls"] == RETRY_MAX_ATTEMPTS