feat(tableau): implement exponential backoff for retry logic on InternalServerError (#15828)

Author: NehaGslab

metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py

@@ -1479,6 +1479,16 @@ def get_connection_object_page(
                 # will be thrown, and we need to re-authenticate and retry.
                 self._re_authenticate()
 
+            # Add exponential backoff to avoid hammering the server
+            backoff_time = min(
+                (self.config.max_retries - retries_remaining + 1) ** 2, 60
+            )
+            logger.info(
+                f"Retrying query due to {e.__class__.__name__} with {retries_remaining} retries remaining, "
+                f"will retry in {backoff_time} seconds"
+            )
+            time.sleep(backoff_time)
+
             return self.get_connection_object_page(
                 query=query,
                 connection_type=connection_type,
@@ -1496,7 +1506,17 @@ def get_connection_object_page(
             if ise.code in RETRIABLE_ERROR_CODES:
                 if retries_remaining <= 0:
                     raise ise
-                logger.info(f"Retrying query due to error {ise.code}")
+
+                # Add exponential backoff to avoid hammering the server
+                backoff_time = min(
+                    (self.config.max_retries - retries_remaining + 1) ** 2, 60
+                )
+                logger.info(
+                    f"Retrying query due to error {ise.code} with {retries_remaining} retries remaining, "
+                    f"will retry in {backoff_time} seconds"
+                )
+                time.sleep(backoff_time)
+
                 return self.get_connection_object_page(
                     query=query,
                     connection_type=connection_type,
@@ -1519,6 +1539,17 @@ def get_connection_object_page(
             # retry logic is basically a bandaid for that.
             if retries_remaining <= 0:
                 raise
+
+            # Add exponential backoff to avoid hammering the server
+            backoff_time = min(
+                (self.config.max_retries - retries_remaining + 1) ** 2, 60
+            )
+            logger.info(
+                f"Retrying query due to OSError with {retries_remaining} retries remaining, "
+                f"will retry in {backoff_time} seconds"
+            )
+            time.sleep(backoff_time)
+
             return self.get_connection_object_page(
                 query=query,
                 connection_type=connection_type,

metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py

@@ -1148,9 +1148,11 @@ def test_retry_on_error(pytestconfig, tmp_path, mock_datahub_graph):
             mock_sdk.return_value = mock_client
 
             reporter = TableauSourceReport()
+            mock_config = mock.MagicMock()
+            mock_config.max_retries = 3  # Set max_retries for backoff calculation
             tableau_source = TableauSiteSource(
                 platform="tableau",
-                config=mock.MagicMock(),
+                config=mock_config,
                 ctx=mock.MagicMock(),
                 site=mock.MagicMock(spec=SiteItem, id="Site1", content_url="site1"),
                 server=mock_sdk.return_value,