retry tao price shovel request on corrupted CMC data

Author: hey-michal

scraper_service/shovel_tao_price/cmc_client.py

@@ -22,12 +22,16 @@ def fetch_cmc_data(params, endpoint):
         try:
             response = requests.get(url, headers=headers, params=params, timeout=30)  # Add timeout
         except requests.Timeout:
+            logging.warning("CMC API request timed out")
             raise ShovelProcessingError("CMC API request timed out")
         except requests.ConnectionError:
+            logging.warning("Failed to connect to CMC API")
             raise ShovelProcessingError("Failed to connect to CMC API")
 
         # Handle rate limiting explicitly
         if response.status_code == 429:
+            retry_after = int(response.headers.get('Retry-After', 60))
+            logging.warning(f"CMC API rate limit exceeded, retry after {retry_after} seconds")
             raise ShovelProcessingError("CMC API rate limit exceeded")
 
         # Handle other common error codes
@@ -36,25 +40,55 @@ def fetch_cmc_data(params, endpoint):
         elif response.status_code == 403:
             raise ShovelProcessingError("CMC API access forbidden")
         elif response.status_code >= 500:
+            logging.warning(f"CMC API server error: {response.status_code}")
             raise ShovelProcessingError(f"CMC API server error: {response.status_code}")
         elif response.status_code != 200:
+            logging.warning(f"CMC API request failed with status code: {response.status_code}")
             raise ShovelProcessingError(f"CMC API request failed with status code: {response.status_code}")
 
         try:
             data = response.json()
         except ValueError:
+            logging.warning("Failed to parse CMC API response as JSON")
             raise ShovelProcessingError("Failed to parse CMC API response as JSON")
 
         # Check for API-level errors
         if 'status' in data and 'error_code' in data['status'] and data['status']['error_code'] != 0:
             error_message = data['status'].get('error_message', 'Unknown API error')
+            logging.warning(f"CMC API error: {error_message}")
             raise ShovelProcessingError(f"CMC API error: {error_message}")
 
+        if 'data' not in data:
+            logging.warning("Invalid CMC API response: missing 'data' field")
+            raise DatabaseConnectionError("Invalid CMC API response: missing 'data' field")
+        if 'quotes' not in data['data']:
+            logging.warning("Invalid CMC API response: missing 'quotes' field")
+            raise DatabaseConnectionError("Invalid CMC API response: missing 'quotes' field")
+        if not data['data']['quotes']:
+            logging.warning(f"No price data available for timestamp {params.get('time_start', 'latest')}")
+            raise DatabaseConnectionError(f"No price data available for timestamp {params.get('time_start', 'latest')}")
+
+        quote = data['data']['quotes'][0] if endpoint == 'historical' else data['data'][str(CMC_TAO_ID)]
+        if 'quote' not in quote or 'USD' not in quote['quote']:
+            logging.warning("Invalid CMC API response: missing USD quote data")
+            raise DatabaseConnectionError("Invalid CMC API response: missing USD quote data")
+
+        usd_quote = quote['quote']['USD']
+        required_fields = ['price', 'market_cap', 'volume_24h']
+        for field in required_fields:
+            if field not in usd_quote:
+                logging.warning(f"Invalid CMC API response: missing {field} field")
+                raise DatabaseConnectionError(f"Invalid CMC API response: missing {field} field")
+            if usd_quote[field] is None:
+                logging.warning(f"Invalid CMC API response: {field} is None")
+                raise DatabaseConnectionError(f"Invalid CMC API response: {field} is None")
+
         return data, response.status_code
-    except requests.exceptions.RequestException as e:
-        raise ShovelProcessingError(f"Failed to make CMC API request: {str(e)}")
+    except ShovelProcessingError:
+        raise
     except Exception as e:
-        raise ShovelProcessingError(f"Unexpected error in CMC API request: {str(e)}")
+        logging.warning(f"Unexpected error in CMC API request: {str(e)}")
+        raise DatabaseConnectionError(f"Unexpected error in CMC API request: {str(e)}")
 
 def get_price_by_time(timestamp):
     if timestamp is None or timestamp <= 0:
@@ -93,23 +127,23 @@ def get_price_by_time(timestamp):
         logging.debug(f"Got response with status code: {status_code}")
 
         if 'data' not in data:
-            raise ShovelProcessingError(f"Invalid CMC API response: missing 'data' field")
+            raise DatabaseConnectionError(f"Invalid CMC API response: missing 'data' field")
         if 'quotes' not in data['data']:
-            raise ShovelProcessingError(f"Invalid CMC API response: missing 'quotes' field")
+            raise DatabaseConnectionError(f"Invalid CMC API response: missing 'quotes' field")
         if not data['data']['quotes']:
-            raise ShovelProcessingError(f"No price data available for timestamp {timestamp}")
+            raise DatabaseConnectionError(f"No price data available for timestamp {timestamp}")
 
         quote = data['data']['quotes'][0]
         if 'quote' not in quote or 'USD' not in quote['quote']:
-            raise ShovelProcessingError(f"Invalid CMC API response: missing USD quote data")
+            raise DatabaseConnectionError(f"Invalid CMC API response: missing USD quote data")
 
         usd_quote = quote['quote']['USD']
         required_fields = ['price', 'market_cap', 'volume_24h']
         for field in required_fields:
             if field not in usd_quote:
-                raise ShovelProcessingError(f"Invalid CMC API response: missing {field} field")
+                raise DatabaseConnectionError(f"Invalid CMC API response: missing {field} field")
             if usd_quote[field] is None:
-                raise ShovelProcessingError(f"Invalid CMC API response: {field} is None")
+                raise DatabaseConnectionError(f"Invalid CMC API response: {field} is None")
 
         price = usd_quote['price']
         market_cap = usd_quote['market_cap']
@@ -125,7 +159,7 @@ def get_price_by_time(timestamp):
     except ShovelProcessingError:
         raise
     except Exception as e:
-        raise ShovelProcessingError(f"Failed to get price data: {str(e)}")
+        raise DatabaseConnectionError(f"Failed to get price data: {str(e)}")
 
 def get_latest_price():
     try:
@@ -137,23 +171,23 @@ def get_latest_price():
         data, status_code = fetch_cmc_data(parameters, 'latest')
 
         if 'data' not in data:
-            raise ShovelProcessingError(f"Invalid CMC API response: missing 'data' field")
+            raise DatabaseConnectionError(f"Invalid CMC API response: missing 'data' field")
 
         tao_id_str = str(CMC_TAO_ID)
         if tao_id_str not in data['data']:
-            raise ShovelProcessingError(f"No data available for TAO (ID: {CMC_TAO_ID})")
+            raise DatabaseConnectionError(f"No data available for TAO (ID: {CMC_TAO_ID})")
 
         tao_data = data['data'][tao_id_str]
         if 'quote' not in tao_data or 'USD' not in tao_data['quote']:
-            raise ShovelProcessingError(f"Invalid CMC API response: missing USD quote data")
+            raise DatabaseConnectionError(f"Invalid CMC API response: missing USD quote data")
 
         usd_quote = tao_data['quote']['USD']
         required_fields = ['price', 'market_cap', 'volume_24h']
         for field in required_fields:
             if field not in usd_quote:
-                raise ShovelProcessingError(f"Invalid CMC API response: missing {field} field")
+                raise DatabaseConnectionError(f"Invalid CMC API response: missing {field} field")
             if usd_quote[field] is None:
-                raise ShovelProcessingError(f"Invalid CMC API response: {field} is None")
+                raise DatabaseConnectionError(f"Invalid CMC API response: {field} is None")
 
         price = usd_quote['price']
         market_cap = usd_quote['market_cap']
@@ -168,4 +202,4 @@ def get_latest_price():
     except ShovelProcessingError:
         raise
     except Exception as e:
-        raise ShovelProcessingError(f"Failed to get latest price data: {str(e)}")
+        raise DatabaseConnectionError(f"Failed to get latest price data: {str(e)}")