Skip to content

Commit 1a01d1b

Browse files
hey-michalhey-michal
committed
restart tao_price shovel on failure
1 parent 9c9bd21 commit 1a01d1b

File tree

2 files changed

+213
-134
lines changed

2 files changed

+213
-134
lines changed

‎scraper_service/shovel_tao_price/cmc_client.py‎

Lines changed: 139 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -2,86 +2,170 @@
22
import logging
33
import os
44
from datetime import datetime, timedelta
5+
from shared.exceptions import DatabaseConnectionError, ShovelProcessingError
56

67
CMC_TAO_ID = 22974
78
CMC_TOKEN = os.getenv("CMC_TOKEN")
89
FIRST_TAO_LISTING_DAY = datetime(2023, 3, 6)
910

1011
def fetch_cmc_data(params, endpoint):
11-
url = f"https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/{endpoint}"
12-
headers = {
13-
'Accepts': 'application/json',
14-
'X-CMC_PRO_API_KEY': CMC_TOKEN
15-
}
12+
try:
13+
if not CMC_TOKEN:
14+
raise ShovelProcessingError("CMC_TOKEN is not set")
15+
16+
url = f"https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/{endpoint}"
17+
headers = {
18+
'Accepts': 'application/json',
19+
'X-CMC_PRO_API_KEY': CMC_TOKEN
20+
}
21+
22+
try:
23+
response = requests.get(url, headers=headers, params=params, timeout=30) # Add timeout
24+
except requests.Timeout:
25+
raise ShovelProcessingError("CMC API request timed out")
26+
except requests.ConnectionError:
27+
raise ShovelProcessingError("Failed to connect to CMC API")
28+
29+
# Handle rate limiting explicitly
30+
if response.status_code == 429:
31+
raise ShovelProcessingError("CMC API rate limit exceeded")
1632

17-
response = requests.get(url, headers=headers, params=params)
18-
return response.json(), response.status_code
33+
# Handle other common error codes
34+
if response.status_code == 401:
35+
raise ShovelProcessingError("Invalid CMC API key")
36+
elif response.status_code == 403:
37+
raise ShovelProcessingError("CMC API access forbidden")
38+
elif response.status_code >= 500:
39+
raise ShovelProcessingError(f"CMC API server error: {response.status_code}")
40+
elif response.status_code != 200:
41+
raise ShovelProcessingError(f"CMC API request failed with status code: {response.status_code}")
42+
43+
try:
44+
data = response.json()
45+
except ValueError:
46+
raise ShovelProcessingError("Failed to parse CMC API response as JSON")
47+
48+
# Check for API-level errors
49+
if 'status' in data and 'error_code' in data['status'] and data['status']['error_code'] != 0:
50+
error_message = data['status'].get('error_message', 'Unknown API error')
51+
raise ShovelProcessingError(f"CMC API error: {error_message}")
52+
53+
return data, response.status_code
54+
except requests.exceptions.RequestException as e:
55+
raise ShovelProcessingError(f"Failed to make CMC API request: {str(e)}")
56+
except Exception as e:
57+
raise ShovelProcessingError(f"Unexpected error in CMC API request: {str(e)}")
1958

2059
def get_price_by_time(timestamp):
21-
logging.info(f"Getting price for timestamp {timestamp}")
22-
23-
# Calculate the time 48 hours ago from now
24-
time_48_hours_ago = datetime.now() - timedelta(hours=48)
25-
logging.info(f"48 hours ago: {time_48_hours_ago}")
26-
27-
# Determine the interval based on the timestamp
28-
timestamp_dt = datetime.fromtimestamp(timestamp)
29-
logging.info(f"Timestamp as datetime: {timestamp_dt}")
30-
31-
if timestamp_dt > time_48_hours_ago:
32-
interval = '5m'
33-
logging.info("Using 5m interval (within last 48 hours)")
34-
else:
35-
interval = '24h'
36-
logging.info("Using 24h interval (older than 48 hours)")
37-
38-
parameters = {
39-
'id': CMC_TAO_ID,
40-
'convert': 'USD',
41-
'interval': interval,
42-
'time_start': timestamp,
43-
'count': 1
44-
}
45-
logging.info(f"Request parameters: {parameters}")
60+
if timestamp is None or timestamp <= 0:
61+
raise ShovelProcessingError("Invalid timestamp provided")
4662

4763
try:
48-
logging.info("Fetching data from CMC API...")
64+
# Calculate the time 48 hours ago from now
65+
time_48_hours_ago = datetime.now() - timedelta(hours=48)
66+
logging.debug(f"48 hours ago: {time_48_hours_ago}")
67+
68+
# Determine the interval based on the timestamp
69+
timestamp_dt = datetime.fromtimestamp(timestamp)
70+
logging.debug(f"Timestamp as datetime: {timestamp_dt}")
71+
72+
# Validate timestamp is not before TAO listing
73+
if timestamp_dt < FIRST_TAO_LISTING_DAY:
74+
raise ShovelProcessingError(f"Timestamp {timestamp_dt} is before TAO listing date {FIRST_TAO_LISTING_DAY}")
75+
76+
if timestamp_dt > time_48_hours_ago:
77+
interval = '5m'
78+
logging.debug("Using 5m interval (within last 48 hours)")
79+
else:
80+
interval = '24h'
81+
logging.debug("Using 24h interval (older than 48 hours)")
82+
83+
parameters = {
84+
'id': CMC_TAO_ID,
85+
'convert': 'USD',
86+
'interval': interval,
87+
'time_start': timestamp,
88+
'count': 1
89+
}
90+
logging.debug(f"Request parameters: {parameters}")
91+
4992
data, status_code = fetch_cmc_data(parameters, 'historical')
50-
logging.info(f"Got response with status code: {status_code}")
51-
except Exception as e:
52-
logging.error("Error fetching CMC data: %s", str(e))
53-
logging.error("Full exception:", exc_info=True)
54-
return None
93+
logging.debug(f"Got response with status code: {status_code}")
94+
95+
if 'data' not in data:
96+
raise ShovelProcessingError(f"Invalid CMC API response: missing 'data' field")
97+
if 'quotes' not in data['data']:
98+
raise ShovelProcessingError(f"Invalid CMC API response: missing 'quotes' field")
99+
if not data['data']['quotes']:
100+
raise ShovelProcessingError(f"No price data available for timestamp {timestamp}")
55101

56-
if status_code == 200 and 'data' in data and 'quotes' in data['data']:
57-
logging.info("Successfully parsed response data")
58102
quote = data['data']['quotes'][0]
103+
if 'quote' not in quote or 'USD' not in quote['quote']:
104+
raise ShovelProcessingError(f"Invalid CMC API response: missing USD quote data")
105+
59106
usd_quote = quote['quote']['USD']
107+
required_fields = ['price', 'market_cap', 'volume_24h']
108+
for field in required_fields:
109+
if field not in usd_quote:
110+
raise ShovelProcessingError(f"Invalid CMC API response: missing {field} field")
111+
if usd_quote[field] is None:
112+
raise ShovelProcessingError(f"Invalid CMC API response: {field} is None")
113+
60114
price = usd_quote['price']
61115
market_cap = usd_quote['market_cap']
62116
volume = usd_quote['volume_24h']
63-
logging.info(f"Returning price={price}, market_cap={market_cap}, volume={volume}")
117+
118+
# Validate values
119+
if price < 0 or market_cap < 0 or volume < 0:
120+
raise ShovelProcessingError(f"Invalid negative values in price data: price={price}, market_cap={market_cap}, volume={volume}")
121+
122+
logging.debug(f"Returning price={price}, market_cap={market_cap}, volume={volume}")
64123
return price, market_cap, volume
65-
else:
66-
logging.error("Failed to fetch TAO price with parameters %s: %s", parameters, data.get('status', {}).get('error_message', 'Unknown error'))
67-
logging.error(f"Full response data: {data}")
68-
return None
124+
125+
except ShovelProcessingError:
126+
raise
127+
except Exception as e:
128+
raise ShovelProcessingError(f"Failed to get price data: {str(e)}")
69129

70130
def get_latest_price():
71-
parameters = {
72-
'id': CMC_TAO_ID,
73-
'convert': 'USD'
74-
}
131+
try:
132+
parameters = {
133+
'id': CMC_TAO_ID,
134+
'convert': 'USD'
135+
}
136+
137+
data, status_code = fetch_cmc_data(parameters, 'latest')
75138

76-
data, status_code = fetch_cmc_data(parameters, 'latest')
139+
if 'data' not in data:
140+
raise ShovelProcessingError(f"Invalid CMC API response: missing 'data' field")
141+
142+
tao_id_str = str(CMC_TAO_ID)
143+
if tao_id_str not in data['data']:
144+
raise ShovelProcessingError(f"No data available for TAO (ID: {CMC_TAO_ID})")
145+
146+
tao_data = data['data'][tao_id_str]
147+
if 'quote' not in tao_data or 'USD' not in tao_data['quote']:
148+
raise ShovelProcessingError(f"Invalid CMC API response: missing USD quote data")
77149

78-
if status_code == 200 and 'data' in data and str(CMC_TAO_ID) in data['data']:
79-
tao_data = data['data'][str(CMC_TAO_ID)]
80150
usd_quote = tao_data['quote']['USD']
151+
required_fields = ['price', 'market_cap', 'volume_24h']
152+
for field in required_fields:
153+
if field not in usd_quote:
154+
raise ShovelProcessingError(f"Invalid CMC API response: missing {field} field")
155+
if usd_quote[field] is None:
156+
raise ShovelProcessingError(f"Invalid CMC API response: {field} is None")
157+
81158
price = usd_quote['price']
82159
market_cap = usd_quote['market_cap']
83160
volume = usd_quote['volume_24h']
161+
162+
# Validate values
163+
if price < 0 or market_cap < 0 or volume < 0:
164+
raise ShovelProcessingError(f"Invalid negative values in price data: price={price}, market_cap={market_cap}, volume={volume}")
165+
84166
return price, market_cap, volume
85-
else:
86-
logging.error("Failed to fetch latest TAO price: %s", data.get('status', {}).get('error_message', 'Unknown error'))
87-
return None, None, None
167+
168+
except ShovelProcessingError:
169+
raise
170+
except Exception as e:
171+
raise ShovelProcessingError(f"Failed to get latest price data: {str(e)}")

0 commit comments

Comments
 (0)