Error handling for exceeding BQ quota

Author: LadyChristina

data_collection_scripts/collect_block_data.py

@@ -32,17 +32,21 @@ def collect_data(ledgers, from_block, to_date):
 
     for ledger in ledgers:
         file = RAW_DATA_DIR / f'{ledger}_raw_data.json'
-        logging.info(f"Querying {ledger}..")
+        logging.info(f"Querying {ledger} from block {from_block[ledger]} until {to_date}..")
 
         query = (queries[ledger]).replace("{{block_number}}", str(from_block[ledger]) if from_block[ledger] else "-1").replace("{{timestamp}}", to_date)
         query_job = client.query(query)
         try:
             rows = query_job.result()
             logging.info(f'Done querying {ledger}')
         except Exception as e:
-            logging.info(f'{ledger} query failed, please make sure it is properly defined.')
-            logging.info(f'The following exception was raised: {repr(e)}')
-            continue
+            if 'Quota exceeded' in repr(e):
+                logging.info(f'Quota exceeded for this service account key. Aborting..')
+                break
+            else:
+                logging.info(f'{ledger} query failed, please make sure it is properly defined.')
+                logging.info(f'The following exception was raised: {repr(e)}\n')
+                continue
 
         logging.info(f"Writing {ledger} data to file..")
         # Append result to file

data_collection_scripts/queries.yaml

@@ -45,7 +45,6 @@ ethereum:
   AND number > {{block_number}}
   ORDER BY timestamp
 
-
 litecoin:
   SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_litecoin.transactions`.outputs
   FROM `bigquery-public-data.crypto_litecoin.transactions`
@@ -56,7 +55,6 @@ litecoin:
   AND timestamp < '{{timestamp}}'
   ORDER BY timestamp
 
-
 tezos:
   SELECT level as number, timestamp, baker as reward_addresses
   FROM `public-data-finance.crypto_tezos.blocks`
@@ -74,4 +72,3 @@ zcash:
   AND timestamp > '2018-01-01'
   AND timestamp < '{{timestamp}}'
   ORDER BY timestamp
-