Better error handling

Author: k9845

databank/management/commands/ingest_acaps.py

@@ -2,6 +2,7 @@
 import datetime as dt
 import xmltodict
 import pandas as pd
+import time
 from dateutil.parser import parse
 
 from django.core.management.base import BaseCommand
@@ -26,6 +27,7 @@ def handle(self, *args, **kwargs):
                     "Authorization": "Token %s" % settings.ACAPS_API_TOKEN
                 }
             )
+            logger.info(f'Importing for country {name}')
             response_data = response.json()
             if 'results' in response_data and len(response_data['results']):
                 df = pd.DataFrame.from_records(response_data["results"])
@@ -41,4 +43,5 @@ def handle(self, *args, **kwargs):
                             'source': df_data[11],
                             'source_date': df_data[12]
                         }
-                        AcapsSeasonalCalender.objects.create(**dict_data)
+                        AcapsSeasonalCalender.objects.create(**dict_data)
+            time.sleep(5)

databank/management/commands/ingest_worldbank.py

@@ -40,16 +40,20 @@ def handle(self, *args, **kwargs):
             independent=True
         ).exclude(iso3="COK"):
             country_iso3 = country.iso3
+            logger.info(f'Importing country {country_iso3}')
             for indicator in world_bank_indicators:
                 page = 1  # Reset the page for each indicator
                 while True:
-                    response = requests.get(f'https://api.worldbank.org/v2/country/{country_iso3}/indicator/{indicator}?date={daterange}', params={
+                    try:
+                        response = requests.get(f'https://api.worldbank.org/v2/country/{country_iso3}/indicator/{indicator}?date={daterange}', params={
                         'format': 'json',
                         'source': 2,
                         'per_page': 5000 - 1,  # World Bank throws error on 5000
                         'page': page,
-                    })
-                    response.raise_for_status()
+                        })
+                    except requests.exceptions.HTTPError as err:
+                        print(err.response.text)
+                        continue
                     try:
                         data_list = response.json()[1]
                         if data_list is not None and len(data_list) > 0: