Add ingest script for country plan public file

Author: thenav56

api/management/commands/ingest_appeals.py

@@ -87,7 +87,7 @@ def get_new_or_modified_appeals(self):
             # get latest BILATERALS
             logger.info('Querying appeals API for new appeals data (bilateral)')
             url = 'http://go-api.ifrc.org/api/appealbilaterals'
-            auth = (os.getenv('APPEALS_USER'), os.getenv('APPEALS_PASS'))
+            auth = (settings.APPEALS_USER, settings.APPEALS_PASS)
 
             adapter = HTTPAdapter(max_retries=settings.RETRY_STRATEGY)
             sess = Session()

api/models.py

@@ -1925,6 +1925,7 @@ def __str__(self):
             return '%s | %s | %s' % (self.name, self.get_status_display(), str(self.created_at)[5:16])  # omit irrelevant 0
 
     # Given a request containing new CronJob log row, validate and add the CronJob log row.
+    @staticmethod
     def sync_cron(body):
         new = []
         errors = []

country_plan/management/commands/ingest_country_plan_file.py

@@ -0,0 +1,112 @@
+import logging
+import requests
+from typing import Union
+from datetime import datetime
+from django.utils.timezone import make_aware
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.db import models
+
+from api.models import CronJob, CronJobStatus
+from country_plan.models import CountryPlan
+
+logger = logging.getLogger(__name__)
+
+NAME = 'ingest_country_plan_file'
+# Ref: https://github.com/IFRCGo/go-api/issues/1614
+SOURCE = 'https://go-api.ifrc.org/api/publicsiteappeals?AppealsTypeID=1851&Hidden=false'
+
+
+class Command(BaseCommand):
+    @staticmethod
+    def parse_date(text: str) -> Union[datetime, None]:
+        """
+        Convert Appeal API datetime into django datetime
+        Parameters
+        ----------
+          text : str
+              Datetime eg: 2022-11-29T11:24:00
+        """
+        if text:
+            return make_aware(
+                # NOTE: Format is assumed by looking at the data from Appeal API
+                datetime.strptime(text, '%Y-%m-%dT%H:%M:%S')
+            )
+
+    def load_for_country(self, country_data):
+        country_iso2 = country_data.get('LocationCountryCode')
+        country_name = country_data.get('LocationCountryName')
+        public_plan_url = country_data.get('BaseDirectory') or '' + country_data.get('BaseFileName') or ''
+        inserted_date = self.parse_date(country_data.get('Inserted'))
+        if (
+            (country_iso2 is None and country_name is None) or
+            public_plan_url is None or
+            inserted_date is None
+        ):
+            return
+        country_plan = CountryPlan.objects.filter(
+            models.Q(country__iso__iexact=country_iso2) |
+            models.Q(country__name__iexact=country_name)
+        ).first()
+        if country_plan is None:
+            logger.warning(f'{NAME} No country_plan found for: {(country_iso2, country_name)}')
+            return
+        if country_plan.appeal_api_inserted_date and country_plan.appeal_api_inserted_date >= inserted_date:
+            # No need to do anything here
+            return
+        public_plan_url = country_data['BaseDirectory'] + country_data['BaseFileName']
+        country_plan.appeal_api_inserted_date = inserted_date
+        country_plan.load_file_to_country_plan(
+            public_plan_url,
+            # NOTE: File provided are PDF,
+            f"public-plan-{country_data['BaseFileName']}.pdf",
+        )
+        country_plan.is_publish = True
+        country_plan.save(
+            update_fields=(
+                'appeal_api_inserted_date',
+                'public_plan_file',  # By load_file_to_country_plan
+                'is_publish',
+            )
+        )
+        return True
+
+    def load(self):
+        updated = 0
+        auth = (settings.APPEALS_USER, settings.APPEALS_PASS)
+        results = requests.get(SOURCE, auth=auth, headers={'Accept': 'application/json'}).json()
+        for result in results:
+            try:
+                if self.load_for_country(result):
+                    updated += 1
+            except Exception as ex:
+                logger.error('Could not Updated countries plan', exc_info=True)
+                country_info = (
+                    result.get('LocationCountryCode'),
+                    result.get('LocationCountryName'),
+                )
+                CronJob.sync_cron({
+                    'name': NAME,
+                    'message': f"Could not updated country plan for {country_info}\n\nException:\n{str(ex)}",
+                    'status': CronJobStatus.ERRONEOUS,
+                })
+        return updated
+
+    def handle(self, *args, **kwargs):
+        try:
+            logger.info('\nFetching data for country plans:: ')
+            countries_plan_updated = self.load()
+            CronJob.sync_cron({
+                'name': NAME,
+                'message': 'Updated countries plan',
+                'num_result': countries_plan_updated,
+                'status': CronJobStatus.SUCCESSFUL,
+            })
+            logger.info('Updated countries plan')
+        except Exception as ex:
+            logger.error('Could not Updated countries plan', exc_info=True)
+            CronJob.sync_cron({
+                'name': NAME,
+                'message': f'Could not Updated countries plan\n\nException:\n{str(ex)}',
+                'status': CronJobStatus.ERRONEOUS,
+            })

country_plan/migrations/0004_countryplan_appeal_api_inserted_date.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2023-01-02 07:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('country_plan', '0003_auto_20221128_0831'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='countryplan',
+            name='appeal_api_inserted_date',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

country_plan/models.py

@@ -3,7 +3,9 @@
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from django.core.validators import FileExtensionValidator
+from django.core.files.base import File
 
+from main.utils import DownloadFileManager
 from api.models import Country
 
 
@@ -80,9 +82,21 @@ class CountryPlan(CountryPlanAbstract):
     people_targeted = models.IntegerField(verbose_name=_('People Targeted'), blank=True, null=True)
     is_publish = models.BooleanField(default=False, verbose_name=_('Published'))
 
+    # NOTE: Used to sync with Appeal API (ingest_country_plan_file.py for more info)
+    appeal_api_inserted_date = models.DateTimeField(blank=True, null=True)
+
     def __str__(self):
         return f'{self.country}'
 
+    def load_file_to_country_plan(self, url: str, filename: str, commit=False):
+        with DownloadFileManager(url, suffix='.pdf') as f:
+            self.public_plan_file.save(
+                filename,
+                File(f),
+            )
+        if commit:
+            self.save()
+
     # NOTE: To be used by CountryPlanSerializer (From CountryPlanViewset)
     def full_country_plan_mc(self):
         all_mc = list(self.country_plan_mc.all())

docker-compose.yml

@@ -24,6 +24,9 @@ x-server: &base_server_setup
     ERP_API_ENDPOINT: ${ERP_API_ENDPOINT:-https://ifrctintapim001.azure-api.net/GoAPI/ExtractGoEmergency}
     ERP_API_SUBSCRIPTION_KEY: ${ERP_API_SUBSCRIPTION_KEY:-abcdef}
     CELERY_REDIS_URL: ${CELERY_REDIS_URL:-redis://redis:6379/0}
+    # Appeal API
+    APPEALS_USER: ${APPEALS_USER}
+    APPEALS_PASS: ${APPEALS_PASS}
   env_file:
     - .env
   volumes:

main/settings.py

@@ -69,6 +69,9 @@
     GO_FTPUSER=(str, None),
     GO_FTPPASS=(str, None),
     GO_DBPASS=(str, None),
+    # Appeal Server Credentials (https://go-api.ifrc.org/api/)
+    APPEALS_USER=(str, None),
+    APPEALS_PASS=(str, None),
 )
 
 
@@ -471,3 +474,7 @@
 
 
 DREF_OP_UPDATE_FINAL_REPORT_UPDATE_ERROR_MESSAGE = "OBSOLETE_PAYLOAD"
+
+# Appeal Server Credentials
+APPEALS_USER = env('APPEALS_USER')
+APPEALS_PASS = env('APPEALS_PASS')

main/utils.py

@@ -1,3 +1,6 @@
+import requests
+
+from tempfile import NamedTemporaryFile, _TemporaryFileWrapper
 from collections import defaultdict
 
 
@@ -24,3 +27,36 @@ def get_merged_items_by_fields(items, fields, seperator=', '):
         field: seperator.join(data[field])
         for field in fields
     }
+
+
+class DownloadFileManager():
+    """
+    Convert Appeal API datetime into django datetime
+    Parameters
+    ----------
+      url : str
+    Return: TemporaryFile
+    On close: Close and Delete the file
+    """
+    def __init__(self, url, dir='/tmp/', **kwargs):
+        self.url = url
+        self.downloaded_file = None
+        # NamedTemporaryFile attributes
+        self.named_temporary_file_args = {
+            'dir': dir,
+            **kwargs,
+        }
+
+    def __enter__(self) -> _TemporaryFileWrapper:
+        file = NamedTemporaryFile(delete=True, **self.named_temporary_file_args)
+        with requests.get(self.url, stream=True) as r:
+            r.raise_for_status()
+            for chunk in r.iter_content(chunk_size=8192):
+                file.write(chunk)
+        file.flush()
+        self.downloaded_file = file
+        return self.downloaded_file
+
+    def __exit__(self, *_):
+        if self.downloaded_file:
+            self.downloaded_file.close()