Merge pull request #1624 from IFRCGo/feature/country-plan-auto-fetch

frozenhelium · web-flow · commit 9eb017ec8b76 · 2023-01-04T10:45:16.000+05:45
Feature/country plan auto fetch
diff --git a/api/management/commands/ingest_appeals.py b/api/management/commands/ingest_appeals.py
@@ -87,7 +87,7 @@ def get_new_or_modified_appeals(self):
             # get latest BILATERALS
             logger.info('Querying appeals API for new appeals data (bilateral)')
             url = 'http://go-api.ifrc.org/api/appealbilaterals'
-            auth = (os.getenv('APPEALS_USER'), os.getenv('APPEALS_PASS'))
+            auth = (settings.APPEALS_USER, settings.APPEALS_PASS)
 
             adapter = HTTPAdapter(max_retries=settings.RETRY_STRATEGY)
             sess = Session()
diff --git a/api/models.py b/api/models.py
@@ -1925,6 +1925,7 @@ def __str__(self):
             return '%s | %s | %s' % (self.name, self.get_status_display(), str(self.created_at)[5:16])  # omit irrelevant 0
 
     # Given a request containing new CronJob log row, validate and add the CronJob log row.
+    @staticmethod
     def sync_cron(body):
         new = []
         errors = []
diff --git a/country_plan/factories.py b/country_plan/factories.py
@@ -0,0 +1,8 @@
+import factory
+
+from .models import CountryPlan
+
+
+class CountryPlanFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = CountryPlan
diff --git a/country_plan/management/__init__.py b/country_plan/management/__init__.py
diff --git a/country_plan/management/commands/__init__.py b/country_plan/management/commands/__init__.py
diff --git a/country_plan/management/commands/ingest_country_plan_file.py b/country_plan/management/commands/ingest_country_plan_file.py
@@ -0,0 +1,112 @@
+import logging
+import requests
+from typing import Union
+from datetime import datetime
+from django.utils.timezone import make_aware
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.db import models
+
+from api.models import CronJob, CronJobStatus
+from country_plan.models import CountryPlan
+
+logger = logging.getLogger(__name__)
+
+NAME = 'ingest_country_plan_file'
+# Ref: https://github.com/IFRCGo/go-api/issues/1614
+SOURCE = 'https://go-api.ifrc.org/api/publicsiteappeals?AppealsTypeID=1851&Hidden=false'
+
+
+class Command(BaseCommand):
+    @staticmethod
+    def parse_date(text: str) -> Union[datetime, None]:
+        """
+        Convert Appeal API datetime into django datetime
+        Parameters
+        ----------
+          text : str
+              Datetime eg: 2022-11-29T11:24:00
+        """
+        if text:
+            return make_aware(
+                # NOTE: Format is assumed by looking at the data from Appeal API
+                datetime.strptime(text, '%Y-%m-%dT%H:%M:%S')
+            )
+
+    def load_for_country(self, country_data):
+        country_iso2 = country_data.get('LocationCountryCode')
+        country_name = country_data.get('LocationCountryName')
+        public_plan_url = country_data.get('BaseDirectory') or '' + country_data.get('BaseFileName') or ''
+        inserted_date = self.parse_date(country_data.get('Inserted'))
+        if (
+            (country_iso2 is None and country_name is None) or
+            public_plan_url is None or
+            inserted_date is None
+        ):
+            return
+        country_plan = CountryPlan.objects.filter(
+            models.Q(country__iso__iexact=country_iso2) |
+            models.Q(country__name__iexact=country_name)
+        ).first()
+        if country_plan is None:
+            logger.warning(f'{NAME} No country_plan found for: {(country_iso2, country_name)}')
+            return
+        if country_plan.appeal_api_inserted_date and country_plan.appeal_api_inserted_date >= inserted_date:
+            # No need to do anything here
+            return
+        public_plan_url = country_data['BaseDirectory'] + country_data['BaseFileName']
+        country_plan.appeal_api_inserted_date = inserted_date
+        country_plan.load_file_to_country_plan(
+            public_plan_url,
+            # NOTE: File provided are PDF,
+            f"public-plan-{country_data['BaseFileName']}.pdf",
+        )
+        country_plan.is_publish = True
+        country_plan.save(
+            update_fields=(
+                'appeal_api_inserted_date',
+                'public_plan_file',  # By load_file_to_country_plan
+                'is_publish',
+            )
+        )
+        return True
+
+    def load(self):
+        updated = 0
+        auth = (settings.APPEALS_USER, settings.APPEALS_PASS)
+        results = requests.get(SOURCE, auth=auth, headers={'Accept': 'application/json'}).json()
+        for result in results:
+            try:
+                if self.load_for_country(result):
+                    updated += 1
+            except Exception as ex:
+                logger.error('Could not Updated countries plan', exc_info=True)
+                country_info = (
+                    result.get('LocationCountryCode'),
+                    result.get('LocationCountryName'),
+                )
+                CronJob.sync_cron({
+                    'name': NAME,
+                    'message': f"Could not updated country plan for {country_info}\n\nException:\n{str(ex)}",
+                    'status': CronJobStatus.ERRONEOUS,
+                })
+        return updated
+
+    def handle(self, *args, **kwargs):
+        try:
+            logger.info('\nFetching data for country plans:: ')
+            countries_plan_updated = self.load()
+            CronJob.sync_cron({
+                'name': NAME,
+                'message': 'Updated countries plan',
+                'num_result': countries_plan_updated,
+                'status': CronJobStatus.SUCCESSFUL,
+            })
+            logger.info('Updated countries plan')
+        except Exception as ex:
+            logger.error('Could not Updated countries plan', exc_info=True)
+            CronJob.sync_cron({
+                'name': NAME,
+                'message': f'Could not Updated countries plan\n\nException:\n{str(ex)}',
+                'status': CronJobStatus.ERRONEOUS,
+            })
diff --git a/country_plan/migrations/0004_countryplan_appeal_api_inserted_date.py b/country_plan/migrations/0004_countryplan_appeal_api_inserted_date.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2023-01-02 07:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('country_plan', '0003_auto_20221128_0831'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='countryplan',
+            name='appeal_api_inserted_date',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]
diff --git a/country_plan/models.py b/country_plan/models.py
@@ -3,7 +3,9 @@
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from django.core.validators import FileExtensionValidator
+from django.core.files.base import File
 
+from main.utils import DownloadFileManager
 from api.models import Country
 
 
@@ -80,9 +82,21 @@ class CountryPlan(CountryPlanAbstract):
     people_targeted = models.IntegerField(verbose_name=_('People Targeted'), blank=True, null=True)
     is_publish = models.BooleanField(default=False, verbose_name=_('Published'))
 
+    # NOTE: Used to sync with Appeal API (ingest_country_plan_file.py for more info)
+    appeal_api_inserted_date = models.DateTimeField(blank=True, null=True)
+
     def __str__(self):
         return f'{self.country}'
 
+    def load_file_to_country_plan(self, url: str, filename: str, commit=False):
+        with DownloadFileManager(url, suffix='.pdf') as f:
+            self.public_plan_file.save(
+                filename,
+                File(f),
+            )
+        if commit:
+            self.save()
+
     # NOTE: To be used by CountryPlanSerializer (From CountryPlanViewset)
     def full_country_plan_mc(self):
         all_mc = list(self.country_plan_mc.all())
diff --git a/country_plan/tests/__init__.py b/country_plan/tests/__init__.py
diff --git a/country_plan/tests/test_commands.py b/country_plan/tests/test_commands.py
@@ -0,0 +1,105 @@
+from unittest import mock
+from django.core.management import call_command
+
+from main.test_case import APITestCase
+from api.factories.country import CountryFactory
+from country_plan.factories import CountryPlanFactory
+from country_plan.models import CountryPlan
+from country_plan.management.commands.ingest_country_plan_file import SOURCE
+
+# NOTE: Only defined used fields
+FILE_BASE_DIRECTORY = 'https://example.org/Download.aspx?FileId='
+APPEAL_COUNTRY_PLAN_MOCK_RESPONSE = [
+    {
+        'BaseDirectory': FILE_BASE_DIRECTORY,
+        'BaseFileName': '000000',
+        'Inserted': '2022-11-29T11:24:00',
+        'LocationCountryCode': 'SY',
+        'LocationCountryName': 'Syrian Arab Republic'
+    },
+    {
+        'BaseDirectory': FILE_BASE_DIRECTORY,
+        'BaseFileName': '000001',
+        'Inserted': '2022-11-29T11:24:00',
+        'LocationCountryCode': 'CD',
+        'LocationCountryName': 'Congo, The Democratic Republic Of The'
+    },
+    {
+        'BaseDirectory': FILE_BASE_DIRECTORY,
+        'BaseFileName': '000002',
+        'Inserted': '2022-11-29T11:24:00',
+        'LocationCountryCode': 'MM',
+        'LocationCountryName': 'Myanmar'
+    },
+    {
+        'BaseDirectory': FILE_BASE_DIRECTORY,
+        'BaseFileName': '000003',
+        'Inserted': '2022-11-29T11:24:00',
+        'LocationCountryCode': 'TM',
+        'LocationCountryName': 'Turkmenistan'
+    },
+    {
+        'BaseDirectory': FILE_BASE_DIRECTORY,
+        'BaseFileName': '000004',
+        'Inserted': '2022-11-29T11:24:00',
+        'LocationCountryCode': 'GR',
+        'LocationCountryName': 'Greece'
+    }
+]
+
+
+class MockResponse():
+    class FileStream():
+        def __init__(self, stream):
+            self.stream = stream
+
+        def raise_for_status(self):
+            pass
+
+        def iter_content(self, **_):
+            return self.stream
+
+    def __init__(self, json=None, stream=None):
+        self._json = json
+        self.stream = stream
+
+    def json(self):
+        return self._json
+
+    def __enter__(self):
+        return MockResponse.FileStream(self.stream)
+
+    def __exit__(self, *_):
+        pass
+
+
+class CountryPlanIngestTest(APITestCase):
+    def mock_request(url, *_, **kwargs):
+        if url == SOURCE:
+            return MockResponse(json=APPEAL_COUNTRY_PLAN_MOCK_RESPONSE)
+        if url.startswith(FILE_BASE_DIRECTORY):
+            return MockResponse(stream=[b''])
+
+    @mock.patch('country_plan.management.commands.ingest_country_plan_file.requests.get', side_effect=mock_request)
+    @mock.patch('main.utils.requests.get', side_effect=mock_request)
+    def test_country_plan_ingest(self, *_):
+        CountryPlanFactory.create(
+            country=CountryFactory.create(
+                name='Random Country',
+                iso='RC',
+            ),
+        )
+        for country_plan_data in APPEAL_COUNTRY_PLAN_MOCK_RESPONSE[:3]:
+            CountryPlanFactory.create(
+                country=CountryFactory.create(
+                    name=country_plan_data['LocationCountryName'],
+                    iso=country_plan_data['LocationCountryCode'],
+                ),
+            )
+        assert CountryPlan.objects.count() == 4
+        assert CountryPlan.objects.filter(is_publish=True).count() == 0
+        assert CountryPlan.objects.exclude(public_plan_file='').count() == 0
+        call_command('ingest_country_plan_file')
+        assert CountryPlan.objects.count() == 4
+        assert CountryPlan.objects.filter(is_publish=True).count() == 3
+        assert CountryPlan.objects.exclude(public_plan_file='').count() == 3
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -24,6 +24,9 @@ x-server: &base_server_setup
     ERP_API_ENDPOINT: ${ERP_API_ENDPOINT:-https://ifrctintapim001.azure-api.net/GoAPI/ExtractGoEmergency}
     ERP_API_SUBSCRIPTION_KEY: ${ERP_API_SUBSCRIPTION_KEY:-abcdef}
     CELERY_REDIS_URL: ${CELERY_REDIS_URL:-redis://redis:6379/0}
+    # Appeal API
+    APPEALS_USER: ${APPEALS_USER}
+    APPEALS_PASS: ${APPEALS_PASS}
   env_file:
     - .env
   volumes:
diff --git a/main/runserver.sh b/main/runserver.sh
@@ -52,6 +52,7 @@ printenv | sed 's/^\([a-zA-Z0-9_]*\)=\(.*\)$/export \1="\2"/g' > $HOME/.env
 (crontab -l 2>/dev/null; echo '*/10 * * * * . /home/ifrc/.env; python /home/ifrc/go-api/manage.py sync_molnix >> /home/ifrc/logs/sync_molnix.log 2>&1') | crontab -
 (crontab -l 2>/dev/null; echo '0 3 * * * . /home/ifrc/.env; python /home/ifrc/go-api/manage.py update_project_status >> /home/ifrc/logs/update_project_status.log 2>&1') | crontab -
 (crontab -l 2>/dev/null; echo '0 9 * * * . /home/ifrc/.env; python /home/ifrc/go-api/manage.py user_registration_reminder >> /home/ifrc/logs/user_registration_reminder.log 2>&1') | crontab -
+(crontab -l 2>/dev/null; echo '0 0 * * * . /home/ifrc/.env; python /home/ifrc/go-api/manage.py ingest_country_plan_file >> /home/ifrc/logs/ingest_country_plan_file.log 2>&1') | crontab -
 service cron start
 
 
diff --git a/main/settings.py b/main/settings.py
@@ -69,6 +69,9 @@
     GO_FTPUSER=(str, None),
     GO_FTPPASS=(str, None),
     GO_DBPASS=(str, None),
+    # Appeal Server Credentials (https://go-api.ifrc.org/api/)
+    APPEALS_USER=(str, None),
+    APPEALS_PASS=(str, None),
 )
 
 
@@ -471,3 +474,7 @@
 
 
 DREF_OP_UPDATE_FINAL_REPORT_UPDATE_ERROR_MESSAGE = "OBSOLETE_PAYLOAD"
+
+# Appeal Server Credentials
+APPEALS_USER = env('APPEALS_USER')
+APPEALS_PASS = env('APPEALS_PASS')
diff --git a/main/utils.py b/main/utils.py
@@ -1,3 +1,6 @@
+import requests
+
+from tempfile import NamedTemporaryFile, _TemporaryFileWrapper
 from collections import defaultdict
 
 
@@ -24,3 +27,36 @@ def get_merged_items_by_fields(items, fields, seperator=', '):
         field: seperator.join(data[field])
         for field in fields
     }
+
+
+class DownloadFileManager():
+    """
+    Convert Appeal API datetime into django datetime
+    Parameters
+    ----------
+      url : str
+    Return: TemporaryFile
+    On close: Close and Delete the file
+    """
+    def __init__(self, url, dir='/tmp/', **kwargs):
+        self.url = url
+        self.downloaded_file = None
+        # NamedTemporaryFile attributes
+        self.named_temporary_file_args = {
+            'dir': dir,
+            **kwargs,
+        }
+
+    def __enter__(self) -> _TemporaryFileWrapper:
+        file = NamedTemporaryFile(delete=True, **self.named_temporary_file_args)
+        with requests.get(self.url, stream=True) as r:
+            r.raise_for_status()
+            for chunk in r.iter_content(chunk_size=8192):
+                file.write(chunk)
+        file.flush()
+        self.downloaded_file = file
+        return self.downloaded_file
+
+    def __exit__(self, *_):
+        if self.downloaded_file:
+            self.downloaded_file.close()
