Merge pull request #1651 from IFRCGo/feature/auto-internal-plan-load

szabozoltan69 · web-flow · commit 9d6de39a82ff · 2023-02-08T09:28:21.000+01:00
Add internal plan to ingest_country_plan_file
diff --git a/country_plan/management/commands/ingest_country_plan_file.py b/country_plan/management/commands/ingest_country_plan_file.py
@@ -4,17 +4,20 @@
 from datetime import datetime
 from django.utils.timezone import make_aware
 from django.core.management.base import BaseCommand
+from django.core.files.base import File
 from django.conf import settings
 from django.db import models
 
-from api.models import CronJob, CronJobStatus
+from main.utils import DownloadFileManager
+from api.models import Country
 from country_plan.models import CountryPlan
 
 logger = logging.getLogger(__name__)
 
-NAME = 'ingest_country_plan_file'
 # Ref: https://github.com/IFRCGo/go-api/issues/1614
-SOURCE = 'https://go-api.ifrc.org/api/publicsiteappeals?AppealsTypeID=1851&Hidden=false'
+PUBLIC_SOURCE = 'https://go-api.ifrc.org/api/publicsiteappeals?AppealsTypeID=1851&Hidden=false'
+# Ref: https://github.com/IFRCGo/go-api/issues/1648
+INTERNAL_SOURCE = 'https://go-api.ifrc.org/Api/FedNetAppeals?AppealsTypeId=1844&Hidden=false'
 
 
 class Command(BaseCommand):
@@ -33,7 +36,15 @@ def parse_date(text: str) -> Union[datetime, None]:
                 datetime.strptime(text, '%Y-%m-%dT%H:%M:%S')
             )
 
-    def load_for_country(self, country_data):
+    @staticmethod
+    def load_file_to_country_plan(country_plan: CountryPlan, url: str, filename: str, field_name: str):
+        with DownloadFileManager(url, suffix='.pdf') as f:
+            getattr(country_plan, field_name).save(
+                filename,
+                File(f),
+            )
+
+    def load_for_country(self, country_data, file_field, field_inserted_date_field):
         country_iso2 = country_data.get('LocationCountryCode')
         country_name = country_data.get('LocationCountryName')
         public_plan_url = country_data.get('BaseDirectory') or '' + country_data.get('BaseFileName') or ''
@@ -44,69 +55,56 @@ def load_for_country(self, country_data):
             inserted_date is None
         ):
             return
-        country_plan = CountryPlan.objects.filter(
-            models.Q(country__iso__iexact=country_iso2) |
-            models.Q(country__name__iexact=country_name)
-        ).first()
+        country_qs = Country.objects.filter(
+            models.Q(iso__iexact=country_iso2) |
+            models.Q(name__iexact=country_name)
+        )
+        country_plan = CountryPlan.objects.filter(country__in=country_qs).first()
         if country_plan is None:
-            logger.warning(f'{NAME} No country_plan found for: {(country_iso2, country_name)}')
-            return
-        if country_plan.appeal_api_inserted_date and country_plan.appeal_api_inserted_date >= inserted_date:
+            country = country_qs.first()
+            # If there is no country as well, show warning and return
+            if not country:
+                logger.warning(f'{file_field} No country found for: {(country_iso2, country_name)}')
+                return
+            # Else create one and continue
+            country_plan = CountryPlan(country=country)
+        existing_inserted_date = getattr(country_plan, field_inserted_date_field, None)
+        if existing_inserted_date and existing_inserted_date >= inserted_date:
             # No need to do anything here
             return
+        self.stdout.write(f'- Saving data for country:: {country_plan.country.name}')
         public_plan_url = country_data['BaseDirectory'] + country_data['BaseFileName']
-        country_plan.appeal_api_inserted_date = inserted_date
-        country_plan.load_file_to_country_plan(
+        setattr(country_plan, field_inserted_date_field, inserted_date)
+        self.load_file_to_country_plan(
+            country_plan,
             public_plan_url,
             # NOTE: File provided are PDF,
-            f"public-plan-{country_data['BaseFileName']}.pdf",
+            f"{file_field.replace('_', '-').replace('file', '')}-{country_data['BaseFileName']}.pdf",
+            file_field,
         )
         country_plan.is_publish = True
         country_plan.save(
             update_fields=(
-                'appeal_api_inserted_date',
-                'public_plan_file',  # By load_file_to_country_plan
+                field_inserted_date_field,
+                file_field,  # By load_file_to_country_plan
                 'is_publish',
             )
         )
         return True
 
-    def load(self):
-        updated = 0
+    def load(self, url: str, file_field: str, field_inserted_date_field: str):
         auth = (settings.APPEALS_USER, settings.APPEALS_PASS)
-        results = requests.get(SOURCE, auth=auth, headers={'Accept': 'application/json'}).json()
+        results = requests.get(url, auth=auth, headers={'Accept': 'application/json'}).json()
         for result in results:
             try:
-                if self.load_for_country(result):
-                    updated += 1
-            except Exception as ex:
+                self.load_for_country(result, file_field, field_inserted_date_field)
+            except Exception:
                 logger.error('Could not Updated countries plan', exc_info=True)
-                country_info = (
-                    result.get('LocationCountryCode'),
-                    result.get('LocationCountryName'),
-                )
-                CronJob.sync_cron({
-                    'name': NAME,
-                    'message': f"Could not updated country plan for {country_info}\n\nException:\n{str(ex)}",
-                    'status': CronJobStatus.ERRONEOUS,
-                })
-        return updated
 
-    def handle(self, *args, **kwargs):
-        try:
-            logger.info('\nFetching data for country plans:: ')
-            countries_plan_updated = self.load()
-            CronJob.sync_cron({
-                'name': NAME,
-                'message': 'Updated countries plan',
-                'num_result': countries_plan_updated,
-                'status': CronJobStatus.SUCCESSFUL,
-            })
-            logger.info('Updated countries plan')
-        except Exception as ex:
-            logger.error('Could not Updated countries plan', exc_info=True)
-            CronJob.sync_cron({
-                'name': NAME,
-                'message': f'Could not Updated countries plan\n\nException:\n{str(ex)}',
-                'status': CronJobStatus.ERRONEOUS,
-            })
+    def handle(self, **_):
+        # Public
+        self.stdout.write('Fetching data for country plans:: PUBLIC')
+        self.load(PUBLIC_SOURCE, 'public_plan_file', 'public_plan_inserted_date')
+        # Private
+        self.stdout.write('\nFetching data for country plans:: PRIVATE')
+        self.load(INTERNAL_SOURCE, 'internal_plan_file', 'internal_plan_inserted_date')
diff --git a/country_plan/migrations/0005_auto_20230207_0840.py b/country_plan/migrations/0005_auto_20230207_0840.py
@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2023-02-07 08:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('country_plan', '0004_countryplan_appeal_api_inserted_date'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='countryplan',
+            old_name='appeal_api_inserted_date',
+            new_name='public_plan_inserted_date',
+        ),
+        migrations.AddField(
+            model_name='countryplan',
+            name='internal_plan_inserted_date',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]
diff --git a/country_plan/models.py b/country_plan/models.py
@@ -3,9 +3,7 @@
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from django.core.validators import FileExtensionValidator
-from django.core.files.base import File
 
-from main.utils import DownloadFileManager
 from api.models import Country
 
 
@@ -83,21 +81,12 @@ class CountryPlan(CountryPlanAbstract):
     is_publish = models.BooleanField(default=False, verbose_name=_('Published'))
 
     # NOTE: Used to sync with Appeal API (ingest_country_plan_file.py for more info)
-    appeal_api_inserted_date = models.DateTimeField(blank=True, null=True)
+    public_plan_inserted_date = models.DateTimeField(blank=True, null=True)
+    internal_plan_inserted_date = models.DateTimeField(blank=True, null=True)
 
     def __str__(self):
         return f'{self.country}'
 
-    def load_file_to_country_plan(self, url: str, filename: str, commit=False):
-        with DownloadFileManager(url, suffix='.pdf') as f:
-            self.public_plan_file.save(
-                filename,
-                File(f),
-            )
-        if commit:
-            self.save()
-
-    # NOTE: To be used by CountryPlanSerializer (From CountryPlanViewset)
     def full_country_plan_mc(self):
         all_mc = list(self.country_plan_mc.all())
         mc_by_ns_sector = {
diff --git a/country_plan/tests/test_commands.py b/country_plan/tests/test_commands.py
@@ -5,11 +5,11 @@
 from api.factories.country import CountryFactory
 from country_plan.factories import CountryPlanFactory
 from country_plan.models import CountryPlan
-from country_plan.management.commands.ingest_country_plan_file import SOURCE
+from country_plan.management.commands.ingest_country_plan_file import PUBLIC_SOURCE, INTERNAL_SOURCE
 
 # NOTE: Only defined used fields
 FILE_BASE_DIRECTORY = 'https://example.org/Download.aspx?FileId='
-APPEAL_COUNTRY_PLAN_MOCK_RESPONSE = [
+PUBLIC_APPEAL_COUNTRY_PLAN_MOCK_RESPONSE = [
     {
         'BaseDirectory': FILE_BASE_DIRECTORY,
         'BaseFileName': '000000',
@@ -24,6 +24,7 @@
         'LocationCountryCode': 'CD',
         'LocationCountryName': 'Congo, The Democratic Republic Of The'
     },
+    # Not included in INTERNAL
     {
         'BaseDirectory': FILE_BASE_DIRECTORY,
         'BaseFileName': '000002',
@@ -47,6 +48,31 @@
     }
 ]
 
+INTERNAL_APPEAL_COUNTRY_PLAN_MOCK_RESPONSE = [
+    {
+        'BaseDirectory': FILE_BASE_DIRECTORY,
+        'BaseFileName': '000000',
+        'Inserted': '2022-11-29T11:24:00',
+        'LocationCountryCode': 'SY',
+        'LocationCountryName': 'Syrian Arab Republic'
+    },
+    {
+        'BaseDirectory': FILE_BASE_DIRECTORY,
+        'BaseFileName': '000001',
+        'Inserted': '2022-11-29T11:24:00',
+        'LocationCountryCode': 'CD',
+        'LocationCountryName': 'Congo, The Democratic Republic Of The'
+    },
+    # Not included in PUBLIC
+    {
+        'BaseDirectory': FILE_BASE_DIRECTORY,
+        'BaseFileName': '000001',
+        'Inserted': '2022-11-29T11:24:00',
+        'LocationCountryCode': 'NP',
+        'LocationCountryName': 'Nepal'
+    },
+]
+
 
 class MockResponse():
     class FileStream():
@@ -75,8 +101,10 @@ def __exit__(self, *_):
 
 class CountryPlanIngestTest(APITestCase):
     def mock_request(url, *_, **kwargs):
-        if url == SOURCE:
-            return MockResponse(json=APPEAL_COUNTRY_PLAN_MOCK_RESPONSE)
+        if url == PUBLIC_SOURCE:
+            return MockResponse(json=PUBLIC_APPEAL_COUNTRY_PLAN_MOCK_RESPONSE)
+        if url == INTERNAL_SOURCE:
+            return MockResponse(json=INTERNAL_APPEAL_COUNTRY_PLAN_MOCK_RESPONSE)
         if url.startswith(FILE_BASE_DIRECTORY):
             return MockResponse(stream=[b''])
 
@@ -89,17 +117,25 @@ def test_country_plan_ingest(self, *_):
                 iso='RC',
             ),
         )
-        for country_plan_data in APPEAL_COUNTRY_PLAN_MOCK_RESPONSE[:3]:
-            CountryPlanFactory.create(
-                country=CountryFactory.create(
-                    name=country_plan_data['LocationCountryName'],
-                    iso=country_plan_data['LocationCountryCode'],
-                ),
-            )
-        assert CountryPlan.objects.count() == 4
+        EXISTING_CP = 1
+        for country_iso in set([
+                item['LocationCountryCode']
+                for item in [
+                    *PUBLIC_APPEAL_COUNTRY_PLAN_MOCK_RESPONSE,
+                    *INTERNAL_APPEAL_COUNTRY_PLAN_MOCK_RESPONSE,
+                ]
+        ]):
+            if country_iso == 'CD':
+                # Not create country for this
+                continue
+            CountryFactory.create(iso=country_iso)
+        # Before
+        assert CountryPlan.objects.count() == EXISTING_CP
         assert CountryPlan.objects.filter(is_publish=True).count() == 0
         assert CountryPlan.objects.exclude(public_plan_file='').count() == 0
         call_command('ingest_country_plan_file')
-        assert CountryPlan.objects.count() == 4
-        assert CountryPlan.objects.filter(is_publish=True).count() == 3
-        assert CountryPlan.objects.exclude(public_plan_file='').count() == 3
+        # After
+        assert CountryPlan.objects.count() == EXISTING_CP + 5
+        assert CountryPlan.objects.filter(is_publish=True).count() == 5
+        assert CountryPlan.objects.exclude(public_plan_file='').count() == 4
+        assert CountryPlan.objects.exclude(internal_plan_file='').count() == 2
