Add internal plan to ingest_country_plan_file

Author: thenav56

country_plan/management/commands/ingest_country_plan_file.py

@@ -4,17 +4,20 @@
 from datetime import datetime
 from django.utils.timezone import make_aware
 from django.core.management.base import BaseCommand
+from django.core.files.base import File
 from django.conf import settings
 from django.db import models
 
-from api.models import CronJob, CronJobStatus
+from main.utils import DownloadFileManager
+from api.models import Country
 from country_plan.models import CountryPlan
 
 logger = logging.getLogger(__name__)
 
-NAME = 'ingest_country_plan_file'
 # Ref: https://github.com/IFRCGo/go-api/issues/1614
-SOURCE = 'https://go-api.ifrc.org/api/publicsiteappeals?AppealsTypeID=1851&Hidden=false'
+PUBLIC_SOURCE = 'https://go-api.ifrc.org/api/publicsiteappeals?AppealsTypeID=1851&Hidden=false'
+# Ref: https://github.com/IFRCGo/go-api/issues/1648
+INTERNAL_SOURCE = 'https://go-api.ifrc.org/Api/FedNetAppeals?AppealsTypeId=1844&Hidden=false'
 
 
 class Command(BaseCommand):
@@ -33,7 +36,15 @@ def parse_date(text: str) -> Union[datetime, None]:
                 datetime.strptime(text, '%Y-%m-%dT%H:%M:%S')
             )
 
-    def load_for_country(self, country_data):
+    @staticmethod
+    def load_file_to_country_plan(country_plan: CountryPlan, url: str, filename: str, field_name: str):
+        with DownloadFileManager(url, suffix='.pdf') as f:
+            getattr(country_plan, field_name).save(
+                filename,
+                File(f),
+            )
+
+    def load_for_country(self, country_data, file_field, field_inserted_date_field):
         country_iso2 = country_data.get('LocationCountryCode')
         country_name = country_data.get('LocationCountryName')
         public_plan_url = country_data.get('BaseDirectory') or '' + country_data.get('BaseFileName') or ''
@@ -44,69 +55,56 @@ def load_for_country(self, country_data):
             inserted_date is None
         ):
             return
-        country_plan = CountryPlan.objects.filter(
-            models.Q(country__iso__iexact=country_iso2) |
-            models.Q(country__name__iexact=country_name)
-        ).first()
+        country_qs = Country.objects.filter(
+            models.Q(iso__iexact=country_iso2) |
+            models.Q(name__iexact=country_name)
+        )
+        country_plan = CountryPlan.objects.filter(country__in=country_qs).first()
         if country_plan is None:
-            logger.warning(f'{NAME} No country_plan found for: {(country_iso2, country_name)}')
-            return
-        if country_plan.appeal_api_inserted_date and country_plan.appeal_api_inserted_date >= inserted_date:
+            country = country_qs.first()
+            # If there is no country as well, show warning and return
+            if not country:
+                logger.warning(f'{file_field} No country found for: {(country_iso2, country_name)}')
+                return
+            # Else create one and continue
+            country_plan = CountryPlan(country=country)
+        existing_inserted_date = getattr(country_plan, field_inserted_date_field, None)
+        if existing_inserted_date and existing_inserted_date >= inserted_date:
             # No need to do anything here
             return
+        self.stdout.write(f'- Saving data for country:: {country_plan.country.name}')
         public_plan_url = country_data['BaseDirectory'] + country_data['BaseFileName']
-        country_plan.appeal_api_inserted_date = inserted_date
-        country_plan.load_file_to_country_plan(
+        setattr(country_plan, field_inserted_date_field, inserted_date)
+        self.load_file_to_country_plan(
+            country_plan,
             public_plan_url,
             # NOTE: File provided are PDF,
-            f"public-plan-{country_data['BaseFileName']}.pdf",
+            f"{file_field.replace('_', '-').replace('file', '')}-{country_data['BaseFileName']}.pdf",
+            file_field,
         )
         country_plan.is_publish = True
         country_plan.save(
             update_fields=(
-                'appeal_api_inserted_date',
-                'public_plan_file',  # By load_file_to_country_plan
+                field_inserted_date_field,
+                file_field,  # By load_file_to_country_plan
                 'is_publish',
             )
         )
         return True
 
-    def load(self):
-        updated = 0
+    def load(self, url: str, file_field: str, field_inserted_date_field: str):
         auth = (settings.APPEALS_USER, settings.APPEALS_PASS)
-        results = requests.get(SOURCE, auth=auth, headers={'Accept': 'application/json'}).json()
+        results = requests.get(url, auth=auth, headers={'Accept': 'application/json'}).json()
         for result in results:
             try:
-                if self.load_for_country(result):
-                    updated += 1
-            except Exception as ex:
+                self.load_for_country(result, file_field, field_inserted_date_field)
+            except Exception:
                 logger.error('Could not Updated countries plan', exc_info=True)
-                country_info = (
-                    result.get('LocationCountryCode'),
-                    result.get('LocationCountryName'),
-                )
-                CronJob.sync_cron({
-                    'name': NAME,
-                    'message': f"Could not updated country plan for {country_info}\n\nException:\n{str(ex)}",
-                    'status': CronJobStatus.ERRONEOUS,
-                })
-        return updated
 
-    def handle(self, *args, **kwargs):
-        try:
-            logger.info('\nFetching data for country plans:: ')
-            countries_plan_updated = self.load()
-            CronJob.sync_cron({
-                'name': NAME,
-                'message': 'Updated countries plan',
-                'num_result': countries_plan_updated,
-                'status': CronJobStatus.SUCCESSFUL,
-            })
-            logger.info('Updated countries plan')
-        except Exception as ex:
-            logger.error('Could not Updated countries plan', exc_info=True)
-            CronJob.sync_cron({
-                'name': NAME,
-                'message': f'Could not Updated countries plan\n\nException:\n{str(ex)}',
-                'status': CronJobStatus.ERRONEOUS,
-            })
+    def handle(self, **_):
+        # Public
+        self.stdout.write('Fetching data for country plans:: PUBLIC')
+        self.load(PUBLIC_SOURCE, 'public_plan_file', 'public_plan_inserted_date')
+        # Private
+        self.stdout.write('\nFetching data for country plans:: PRIVATE')
+        self.load(INTERNAL_SOURCE, 'internal_plan_file', 'internal_plan_inserted_date')

country_plan/migrations/0005_auto_20230207_0840.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2023-02-07 08:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('country_plan', '0004_countryplan_appeal_api_inserted_date'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='countryplan',
+            old_name='appeal_api_inserted_date',
+            new_name='public_plan_inserted_date',
+        ),
+        migrations.AddField(
+            model_name='countryplan',
+            name='internal_plan_inserted_date',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

country_plan/models.py

@@ -3,9 +3,7 @@
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from django.core.validators import FileExtensionValidator
-from django.core.files.base import File
 
-from main.utils import DownloadFileManager
 from api.models import Country
 
 
@@ -83,21 +81,12 @@ class CountryPlan(CountryPlanAbstract):
     is_publish = models.BooleanField(default=False, verbose_name=_('Published'))
 
     # NOTE: Used to sync with Appeal API (ingest_country_plan_file.py for more info)
-    appeal_api_inserted_date = models.DateTimeField(blank=True, null=True)
+    public_plan_inserted_date = models.DateTimeField(blank=True, null=True)
+    internal_plan_inserted_date = models.DateTimeField(blank=True, null=True)
 
     def __str__(self):
         return f'{self.country}'
 
-    def load_file_to_country_plan(self, url: str, filename: str, commit=False):
-        with DownloadFileManager(url, suffix='.pdf') as f:
-            self.public_plan_file.save(
-                filename,
-                File(f),
-            )
-        if commit:
-            self.save()
-
-    # NOTE: To be used by CountryPlanSerializer (From CountryPlanViewset)
     def full_country_plan_mc(self):
         all_mc = list(self.country_plan_mc.all())
         mc_by_ns_sector = {