Merge pull request #1760 from IFRCGo/feature/appealdocs-from-api

Adding type and description to appealdocuments

Author: szabozoltan69

api/management/commands/ingest_appealdocs.py

@@ -18,6 +18,9 @@
 GEC_CODES = GECCode.objects.select_related('country').all()
 APPEAL_DOCUMENT_TYPES = AppealDocumentType.objects.all()  # used only for read in appealdocument locations from api
 
+PUBLIC_SOURCE = 'https://go-api.ifrc.org/api/publicsiteappeals?Hidden=false&AppealsTypeID='
+FEDNET_SOURCE = 'https://go-api.ifrc.org/Api/FedNetAppeals?Hidden=false&AppealsTypeId='
+
 # WORK IN PROGRESS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 class Command(BaseCommand):
@@ -38,15 +41,17 @@ def create_bilaterals_dict(self, records):
                     bilaterals[rec['APP_Code']] = rec['AmountCHF']
         return bilaterals
 
-    def load(self, url: str, file_field: str, field_inserted_date_field: str):
+    def load(self, url: str, is_fednet):
         auth = (settings.APPEALS_USER, settings.APPEALS_PASS)
-        results = requests.get(url, auth=auth, headers={'Accept': 'application/json'}).json()
-        for result in results:
-            try:
-                pass
-                # self.load_for_country(result, file_field, field_inserted_date_field)
-            except Exception:
-                logger.error('Could not update AppealDocuments', exc_info=True)
+        for adtype in APPEAL_DOCUMENT_TYPES.filter(public_site_or_fednet=is_fednet):
+            results = requests.get(url + str(adtype.id), auth=auth, headers={'Accept': 'application/json'}).json()
+            # import pdb; pdb.set_trace()
+            for result in results:
+                try:
+                    pass
+                    # self.load_for_country(result, file_field, field_inserted_date_field)
+                except Exception:
+                    logger.error('Could not update AppealDocuments', exc_info=True)
 
     def get_public_or_fednet_appealdocs(self):
         PUBLIC_SOURCE = 'https://go-api.ifrc.org/api/publicsiteappeals?AppealsTypeID=1851&Hidden=false'
@@ -301,22 +306,29 @@ def parse_appeal_record(self, r, **options):
 
         return fields
 
-    def handle(self, *args, **options):
-        logger.info('Starting appeals ingest')
-        start_appeals_count = Appeal.objects.all().count()
-        try:
-            getFromAPI = self.get_public_or_fednet_appealdocs()
-        except Exception as ex:
-            logger.error(f'Getting Appealdocs failed: {str(ex)}')
-            return
-        try:
-            new, modified, bilaterals = self.get_new_or_modified_appeals()
-        except Exception as ex:
-            logger.error(f'Getting Appeals and AppealBilaterals failed: {str(ex)}')
-            return
-        logger.info(f'{start_appeals_count} current appeals')
-        logger.info(f'Creating {len(new)} new appeals')
-        logger.info(f'Updating {len(modified)} existing appeals that MIGHT have been modified')
+    def handle(self, **_):
+        # Public
+        self.stdout.write('Fetching data for appealdocs:: PUBLIC')
+        self.load(PUBLIC_SOURCE, False)
+        # Private
+        self.stdout.write('\nFetching data for appealdocs:: FEDNET')
+        self.load(FEDNET_SOURCE, True)
+
+#        logger.info('Starting appeals ingest')
+#        start_appeals_count = Appeal.objects.all().count()
+#        try:
+#            getFromAPI = self.get_public_or_fednet_appealdocs()
+#        except Exception as ex:
+#            logger.error(f'Getting Appealdocs failed: {str(ex)}')
+#            return
+#        try:
+#            new, modified, bilaterals = self.get_new_or_modified_appeals()
+#        except Exception as ex:
+#            logger.error(f'Getting Appeals and AppealBilaterals failed: {str(ex)}')
+#            return
+#        logger.info(f'{start_appeals_count} current appeals')
+#        logger.info(f'Creating {len(new)} new appeals')
+#        logger.info(f'Updating {len(modified)} existing appeals that MIGHT have been modified')
 #
 #        errors = []
 #        num_created = 0

api/migrations/0165_auto_20230424_1725.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.18 on 2023-04-24 17:25
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0164_appealdocumenttype'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='appealdocument',
+            name='description',
+            field=models.CharField(blank=True, max_length=200, null=True, verbose_name='description'),
+        ),
+        migrations.AddField(
+            model_name='appealdocument',
+            name='type',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.appealdocumenttype', verbose_name='type'),
+        ),
+        migrations.AlterField(
+            model_name='country',
+            name='iso3',
+            field=models.CharField(blank=True, max_length=3, null=True, unique=True, validators=[django.core.validators.RegexValidator('^[A-Z]*$', 'ISO must be uppercase')], verbose_name='ISO3'),
+        ),
+    ]

api/migrations/0166_auto_20230424_1742.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2.18 on 2023-04-24 17:42
+
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0165_auto_20230424_1725'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='appealdocument',
+            name='iso3',
+            field=models.ForeignKey(db_column='iso3', null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.country', to_field='iso3', verbose_name='iso3'),
+        ),
+    ]

api/models.py

@@ -130,7 +130,7 @@ class Country(models.Model):
         verbose_name=_('ISO'), max_length=2, null=True, blank=True,
         validators=[RegexValidator('^[A-Z]*$', 'ISO must be uppercase')])
     iso3 = models.CharField(
-        verbose_name=_('ISO3'), max_length=3, null=True, blank=True,
+        verbose_name=_('ISO3'), max_length=3, null=True, blank=True, unique=True,
         validators=[RegexValidator('^[A-Z]*$', 'ISO must be uppercase')])
     fdrs = models.CharField(verbose_name=_('FDRS'), max_length=6, null=True, blank=True)
     society_name = models.TextField(verbose_name=_('society name'), blank=True)
@@ -1025,8 +1025,10 @@ class AppealDocument(models.Model):
     name = models.CharField(verbose_name=_('name'), max_length=100)
     document = models.FileField(verbose_name=_('document'), null=True, blank=True, upload_to=appeal_document_path)
     document_url = models.URLField(verbose_name=_('document url'), blank=True)
-
     appeal = models.ForeignKey(Appeal, verbose_name=_('appeal'), on_delete=models.CASCADE)
+    type = models.ForeignKey(AppealDocumentType, verbose_name=_('type'), null=True, on_delete=models.SET_NULL)
+    description = models.CharField(verbose_name=_('description'), max_length=200, null=True, blank=True)
+    iso3 = models.ForeignKey(Country, to_field="iso3", db_column="iso3", verbose_name=_('iso3'), null=True, on_delete=models.SET_NULL)
 
     class Meta:
         verbose_name = _('appeal document')