Merge pull request #1670 from IFRCGo/develop

2023.02 Release

Author: szabozoltan69

.github/workflows/add-issue-to-backlog.yml

@@ -0,0 +1,16 @@
+name: Add issues to Backlog
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+        with:
+          project-url: https://github.com/orgs/IFRCGo/projects/12
+          github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}

CHANGELOG.md

@@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+## 1.1.469
+
+### Added
+ - Ingest country plan and internal plan files
+ - Bump up cryptography and django modules
+ - Surge Alert statuses: Open, Closed, Stood down
+
 ## 1.1.468
 
 ### Added
@@ -2143,7 +2150,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## 0.1.20
 
-[Unreleased]: https://github.com/IFRCGo/go-api/compare/1.1.468...HEAD
+[Unreleased]: https://github.com/IFRCGo/go-api/compare/1.1.469...HEAD
+[1.1.469]: https://github.com/IFRCGo/go-api/compare/1.1.468...1.1.469
 [1.1.468]: https://github.com/IFRCGo/go-api/compare/1.1.467...1.1.468
 [1.1.467]: https://github.com/IFRCGo/go-api/compare/1.1.466...1.1.467
 [1.1.466]: https://github.com/IFRCGo/go-api/compare/1.1.465...1.1.466

api/management/commands/sync_molnix.py

@@ -107,6 +107,7 @@ def get_datetime(datetime_string):
         return None
     return date_parser.parse(datetime_string)
 
+
 def get_status_message(positions_messages, deployments_messages, positions_warnings, deployments_warnings):
     msg = ''
     msg += 'Summary of Open Positions Import:\n\n'
@@ -125,26 +126,21 @@ def get_status_message(positions_messages, deployments_messages, positions_warni
         msg += '\n\n'
     return msg
 
-def add_tags_to_obj(obj, tags):
-    # We clear all tags first, and then re-add them
-    tag_molnix_ids = [t['id'] for t in tags]
-    obj.molnix_tags.clear()
-    for molnix_id in tag_molnix_ids:
-        try:
-            t = MolnixTag.objects.get(molnix_id=molnix_id)
-        except:
-            logger.error('ERROR - tag not found: %d' % molnix_id)
-            continue
-        obj.molnix_tags.add(t)
-    obj.save()
 
+def add_tags_to_obj(obj, tags):
+    _ids = [int(t['id']) for t in tags]
+    tags = list(  # Fetch all at once
+        MolnixTag.objects.filter(molnix_id__in=_ids)
+    )
+    if len(tags) != len(_ids):  # Show warning if all tags are not available
+        missing_tag_ids = list(set(_ids) - set([tag.id for tag in tags]))
+        logger.warning(f'Missing _ids: {missing_tag_ids}')
+        # or   ^^^^^^^ logger.error if we need to add molnix tags manually.
+    obj.molnix_tags.set(tags)  # Add new ones, remove old ones
 
 
 def sync_deployments(molnix_deployments, molnix_api, countries):
-    #import json
-    #print(json.dumps(molnix_deployments, indent=2))
     molnix_ids = [d['id'] for d in molnix_deployments]
-
     warnings = []
     messages = []
     successful_creates = 0

country_plan/management/commands/ingest_country_plan_file.py

@@ -4,17 +4,20 @@
 from datetime import datetime
 from django.utils.timezone import make_aware
 from django.core.management.base import BaseCommand
+from django.core.files.base import File
 from django.conf import settings
 from django.db import models
 
-from api.models import CronJob, CronJobStatus
+from main.utils import DownloadFileManager
+from api.models import Country
 from country_plan.models import CountryPlan
 
 logger = logging.getLogger(__name__)
 
-NAME = 'ingest_country_plan_file'
 # Ref: https://github.com/IFRCGo/go-api/issues/1614
-SOURCE = 'https://go-api.ifrc.org/api/publicsiteappeals?AppealsTypeID=1851&Hidden=false'
+PUBLIC_SOURCE = 'https://go-api.ifrc.org/api/publicsiteappeals?AppealsTypeID=1851&Hidden=false'
+# Ref: https://github.com/IFRCGo/go-api/issues/1648
+INTERNAL_SOURCE = 'https://go-api.ifrc.org/Api/FedNetAppeals?AppealsTypeId=1844&Hidden=false'
 
 
 class Command(BaseCommand):
@@ -33,7 +36,15 @@ def parse_date(text: str) -> Union[datetime, None]:
                 datetime.strptime(text, '%Y-%m-%dT%H:%M:%S')
             )
 
-    def load_for_country(self, country_data):
+    @staticmethod
+    def load_file_to_country_plan(country_plan: CountryPlan, url: str, filename: str, field_name: str):
+        with DownloadFileManager(url, suffix='.pdf') as f:
+            getattr(country_plan, field_name).save(
+                filename,
+                File(f),
+            )
+
+    def load_for_country(self, country_data, file_field, field_inserted_date_field):
         country_iso2 = country_data.get('LocationCountryCode')
         country_name = country_data.get('LocationCountryName')
         public_plan_url = country_data.get('BaseDirectory') or '' + country_data.get('BaseFileName') or ''
@@ -44,69 +55,56 @@ def load_for_country(self, country_data):
             inserted_date is None
         ):
             return
-        country_plan = CountryPlan.objects.filter(
-            models.Q(country__iso__iexact=country_iso2) |
-            models.Q(country__name__iexact=country_name)
-        ).first()
+        country_qs = Country.objects.filter(
+            models.Q(iso__iexact=country_iso2) |
+            models.Q(name__iexact=country_name)
+        )
+        country_plan = CountryPlan.objects.filter(country__in=country_qs).first()
         if country_plan is None:
-            logger.warning(f'{NAME} No country_plan found for: {(country_iso2, country_name)}')
-            return
-        if country_plan.appeal_api_inserted_date and country_plan.appeal_api_inserted_date >= inserted_date:
+            country = country_qs.first()
+            # If there is no country as well, show warning and return
+            if not country:
+                logger.warning(f'{file_field} No country found for: {(country_iso2, country_name)}')
+                return
+            # Else create one and continue
+            country_plan = CountryPlan(country=country)
+        existing_inserted_date = getattr(country_plan, field_inserted_date_field, None)
+        if existing_inserted_date and existing_inserted_date >= inserted_date:
             # No need to do anything here
             return
+        self.stdout.write(f'- Saving data for country:: {country_plan.country.name}')
         public_plan_url = country_data['BaseDirectory'] + country_data['BaseFileName']
-        country_plan.appeal_api_inserted_date = inserted_date
-        country_plan.load_file_to_country_plan(
+        setattr(country_plan, field_inserted_date_field, inserted_date)
+        self.load_file_to_country_plan(
+            country_plan,
             public_plan_url,
             # NOTE: File provided are PDF,
-            f"public-plan-{country_data['BaseFileName']}.pdf",
+            f"{file_field.replace('_', '-').replace('file', '')}-{country_data['BaseFileName']}.pdf",
+            file_field,
         )
         country_plan.is_publish = True
         country_plan.save(
             update_fields=(
-                'appeal_api_inserted_date',
-                'public_plan_file',  # By load_file_to_country_plan
+                field_inserted_date_field,
+                file_field,  # By load_file_to_country_plan
                 'is_publish',
             )
         )
         return True
 
-    def load(self):
-        updated = 0
+    def load(self, url: str, file_field: str, field_inserted_date_field: str):
         auth = (settings.APPEALS_USER, settings.APPEALS_PASS)
-        results = requests.get(SOURCE, auth=auth, headers={'Accept': 'application/json'}).json()
+        results = requests.get(url, auth=auth, headers={'Accept': 'application/json'}).json()
         for result in results:
             try:
-                if self.load_for_country(result):
-                    updated += 1
-            except Exception as ex:
+                self.load_for_country(result, file_field, field_inserted_date_field)
+            except Exception:
                 logger.error('Could not Updated countries plan', exc_info=True)
-                country_info = (
-                    result.get('LocationCountryCode'),
-                    result.get('LocationCountryName'),
-                )
-                CronJob.sync_cron({
-                    'name': NAME,
-                    'message': f"Could not updated country plan for {country_info}\n\nException:\n{str(ex)}",
-                    'status': CronJobStatus.ERRONEOUS,
-                })
-        return updated
 
-    def handle(self, *args, **kwargs):
-        try:
-            logger.info('\nFetching data for country plans:: ')
-            countries_plan_updated = self.load()
-            CronJob.sync_cron({
-                'name': NAME,
-                'message': 'Updated countries plan',
-                'num_result': countries_plan_updated,
-                'status': CronJobStatus.SUCCESSFUL,
-            })
-            logger.info('Updated countries plan')
-        except Exception as ex:
-            logger.error('Could not Updated countries plan', exc_info=True)
-            CronJob.sync_cron({
-                'name': NAME,
-                'message': f'Could not Updated countries plan\n\nException:\n{str(ex)}',
-                'status': CronJobStatus.ERRONEOUS,
-            })
+    def handle(self, **_):
+        # Public
+        self.stdout.write('Fetching data for country plans:: PUBLIC')
+        self.load(PUBLIC_SOURCE, 'public_plan_file', 'public_plan_inserted_date')
+        # Private
+        self.stdout.write('\nFetching data for country plans:: PRIVATE')
+        self.load(INTERNAL_SOURCE, 'internal_plan_file', 'internal_plan_inserted_date')

country_plan/migrations/0005_auto_20230207_0840.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.16 on 2023-02-07 08:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('country_plan', '0004_countryplan_appeal_api_inserted_date'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='countryplan',
+            old_name='appeal_api_inserted_date',
+            new_name='public_plan_inserted_date',
+        ),
+        migrations.AddField(
+            model_name='countryplan',
+            name='internal_plan_inserted_date',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

country_plan/models.py

@@ -3,9 +3,7 @@
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from django.core.validators import FileExtensionValidator
-from django.core.files.base import File
 
-from main.utils import DownloadFileManager
 from api.models import Country
 
 
@@ -83,21 +81,12 @@ class CountryPlan(CountryPlanAbstract):
     is_publish = models.BooleanField(default=False, verbose_name=_('Published'))
 
     # NOTE: Used to sync with Appeal API (ingest_country_plan_file.py for more info)
-    appeal_api_inserted_date = models.DateTimeField(blank=True, null=True)
+    public_plan_inserted_date = models.DateTimeField(blank=True, null=True)
+    internal_plan_inserted_date = models.DateTimeField(blank=True, null=True)
 
     def __str__(self):
         return f'{self.country}'
 
-    def load_file_to_country_plan(self, url: str, filename: str, commit=False):
-        with DownloadFileManager(url, suffix='.pdf') as f:
-            self.public_plan_file.save(
-                filename,
-                File(f),
-            )
-        if commit:
-            self.save()
-
-    # NOTE: To be used by CountryPlanSerializer (From CountryPlanViewset)
     def full_country_plan_mc(self):
         all_mc = list(self.country_plan_mc.all())
         mc_by_ns_sector = {

country_plan/tests/test_commands.py

@@ -5,11 +5,11 @@
 from api.factories.country import CountryFactory
 from country_plan.factories import CountryPlanFactory
 from country_plan.models import CountryPlan
-from country_plan.management.commands.ingest_country_plan_file import SOURCE
+from country_plan.management.commands.ingest_country_plan_file import PUBLIC_SOURCE, INTERNAL_SOURCE
 
 # NOTE: Only defined used fields
 FILE_BASE_DIRECTORY = 'https://example.org/Download.aspx?FileId='
-APPEAL_COUNTRY_PLAN_MOCK_RESPONSE = [
+PUBLIC_APPEAL_COUNTRY_PLAN_MOCK_RESPONSE = [
     {
         'BaseDirectory': FILE_BASE_DIRECTORY,
         'BaseFileName': '000000',
@@ -24,6 +24,7 @@
         'LocationCountryCode': 'CD',
         'LocationCountryName': 'Congo, The Democratic Republic Of The'
     },
+    # Not included in INTERNAL
     {
         'BaseDirectory': FILE_BASE_DIRECTORY,
         'BaseFileName': '000002',
@@ -47,6 +48,31 @@
     }
 ]
 
+INTERNAL_APPEAL_COUNTRY_PLAN_MOCK_RESPONSE = [
+    {
+        'BaseDirectory': FILE_BASE_DIRECTORY,
+        'BaseFileName': '000000',
+        'Inserted': '2022-11-29T11:24:00',
+        'LocationCountryCode': 'SY',
+        'LocationCountryName': 'Syrian Arab Republic'
+    },
+    {
+        'BaseDirectory': FILE_BASE_DIRECTORY,
+        'BaseFileName': '000001',
+        'Inserted': '2022-11-29T11:24:00',
+        'LocationCountryCode': 'CD',
+        'LocationCountryName': 'Congo, The Democratic Republic Of The'
+    },
+    # Not included in PUBLIC
+    {
+        'BaseDirectory': FILE_BASE_DIRECTORY,
+        'BaseFileName': '000001',
+        'Inserted': '2022-11-29T11:24:00',
+        'LocationCountryCode': 'NP',
+        'LocationCountryName': 'Nepal'
+    },
+]
+
 
 class MockResponse():
     class FileStream():
@@ -75,8 +101,10 @@ def __exit__(self, *_):
 
 class CountryPlanIngestTest(APITestCase):
     def mock_request(url, *_, **kwargs):
-        if url == SOURCE:
-            return MockResponse(json=APPEAL_COUNTRY_PLAN_MOCK_RESPONSE)
+        if url == PUBLIC_SOURCE:
+            return MockResponse(json=PUBLIC_APPEAL_COUNTRY_PLAN_MOCK_RESPONSE)
+        if url == INTERNAL_SOURCE:
+            return MockResponse(json=INTERNAL_APPEAL_COUNTRY_PLAN_MOCK_RESPONSE)
         if url.startswith(FILE_BASE_DIRECTORY):
             return MockResponse(stream=[b''])
 
@@ -89,17 +117,25 @@ def test_country_plan_ingest(self, *_):
                 iso='RC',
             ),
         )
-        for country_plan_data in APPEAL_COUNTRY_PLAN_MOCK_RESPONSE[:3]:
-            CountryPlanFactory.create(
-                country=CountryFactory.create(
-                    name=country_plan_data['LocationCountryName'],
-                    iso=country_plan_data['LocationCountryCode'],
-                ),
-            )
-        assert CountryPlan.objects.count() == 4
+        EXISTING_CP = 1
+        for country_iso in set([
+                item['LocationCountryCode']
+                for item in [
+                    *PUBLIC_APPEAL_COUNTRY_PLAN_MOCK_RESPONSE,
+                    *INTERNAL_APPEAL_COUNTRY_PLAN_MOCK_RESPONSE,
+                ]
+        ]):
+            if country_iso == 'CD':
+                # Not create country for this
+                continue
+            CountryFactory.create(iso=country_iso)
+        # Before
+        assert CountryPlan.objects.count() == EXISTING_CP
         assert CountryPlan.objects.filter(is_publish=True).count() == 0
         assert CountryPlan.objects.exclude(public_plan_file='').count() == 0
         call_command('ingest_country_plan_file')
-        assert CountryPlan.objects.count() == 4
-        assert CountryPlan.objects.filter(is_publish=True).count() == 3
-        assert CountryPlan.objects.exclude(public_plan_file='').count() == 3
+        # After
+        assert CountryPlan.objects.count() == EXISTING_CP + 5
+        assert CountryPlan.objects.filter(is_publish=True).count() == 5
+        assert CountryPlan.objects.exclude(public_plan_file='').count() == 4
+        assert CountryPlan.objects.exclude(internal_plan_file='').count() == 2