Merge pull request #1537 from IFRCGo/fix/dref-import

Fix/dref import

Author: szabozoltan69

dref/assessment_utils.py

@@ -12,6 +12,7 @@
 from api.models import (
     DisasterType,
     Country,
+    District,
 )
 
 from .common_utils import (
@@ -23,6 +24,10 @@
     get_table_data,
     get_paragraphs_data,
     parse_disaster_category,
+    parse_contact_information,
+    parse_people,
+    parse_country,
+    parse_currency,
 )
 
 
@@ -117,6 +122,7 @@ def get_nth(items: List[Any], index=0):
 
     def get_table_cells(table_idx):
         table = tables[table_idx]
+
         def f(r, c, i=0, as_list=False):
             if as_list:
                 return table[r][c]
@@ -131,24 +137,42 @@ def f(r, c, i=0, as_list=False):
 
     cells = get_table_cells(0)
     data['appeal_code'] = cells(1, 0)
-    data['amount_requested'] = parse_string_to_int(cells(1, 1, 1))
+    data['amount_requested'] = parse_currency(cells(1, 1))
     data['disaster_category'] = parse_disaster_category(cells(1, 2))
     data['disaster_type'] = parse_disaster_type(cells(1, 4))
     data['glide_code'] = cells(3, 0)
-    data['num_affected'] = parse_string_to_int(cells(3, 1))
-    data['num_assisted'] = parse_string_to_int(cells(3, 2))
+    data['num_affected'] = parse_people(cells(3, 1))
+    data['num_assisted'] = parse_people(cells(3, 2))
     data['type_of_onset'] = parse_type_of_onset(cells(5, 0))
     data['date_of_approval'] = parse_date(cells(5, 1))
     data['end_date'] = parse_date(cells(5, 2))
-    data['operation_timeframe'] = parse_int(cells(5, 3))
-    country = Country.objects.filter(name__icontains=cells(6, 0, 1)).first()
+    data['operation_timeframe'] = parse_people(cells(5, 3))
+    country = Country.objects.filter(name__icontains=parse_country(cells(6, 0))).first()
     if country is None:
         raise serializers.ValidationError('A valid country name is required')
     data['country'] = country
 
-    # TODO: Check for District
-    description = paragraphs[7] or []
-    data['event_description'] = ''.join(description) if description else None
+    district = cells(6, 2)
+    district_list = []
+    if district:
+        new_district = district.split(',')
+        for d in new_district:
+            try:
+                district = District.objects.filter(
+                    country__name__icontains=parse_country(cells(6, 0)),
+                    name__icontains=d
+                ).first()
+            except District.DoesNotExist:
+                pass
+            if district is None:
+                continue
+            district_list.append(district)
+    try:
+        if paragraphs[6][0] == 'What happened, where and when?':
+            description = paragraphs[7] or []
+            data['event_description'] = ''.join(description) if description else None
+    except IndexError:
+        pass
 
     # National Society Actions
     cells = get_table_cells(1)
@@ -207,26 +231,42 @@ def f(r, c, i=0, as_list=False):
     national_authorities = cells(1, 1, as_list=True)
     data['national_authorities'] = ''.join(national_authorities) if national_authorities else None
 
-    un_and_other_actors = cells(2, 1, as_list=True)
+    un_and_other_actors = cells(2, 0, as_list=True)
     data['un_or_other_actor'] = ''.join(un_and_other_actors) if un_and_other_actors else None
 
     coordination_mechanism = cells(3, 1, as_list=True)
     data['major_coordination_mechanism'] = ''.join(coordination_mechanism) if coordination_mechanism else None
 
     # Operational Strategy
-    operation_description = paragraphs[20] or []
-    data['operation_objective'] = ''.join(operation_description) if operation_description else None
+    try:
+        if paragraphs[18][0] == 'Overall objective of the operation':
+            operation_description = paragraphs[19] or []
+            data['operation_objective'] = ''.join(operation_description) if operation_description else None
+    except IndexError:
+        pass
 
     # targeting strategy
-    response_description = paragraphs[23] or []
-    data['response_strategy'] = ''.join(response_description) if response_description else None
+    try:
+        if paragraphs[20][0] == 'Response strategy rationale':
+            response_description = paragraphs[21] or []
+            data['response_strategy'] = ''.join(response_description) if response_description else None
+    except IndexError:
+        pass
 
     # Targeting Strategy
-    people_assisted_description = paragraphs[27] or []
-    data['people_assisted'] = ''.join(people_assisted_description) if people_assisted_description else None
+    try:
+        if paragraphs[23][0] == 'Who will be targeted through this operation?':
+            people_assisted_description = paragraphs[24] or []
+            data['people_assisted'] = ''.join(people_assisted_description) if people_assisted_description else None
+    except IndexError:
+        pass
 
-    selection_criteria_description = paragraphs[31] or []
-    data['selection_criteria'] = ''.join(selection_criteria_description) if selection_criteria_description else None
+    try:
+        if paragraphs[26][0] == 'Explain the selection criteria for the targeted population':
+            selection_criteria_description = paragraphs[27] or []
+            data['selection_criteria'] = ''.join(selection_criteria_description) if selection_criteria_description else None
+    except IndexError:
+        pass
 
     # Targeting Population
     cells = get_table_cells(4)
@@ -291,45 +331,62 @@ def f(r, c, i=0, as_list=False):
             planned_intervention.append(planned)
 
     # About Support Service
-    human_resource_description = paragraphs[55] or []
-    data['human_resource'] = ''.join(human_resource_description) if human_resource_description else None
-
-    surge_personnel_deployed_description = paragraphs[57] or []
-    data['surge_personnel_deployed'] = ''.join(surge_personnel_deployed_description) if surge_personnel_deployed_description else None
-
-    national_society_contact = paragraphs[63] or []
-    data['national_society_contact_title'] = get_nth(national_society_contact, 3)
-    data['national_society_contact_email'] = get_nth(national_society_contact, 5)
-    data['national_society_contact_phone_number'] = get_nth(national_society_contact, 7)
-    data['national_society_contact_name'] = get_nth(national_society_contact, 0)
-
-    ifrc_appeal_manager = paragraphs[64] or []
-    data['ifrc_appeal_manager_title'] = get_nth(ifrc_appeal_manager, 3)
-    data['ifrc_appeal_manager_email'] = get_nth(ifrc_appeal_manager, 5)
-    data['ifrc_appeal_manager_phone_number'] = get_nth(ifrc_appeal_manager, 7)
-    data['ifrc_appeal_manager_name'] = get_nth(ifrc_appeal_manager, 0)
-
-    ifrc_project_manager = paragraphs[65] or []
-    data['ifrc_project_manager_title'] = get_nth(ifrc_project_manager, 3)
-    data['ifrc_project_manager_email'] = get_nth(ifrc_project_manager, 5)
-    data['ifrc_project_manager_phone_number'] = get_nth(ifrc_project_manager, 7)
-    data['ifrc_project_manager_name'] = get_nth(ifrc_project_manager, 0)
+    try:
+        if paragraphs[50][0] == 'How many volunteers and staff involved in the response? Briefly describe their role.':
+            human_resource_description = paragraphs[51] or []
+            data['human_resource'] = ''.join(human_resource_description) if human_resource_description else None
+    except IndexError:
+        pass
+
+    try:
+        if paragraphs[52][0] == 'Will surge personnel be deployed? Please provide the role profile needed.':
+            surge_personnel_deployed_description = paragraphs[53] or []
+            data['surge_personnel_deployed'] = ''.join(surge_personnel_deployed_description) if surge_personnel_deployed_description else None
+    except IndexError:
+        pass
+
+    try:
+        national_society_contact = parse_contact_information(paragraphs[60] or [])
+        data['national_society_contact_title'] = get_nth(national_society_contact, 2)
+        data['national_society_contact_email'] = get_nth(national_society_contact, 4)
+        data['national_society_contact_phone_number'] = get_nth(national_society_contact, 6)
+        data['national_society_contact_name'] = get_nth(national_society_contact, 0)
+    except IndexError:
+        pass
+
+    try:
+        ifrc_appeal_manager = parse_contact_information(paragraphs[61] or [])
+        data['ifrc_appeal_manager_title'] = get_nth(ifrc_appeal_manager, 2)
+        data['ifrc_appeal_manager_email'] = get_nth(ifrc_appeal_manager, 4)
+        data['ifrc_appeal_manager_phone_number'] = get_nth(ifrc_appeal_manager, 6)
+        data['ifrc_appeal_manager_name'] = get_nth(ifrc_appeal_manager, 0)
+    except IndexError:
+        pass
+
+    try:
+        ifrc_project_manager = parse_contact_information(paragraphs[62] or [])
+        data['ifrc_project_manager_title'] = get_nth(ifrc_project_manager, 2)
+        data['ifrc_project_manager_email'] = get_nth(ifrc_project_manager, 4)
+        data['ifrc_project_manager_phone_number'] = get_nth(ifrc_project_manager, 6)
+        data['ifrc_project_manager_name'] = get_nth(ifrc_project_manager, 0)
+    except IndexError:
+        pass
 
     try:
-        ifrc_emergency = paragraphs[66] or []
-        data['ifrc_emergency_title'] = get_nth(ifrc_emergency, 3)
-        data['ifrc_emergency_email'] = get_nth(ifrc_emergency, 5)
-        data['ifrc_emergency_phone_number'] = get_nth(ifrc_emergency, 7)
+        ifrc_emergency = parse_contact_information(paragraphs[63] or [])
+        data['ifrc_emergency_title'] = get_nth(ifrc_emergency, 2)
+        data['ifrc_emergency_email'] = get_nth(ifrc_emergency, 4)
+        data['ifrc_emergency_phone_number'] = get_nth(ifrc_emergency, 6)
         data['ifrc_emergency_name'] = get_nth(ifrc_emergency, 0)
     except IndexError:
         # FIXME: raise validation error
         pass
 
     try:
-        media = paragraphs[67] or []
-        data['media_contact_title'] = get_nth(media, 3)
-        data['media_contact_email'] = get_nth(media, 5)
-        data['media_contact_phone_number'] = get_nth(media, 7)
+        media = parse_contact_information(paragraphs[64] or [])
+        data['media_contact_title'] = get_nth(media, 2)
+        data['media_contact_email'] = get_nth(media, 4)
+        data['media_contact_phone_number'] = get_nth(media, 6)
         data['media_contact_name'] = get_nth(media, 0)
     except IndexError:
         # FIXME: raise validation error
@@ -343,4 +400,6 @@ def f(r, c, i=0, as_list=False):
     dref.planned_interventions.add(*planned_intervention)
     dref.national_society_actions.add(*national_societies)
     dref.risk_security.add(*mitigation_list)
+    if len(district_list) > 0 and None not in district_list:
+        dref.district.add(*district_list)
     return dref

dref/common_utils.py

@@ -73,6 +73,7 @@ def get_table_data(doc):
         tables.append(rowdata)
     return tables
 
+
 def get_paragraphs_data(doc):
     document = docx.Document(doc)
     return [[y.text for y in x._element.xpath('.//w:t')] for x in document.paragraphs]
@@ -86,3 +87,50 @@ def parse_disaster_category(disaster_category):
     elif disaster_category == 'Red':
         return Dref.DisasterCategory.RED
     return None
+
+
+def parse_contact_information(items: List[Any]):
+    result = []
+    for data in items:
+        if data in [
+            'National Society contact: ',
+            'IFRC Appeal Manager: ',
+            'IFRC Project Manager: ',
+            'IFRC focal point for the emergency: ',
+            'Media Contact: '
+            ', ',
+            '• ',
+            'Media Contact: ',
+            '• IFRC Project Manager: ',
+            '• IFRC Project Manager:',
+        ]:
+            continue
+        result.append(data)
+    return result
+
+
+def parse_people(string):
+    try:
+        if string:
+            people = string.split(' ')
+            return int(people[0])
+    except (IndexError, ValueError):
+        pass
+
+
+def parse_country(country):
+    try:
+        if country:
+            new_country = country.split(' ')
+            return new_country[1]
+    except (IndexError, ValueError):
+        pass
+
+
+def parse_currency(currency):
+    try:
+        if currency:
+            new_currency = currency.split(' ')
+            return int(new_currency[1])
+    except (IndexError, ValueError):
+        pass