Merge pull request #1487 from IFRCGo/develop

2022 end-July release

Author: szabozoltan69

CHANGELOG.md

@@ -6,6 +6,32 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+## 1.1.446
+
+### Added
+ - Indexing ElasticSearch results by visibilities
+ - Surge map performance tuning:
+   - eliminated FR queries
+   - molnix and country stuff to prefetch
+   - so there are only 10 queries now by personnel end-point call
+   - end_date index matters (is_active not), so end_date index added
+ - Accounts with identical email addresses - use the active one.
+ - PER Overview creation fix
+
+## 1.1.445
+## 1.1.444
+## 1.1.443
+ - DREF final report
+ - Flash update notification
+ - Notification GUID details - showing "created_at"
+ - Real_data_update date comparison to appeal endpoint
+ - Rich Text Editor enhancements
+ - Adding Notes to AppealFilter
+ - Fixing flash update PDF
+ - Logging frontend login attempts
+ - Search output enhanced with visibility
+ - Contacts to FR notification
+
 ## 1.1.442
 
 ### Added
@@ -1988,7 +2014,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## 0.1.20
 
-[Unreleased]: https://github.com/IFRCGo/go-api/compare/1.1.442...HEAD
+[Unreleased]: https://github.com/IFRCGo/go-api/compare/1.1.446...HEAD
+[1.1.446]: https://github.com/IFRCGo/go-api/compare/1.1.445...1.1.446
+[1.1.445]: https://github.com/IFRCGo/go-api/compare/1.1.444...1.1.445
+[1.1.444]: https://github.com/IFRCGo/go-api/compare/1.1.443...1.1.444
+[1.1.443]: https://github.com/IFRCGo/go-api/compare/1.1.442...1.1.443
 [1.1.442]: https://github.com/IFRCGo/go-api/compare/1.1.441...1.1.442
 [1.1.441]: https://github.com/IFRCGo/go-api/compare/1.1.440...1.1.441
 [1.1.440]: https://github.com/IFRCGo/go-api/compare/1.1.439...1.1.440

api/drf_views.py

@@ -384,13 +384,13 @@ class EventViewset(ReadOnlyVisibilityViewset):
     search_fields = ('name', 'countries__name', 'dtype__name',)  # for /docs
 
     def get_queryset(self, *args, **kwargs):
-        #import pdb; pdb.set_trace();
+        # import pdb; pdb.set_trace();
         qset = super().get_queryset()
         if self.action == 'mini_events':
-            #return Event.objects.filter(parent_event__isnull=True).select_related('dtype')
+            # return Event.objects.filter(parent_event__isnull=True).select_related('dtype')
             return qset.filter(parent_event__isnull=True).select_related('dtype')
         return (
-            #Event.objects.filter(parent_event__isnull=True)
+            # Event.objects.filter(parent_event__isnull=True)
             qset.filter(parent_event__isnull=True).select_related('dtype')
             .prefetch_related(
                 'regions',
@@ -531,6 +531,7 @@ class Meta:
             'end_date': ('exact', 'gt', 'gte', 'lt', 'lte'),
             'valid_from': ('exact', 'gt', 'gte', 'lt', 'lte'),
             'valid_to': ('exact', 'gt', 'gte', 'lt', 'lte'),
+            'appeal__real_data_update': ('exact', 'gt', 'gte', 'lt', 'lte'),
         }
 
 
@@ -542,7 +543,7 @@ class AppealViewset(mixins.ListModelMixin, viewsets.GenericViewSet):
     queryset = AppealHistory.objects.select_related('appeal__event', 'dtype', 'country', 'region').filter(appeal__code__isnull=False)
     # serializer_class = AppealSerializer
     serializer_class = AppealHistorySerializer
-    ordering_fields = ('start_date', 'end_date', 'name', 'aid', 'dtype', 'num_beneficiaries',
+    ordering_fields = ('start_date', 'end_date', 'appeal__name', 'aid', 'dtype', 'num_beneficiaries',
                        'amount_requested', 'amount_funded', 'status', 'atype', 'event',)
     # filterset_class = AppealFilter
     filterset_class = AppealHistoryFilter
@@ -895,6 +896,7 @@ def create_event(self, report):
         event = Event.objects.create(
             name=report.summary,
             dtype=report.dtype,
+            summary=report.description or '',
             disaster_start_date=report.start_date,
             auto_generated=True,
             auto_generated_source=SOURCES['new_report'],

api/indexes.py

@@ -8,6 +8,8 @@
             'type': 'keyword',
             'normalizer': 'lowercase'
         },
+        'visibility': {'type': 'text'},
+        'ns': {'type': 'text'},
         'body': {
             'type': 'text',
             'analyzer': 'autocomplete'

api/management/commands/index_and_notify.py

@@ -7,6 +7,7 @@
 from django.contrib.auth.models import User
 from django.conf import settings
 from django.template.loader import render_to_string
+from django.utils.html import strip_tags
 
 from elasticsearch.helpers import bulk
 from utils.elasticsearch import construct_es_data
@@ -23,7 +24,7 @@
 time_1_week = timedelta(days=7)  # for digest mode
 digest_time = int(10314)  # weekday - hour - min for digest timing (5 minutes once a week, Monday dawn)
 daily_retro = int(654)  # hour - min for daily retropective email timing (5 minutes a day) | Should not contain a leading 0!
-max_length = 860  # after this length (at the first space) we cut the sent content
+max_length = 860  # after this length (at the first space) we cut the sent content. In case of HTML tags we remove tags (to avoid chunked tags)
 events_sent_to = {}  # to document sent events before re-sending them via specific following
 template_types = {
     99: 'design/generic_notification.html',
@@ -36,13 +37,19 @@
 
 
 # Used for 'Notification GUID' record creation in 'send_notification(4th param)'
+# Should be ~ github.com/IFRCGo/go-frontend/blob/develop/src/root/views/account.js#L80
 RTYPE_NAMES = {
-    0: 'Emergency',
-    1: 'Appeal',
-    2: 'Field Report',
+    0: 'Emergency',     # not to use in rtype_of_subscr
+    1: 'Appeal',        # not to use in rtype_of_subscr
+    2: 'Field Report',  # not to use in rtype_of_subscr – L752 depends: RTYPE_NAMES[rtype]
     3: 'Surge Alert',
+    4: 'Country',
+    5: 'Region',
+    6: 'Disaster Type',
+    7: 'PER Due Date',
     8: 'Followed Emergency',
     9: 'Surge Deployment',
+    10: 'Surge Approaching end of Mission',
     11: 'Weekly Digest',
     12: 'Field Report/Emergency',
     13: 'New Operation',
@@ -434,6 +441,7 @@ def construct_template_record(self, rtype, record):
         if rtype != RecordType.WEEKLY_DIGEST:
             shortened = self.get_record_content(record, rtype)
             if len(shortened) > max_length:
+                shortened = strip_tags(shortened)
                 shortened = shortened[:max_length] + \
                     shortened[max_length:].split(' ', 1)[0] + '...'  # look for the first space
 
@@ -442,7 +450,7 @@ def construct_template_record(self, rtype, record):
                 'resource_uri': self.get_resource_uri(record, rtype),
                 'admin_uri': self.get_admin_uri(record, rtype),
                 'title': self.get_record_title(record, rtype),
-                'description': shortened,
+                'description': shortened if shortened else None,
                 'key_figures': {
                     'affected': self.get_fieldreport_keyfigures(
                         [record.num_affected, record.gov_num_affected, record.other_num_affected]),
@@ -467,11 +475,87 @@ def construct_template_record(self, rtype, record):
                 'epi_figures_source': self.get_epi_figures_source_name(record.epi_figures_source),
                 'sit_fields_date': record.sit_fields_date,
                 'actions_taken': self.get_actions_taken(record.id),
-                'actions_others': record.actions_others,
+                'actions_others': record.actions_others.split("\n") if record.actions_others else None,
                 'gov_assistance': 'Yes' if record.request_assistance else 'No',
                 'ns_assistance': 'Yes' if record.ns_request_assistance else 'No',
                 'dtype_id': record.dtype_id,
                 'visibility': record.visibility,
+                'other_fields': {
+                    'appeal': record.appeal,
+                    'appeal_amount': record.appeal_amount,
+                    'bulletin': record.bulletin,
+                    'countries': ', '.join(i.name for i in record.countries.all()),
+                    'contacts': [{'ctype': c.ctype,
+                                  'name': c.name,
+                                  'title': c.title,
+                                  'email': c.email,
+                                  'phone': c.phone} for c in record.contacts.all()[::-1]],
+                    #not used this way, but shortened: 'description': record.description,
+                    'districts': ', '.join(i.name for i in record.districts.all()),
+                    'dref': record.dref,
+                    'dref_amount': record.dref_amount,
+                    'epi_cases_since_last_fr': record.epi_cases_since_last_fr,
+                    'epi_deaths_since_last_fr': record.epi_deaths_since_last_fr,
+                    'epi_notes_since_last_fr': record.epi_notes_since_last_fr.split("\n") if record.epi_notes_since_last_fr else None,
+                    'eru_base_camp': record.eru_base_camp,
+                    'eru_base_camp_units': record.eru_base_camp_units,
+                    'eru_basic_health_care': record.eru_basic_health_care,
+                    'eru_basic_health_care_units': record.eru_basic_health_care_units,
+                    'eru_deployment_hospital': record.eru_deployment_hospital,
+                    'eru_deployment_hospital_units': record.eru_deployment_hospital_units,
+                    'eru_it_telecom': record.eru_it_telecom,
+                    'eru_it_telecom_units': record.eru_it_telecom_units,
+                    'eru_logistics': record.eru_logistics,
+                    'eru_logistics_units': record.eru_logistics_units,
+                    'eru_referral_hospital': record.eru_referral_hospital,
+                    'eru_referral_hospital_units': record.eru_referral_hospital_units,
+                    'eru_relief': record.eru_relief,
+                    'eru_relief_units': record.eru_relief_units,
+                    'eru_water_sanitation_15': record.eru_water_sanitation_15,
+                    'eru_water_sanitation_15_units': record.eru_water_sanitation_15_units,
+                    'eru_water_sanitation_20': record.eru_water_sanitation_20,
+                    'eru_water_sanitation_20_units': record.eru_water_sanitation_20_units,
+                    'eru_water_sanitation_40': record.eru_water_sanitation_40,
+                    'eru_water_sanitation_40_units': record.eru_water_sanitation_40_units,
+                    'event': record.event,
+                    'external_partners': ', '.join(i.name for i in record.external_partners.all()),
+                    'forecast_based_action': record.forecast_based_action,
+                    'forecast_based_action_amount': record.forecast_based_action_amount,
+                    'imminent_dref': record.imminent_dref,
+                    'imminent_dref_amount': record.imminent_dref_amount,
+                    'notes_health': record.notes_health,
+                    'notes_ns': record.notes_ns,
+                    'notes_socioeco': record.notes_socioeco,
+                    'other_sources': record.other_sources.split("\n") if record.other_sources else None,
+                    'start_date': record.start_date,
+                    'supported_activities': ', '.join(i.name for i in record.supported_activities.all()),
+                    # not used: 'fact': record.fact,
+                    # not used: 'affected_pop_centres': record.affected_pop_centres,
+                    # not used: 'gov_affected_pop_centres': record.gov_affected_pop_centres,
+                    # not used: 'gov_num_highest_risk': record.gov_num_highest_risk,
+                    # not used: 'gov_num_potentially_affected': record.gov_num_potentially_affected,
+                    # not used: 'health_min_num_assisted': record.health_min_num_assisted,
+                    # not used: 'who_num_assisted': record.who_num_assisted,
+                    # not used: 'ifrc_staff': record.ifrc_staff,
+                    # not used: 'is_covid_report': record.is_covid_report,
+                    # not used: 'num_expats_delegates': record.num_expats_delegates,
+                    # not used: 'num_fact': record.num_fact,
+                    # not used: 'num_highest_risk': record.num_highest_risk,
+                    # not used: 'num_ifrc_staff': record.num_ifrc_staff,
+                    # not used: 'num_localstaff': record.num_localstaff,
+                    # not used: 'num_potentially_affected': record.num_potentially_affected,
+                    # not used: 'rdrt': record.rdrt,
+                    # not used: 'num_rdrt': record.num_rdrt,
+                    # not used: 'num_volunteers': record.num_volunteers,
+                    # not used: 'other_affected_pop_centres': record.other_affected_pop_centres,
+                    # not used: 'other_num_highest_risk': record.other_num_highest_risk,
+                    # not used: 'other_num_potentially_affected': record.other_num_potentially_affected,
+                    # not used: 'regions': ', '.join(i.label for i in record.regions.all()),
+                    # not used: 'report_date': record.report_date,
+                    # not used: 'rid': record.rid,
+                    # not used: 'status': 'Early Warning' if record.status == 8 else 'Event-related',
+                    # not used: 'summary' here, but in the title via "get_record_title"
+                },
             }
         elif rtype == RecordType.APPEAL:
             optypes = {

api/migrations/0153_auto_20220510_0730.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.27 on 2022-05-10 07:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0152_countryoffieldreporttoreview'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='fieldreport',
+            name='status',
+            field=models.IntegerField(choices=[(0, 'Unknown'), (2, 'Two'), (3, 'Three'), (8, 'Early Warning'), (9, 'Event-related'), (10, 'Ten')], default=0, verbose_name='type'),
+        ),
+    ]

api/migrations/0153_auto_20220513_0633.py

@@ -0,0 +1,24 @@
+# Generated by Django 2.2.28 on 2022-05-13 06:33
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0152_countryoffieldreporttoreview'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='action',
+            name='organizations',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(choices=[('NTLS', 'National Society'), ('PNS', 'RCRC'), ('FDRN', 'Federation'), ('GOV', 'Government')], max_length=4), blank=True, default=list, size=None, verbose_name='organizations'),
+        ),
+        migrations.AlterField(
+            model_name='actionstaken',
+            name='organization',
+            field=models.CharField(choices=[('NTLS', 'National Society'), ('PNS', 'RCRC'), ('FDRN', 'Federation'), ('GOV', 'Government')], max_length=16, verbose_name='organization'),
+        ),
+    ]

api/migrations/0154_merge_20220513_0934.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.27 on 2022-05-13 09:34
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0153_auto_20220513_0633'),
+        ('api', '0153_auto_20220510_0730'),
+    ]
+
+    operations = [
+    ]

api/migrations/0155_auto_20220621_0757.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.27 on 2022-06-21 07:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0154_merge_20220513_0934'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='fieldreport',
+            name='health_min_num_assisted',
+            field=models.IntegerField(blank=True, help_text='not used any more', null=True, verbose_name='number of assisted (ministry of health)'),
+        ),
+        migrations.AlterField(
+            model_name='fieldreport',
+            name='who_num_assisted',
+            field=models.IntegerField(blank=True, help_text='not used any more', null=True, verbose_name='number of assisted (who)'),
+        ),
+    ]

api/migrations/0156_appealfilter_comment.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.27 on 2022-06-21 09:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0155_auto_20220621_0757'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='appealfilter',
+            name='notes',
+            field=models.TextField(blank=True, null=True, verbose_name='notes'),
+        ),
+    ]