Merge pull request #1741 from IFRCGo/develop

Release 1.1.473

Author: szabozoltan69

.dockerignore

@@ -1 +1,6 @@
 .db/
+.mypy_cache/
+.pytest_cache/
+go-logs/
+media/
+go.log

CHANGELOG.md

@@ -6,10 +6,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+## 1.1.473
+
+### Added
+ - Django_haystack based, full page search
+ - Sector and SectorTag not hardwired
+ - Bump up modules (redis), snapshot test fixes
+
+## 1.1.472
 ## 1.1.471
 
 ### Added
- - Hotfix for DREF permissions
+ - Hotfix for DREF: image fields
+ - Hotfix for DREF: permissions
 
 ## 1.1.470
 
@@ -2163,7 +2172,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## 0.1.20
 
-[Unreleased]: https://github.com/IFRCGo/go-api/compare/1.1.471...HEAD
+[Unreleased]: https://github.com/IFRCGo/go-api/compare/1.1.473...HEAD
+[1.1.473]: https://github.com/IFRCGo/go-api/compare/1.1.472...1.1.473
+[1.1.472]: https://github.com/IFRCGo/go-api/compare/1.1.471...1.1.472
 [1.1.471]: https://github.com/IFRCGo/go-api/compare/1.1.470...1.1.471
 [1.1.470]: https://github.com/IFRCGo/go-api/compare/1.1.469...1.1.470
 [1.1.469]: https://github.com/IFRCGo/go-api/compare/1.1.468...1.1.469

README.md

@@ -177,13 +177,27 @@ set. Check the script for the specific variables in your environment.
 ## Deployment command
 
 ```(bash)
-docker run -p 80:80 --env-file .env -d -t ifrcgo/go-api:{TAG_NUMBER}
+docker-compose up serve celery
+```
+or (just the base serve command):
+```(bash)
+docker-compose run --rm --service-ports serve
 ```
-
 ## Comment for loading data
 
 In `main/runserver.sh` the line containing the `loaddata` command is only necessary when creating a new database. In other cases it might be causing the conflict, so it is commented. 
 
+## Initializing ElasticSearch
+
+For the initial creation of an index
+```(bash)
+docker-compose exec serve bash python manage.py rebuild_index
+```
+For updating the index
+```(bash)
+docker-compose exec serve bash python manage.py update_index
+```
+
 # Management commands to update and import admin0 and admin1 data
 
 There are two Django management commands that helps to work with ICRC admin0 and admin1 shapefiles. These commands should be used only when you want to update geometries, or import new ones from a shapefile. The structure of the shapefile is not very flexible, but can be adjusted easily in the scripts. 

api/drf_views.py

@@ -131,14 +131,15 @@
 
 class DeploymentsByEventViewset(viewsets.ReadOnlyModelViewSet):
     serializer_class = DeploymentsByEventSerializer
+    today = timezone.now().date().strftime("%Y-%m-%d")
     queryset = Event.objects.prefetch_related('personneldeployment_set__personnel_set__country_from') \
                             .annotate(
                                 personnel_count=Count(
                                     'personneldeployment__personnel',
                                     filter=Q(
                                         personneldeployment__personnel__type=Personnel.TypeChoices.RR,
-                                        personneldeployment__personnel__start_date__lte=timezone.now(),
-                                        personneldeployment__personnel__end_date__gte=timezone.now(),
+                                        personneldeployment__personnel__start_date__date__lte=today,
+                                        personneldeployment__personnel__end_date__date__gte=today,
                                         personneldeployment__personnel__is_active=True
                                     )
                                 )
@@ -151,9 +152,10 @@ class EventDeploymentsViewset(viewsets.ReadOnlyModelViewSet):
     serializer_class = ListEventDeploymentsSerializer
 
     def get_queryset(self):
+        today = timezone.now().date().strftime("%Y-%m-%d")
         return Personnel.objects.filter(
-            start_date__lte=timezone.now(),
-            end_date__gte=timezone.now(),
+            start_date__date__lte=today,
+            end_date__date__gte=today,
             is_active=True
         ).order_by().values(
             'deployment__event_deployed_to', 'type',

api/management/commands/sync_molnix.py

@@ -55,6 +55,9 @@ def add_tags(molnix_tags):
         tag.molnix_id = molnix_tag['id']
         tag.name = n = molnix_tag['name']
         tag.description = molnix_tag['description']
+        if tag.description is None:
+            tag.description = ''
+            logger.warning('%s named tag has no description.' % tag.name)
         tag.tag_type = molnix_tag['type']
         tag.tag_category = 'molnix_language' if n.startswith('L-') else \
             'molnix_operation' if n.startswith('OP-') else \

api/migrations/0162_admin2_created_at.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2.18 on 2023-03-31 11:42
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0161_alter_event_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='admin2',
+            name='created_at',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now, verbose_name='Created at'),
+            preserve_default=False,
+        ),
+    ]

api/models.py

@@ -273,7 +273,7 @@ class Admin2(models.Model):
     local_name_code = models.CharField(verbose_name=_('Local Name Language Code'), max_length=10, blank=True, null=True)
     alternate_name = models.CharField(verbose_name=_('Alternate Name'), max_length=100, blank=True, null=True)
     alternate_name_code = models.CharField(verbose_name=_('Alternate Name Language Code'), max_length=10, blank=True, null=True)
-
+    created_at = models.DateTimeField(verbose_name=_('Created at'), auto_now_add=True)
 
     class Meta:
         verbose_name = _('admin2')
@@ -365,7 +365,7 @@ class TabNumber(models.IntegerChoices):
     TAB_3 = 3, _('Tab 3')
 
 
-@reversion.register()
+@reversion.register(follow=('region',))
 class RegionSnippet(models.Model):
     region = models.ForeignKey(Region, verbose_name=_('region'), related_name='snippets', on_delete=models.CASCADE)
     snippet = HTMLField(verbose_name=_('snippet'), null=True, blank=True)
@@ -382,7 +382,7 @@ def __str__(self):
         return self.snippet
 
 
-@reversion.register()
+@reversion.register(follow=('region',))
 class RegionEmergencySnippet(models.Model):
     region = models.ForeignKey(Region, verbose_name=_('region'), related_name='emergency_snippets', on_delete=models.CASCADE)
     title = models.CharField(max_length=255, blank=True)
@@ -399,7 +399,7 @@ def __str__(self):
         return self.snippet
 
 
-@reversion.register()
+@reversion.register(follow=('region',))
 class RegionPreparednessSnippet(models.Model):
     region = models.ForeignKey(Region, verbose_name=_('region'), related_name='preparedness_snippets', on_delete=models.CASCADE)
     title = models.CharField(max_length=255, blank=True)
@@ -416,7 +416,7 @@ def __str__(self):
         return self.snippet
 
 
-@reversion.register()
+@reversion.register(follow=('region',))
 class RegionProfileSnippet(models.Model):
     region = models.ForeignKey(Region, verbose_name=_('region'), related_name='profile_snippets', on_delete=models.CASCADE)
     title = models.CharField(max_length=255, blank=True)
@@ -442,7 +442,7 @@ def __str__(self):
 #         return self.title
 
 
-@reversion.register()
+@reversion.register(follow=('country',))
 class CountrySnippet(models.Model):
     country = models.ForeignKey(Country, verbose_name=_('country'), related_name='snippets', on_delete=models.CASCADE)
     snippet = HTMLField(verbose_name=_('snippet'), null=True, blank=True)
@@ -736,7 +736,7 @@ class Meta:
         ordering = ('id',)
 
 
-@reversion.register()
+@reversion.register(follow=('event',))
 class Snippet(models.Model):
     """ Snippet of text """
     snippet = HTMLField(verbose_name=_('snippet'), null=True, blank=True)

api/receivers.py

@@ -1,9 +1,13 @@
 import json
+from datetime import datetime
 from dateutil.relativedelta import relativedelta
+
 from django.db import transaction
 from django.db.models import Q
 from django.db.models.signals import pre_delete, pre_save, post_save, post_delete
 from django.dispatch import receiver
+from django.utils import timezone
+
 from reversion.models import Version
 from reversion.signals import post_revision_commit
 from api.models import ReversionDifferenceLog, Event, Country, FieldReport
@@ -12,8 +16,7 @@
 from utils.erp import push_fr_data
 from api.logger import logger
 from .models import Appeal, AppealHistory, AppealFilter
-from django.utils import timezone
-from datetime import datetime, timedelta
+from main.suspend_receivers import suspendingreceiver
 
 
 MODEL_TYPES = {
@@ -136,7 +139,8 @@ def log_deletion(sender, instance, using, **kwargs):
     delete_es_index(instance)
 
 
-@receiver(pre_save)
+# NOTE: Adding this to disable indexing in testcases
+@suspendingreceiver(pre_save)
 def remove_child_events_from_es(sender, instance, using, **kwargs):
     ''' Handle Emergency Elasticsearch indexes '''
     model = instance.__class__.__name__
@@ -202,7 +206,6 @@ def add_update_appeal_history(sender, instance, created, **kwargs):
         if field.name not in ['id', 'appeal', 'valid_from', 'valid_to','aid','amount_funded']
     ]
     now = timezone.now()
-    valid_to = timezone.now() + relativedelta(years=+10)
     changed = False
 
     if created:  # Appeal Insert
@@ -220,12 +223,12 @@ def add_update_appeal_history(sender, instance, created, **kwargs):
             atype=instance.atype,
             country=instance.country,
             region=instance.region,
-            dtype = instance.dtype,
-            needs_confirmation = instance.needs_confirmation,
-            status = instance.status,
-            code = instance.code,
-            triggering_amount = instance.triggering_amount
-            #deleted_at = instance.deleted_at
+            dtype=instance.dtype,
+            needs_confirmation=instance.needs_confirmation,
+            status=instance.status,
+            code=instance.code,
+            triggering_amount=instance.triggering_amount
+            # deleted_at = instance.deleted_at
         )
 
     else:  # Appeal Update
@@ -258,22 +261,22 @@ def add_update_appeal_history(sender, instance, created, **kwargs):
             atype=instance.atype,
             country=instance.country,
             region=instance.region,
-            dtype = instance.dtype,
-            needs_confirmation = instance.needs_confirmation,
-            status = instance.status,
-            code = instance.code,
-            triggering_amount = instance.triggering_amount
-            #deleted_at = instance.deleted_at
+            dtype=instance.dtype,
+            needs_confirmation=instance.needs_confirmation,
+            status=instance.status,
+            code=instance.code,
+            triggering_amount=instance.triggering_amount
+            # deleted_at = instance.deleted_at
         )
 
 
 @receiver(post_delete, sender=Appeal)
 def remove_appeal_filter(sender, instance, using, **kwargs):
     appealFilter = AppealFilter.objects.get(name='ingestAppealFilter')
     lstCodesToSkip = appealFilter.value.split(",")
-    if instance.code not in lstCodesToSkip: 
-            lstCodesToSkip.append(instance.code)
+    if instance.code not in lstCodesToSkip:
+        lstCodesToSkip.append(instance.code)
 
     appealFilter.value = ",".join(lstCodesToSkip)
-        #appealFilter.value = ",".join(appealFilter.value.split(",").remove(self.code))
-    appealFilter.save()
+    # appealFilter.value = ",".join(appealFilter.value.split(",").remove(self.code))
+    appealFilter.save()