Merge pull request #2312 from IFRCGo/feature/new-notif-strategy

Use dtype, country and region to filter notifications

Author: szabozoltan69

api/management/commands/index_and_notify.py

@@ -116,6 +116,32 @@ def gather_countries_and_regions(self, records):
         countries = ["c%s" % id for id in countries]
         return countries, regions
 
+    def gather_event_countries_and_regions(self, records):
+        # Applies to surgealerts, which have a
+        # many-to-many relationship to countries and regions through event table
+        countries = []
+        for record in records:
+            if record.event.countries is not None:
+                countries += [country.id for country in record.event.countries.all()]
+        countries = list(set(countries))
+        qs = Country.objects.filter(pk__in=countries)
+        regions = ["r%s" % country.region.id for country in qs if country.region is not None]
+        countries = ["c%s" % id for id in countries]
+        return countries, regions
+
+    def gather_eventdt_countries_and_regions(self, records):
+        # Applies to deployments_personneldeployments, which have a
+        # many-to-many relationship to countries and regions through event_deployed_to
+        countries = []
+        for record in records:
+            if record.event_deployed_to.countries is not None:
+                countries += [country.id for country in record.event_deployed_to.countries.all()]
+        countries = list(set(countries))
+        qs = Country.objects.filter(pk__in=countries)
+        regions = ["r%s" % country.region.id for country in qs if country.region is not None]
+        countries = ["c%s" % id for id in countries]
+        return countries, regions
+
     def fix_types_for_subs(self, rtype, stype=SubscriptionType.NEW):
         # Correction for the new notification types:
         if rtype == RecordType.EVENT or rtype == RecordType.FIELD_REPORT:
@@ -133,34 +159,68 @@ def gather_subscribers(self, records, rtype, stype):
         if self.is_digest_mode():
             subscribers = User.objects.filter(subscription__rtype=RecordType.WEEKLY_DIGEST, is_active=True).values("email")
             # In digest mode we do not care about other circumstances, just get every subscriber's email.
-            emails = [subscriber["email"] for subscriber in subscribers]
+            emails = list(set([subscriber["email"] for subscriber in subscribers]))
             return emails
         else:
             # Start with any users subscribed directly to this record type.
             subscribers = User.objects.filter(
                 subscription__rtype=rtype_of_subscr, subscription__stype=stype, is_active=True
             ).values("email")
+            emails = list(set([subscriber["email"] for subscriber in subscribers]))
+
+        # For FOLLOWED_EVENTs we do not collect other generic (d*, country, region) subscriptions.
+        if rtype_of_subscr != RecordType.FOLLOWED_EVENT:
+            subscribers_no_geo_dtype = (
+                User.objects.filter(subscription__rtype=rtype_of_subscr, subscription__stype=stype, is_active=True)
+                .exclude(subscription__rtype__in=[RecordType.COUNTRY, RecordType.REGION, RecordType.DTYPE])
+                .values("email")
+            )
+            emailset_no_geo_dtype = {subscriber["email"] for subscriber in subscribers_no_geo_dtype}
 
-        # For FOLLOWED_EVENTs and DEPLOYMENTs we do not collect other generic (d*, country, region) subscriptions, just one.
-        # This part is not called.
-        if (
-            rtype_of_subscr != RecordType.FOLLOWED_EVENT
-            and rtype_of_subscr != RecordType.SURGE_ALERT
-            and rtype_of_subscr != RecordType.SURGE_DEPLOYMENT_MESSAGES
-        ):
-            dtypes = list(set(["d%s" % record.dtype.id for record in records if record.dtype is not None]))
+            subscribers_geo = (
+                User.objects.filter(subscription__rtype=rtype_of_subscr, subscription__stype=stype, is_active=True)
+                .filter(subscription__rtype__in=[RecordType.COUNTRY, RecordType.REGION])
+                .values("email")
+            )
+            emailset_geo = {subscriber["email"] for subscriber in subscribers_geo}
+
+            subscribers_dtype = (
+                User.objects.filter(subscription__rtype=rtype_of_subscr, subscription__stype=stype, is_active=True)
+                .filter(subscription__rtype=RecordType.DTYPE)
+                .values("email")
+            )
+            emailset_dtype = {subscriber["email"] for subscriber in subscribers_dtype}
 
             if rtype_of_subscr == RecordType.NEW_OPERATIONS:
                 countries, regions = self.gather_country_and_region(records)
+            elif rtype_of_subscr == RecordType.SURGE_ALERT:
+                countries, regions = self.gather_event_countries_and_regions(records)
+            elif rtype_of_subscr == RecordType.SURGE_DEPLOYMENT_MESSAGES:
+                countries, regions = self.gather_eventdt_countries_and_regions(records)
             else:
                 countries, regions = self.gather_countries_and_regions(records)
 
-            lookups = dtypes + countries + regions
-            if len(lookups):
-                subscribers = (
-                    subscribers | User.objects.filter(subscription__lookup_id__in=lookups, is_active=True).values("email")
-                ).distinct()
-        emails = list(set([subscriber["email"] for subscriber in subscribers]))
+            if rtype_of_subscr == RecordType.SURGE_ALERT:
+                dtypes = list(set(["d%s" % record.event.dtype.id for record in records if record.event.dtype is not None]))
+            elif rtype_of_subscr == RecordType.SURGE_DEPLOYMENT_MESSAGES:
+                dtypes = list(set(["d%s" % rec.event_deployed_to.dtype.id for rec in records if rec.event_deployed_to.dtype]))
+            else:
+                dtypes = list(set(["d%s" % record.dtype.id for record in records if record.dtype is not None]))
+
+            geo = countries + regions
+            if len(geo):
+                emailset_geo = emailset_geo & {
+                    subscriber["email"]
+                    for subscriber in User.objects.filter(subscription__lookup_id__in=geo, is_active=True).values("email")
+                }
+
+            if len(dtypes):
+                emailset_dtype = emailset_dtype & {
+                    subscriber["email"]
+                    for subscriber in User.objects.filter(subscription__lookup_id__in=dtypes, is_active=True).values("email")
+                }
+
+            emails = list(emailset_no_geo_dtype | emailset_geo | emailset_dtype)
         return emails
 
     def get_template(self, rtype=99):

api/test_scrapers.py

@@ -57,6 +57,11 @@ def test_new_record_subscription(self):
     def test_country_subscription(self):
         # Subscription to a country
         user = get_user()
+        Subscription.objects.create(
+            user=user,
+            rtype=RecordType.NEW_EMERGENCIES,  # FIELD_REPORT,
+            stype=SubscriptionType.NEW,
+        )
         c = Country.objects.get(name="c2")
         Subscription.objects.create(
             user=user,
@@ -66,7 +71,7 @@ def test_country_subscription(self):
         notify = Notify()
         emails = notify.gather_subscribers(
             FieldReport.objects.filter(created_at__gte=notify.diff_5_minutes()),
-            RecordType.FIELD_REPORT,
+            RecordType.NEW_EMERGENCIES,  # FIELD_REPORT,
             SubscriptionType.NEW,
         )
         self.assertEqual(len(emails), 1)
@@ -75,6 +80,11 @@ def test_country_subscription(self):
     def test_region_subscription(self):
         # Subscription to a region
         user = get_user()
+        Subscription.objects.create(
+            user=user,
+            rtype=RecordType.NEW_EMERGENCIES,  # FIELD_REPORT,
+            stype=SubscriptionType.NEW,
+        )
         r = Region.objects.get(name=1)
         Subscription.objects.create(
             user=user,
@@ -84,7 +94,7 @@ def test_region_subscription(self):
         notify = Notify()
         emails = notify.gather_subscribers(
             FieldReport.objects.filter(created_at__gte=notify.diff_5_minutes()),
-            RecordType.FIELD_REPORT,
+            RecordType.NEW_EMERGENCIES,  # FIELD_REPORT,
             SubscriptionType.NEW,
         )
         self.assertEqual(len(emails), 1)
@@ -93,6 +103,11 @@ def test_region_subscription(self):
     def test_dtype_subscription(self):
         # Subscription to a disaster type
         user = get_user()
+        Subscription.objects.create(
+            user=user,
+            rtype=RecordType.NEW_EMERGENCIES,  # FIELD_REPORT,
+            stype=SubscriptionType.NEW,
+        )
         d = DisasterType.objects.get(name="d1")
         Subscription.objects.create(
             user=user,
@@ -102,7 +117,7 @@ def test_dtype_subscription(self):
         notify = Notify()
         emails = notify.gather_subscribers(
             FieldReport.objects.filter(created_at__gte=notify.diff_5_minutes()),
-            RecordType.FIELD_REPORT,
+            RecordType.NEW_EMERGENCIES,  # FIELD_REPORT,
             SubscriptionType.NEW,
         )
         self.assertEqual(len(emails), 1)
@@ -112,6 +127,18 @@ def test_multiple_subscription(self):
         user1 = get_user()
         user2 = get_user()
 
+        Subscription.objects.create(
+            user=user1,
+            rtype=RecordType.NEW_EMERGENCIES,  # FIELD_REPORT,
+            stype=SubscriptionType.NEW,
+        )
+
+        Subscription.objects.create(
+            user=user2,
+            rtype=RecordType.NEW_EMERGENCIES,  # FIELD_REPORT,
+            stype=SubscriptionType.NEW,
+        )
+
         d = DisasterType.objects.get(name="d1")
         r = Region.objects.get(name=1)
 
@@ -139,7 +166,7 @@ def test_multiple_subscription(self):
         notify = Notify()
         emails = notify.gather_subscribers(
             FieldReport.objects.filter(created_at__gte=notify.diff_5_minutes()),
-            RecordType.FIELD_REPORT,
+            RecordType.NEW_EMERGENCIES,  # FIELD_REPORT,
             SubscriptionType.NEW,
         )
         self.assertEqual(len(emails), 2)
@@ -161,6 +188,11 @@ def setUp(self):
 
     def test_region_subscription(self):
         user = get_user()
+        Subscription.objects.create(
+            user=user,
+            rtype=RecordType.NEW_OPERATIONS,
+            stype=SubscriptionType.NEW,
+        )
         r = Region.objects.get(name="1")
         Subscription.objects.create(user=user, region=r, lookup_id="r%s" % r.id)
         notify = Notify()
@@ -174,6 +206,11 @@ def test_region_subscription(self):
 
     def test_region_and_country_subscription(self):
         user = get_user()
+        Subscription.objects.create(
+            user=user,
+            rtype=RecordType.NEW_OPERATIONS,
+            stype=SubscriptionType.NEW,
+        )
         r = Region.objects.get(name="1")
         c = Country.objects.get(name="2")
         Subscription.objects.create(user=user, region=r, lookup_id="r%s" % r.id)