2024. December release

api/management/commands/index_and_notify.py

@@ -116,6 +116,32 @@ def gather_countries_and_regions(self, records):
         countries = ["c%s" % id for id in countries]
         return countries, regions
 
+    def gather_event_countries_and_regions(self, records):
+        # Applies to surgealerts, which have a
+        # many-to-many relationship to countries and regions through event table
+        countries = []
+        for record in records:
+            if record.event.countries is not None:
+                countries += [country.id for country in record.event.countries.all()]
+        countries = list(set(countries))
+        qs = Country.objects.filter(pk__in=countries)
+        regions = ["r%s" % country.region.id for country in qs if country.region is not None]
+        countries = ["c%s" % id for id in countries]
+        return countries, regions
+
+    def gather_eventdt_countries_and_regions(self, records):
+        # Applies to deployments_personneldeployments, which have a
+        # many-to-many relationship to countries and regions through event_deployed_to
+        countries = []
+        for record in records:
+            if record.event_deployed_to.countries is not None:
+                countries += [country.id for country in record.event_deployed_to.countries.all()]
+        countries = list(set(countries))
+        qs = Country.objects.filter(pk__in=countries)
+        regions = ["r%s" % country.region.id for country in qs if country.region is not None]
+        countries = ["c%s" % id for id in countries]
+        return countries, regions
+
     def fix_types_for_subs(self, rtype, stype=SubscriptionType.NEW):
         # Correction for the new notification types:
         if rtype == RecordType.EVENT or rtype == RecordType.FIELD_REPORT:
@@ -133,34 +159,68 @@ def gather_subscribers(self, records, rtype, stype):
         if self.is_digest_mode():
             subscribers = User.objects.filter(subscription__rtype=RecordType.WEEKLY_DIGEST, is_active=True).values("email")
             # In digest mode we do not care about other circumstances, just get every subscriber's email.
-            emails = [subscriber["email"] for subscriber in subscribers]
+            emails = list(set([subscriber["email"] for subscriber in subscribers]))
             return emails
         else:
             # Start with any users subscribed directly to this record type.
             subscribers = User.objects.filter(
                 subscription__rtype=rtype_of_subscr, subscription__stype=stype, is_active=True
             ).values("email")
+            emails = list(set([subscriber["email"] for subscriber in subscribers]))
+
+        # For FOLLOWED_EVENTs we do not collect other generic (d*, country, region) subscriptions.
+        if rtype_of_subscr != RecordType.FOLLOWED_EVENT:
+            subscribers_no_geo_dtype = (
+                User.objects.filter(subscription__rtype=rtype_of_subscr, subscription__stype=stype, is_active=True)
+                .exclude(subscription__rtype__in=[RecordType.COUNTRY, RecordType.REGION, RecordType.DTYPE])
+                .values("email")
+            )
+            emailset_no_geo_dtype = {subscriber["email"] for subscriber in subscribers_no_geo_dtype}
 
-        # For FOLLOWED_EVENTs and DEPLOYMENTs we do not collect other generic (d*, country, region) subscriptions, just one.
-        # This part is not called.
-        if (
-            rtype_of_subscr != RecordType.FOLLOWED_EVENT
-            and rtype_of_subscr != RecordType.SURGE_ALERT
-            and rtype_of_subscr != RecordType.SURGE_DEPLOYMENT_MESSAGES
-        ):
-            dtypes = list(set(["d%s" % record.dtype.id for record in records if record.dtype is not None]))
+            subscribers_geo = (
+                User.objects.filter(subscription__rtype=rtype_of_subscr, subscription__stype=stype, is_active=True)
+                .filter(subscription__rtype__in=[RecordType.COUNTRY, RecordType.REGION])
+                .values("email")
+            )
+            emailset_geo = {subscriber["email"] for subscriber in subscribers_geo}
+
+            subscribers_dtype = (
+                User.objects.filter(subscription__rtype=rtype_of_subscr, subscription__stype=stype, is_active=True)
+                .filter(subscription__rtype=RecordType.DTYPE)
+                .values("email")
+            )
+            emailset_dtype = {subscriber["email"] for subscriber in subscribers_dtype}
 
             if rtype_of_subscr == RecordType.NEW_OPERATIONS:
                 countries, regions = self.gather_country_and_region(records)
+            elif rtype_of_subscr == RecordType.SURGE_ALERT:
+                countries, regions = self.gather_event_countries_and_regions(records)
+            elif rtype_of_subscr == RecordType.SURGE_DEPLOYMENT_MESSAGES:
+                countries, regions = self.gather_eventdt_countries_and_regions(records)
             else:
                 countries, regions = self.gather_countries_and_regions(records)
 
-            lookups = dtypes + countries + regions
-            if len(lookups):
-                subscribers = (
-                    subscribers | User.objects.filter(subscription__lookup_id__in=lookups, is_active=True).values("email")
-                ).distinct()
-        emails = list(set([subscriber["email"] for subscriber in subscribers]))
+            if rtype_of_subscr == RecordType.SURGE_ALERT:
+                dtypes = list(set(["d%s" % record.event.dtype.id for record in records if record.event.dtype is not None]))
+            elif rtype_of_subscr == RecordType.SURGE_DEPLOYMENT_MESSAGES:
+                dtypes = list(set(["d%s" % rec.event_deployed_to.dtype.id for rec in records if rec.event_deployed_to.dtype]))
+            else:
+                dtypes = list(set(["d%s" % record.dtype.id for record in records if record.dtype is not None]))
+
+            geo = countries + regions
+            if len(geo):
+                emailset_geo = emailset_geo & {
+                    subscriber["email"]
+                    for subscriber in User.objects.filter(subscription__lookup_id__in=geo, is_active=True).values("email")
+                }
+
+            if len(dtypes):
+                emailset_dtype = emailset_dtype & {
+                    subscriber["email"]
+                    for subscriber in User.objects.filter(subscription__lookup_id__in=dtypes, is_active=True).values("email")
+                }
+
+            emails = list(emailset_no_geo_dtype | emailset_geo | emailset_dtype)
         return emails
 
     def get_template(self, rtype=99):

api/management/commands/ingest_icrc.py

@@ -43,7 +43,23 @@ def handle(self, *args, **kwargs):
         key_operations_country_list = soup.find("div", {"class": "key-operations-content"}).find_all("div", class_="title")
 
         # NOTE: Mapping this Country, as it doesnot match with the name in the database
-        country_name_mapping = {"Syria": "Syrian Arab Republic", "Israel and the occupied territories": "Israel"}
+        country_name_mapping = {
+            "Syria": "Syrian Arab Republic",
+            "Israel and the occupied territories": "Israel",
+            "Republic of Moldova": "Moldova, Republic of",
+            "United Arab Emirates (UAE) delegation": "United Arab Emirates",
+            "The Republic of South Sudan": "South Sudan",
+            "United States of America": "United States",
+            "Russia": "Russian Federation",
+            "Iran": "Iran, Islamic Republic of",
+            "Democratic Republic of the Congo": "Democratic Republic of Congo",
+        }
+
+        # NOTE: Group country names
+        countries_group = {
+            "Indonesia and Timor-Leste",
+            "Gulf Cooperation Council (GCC) Countries",
+        }
 
         country_operations_list = [
             country.text.strip() for key_operation in key_operations_country_list for country in key_operation.find_all("a")
@@ -67,6 +83,25 @@ def handle(self, *args, **kwargs):
                         country_soup = BeautifulSoup(country_page.content, "html.parser")
                         description_tag = country_soup.find("div", class_="description").find("div", class_="ck-text")
                         description = description_tag.text.strip() if description_tag else None
+
+                        # NOTE: Getting the countries that are part of the country_group
+                        if name in countries_group:
+                            country_contact_soup = (
+                                country_soup.find(id="contact-country")
+                                .find("div", class_="icrc-container")
+                                .find_all("div", class_="contact-row")
+                            )
+                            for contact in country_contact_soup:
+                                country_name = contact.find("div", class_="title").text.strip()
+                                countries.append(
+                                    {
+                                        "Country": country_name,
+                                        "ICRC presence": presence,
+                                        "URL": country_url,
+                                        "Key operation": key_operation,
+                                        "Description": description,
+                                    }
+                                )
                     except Exception:
                         pass
 
@@ -81,6 +116,9 @@ def handle(self, *args, **kwargs):
                     }
                 )
 
+        # Remove the countries_group countries from the countries list
+        countries = [country for country in countries if country["Country"] not in countries_group]
+
         added = 0
         created_ns_presence_pk = []
         for data in countries:

api/migrations/0216_district_emma_id_district_fips_code_district_nuts1_and_more.py

@@ -0,0 +1,56 @@
+# Generated by Django 4.2.16 on 2024-11-28 09:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("api", "0215_alter_generaldocument_document_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="district",
+            name="emma_id",
+            field=models.CharField(blank=True, help_text="Meteoalarm EMMA_ID", max_length=10, null=True, verbose_name="emma_id"),
+        ),
+        migrations.AddField(
+            model_name="district",
+            name="fips_code",
+            field=models.PositiveIntegerField(blank=True, help_text="USA FIPS Code", null=True, verbose_name="fips_code"),
+        ),
+        migrations.AddField(
+            model_name="district",
+            name="nuts1",
+            field=models.CharField(
+                blank=True,
+                help_text="Nomenclature of Territorial Units for Statistics 1",
+                max_length=3,
+                null=True,
+                verbose_name="nuts1",
+            ),
+        ),
+        migrations.AddField(
+            model_name="district",
+            name="nuts2",
+            field=models.CharField(
+                blank=True,
+                help_text="Nomenclature of Territorial Units for Statistics 2",
+                max_length=4,
+                null=True,
+                verbose_name="nuts2",
+            ),
+        ),
+        migrations.AddField(
+            model_name="district",
+            name="nuts3",
+            field=models.CharField(
+                blank=True,
+                help_text="Nomenclature of Territorial Units for Statistics 3",
+                max_length=5,
+                null=True,
+                verbose_name="nuts3",
+            ),
+        ),
+    ]

api/models.py

@@ -298,6 +298,29 @@ class District(models.Model):
     wb_year = models.CharField(
         verbose_name=_("WB year"), max_length=4, null=True, blank=True, help_text=_("population data year from WB API")
     )
+    nuts1 = models.CharField(
+        verbose_name=_("nuts1"),
+        max_length=3,
+        blank=True,
+        null=True,
+        help_text=_("Nomenclature of Territorial Units for Statistics 1"),
+    )
+    nuts2 = models.CharField(
+        verbose_name=_("nuts2"),
+        max_length=4,
+        blank=True,
+        null=True,
+        help_text=_("Nomenclature of Territorial Units for Statistics 2"),
+    )
+    nuts3 = models.CharField(
+        verbose_name=_("nuts3"),
+        max_length=5,
+        blank=True,
+        null=True,
+        help_text=_("Nomenclature of Territorial Units for Statistics 3"),
+    )
+    emma_id = models.CharField(verbose_name=_("emma_id"), max_length=10, blank=True, null=True, help_text=_("Meteoalarm EMMA_ID"))
+    fips_code = models.PositiveIntegerField(verbose_name=_("fips_code"), blank=True, null=True, help_text=_("USA FIPS Code"))
 
     class Meta:
         ordering = ("code",)

api/serializers.py

@@ -332,7 +332,22 @@ class DistrictSerializer(ModelSerializer):
 
     class Meta:
         model = District
-        fields = ("name", "code", "country", "id", "is_deprecated", "bbox", "centroid", "wb_population", "wb_year")
+        fields = (
+            "name",
+            "code",
+            "country",
+            "id",
+            "is_deprecated",
+            "bbox",
+            "centroid",
+            "wb_population",
+            "wb_year",
+            "nuts1",
+            "nuts2",
+            "nuts3",
+            "emma_id",
+            "fips_code",
+        )
 
     @staticmethod
     def get_bbox(district) -> dict:
@@ -387,6 +402,11 @@ class MiniDistrictGeoSerializer(GeoSerializerMixin, ModelSerializer):
     country_name = serializers.CharField(source="country.name", read_only=True)
     country_iso = serializers.CharField(source="country.iso", read_only=True)
     country_iso3 = serializers.CharField(source="country.iso3", read_only=True)
+    nuts1 = serializers.CharField(read_only=True)
+    nuts2 = serializers.CharField(read_only=True)
+    nuts3 = serializers.CharField(read_only=True)
+    emma_id = serializers.CharField(read_only=True)
+    fips_code = serializers.IntegerField(read_only=True)
 
     @staticmethod
     def get_bbox(district) -> Union[dict, None]:
@@ -417,6 +437,11 @@ class Meta:
             "is_deprecated",
             "wb_population",
             "wb_year",
+            "nuts1",
+            "nuts2",
+            "nuts3",
+            "emma_id",
+            "fips_code",
         )
 
 
@@ -1561,6 +1586,7 @@ class Meta:
             "id",
             "code",
             "event",
+            "start_date",
         )