Merge pull request #2564 from IFRCGo/feature/ns-initiatives-categories

szabozoltan69 · web-flow · commit 16f7a2e83a88 · 2025-09-26T14:41:45.000+02:00
NS initiatives – 4 lang categories
diff --git a/api/management/commands/ingest_ns_initiatives.py b/api/management/commands/ingest_ns_initiatives.py
@@ -5,7 +5,13 @@
 from sentry_sdk.crons import monitor
 
 from api.logger import logger
-from api.models import Country, CronJob, CronJobStatus, NSDInitiatives
+from api.models import (
+    Country,
+    CronJob,
+    CronJobStatus,
+    NSDInitiatives,
+    NSDInitiativesCategory,
+)
 from main.sentry import SentryMonitor
 
 DEFAULT_COUNTRY_ID = 289  # IFRC
@@ -32,7 +38,6 @@ def get_defaults(element, country, funding_period, lang):
         "fund_type": (
             f"{element.get('Fund')} – {element.get('FundingType')}" if element.get("FundingType") else element.get("Fund")
         ),
-        "categories": element.get("Categories"),
         "allocation": element.get("AllocationInCHF"),
         "funding_period": funding_period,
         "translation_module_original_language": lang,
@@ -89,6 +94,10 @@ def handle(self, *args, **kwargs):
         updated_remote_ids = set()
         created_ns_initiatives_pk = []
 
+        # In-memory alignment helpers
+        category_by_remote_index = {}  # remote_id -> list[NSDInitiativesCategory]
+        pending_translations = {}  # remote_id -> list[(lang, index, label)]
+
         for lang, resp in responses:
             for element in resp:
                 try:
@@ -114,24 +123,99 @@ def handle(self, *args, **kwargs):
                         for field, value in defaults.items():
                             setattr(ni, field, value)
                         ni.save(update_fields=defaults.keys())
-                        updated_remote_ids.add(remote_id)  # Mark as updated, only for EN entries
+                        updated_remote_ids.add(remote_id)
                     created_ns_initiatives_pk.append(ni.pk)
-                else:
-                    try:
-                        # We could use ISO also to identify the entry, but remote_id is more robust
-                        ni = NSDInitiatives.objects.get(remote_id=remote_id)
-                        setattr(ni, title_field, element.get("InitiativeTitle"))
-                        setattr(ni, risk_field, element.get("Risk"))
-                        ni.save(update_fields=[title_field, risk_field])
-                    except NSDInitiatives.DoesNotExist:
-                        # Should not happen – only if EN entry is missing
-                        ni = NSDInitiatives.objects.create(
-                            remote_id=remote_id,
-                            **defaults,
-                        )
-                        added += 1
-                        created_ns_initiatives_pk.append(ni.pk)
-                        logger.warning(f"Created non-EN entry: {remote_id} / {lang}")
+
+                    # Establish baseline categories from EN by index
+                    raw_categories = element.get("Categories") or []
+                    cats_en = []
+                    cat_objs = []
+                    if isinstance(raw_categories, (list, tuple)):
+                        for idx, raw in enumerate(raw_categories):
+                            label = (raw or "").strip()
+                            if not label:
+                                cats_en.append(None)
+                                continue
+                            # Reuse/create a global category by English label
+                            cat = NSDInitiativesCategory.objects.filter(name_en__iexact=label).first()
+                            if not cat:
+                                cat = NSDInitiativesCategory.objects.create(name=label, name_en=label)
+                            else:
+                                # Ensure plain 'name' mirrors EN for convenience
+                                to_update = []
+                                if getattr(cat, "name_en", None) != label:
+                                    cat.name_en = label
+                                    to_update.append("name_en")
+                                if getattr(cat, "name", None) != label:
+                                    cat.name = label
+                                    to_update.append("name")
+                                if to_update:
+                                    cat.save(update_fields=to_update)
+                            cats_en.append(cat)
+                            cat_objs.append(cat)
+
+                        if cat_objs:
+                            ni.categories.set(cat_objs)
+                        else:
+                            ni.categories.clear()
+
+                        # Save baseline for this remote_id
+                        category_by_remote_index[remote_id] = cats_en
+
+                        # Apply any pending translations queued before EN
+                        for item in pending_translations.pop(remote_id, []):
+                            plang, pidx, plabel = item
+                            if 0 <= pidx < len(cats_en) and cats_en[pidx]:
+                                field = f"name_{plang}"
+                                cat = cats_en[pidx]
+                                if getattr(cat, field, None) != plabel:
+                                    setattr(cat, field, plabel)
+                                    cat.save(update_fields=[field])
+                    continue  # Done with EN row; go next element
+
+                # Non-EN branch: update fields and queue/apply category translations
+                try:
+                    ni = NSDInitiatives.objects.get(remote_id=remote_id)
+                    setattr(ni, title_field, element.get("InitiativeTitle"))
+                    setattr(ni, risk_field, element.get("Risk"))
+                    ni.save(update_fields=[title_field, risk_field])
+                except NSDInitiatives.DoesNotExist:
+                    # Should not happen – only if EN entry is missing
+                    ni = NSDInitiatives.objects.create(
+                        remote_id=remote_id,
+                        **defaults,
+                    )
+                    added += 1
+                    created_ns_initiatives_pk.append(ni.pk)
+                    logger.warning(f"Created non-EN entry: {remote_id} / {lang}")
+
+                # Align categories by index to the EN baseline for this remote_id
+                raw_categories = element.get("Categories") or []
+                if not isinstance(raw_categories, (list, tuple)):
+                    continue
+
+                cats_en = category_by_remote_index.get(remote_id)
+                if cats_en is None:
+                    # Baseline not processed yet; queue these translations
+                    q = pending_translations.setdefault(remote_id, [])
+                    for idx, raw in enumerate(raw_categories):
+                        label = (raw or "").strip()
+                        if label:
+                            q.append((lang, idx, label))
+                    continue
+
+                # Baseline exists: update translated fields by index
+                field = f"name_{lang}"
+                for idx, raw in enumerate(raw_categories):
+                    label = (raw or "").strip()
+                    if not label or idx >= len(cats_en):
+                        continue
+                    cat = cats_en[idx]
+                    if not cat:
+                        continue
+                    if getattr(cat, field, None) != label:
+                        setattr(cat, field, label)
+                        cat.save(update_fields=[field])
 
         # Remove old entries not present in the latest fetch
         NSDInitiatives.objects.exclude(id__in=created_ns_initiatives_pk).delete()
diff --git a/api/migrations/0226_nsdinitiativescategory_and_more.py b/api/migrations/0226_nsdinitiativescategory_and_more.py
@@ -0,0 +1,65 @@
+# Generated by Django 4.2.19 on 2025-09-25 12:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("api", "0225_nsdinitiatives_nsia_risk_ar_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="NSDInitiativesCategory",
+            fields=[
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("name", models.CharField(max_length=255, verbose_name="Name")),
+                ("name_en", models.CharField(max_length=255, null=True)),
+                ("name_es", models.CharField(max_length=255, null=True)),
+                ("name_fr", models.CharField(max_length=255, null=True)),
+                ("name_ar", models.CharField(max_length=255, null=True)),
+                (
+                    "translation_module_original_language",
+                    models.CharField(
+                        choices=[("en", "English"), ("es", "Spanish"), ("fr", "French"), ("ar", "Arabic")],
+                        default="en",
+                        help_text="Language used to create this entity",
+                        max_length=2,
+                        verbose_name="Entity Original language",
+                    ),
+                ),
+                (
+                    "translation_module_skip_auto_translation",
+                    models.BooleanField(
+                        default=False,
+                        help_text="Skip auto translation operation for this entity?",
+                        verbose_name="Skip auto translation",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "NSD initiative category",
+                "verbose_name_plural": "NSD initiative categories",
+                "ordering": ("name",),
+            },
+        ),
+        migrations.RemoveField(
+            model_name="nsdinitiatives",
+            name="categories",  # remove old ArrayField
+        ),
+        migrations.AddField(
+            model_name="nsdinitiatives",
+            name="categories",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="initiatives",
+                to="api.nsdinitiativescategory",
+                verbose_name="Funding categories",
+            ),
+        ),
+        migrations.AlterField(
+            model_name='nsdinitiativescategory',
+            name='name',
+            field=models.CharField(max_length=255),
+        ),
+    ]
diff --git a/api/models.py b/api/models.py
@@ -419,14 +419,31 @@ class CountryOrganizationalCapacity(models.Model):
     financial_capacity = models.TextField(verbose_name=_("Financial Capacity"), null=True, blank=True)
 
 
+class NSDInitiativesCategory(models.Model):
+    name = models.CharField(max_length=255)
+
+    class Meta:
+        ordering = ("name",)
+        verbose_name = _("NSD initiative category")
+        verbose_name_plural = _("NSD initiative categories")
+
+    def __str__(self):
+        return f"{self.name}"
+
+
 class NSDInitiatives(models.Model):
     country = models.ForeignKey(Country, verbose_name=_("Country"), on_delete=models.CASCADE)
     title = models.CharField(verbose_name=_("Title"), max_length=255)
     fund_type = models.CharField(verbose_name=_("Fund Type"), max_length=255)
     allocation = models.IntegerField(verbose_name=_("Allocation in CHF"))
     year = models.CharField(verbose_name=_("Year"), max_length=20)
     funding_period = models.IntegerField(verbose_name=_("Funding Period in Month"))
-    categories = ArrayField(models.CharField(max_length=255), verbose_name=_("Funding categories"), default=list, null=True)
+    categories = models.ManyToManyField(
+        NSDInitiativesCategory,
+        blank=True,
+        related_name="initiatives",
+        verbose_name=_("Funding categories"),
+    )
     remote_id = models.IntegerField(db_index=True, unique=True, null=True, blank=True)
     nsia_risk = models.CharField(verbose_name=_("NSIA Risk"), max_length=30, null=True, blank=True)
 
diff --git a/api/serializers.py b/api/serializers.py
@@ -657,11 +657,56 @@ class Meta:
         fields = ("id", "first_name", "last_name", "position")
 
 
-class NSDInitiativesSerialzier(ModelSerializer):
+class NSDInitiativesSerializer(ModelSerializer):
+    # Return translated category names (current active language) – like title does.
+    categories = serializers.SerializerMethodField()
+
     class Meta:
         model = NSDInitiatives
         fields = "__all__"
 
+    @staticmethod
+    def get_categories(obj):
+        """
+        Return category names in the currently active language (like title does),
+        avoiding duplicates and avoiding showing the same semantic category in
+        multiple languages (caused by legacy per-language rows).
+
+        Strategy:
+        - Active language value (name_<lang>) wins.
+        - If missing, fall back to English ONLY if no row already provided a
+          translated value for that semantic slot.
+        - Ignore rows that have neither an active-language value nor an English fallback.
+        """
+        from django.utils.translation import get_language
+
+        lang = (get_language() or "en")[:2]
+        lang_field = f"name_{lang}"
+
+        cats = obj.categories.all()
+        # First collect all explicit translations in the active language
+        explicit_lang_values = {
+            getattr(c, lang_field).strip() for c in cats if getattr(c, lang_field, None) and getattr(c, lang_field).strip()
+        }
+
+        seen = set()
+        result = []
+        for c in cats:
+            val_lang = getattr(c, lang_field, None)
+            val_lang = val_lang.strip() if val_lang else ""
+            if val_lang:
+                if val_lang not in seen:
+                    seen.add(val_lang)
+                    result.append(val_lang)
+                continue
+            # fallback to English only if there is NO active-language version in any row
+            val_en = getattr(c, "name_en", None)
+            val_en = val_en.strip() if val_en else ""
+            if val_en and not explicit_lang_values and val_en not in seen:
+                seen.add(val_en)
+                result.append(val_en)
+        return result
+
 
 class CountryCapacityStrengtheningSerializer(ModelSerializer):
     assessment_type_display = serializers.CharField(read_only=True, source="get_assessment_type_display")
@@ -691,7 +736,7 @@ class CountryRelationSerializer(ModelSerializer):
     centroid = serializers.SerializerMethodField()
     regions_details = RegionSerializer(source="region", read_only=True)
     directory = CountryDirectorySerializer(source="countrydirectory_set", read_only=True, many=True)
-    initiatives = NSDInitiativesSerialzier(source="nsdinitiatives_set", read_only=True, many=True)
+    initiatives = NSDInitiativesSerializer(source="nsdinitiatives_set", read_only=True, many=True)
     capacity = CountryCapacityStrengtheningSerializer(source="countrycapacitystrengthening_set", read_only=True, many=True)
     organizational_capacity = CountryOrganizationalCapacitySerializer(
         source="countryorganizationalcapacity",
@@ -2521,6 +2566,7 @@ def create(self, validated_data):
         else:
             title = "Export"
         user = self.context["request"].user
+
         if export_type == Export.ExportType.PER:
             validated_data["url"] = f"{settings.GO_WEB_INTERNAL_URL}/countries/{country_id}/{export_type}/{export_id}/export/"
         else:
diff --git a/api/translation.py b/api/translation.py
@@ -19,6 +19,7 @@
     KeyFigure,
     MainContact,
     NSDInitiatives,
+    NSDInitiativesCategory,
     Region,
     RegionEmergencySnippet,
     RegionKeyFigure,
@@ -103,6 +104,12 @@ class MainContactTO(TranslationOptions):
     fields = ("extent",)
 
 
+@register(NSDInitiativesCategory)
+class NSDInitiativesCategoryTO(TranslationOptions):
+    fields = ("name",)
+    skip_fields = ("name",)
+
+
 @register(NSDInitiatives)
 class NSDInitiativesTO(TranslationOptions):
     fields = ("title", "nsia_risk")
