NS initiatives categories translated

szabozoltan69 · szabozoltan69 · commit 0ab922ca9e5b · 2025-09-26T14:31:20.000+02:00
diff --git a/api/management/commands/ingest_ns_initiatives.py b/api/management/commands/ingest_ns_initiatives.py
@@ -94,6 +94,10 @@ def handle(self, *args, **kwargs):
         updated_remote_ids = set()
         created_ns_initiatives_pk = []
 
+        # In-memory alignment helpers
+        category_by_remote_index = {}  # remote_id -> list[NSDInitiativesCategory]
+        pending_translations = {}  # remote_id -> list[(lang, index, label)]
+
         for lang, resp in responses:
             for element in resp:
                 try:
@@ -119,44 +123,99 @@ def handle(self, *args, **kwargs):
                         for field, value in defaults.items():
                             setattr(ni, field, value)
                         ni.save(update_fields=defaults.keys())
-                        updated_remote_ids.add(remote_id)  # Mark as updated, only for EN entries
+                        updated_remote_ids.add(remote_id)
                     created_ns_initiatives_pk.append(ni.pk)
-                else:
-                    try:
-                        # We could use ISO also to identify the entry, but remote_id is more robust
-                        ni = NSDInitiatives.objects.get(remote_id=remote_id)
-                        setattr(ni, title_field, element.get("InitiativeTitle"))
-                        setattr(ni, risk_field, element.get("Risk"))
-                        ni.save(update_fields=[title_field, risk_field])
-                    except NSDInitiatives.DoesNotExist:
-                        # Should not happen – only if EN entry is missing
-                        ni = NSDInitiatives.objects.create(
-                            remote_id=remote_id,
-                            **defaults,
-                        )
-                        added += 1
-                        created_ns_initiatives_pk.append(ni.pk)
-                        logger.warning(f"Created non-EN entry: {remote_id} / {lang}")
 
-                # Handle categories (language-aware)
-                raw_categories = element.get("Categories") or []
-                if isinstance(raw_categories, (list, tuple)):
+                    # Establish baseline categories from EN by index
+                    raw_categories = element.get("Categories") or []
+                    cats_en = []
                     cat_objs = []
-                    for c in raw_categories:
-                        if not c:
-                            continue
-                        name = str(c).strip()
-                        if not name:
-                            continue
-                        # Create / fetch category for this specific language
-                        obj, _ = NSDInitiativesCategory.objects.get_or_create(
-                            name=name,
-                            lang=lang,
-                        )
-                        cat_objs.append(obj)
-
-                    if cat_objs:
-                        ni.categories.add(*cat_objs)
+                    if isinstance(raw_categories, (list, tuple)):
+                        for idx, raw in enumerate(raw_categories):
+                            label = (raw or "").strip()
+                            if not label:
+                                cats_en.append(None)
+                                continue
+                            # Reuse/create a global category by English label
+                            cat = NSDInitiativesCategory.objects.filter(name_en__iexact=label).first()
+                            if not cat:
+                                cat = NSDInitiativesCategory.objects.create(name=label, name_en=label)
+                            else:
+                                # Ensure plain 'name' mirrors EN for convenience
+                                to_update = []
+                                if getattr(cat, "name_en", None) != label:
+                                    cat.name_en = label
+                                    to_update.append("name_en")
+                                if getattr(cat, "name", None) != label:
+                                    cat.name = label
+                                    to_update.append("name")
+                                if to_update:
+                                    cat.save(update_fields=to_update)
+                            cats_en.append(cat)
+                            cat_objs.append(cat)
+
+                        if cat_objs:
+                            ni.categories.set(cat_objs)
+                        else:
+                            ni.categories.clear()
+
+                        # Save baseline for this remote_id
+                        category_by_remote_index[remote_id] = cats_en
+
+                        # Apply any pending translations queued before EN
+                        for item in pending_translations.pop(remote_id, []):
+                            plang, pidx, plabel = item
+                            if 0 <= pidx < len(cats_en) and cats_en[pidx]:
+                                field = f"name_{plang}"
+                                cat = cats_en[pidx]
+                                if getattr(cat, field, None) != plabel:
+                                    setattr(cat, field, plabel)
+                                    cat.save(update_fields=[field])
+                    continue  # Done with EN row; go next element
+
+                # Non-EN branch: update fields and queue/apply category translations
+                try:
+                    ni = NSDInitiatives.objects.get(remote_id=remote_id)
+                    setattr(ni, title_field, element.get("InitiativeTitle"))
+                    setattr(ni, risk_field, element.get("Risk"))
+                    ni.save(update_fields=[title_field, risk_field])
+                except NSDInitiatives.DoesNotExist:
+                    # Should not happen – only if EN entry is missing
+                    ni = NSDInitiatives.objects.create(
+                        remote_id=remote_id,
+                        **defaults,
+                    )
+                    added += 1
+                    created_ns_initiatives_pk.append(ni.pk)
+                    logger.warning(f"Created non-EN entry: {remote_id} / {lang}")
+
+                # Align categories by index to the EN baseline for this remote_id
+                raw_categories = element.get("Categories") or []
+                if not isinstance(raw_categories, (list, tuple)):
+                    continue
+
+                cats_en = category_by_remote_index.get(remote_id)
+                if cats_en is None:
+                    # Baseline not processed yet; queue these translations
+                    q = pending_translations.setdefault(remote_id, [])
+                    for idx, raw in enumerate(raw_categories):
+                        label = (raw or "").strip()
+                        if label:
+                            q.append((lang, idx, label))
+                    continue
+
+                # Baseline exists: update translated fields by index
+                field = f"name_{lang}"
+                for idx, raw in enumerate(raw_categories):
+                    label = (raw or "").strip()
+                    if not label or idx >= len(cats_en):
+                        continue
+                    cat = cats_en[idx]
+                    if not cat:
+                        continue
+                    if getattr(cat, field, None) != label:
+                        setattr(cat, field, label)
+                        cat.save(update_fields=[field])
 
         # Remove old entries not present in the latest fetch
         NSDInitiatives.objects.exclude(id__in=created_ns_initiatives_pk).delete()
diff --git a/api/migrations/0226_nsdinitiativescategory_and_more.py b/api/migrations/0226_nsdinitiativescategory_and_more.py
@@ -4,45 +4,62 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
-        ('api', '0225_nsdinitiatives_nsia_risk_ar_and_more'),
+        ("api", "0225_nsdinitiatives_nsia_risk_ar_and_more"),
     ]
 
     operations = [
         migrations.CreateModel(
-            name='NSDInitiativesCategory',
+            name="NSDInitiativesCategory",
             fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('name', models.CharField(max_length=255, unique=True, verbose_name='Name')),
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("name", models.CharField(max_length=255, verbose_name="Name")),
+                ("name_en", models.CharField(max_length=255, null=True)),
+                ("name_es", models.CharField(max_length=255, null=True)),
+                ("name_fr", models.CharField(max_length=255, null=True)),
+                ("name_ar", models.CharField(max_length=255, null=True)),
+                (
+                    "translation_module_original_language",
+                    models.CharField(
+                        choices=[("en", "English"), ("es", "Spanish"), ("fr", "French"), ("ar", "Arabic")],
+                        default="en",
+                        help_text="Language used to create this entity",
+                        max_length=2,
+                        verbose_name="Entity Original language",
+                    ),
+                ),
+                (
+                    "translation_module_skip_auto_translation",
+                    models.BooleanField(
+                        default=False,
+                        help_text="Skip auto translation operation for this entity?",
+                        verbose_name="Skip auto translation",
+                    ),
+                ),
             ],
             options={
-                'verbose_name': 'NSD initiative category',
-                'verbose_name_plural': 'NSD initiative categories',
-                'ordering': ('name',),
+                "verbose_name": "NSD initiative category",
+                "verbose_name_plural": "NSD initiative categories",
+                "ordering": ("name",),
             },
         ),
         migrations.RemoveField(
-            model_name='nsdinitiatives',
-            name='categories',
+            model_name="nsdinitiatives",
+            name="categories",  # remove old ArrayField
         ),
         migrations.AddField(
-            model_name='nsdinitiatives',
-            name='categories',
-            field=models.ManyToManyField(blank=True, related_name='initiatives', to='api.nsdinitiativescategory', verbose_name='Funding categories'),
-        ),
-        migrations.AddField(
-            model_name='nsdinitiativescategory',
-            name='lang',
-            field=models.CharField(choices=[('en', 'English'), ('es', 'Spanish'), ('fr', 'French'), ('ar', 'Arabic')], db_index=True, default='en', max_length=5),
+            model_name="nsdinitiatives",
+            name="categories",
+            field=models.ManyToManyField(
+                blank=True,
+                related_name="initiatives",
+                to="api.nsdinitiativescategory",
+                verbose_name="Funding categories",
+            ),
         ),
         migrations.AlterField(
             model_name='nsdinitiativescategory',
             name='name',
             field=models.CharField(max_length=255),
         ),
-        migrations.AlterUniqueTogether(
-            name='nsdinitiativescategory',
-            unique_together={('name', 'lang')},
-        ),
     ]
diff --git a/api/models.py b/api/models.py
@@ -419,26 +419,16 @@ class CountryOrganizationalCapacity(models.Model):
     financial_capacity = models.TextField(verbose_name=_("Financial Capacity"), null=True, blank=True)
 
 
-LANG_CHOICES = (
-    ("en", "English"),
-    ("es", "Spanish"),
-    ("fr", "French"),
-    ("ar", "Arabic"),
-)
-
-
 class NSDInitiativesCategory(models.Model):
     name = models.CharField(max_length=255)
-    lang = models.CharField(max_length=5, choices=LANG_CHOICES, db_index=True, default="en")
 
     class Meta:
         ordering = ("name",)
-        unique_together = ("name", "lang")
         verbose_name = _("NSD initiative category")
         verbose_name_plural = _("NSD initiative categories")
 
     def __str__(self):
-        return f"{self.name} ({self.lang})"
+        return f"{self.name}"
 
 
 class NSDInitiatives(models.Model):
diff --git a/api/serializers.py b/api/serializers.py
@@ -658,6 +658,7 @@ class Meta:
 
 
 class NSDInitiativesSerializer(ModelSerializer):
+    # Return translated category names (current active language) – like title does.
     categories = serializers.SerializerMethodField()
 
     class Meta:
@@ -666,10 +667,45 @@ class Meta:
 
     @staticmethod
     def get_categories(obj):
-        out = {}
-        for lang, name in obj.categories.values_list("lang", "name"):
-            out.setdefault(lang, []).append(name)
-        return out
+        """
+        Return category names in the currently active language (like title does),
+        avoiding duplicates and avoiding showing the same semantic category in
+        multiple languages (caused by legacy per-language rows).
+
+        Strategy:
+        - Active language value (name_<lang>) wins.
+        - If missing, fall back to English ONLY if no row already provided a
+          translated value for that semantic slot.
+        - Ignore rows that have neither an active-language value nor an English fallback.
+        """
+        from django.utils.translation import get_language
+
+        lang = (get_language() or "en")[:2]
+        lang_field = f"name_{lang}"
+
+        cats = obj.categories.all()
+        # First collect all explicit translations in the active language
+        explicit_lang_values = {
+            getattr(c, lang_field).strip() for c in cats if getattr(c, lang_field, None) and getattr(c, lang_field).strip()
+        }
+
+        seen = set()
+        result = []
+        for c in cats:
+            val_lang = getattr(c, lang_field, None)
+            val_lang = val_lang.strip() if val_lang else ""
+            if val_lang:
+                if val_lang not in seen:
+                    seen.add(val_lang)
+                    result.append(val_lang)
+                continue
+            # fallback to English only if there is NO active-language version in any row
+            val_en = getattr(c, "name_en", None)
+            val_en = val_en.strip() if val_en else ""
+            if val_en and not explicit_lang_values and val_en not in seen:
+                seen.add(val_en)
+                result.append(val_en)
+        return result
 
 
 class CountryCapacityStrengtheningSerializer(ModelSerializer):
diff --git a/api/translation.py b/api/translation.py
@@ -19,6 +19,7 @@
     KeyFigure,
     MainContact,
     NSDInitiatives,
+    NSDInitiativesCategory,
     Region,
     RegionEmergencySnippet,
     RegionKeyFigure,
@@ -103,6 +104,12 @@ class MainContactTO(TranslationOptions):
     fields = ("extent",)
 
 
+@register(NSDInitiativesCategory)
+class NSDInitiativesCategoryTO(TranslationOptions):
+    fields = ("name",)
+    skip_fields = ("name",)
+
+
 @register(NSDInitiatives)
 class NSDInitiativesTO(TranslationOptions):
     fields = ("title", "nsia_risk")
