Merge pull request #2527 from IFRCGo/feature/translation-field-stats

Add table_field to translation cache for stats

Author: szabozoltan69

lang/migrations/0008_translationcache_table_field.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.2.19 on 2025-08-05 12:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("lang", "0007_translationcache"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="translationcache",
+            name="table_field",
+            field=models.CharField(blank=True, default="", max_length=128),
+        ),
+        migrations.AddField(
+            model_name="translationcache",
+            name="other_fields",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="translationcache",
+            name="num_calls",
+            field=models.IntegerField(default=0),
+        ),
+    ]

lang/migrations/0009_remove_translationcache_lang_transl_text_4a497b_idx_and_more.py

@@ -0,0 +1,50 @@
+# Generated by Django 4.2.19 on 2025-08-11 08:18
+
+import hashlib
+
+from django.db import migrations, models
+
+
+def fill_text_hash(apps, schema_editor):
+    TranslationCache = apps.get_model("lang", "TranslationCache")
+    for obj in TranslationCache.objects.all():
+        if not obj.text_hash:
+            obj.text_hash = hashlib.sha256(obj.text.encode("utf-8")).hexdigest()
+            obj.save(update_fields=["text_hash"])
+    print(" Hash field is populated in lang_translationcache.")
+
+
+def fill_last_used(apps, schema_editor):
+    TranslationCache = apps.get_model("lang", "TranslationCache")
+    TranslationCache.objects.update(last_used=models.F("created_at"))
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("lang", "0008_translationcache_table_field"),
+    ]
+
+    operations = [
+        migrations.RemoveIndex(
+            model_name="translationcache",
+            name="lang_transl_text_4a497b_idx",
+        ),
+        migrations.AddField(
+            model_name="translationcache",
+            name="text_hash",
+            field=models.CharField(default="", max_length=64),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="translationcache",
+            name="last_used",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddIndex(
+            model_name="translationcache",
+            index=models.Index(fields=["text_hash", "source_language", "dest_language"], name="lang_transl_text_ha_7b6786_idx"),
+        ),
+        migrations.RunPython(fill_text_hash, reverse_code=migrations.RunPython.noop),
+        migrations.RunPython(fill_last_used, reverse_code=migrations.RunPython.noop),
+    ]

lang/models.py

@@ -37,16 +37,21 @@ def get_user_permissions_per_language(cls, user):
 
 class TranslationCache(models.Model):
     text = models.TextField()
+    text_hash = models.CharField(max_length=64)
     source_language = models.CharField(max_length=16)
     dest_language = models.CharField(max_length=16)
     translated_text = models.TextField()
+    table_field = models.CharField(max_length=128, blank=True, default="")  # for stats only
+    other_fields = models.BooleanField(default=False)  # for stats only
+    num_calls = models.IntegerField(default=0)  # for stats only
     created_at = models.DateTimeField(auto_now_add=True)
+    last_used = models.DateTimeField(null=True, blank=True)
 
     class Meta:
         unique_together = ("text", "source_language", "dest_language")
         indexes = [
-            models.Index(fields=["text", "source_language", "dest_language"]),
+            models.Index(fields=["text_hash", "source_language", "dest_language"]),
         ]
 
     def __str__(self):
-        return f"{self.source_language}>{self.dest_language}: {self.text[:30]}..."
+        return f"{self.source_language}>{self.dest_language} – {self.table_field}: {self.text[:30]}..."

lang/tasks.py

@@ -44,13 +44,19 @@ def translate_fields_object(self, obj, field):
             if value:
                 continue
 
+            model = type(obj)
+            app_label = model._meta.app_label
+            model_name = model._meta.model_name
+            table_field = f"{app_label}:{model_name}:{field}"
+
             new_value = self.translator.translate_text(
                 initial_value,
                 lang,
                 source_language=initial_lang,
+                table_field=table_field,
             )
 
-            field_max_length = type(obj)._meta.get_field(field).max_length
+            field_max_length = model._meta.get_field(field).max_length
             if field_max_length and len(new_value) > field_max_length:
                 logger.warning(f"Greater then max_length found for Model ({type(obj)}<{lang_field}>) pk: ({obj.pk})")
                 new_value = new_value[:field_max_length]

lang/translation.py

@@ -1,10 +1,13 @@
+import hashlib
 import logging
 import threading
 
 import boto3
 import requests
 from bs4 import BeautifulSoup
 from django.conf import settings
+from django.db.models import BooleanField, Case, F, Value, When
+from django.utils import timezone
 from django.utils.module_loading import import_string
 
 from .models import TranslationCache
@@ -19,16 +22,20 @@
 IFRC_TRANSLATION_CALL_LOCK = threading.Lock()
 
 
+def sha256_hash(text):
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
 class BaseTranslator:
-    def _fake_translation(self, text, dest_language, source_language):
+    def _fake_translation(self, text, dest_language, source_language, table_field=""):
         """
         This is only used for test
         """
         return text + f' translated to "{dest_language}" using source language "{source_language}"'
 
 
 class DummyTranslator(BaseTranslator):
-    def translate_text(self, text, dest_language, source_language="auto"):
+    def translate_text(self, text, dest_language, source_language="auto", table_field=""):
         return self._fake_translation(text, dest_language, source_language)
 
 
@@ -100,14 +107,14 @@ def find_last_slashp(cls, text, limit):
             truncate_here += len(tag)
         return truncate_here
 
-    def translate_text(self, text, dest_language, source_language=None):
+    def translate_text(self, text, dest_language, source_language=None, table_field=""):
         if settings.TESTING:
             # NOTE: Mocking for test purpose
             return self._fake_translation(text, dest_language, source_language)
 
         global IFRC_TRANSLATION_CALL_COUNT
 
-        # A dirty workaround to handle oversized HTML+CSS texts, usually tables:
+        # A workaround to handle oversized HTML+CSS texts, usually tables:
         textTail = ""
         if len(text) > settings.AZURE_TRANSL_LIMIT:
             truncate_here = self.find_last_slashtable(text, settings.AZURE_TRANSL_LIMIT)
@@ -134,22 +141,35 @@ def translate_text(self, text, dest_language, source_language=None):
             payload["textType"] = "html"
 
         # Try cache at first (for shorter texts)
-        use_cache = len(text) < 200
+        use_cache = len(text) < 300
 
         if use_cache:
+            text_hash = sha256_hash(text)
             cache = TranslationCache.objects.filter(
-                text=text,
+                text_hash=text_hash,
                 source_language=source_language or "",  # source_language can be "detected"
                 dest_language=dest_language,
             ).first()
             if cache:
-                logger.info(f"IFRC translation cache hit: {text[:30]}... {source_language}>{dest_language}")
+                cache_other_fields = cache.table_field != table_field
+                TranslationCache.objects.filter(id=cache.pk).update(
+                    last_used=timezone.now(),
+                    num_calls=F("num_calls") + 1,
+                    other_fields=Case(
+                        When(other_fields=True, then=Value(True)),
+                        default=Value(cache_other_fields),
+                        output_field=BooleanField(),
+                    ),
+                )
+                logger.info(
+                    f"Translation cache hit, {source_language}>{dest_language} {table_field} – {cache.num_calls}: {text[:30]}... "
+                )
                 return cache.translated_text
 
         with IFRC_TRANSLATION_CALL_LOCK:
             IFRC_TRANSLATION_CALL_COUNT += 1
             logger.info(f"IFRC translation API call count: {IFRC_TRANSLATION_CALL_COUNT}")
-        logger.info(f"IFRC translation API call: {text[:30]}... {source_language}>{dest_language}")
+        logger.info(f"IFRC translation API call – {source_language}>{dest_language} – {table_field}: {text[:30]}... ")
         response = requests.post(
             self.url,
             headers=self.headers,
@@ -164,9 +184,12 @@ def translate_text(self, text, dest_language, source_language=None):
             if use_cache:
                 TranslationCache.objects.create(
                     text=text,
+                    text_hash=text_hash,
                     source_language=source_language or "",  # source_language can be "detected"
                     dest_language=dest_language,
                     translated_text=translated,
+                    table_field=table_field or "",
+                    last_used=timezone.now(),
                 )
             return translated + textTail