[Fixes #13800] Refact metadata i18n mechanism

* [Fixes #13800] Refact metadata i18n mechanism

* [Fixes #13800] Refact metadata i18n mechanism - gemini fixes

* [Fixes #13800] Refact metadata i18n mechanism - use new gettext

Author: etj

geonode/base/apps.py

@@ -42,3 +42,11 @@ class BaseAppConfig(NotificationsAppConfigBase, AppConfig):
             _("Resource is created"),
         ),
     )
+
+    def ready(self):
+        """Finalize setup"""
+        from geonode.base.signals import connect_signals
+
+        connect_signals()
+
+        super(BaseAppConfig, self).ready()

geonode/metadata/i18n.py geonode/base/i18n.pygeonode/metadata/i18n.py renamed to geonode/base/i18n.py

@@ -1,12 +1,10 @@
 import logging
-from datetime import datetime
 
-from cachetools import FIFOCache
 from django.db import connection
+from django.utils.translation import get_language, gettext as _
 
 from geonode.base.models import ThesaurusKeywordLabel, Thesaurus
 
-
 logger = logging.getLogger(__name__)
 
 I18N_THESAURUS_IDENTIFIER = "labels-i18n"
@@ -47,6 +45,7 @@ def get_localized_tkeywords(lang, thesaurus_identifier: str):
     return sorted(ret.values(), key=lambda i: i["about"].lower())
 
 
+# TODO: deprecate and use LabelResolver.gettext()
 def get_localized_label(lang, about):
     # look for override
     ovr_qs = ThesaurusKeywordLabel.objects.filter(
@@ -65,41 +64,46 @@ def get_localized_label(lang, about):
     )
 
 
-class I18nCache:
+class I18nCacheEntry:
+    def __init__(self):
+        # the date field of the thesaurus when it was last loaded, it's used for the expiration check
+        self.date: str | None = None
+        self.caches: dict = {}  # the caches for this language
 
-    DATA_KEY_SCHEMA = "schema"
-    DATA_KEY_LABELS = "labels"
+
+class I18nCache:
+    """
+    Caches language related data.
+    Synch is performed via date field in the "labels-i18n" thesaurus.
+    """
 
     def __init__(self):
-        # the cache has the lang as key, and various info in the dict value:
-        # - date: the date field of the thesaurus when it was last loaded, it's used for the expiration check
-        # - labels: the keyword labels from the i18n thesaurus
-        # - schema: the localized json schema
-        # FIFO bc we want to renew the data once in a while
-        self.cache = FIFOCache(16)
+        # the cache has the lang as key, and I18nCacheEntry as a value:
+        self.lang_cache = {}
 
     def get_entry(self, lang, data_key):
         """
         returns date:str, data
         date is needed for checking the entry freshness when setting info
         data may be None if not cached or expired
         """
-        cached_entry = self.cache.get(lang, None)
+        cached_entry: I18nCacheEntry = self.lang_cache.get(lang, None)
 
+        # TODO: thesaurus date check should be done only after a given time interval from last check
         thesaurus_date = (  # may be none if thesaurus does not exist
             Thesaurus.objects.filter(identifier=I18N_THESAURUS_IDENTIFIER).values_list("date", flat=True).first()
         )
         if cached_entry:
-            if thesaurus_date == cached_entry["date"]:
+            if thesaurus_date == cached_entry.date:
                 # only return cached data if thesaurus has not been modified
-                return thesaurus_date, cached_entry.get(data_key, None)
+                return thesaurus_date, cached_entry.caches.get(data_key, None)
             else:
                 logger.info(f"Schema for {lang}:{data_key} needs to be recreated")
 
         return thesaurus_date, None
 
     def set(self, lang: str, data_key: str, data: dict, request_date: str):
-        cached_entry: dict = self.cache.setdefault(lang, {})
+        cached_entry: I18nCacheEntry = self.lang_cache.setdefault(lang, I18nCacheEntry())
 
         latest_date = (
             Thesaurus.objects.filter(identifier=I18N_THESAURUS_IDENTIFIER).values_list("date", flat=True).first()
@@ -108,60 +112,48 @@ def set(self, lang: str, data_key: str, data: dict, request_date: str):
         if request_date == latest_date:
             # no changes after processing, set the info right away
             logger.debug(f"Caching lang:{lang} key:{data_key} date:{request_date}")
-            cached_entry.update({"date": latest_date, data_key: data})
+            cached_entry.date = latest_date
+            cached_entry.caches[data_key] = data
         else:
             logger.warning(
                 f"Cache will not be updated for lang:{lang} key:{data_key} reqdate:{request_date} latest:{latest_date}"
             )
 
-    def get_labels(self, lang):
-        date, labels = self.get_entry(lang, self.DATA_KEY_LABELS)
-        if labels is None:
-            labels = {}
-            for i in get_localized_tkeywords(lang, I18N_THESAURUS_IDENTIFIER):
-                about = i["about"]
-                if about.endswith(OVR_SUFFIX) and not i["label"]:
-                    # we don't want default values for override entries
-                    continue
-                labels[about] = i["label"] or i["default"]
-            self.set(lang, self.DATA_KEY_LABELS, labels, date)
-        return labels
-
-    def clear_schema_cache(self):
+    def clear(self):
         logger.info("Clearing schema cache")
-        while True:
-            try:
-                self.cache.popitem()
-            except KeyError:
-                return
+        self.lang_cache.clear()
 
 
-def thesaurus_changed(sender, instance, **kwargs):
-    if instance.identifier == I18N_THESAURUS_IDENTIFIER:
-        if hasattr(instance, "_signal_handled"):  # avoid signal recursion
-            return
-        logger.debug(f"Thesaurus changed: {instance.identifier}")
-        _update_thesaurus_date()
+class LabelResolver:
+    CACHE_KEY_LABELS = "labels"
 
+    def gettext(self, key, lang=None, fallback=True):
+        """
+        Return the translated text in the label Thesaurus, falling back to the PO/MO translation.
+        If fallback=False only search in the label Thesaurus, and may return None if not found
+        """
+        lang = lang or get_language()
+        # TODO: implement the OVR search
+        return self.get_labels(lang).get(key, None) or (_(key) if fallback else None)
 
-def thesaurusk_changed(sender, instance, **kwargs):
-    if instance.thesaurus.identifier == I18N_THESAURUS_IDENTIFIER:
-        logger.debug(f"ThesaurusKeyword changed: {instance.about} ALT:{instance.alt_label}")
-        _update_thesaurus_date()
-
+    def get_labels(self, lang):
+        date, labels = i18nCache.get_entry(lang, self.CACHE_KEY_LABELS)
+        if labels is None:
+            labels = self._create_labels_cache(lang)
+            i18nCache.set(lang, self.CACHE_KEY_LABELS, labels, date)
+        return labels
 
-def thesauruskl_changed(sender, instance, **kwargs):
-    if instance.keyword.thesaurus.identifier == I18N_THESAURUS_IDENTIFIER:
-        logger.debug(
-            f"ThesaurusKeywordLabel changed: {instance.keyword.about} ALT:{instance.keyword.alt_label} L:{instance.lang}"
-        )
-        _update_thesaurus_date()
+    def _create_labels_cache(self, lang):
+        labels = {}
+        for i in get_localized_tkeywords(lang, I18N_THESAURUS_IDENTIFIER):
+            about = i["about"]
+            if about.endswith(OVR_SUFFIX) and not i["label"]:
+                # we don't want default values for override entries
+                continue
+            labels[about] = i["label"] or i["default"]
+        return labels
 
 
-def _update_thesaurus_date():
-    logger.debug("Updating label thesaurus date")
-    # update timestamp to invalidate other processes also
-    i18n_thesaurus = Thesaurus.objects.get(identifier=I18N_THESAURUS_IDENTIFIER)
-    i18n_thesaurus.date = datetime.now().replace(microsecond=0).isoformat()
-    i18n_thesaurus._signal_handled = True
-    i18n_thesaurus.save()
+i18nCache = I18nCache()
+labelResolver = LabelResolver()
+gettext = labelResolver.gettext

geonode/base/signals.py

@@ -0,0 +1,48 @@
+import logging
+from datetime import datetime
+
+from django.db.models.signals import post_save
+
+from geonode.base.i18n import I18N_THESAURUS_IDENTIFIER
+from geonode.base.models import Thesaurus, ThesaurusKeyword, ThesaurusKeywordLabel
+
+logger = logging.getLogger(__name__)
+
+
+def connect_signals():
+    logger.debug("Setting up signal connections...")
+    post_save.connect(thesaurus_changed, sender=Thesaurus, weak=False, dispatch_uid="metadata_reset_t")
+    post_save.connect(thesaurusk_changed, sender=ThesaurusKeyword, weak=False, dispatch_uid="metadata_reset_tk")
+    post_save.connect(thesauruskl_changed, sender=ThesaurusKeywordLabel, weak=False, dispatch_uid="metadata_reset_tkl")
+    logger.debug("Signal connections set")
+
+
+def thesaurus_changed(sender, instance, **kwargs):
+    if instance.identifier == I18N_THESAURUS_IDENTIFIER:
+        if hasattr(instance, "_signal_handled"):  # avoid signal recursion
+            return
+        logger.debug(f"Thesaurus changed: {instance.identifier}")
+        _update_thesaurus_date()
+
+
+def thesaurusk_changed(sender, instance, **kwargs):
+    if instance.thesaurus.identifier == I18N_THESAURUS_IDENTIFIER:
+        logger.debug(f"ThesaurusKeyword changed: {instance.about} ALT:{instance.alt_label}")
+        _update_thesaurus_date()
+
+
+def thesauruskl_changed(sender, instance, **kwargs):
+    if instance.keyword.thesaurus.identifier == I18N_THESAURUS_IDENTIFIER:
+        logger.debug(
+            f"ThesaurusKeywordLabel changed: {instance.keyword.about} ALT:{instance.keyword.alt_label} L:{instance.lang}"
+        )
+        _update_thesaurus_date()
+
+
+def _update_thesaurus_date():
+    logger.debug("Updating label thesaurus date")
+    # update timestamp to invalidate other processes also
+    i18n_thesaurus = Thesaurus.objects.get(identifier=I18N_THESAURUS_IDENTIFIER)
+    i18n_thesaurus.date = datetime.now().replace(microsecond=0).isoformat()
+    i18n_thesaurus._signal_handled = True
+    i18n_thesaurus.save()

geonode/indexing/tests/test_invoke.py

@@ -25,6 +25,7 @@
 
 from django.test import override_settings
 
+from geonode.base.i18n import i18nCache
 from geonode.base.models import ResourceBase
 from geonode.metadata.handlers.multilang import MultiLangHandler
 from geonode.metadata.handlers.sparse import SparseHandler, SparseFieldRegistry
@@ -40,7 +41,7 @@
 class IndexingInvocationTests(GeoNodeBaseTestSupport):
 
     def setUp(self):
-        pass
+        i18nCache.clear()
 
     def tearDown(self):
         super().tearDown()

geonode/indexing/tests/test_tsindex.py

@@ -23,6 +23,7 @@
 
 from django.test import override_settings
 
+from geonode.base.i18n import i18nCache
 from geonode.indexing.manager import TSVectorIndexManager
 from geonode.tests.base import GeoNodeBaseTestSupport
 
@@ -33,7 +34,7 @@
 class IndexingTests(GeoNodeBaseTestSupport):
 
     def setUp(self):
-        pass
+        i18nCache.clear()
 
     def tearDown(self):
         super().tearDown()

geonode/metadata/api/views.py

@@ -37,7 +37,7 @@
 from geonode.base.views import LinkedResourcesAutocomplete, RegionAutocomplete, HierarchicalKeywordAutocomplete
 from geonode.groups.models import GroupProfile
 from geonode.metadata.handlers.abstract import MetadataHandler
-from geonode.metadata.i18n import get_localized_label
+from geonode.base.i18n import get_localized_label
 from geonode.metadata.manager import metadata_manager
 from geonode.metadata.multilang import utils as multi
 from geonode.people.utils import get_available_users

geonode/metadata/apps.py

@@ -19,10 +19,6 @@ def ready(self):
 def run_setup_hooks(*args, **kwargs):
     setup_metadata_handlers()
 
-    from geonode.metadata.signals import connect_signals
-
-    connect_signals()
-
 
 def setup_metadata_handlers():
     from geonode.metadata.manager import metadata_manager

geonode/metadata/handlers/abstract.py

@@ -23,10 +23,8 @@
 
 from typing_extensions import deprecated
 
-from django.utils.translation import gettext as _
-
 from geonode.base.models import ResourceBase
-from geonode.metadata.i18n import OVR_SUFFIX
+from geonode.base.i18n import OVR_SUFFIX, labelResolver
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +49,7 @@ def update_schema(self, jsonschema: dict, context: dict, lang=None):
         It adds the subschema handled by the handler, and returns the
         augmented instance of the JSON Schema.
         Context is populated by the manager with some common info:
-         - key "labels": contains the localized label loaded from the db as a dict, where key is the ThesaurusKeyword about
+         - key CONTEXT_KEY_LABELS: contains the localized label loaded from the db as a dict, where key is the ThesaurusKeyword about
            and value is the localized ThesaurusKeywordLabel, or the AltLabel if the localized label does not exist.
         """
         pass
@@ -72,7 +70,7 @@ def update_resource(
     ):
         """
         Called when persisting data, updates the field `field_name` of the resource
-        with the content content, where json_instance is  the full JSON Schema instance,
+        with the content, where json_instance is the full JSON Schema instance,
         in case the handler needs some cross related data contained in the resource.
         """
         pass
@@ -162,35 +160,26 @@ def _set_error(errors: dict, path: list, msg: str):
 
     @staticmethod
     def localize_message(context: dict, msg_code: str, msg_info: dict):
-        msg_loc: str = MetadataHandler._get_tkl_labels(context, None, msg_code)
+        msg_loc: str = labelResolver.gettext(msg_code)
         if msg_loc:
             tokens = defaultdict(lambda: "N/A", msg_info or {})
             return msg_loc.format_map(tokens)
         else:
             logger.warning(f"Missing i18n entry for key '{msg_code}' -- info is {msg_info}")
             return f"{msg_code}:{msg_info}"
 
-    @staticmethod
-    def _localize_label(context, lang: str, text: str):
-        label = MetadataHandler._get_tkl_labels(context, lang, text)
-        return label or _(text)
-
-    @staticmethod
-    def _get_tkl_labels(context, lang: str | None, text: str):
-        return context.get("labels", {}).get(text, None)
-
     @staticmethod
     def _localize_subschema_labels(context, subschema: dict, lang: str, property_name: str = None):
         for annotation_name, synt in (
             ("title", ""),
             ("description", "__descr"),
         ):
-            if ovr := MetadataHandler._get_tkl_labels(context, lang, f"{property_name}{synt}{OVR_SUFFIX}"):
+            if ovr := labelResolver.gettext(f"{property_name}{synt}{OVR_SUFFIX}", lang, fallback=False):
                 subschema[annotation_name] = ovr
             elif annotation_name in subschema:
-                subschema[annotation_name] = MetadataHandler._localize_label(context, lang, subschema[annotation_name])
+                subschema[annotation_name] = labelResolver.gettext(subschema[annotation_name], lang)
             elif property_name:  # arrays may not have a name
-                label = MetadataHandler._get_tkl_labels(context, lang, f"{property_name}{synt}")
+                label = labelResolver.gettext(f"{property_name}{synt}", lang, fallback=False)
                 if label:
                     subschema[annotation_name] = label
 

geonode/metadata/handlers/contact.py

@@ -24,6 +24,7 @@
 from django.contrib.auth import get_user_model
 from django.utils.translation import gettext as _
 
+from geonode.base.i18n import labelResolver
 from geonode.metadata.handlers.abstract import MetadataHandler
 from geonode.people import Roles
 from geonode.resource.manager import resource_manager
@@ -66,7 +67,7 @@ def update_schema(self, jsonschema, context, lang=None):
             if role.is_multivalue:
                 contact = {
                     "type": "array",
-                    "title": self._localize_label(context, lang, role.label) + card,
+                    "title": labelResolver.gettext(role.label, lang) + card,
                     "minItems": minitems,
                     "items": {
                         "type": "object",
@@ -86,7 +87,7 @@ def update_schema(self, jsonschema, context, lang=None):
             else:
                 contact = {
                     "type": "object",
-                    "title": self._localize_label(context, lang, role.label) + card,
+                    "title": labelResolver.gettext(role.label, lang) + card,
                     "properties": {
                         "id": {
                             "type": "string",