Fix translation references to unverified translations (#155314)

Co-authored-by: Martin Hjelmare <[email protected]>

Author: arturpragacz

script/translations/develop.py

@@ -3,12 +3,11 @@
 import argparse
 import json
 from pathlib import Path
-import re
 import sys
 
 from . import download, upload
 from .const import INTEGRATIONS_DIR
-from .util import flatten_translations, get_base_arg_parser
+from .util import flatten_translations, get_base_arg_parser, substitute_references
 
 
 def valid_integration(integration):
@@ -31,42 +30,6 @@ def get_arguments() -> argparse.Namespace:
     return parser.parse_args()
 
 
-def substitute_translation_references(integration_strings, flattened_translations):
-    """Recursively processes all translation strings for the integration."""
-    result = {}
-    for key, value in integration_strings.items():
-        if isinstance(value, dict):
-            sub_dict = substitute_translation_references(value, flattened_translations)
-            result[key] = sub_dict
-        elif isinstance(value, str):
-            result[key] = substitute_reference(value, flattened_translations)
-
-    return result
-
-
-def substitute_reference(value, flattened_translations):
-    """Substitute localization key references in a translation string."""
-    matches = re.findall(r"\[\%key:([a-z0-9_]+(?:::(?:[a-z0-9-_])+)+)\%\]", value)
-    if not matches:
-        return value
-
-    new = value
-    for key in matches:
-        if key in flattened_translations:
-            new = new.replace(
-                f"[%key:{key}%]",
-                # New value can also be a substitution reference
-                substitute_reference(
-                    flattened_translations[key], flattened_translations
-                ),
-            )
-        else:
-            print(f"Invalid substitution key '{key}' found in string '{value}'")
-            sys.exit(1)
-
-    return new
-
-
 def run_single(translations, flattened_translations, integration):
     """Run the script for a single integration."""
     print(f"Generating translations for {integration}")
@@ -77,8 +40,8 @@ def run_single(translations, flattened_translations, integration):
 
     integration_strings = translations["component"][integration]
 
-    translations["component"][integration] = substitute_translation_references(
-        integration_strings, flattened_translations
+    translations["component"][integration] = substitute_references(
+        integration_strings, flattened_translations, fail_on_missing=True
     )
 
     if download.DOWNLOAD_DIR.is_dir():

script/translations/download.py

@@ -10,12 +10,17 @@
 
 from .const import CLI_2_DOCKER_IMAGE, CORE_PROJECT_ID, INTEGRATIONS_DIR
 from .error import ExitApp
-from .util import get_lokalise_token, load_json_from_path
+from .util import (
+    flatten_translations,
+    get_lokalise_token,
+    load_json_from_path,
+    substitute_references,
+)
 
 DOWNLOAD_DIR = Path("build/translations-download").absolute()
 
 
-def run_download_docker():
+def run_download_docker() -> None:
     """Run the Docker image to download the translations."""
     print("Running Docker to download latest translations.")
     result = subprocess.run(
@@ -39,6 +44,7 @@ def run_download_docker():
             "--replace-breaks=false",
             "--filter-data",
             "nonfuzzy",
+            "--disable-references",
             "--export-empty-as",
             "skip",
             "--format",
@@ -76,9 +82,12 @@ def filter_translations(translations: dict[str, Any], strings: dict[str, Any]) -
                 continue
 
 
-def save_language_translations(lang, translations):
+def save_language_translations(lang: str, translations: dict[str, Any]) -> None:
     """Save translations for a single language."""
     components = translations.get("component", {})
+
+    flattened_translations = flatten_translations(translations)
+
     for component, component_translations in components.items():
         # Remove legacy platform translations
         component_translations.pop("platform", None)
@@ -104,26 +113,30 @@ def save_language_translations(lang, translations):
         path = component_path / "translations" / f"{lang}.json"
         path.parent.mkdir(parents=True, exist_ok=True)
 
+        component_translations = substitute_references(
+            component_translations, flattened_translations, fail_on_missing=False
+        )
+
         filter_translations(component_translations, strings)
 
         save_json(path, component_translations)
 
 
-def save_integrations_translations():
+def save_integrations_translations() -> None:
     """Save integrations translations."""
     for lang_file in DOWNLOAD_DIR.glob("*.json"):
         lang = lang_file.stem
         translations = load_json_from_path(lang_file)
         save_language_translations(lang, translations)
 
 
-def delete_old_translations():
+def delete_old_translations() -> None:
     """Delete old translations."""
     for fil in INTEGRATIONS_DIR.glob("*/translations/*"):
         fil.unlink()
 
 
-def run():
+def run() -> None:
     """Run the script."""
     DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True)
 

script/translations/error.py

@@ -12,6 +12,15 @@ def __init__(self, reason, exit_code=1):
         self.exit_code = exit_code
 
 
+class MissingReference(Exception):
+    """Exception to indicate a missing reference in translations."""
+
+    def __init__(self, reference_key: str):
+        """Initialize the missing reference exception."""
+        self.reference_key = reference_key
+        super().__init__(f"Missing reference key: {reference_key}")
+
+
 class JSONDecodeErrorWithPath(json.JSONDecodeError):
     """Subclass of JSONDecodeError with additional properties.
 

script/translations/util.py

@@ -4,10 +4,11 @@
 import json
 import os
 import pathlib
+import re
 import subprocess
 from typing import Any
 
-from .error import ExitApp, JSONDecodeErrorWithPath
+from .error import ExitApp, JSONDecodeErrorWithPath, MissingReference
 
 
 def get_base_arg_parser() -> argparse.ArgumentParser:
@@ -89,3 +90,52 @@ def flatten_translations(translations):
                 key_stack.pop()
 
     return flattened_translations
+
+
+def substitute_reference(value: str, flattened_translations: dict[str, str]) -> str:
+    """Substitute localization key references in a translation string."""
+    matches = re.findall(r"\[\%key:([a-z0-9_]+(?:::(?:[a-z0-9-_])+)+)\%\]", value)
+    if not matches:
+        return value
+
+    new = value
+    for key in matches:
+        if key in flattened_translations:
+            # New value can also be a substitution reference
+            substituted = substitute_reference(
+                flattened_translations[key], flattened_translations
+            )
+            new = new.replace(f"[%key:{key}%]", substituted)
+        else:
+            raise MissingReference(key)
+
+    return new
+
+
+def substitute_references(
+    translations: dict[str, Any],
+    substitutions: dict[str, str],
+    *,
+    fail_on_missing: bool,
+) -> dict[str, Any]:
+    """Recursively substitute references for all translation strings."""
+    result = {}
+    for key, value in translations.items():
+        if isinstance(value, dict):
+            sub_dict = substitute_references(
+                value, substitutions, fail_on_missing=fail_on_missing
+            )
+            if sub_dict:
+                result[key] = sub_dict
+        elif isinstance(value, str):
+            try:
+                substituted = substitute_reference(value, substitutions)
+            except MissingReference as err:
+                if fail_on_missing:
+                    raise ExitApp(
+                        f"Missing reference '{err.reference_key}' in translation for key '{key}'"
+                    ) from err
+                continue
+            result[key] = substituted
+
+    return result