Deduplication with optional fallback to default

python_twine/tests/fixtures/twine_preoptimized.txt

@@ -0,0 +1,14 @@
+[[Main]]
+
+[value_with_local_lang_codes]
+en = Bar
+es = Bar
+es-MX = Bar
+pt = Bar
+pt-BR = Bar
+
+[value_with_duplicated]
+en = NULL
+de = NULL
+fr = Avant
+

python_twine/tests/test_twine_file.py

@@ -203,7 +203,7 @@ def test_write_file(self):
             Path(temp_path).unlink()
 
 
-class TestWriter:
+class TestTwineFileReader:
     @pytest.fixture
     def fixtures_dir(self):
         """Get fixtures directory path."""
@@ -219,5 +219,42 @@ def test_accent_symbol(self, fixtures_dir):
         assert twine_file.definitions_by_key["value_wrapped_by_spaces"].translations["en"] == ' value '
         assert twine_file.definitions_by_key["value_wrapped_by_accents"].translations["en"] == '`value`'
 
+class TestTwineFileOptimizations:
+    @pytest.fixture
+    def fixture_twine_file(self) -> TwineFile:
+        twine_file = TwineFile()
+        fixtures_dir = Path(__file__).parent / "fixtures"
+        twine_file.read(str(fixtures_dir / "twine_preoptimized.txt"))
+        return twine_file
+
+    def test_deduplication_local_langs(self, fixture_twine_file: TwineFile):
+        definitionA = fixture_twine_file.definitions_by_key['value_with_local_lang_codes']
+        assert definitionA.translations['pt'] == definitionA.translations['pt-BR']
+        assert definitionA.translations['es'] == definitionA.translations['es-MX']
+
+        fixture_twine_file.fallback_to_default = False
+        fixture_twine_file.optimize_duplicates()
+
+        # Check that 'pt-BR' and 'es-MX' are removed from the TwineFile
+        definitionA = fixture_twine_file.definitions_by_key['value_with_local_lang_codes']
+        assert 'pt-BR' not in definitionA.translations
+        assert 'pt' in definitionA.translations
+        assert 'es-MX' not in definitionA.translations
+        assert 'es' in definitionA.translations
+        assert definitionA.translations['es'] == definitionA.translations['en']
+
+    def test_deduplication_default_lang(self, fixture_twine_file: TwineFile):
+        definitionB = fixture_twine_file.definitions_by_key['value_with_duplicated']
+        assert definitionB.translations['en'] == definitionB.translations['de']
+
+        fixture_twine_file.fallback_to_default = True
+        fixture_twine_file.optimize_duplicates()
+
+        # Check that 'de' lang is removed from the TwineFile because it matches 'en'
+        definitionB = fixture_twine_file.definitions_by_key['value_with_duplicated']
+        assert 'de' not in definitionB.translations
+        assert 'fr' in definitionB.translations
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])

python_twine/twine/cli.py

@@ -63,6 +63,11 @@ def create_parser() -> argparse.ArgumentParser:
         )
         consume_file.add_argument("twine_file", help="Path to Twine data file")
         consume_file.add_argument("input_path", help="Input file path")
+        consume_file.add_argument(
+            "--fallback-to-default",
+            action="store_true",
+            help="Remove translations matching default language",
+        )
         CLI._add_common_arguments(consume_file)
         CLI._add_language_argument(consume_file)
         CLI._add_consume_arguments(consume_file)
@@ -74,6 +79,11 @@ def create_parser() -> argparse.ArgumentParser:
         )
         consume_all.add_argument("twine_file", help="Path to Twine data file")
         consume_all.add_argument("input_path", help="Input directory path")
+        consume_all.add_argument(
+            "--fallback-to-default",
+            action="store_true",
+            help="Remove translations matching default language",
+        )
         consume_all.add_argument(
             "-n", "--file-name", help="Input file name (default: format-specific)"
         )

python_twine/twine/runner.py

@@ -172,6 +172,9 @@ def consume_localization_file(self):
         if self.options.get("languages"):
             lang = self.options["languages"][0]
 
+        if bool(self.options.get("fallback_to_default")):
+            self.twine_file.fallback_to_default = True
+
         formatter, lang = self._prepare_read_write(self.options["input_path"], lang)
 
         with open(self.options["input_path"], "r", encoding="UTF-8") as f:
@@ -191,6 +194,9 @@ def consume_all_localization_files(self):
         if not input_path.is_dir():
             raise TwineError(f"Directory does not exist: {input_path}")
 
+        if bool(self.options.get("fallback_to_default")):
+            self.twine_file.fallback_to_default = True
+
         formatter = self._get_formatter()
 
         files_consumed = 0

python_twine/twine/twine_file.py

@@ -170,10 +170,27 @@ def __init__(self, name: str):
 class TwineFile:
     """Main Twine data file containing sections and definitions."""
 
-    def __init__(self):
+    def __init__(self, fallback_to_default:bool = False):
+        """
+        :param fallback_to_default: indicates if TwineFile optimization should deduplicate
+               languages matching default (English) value.
+
+               For example in next strings 'es' should not be removed (fallback_to_default = False)
+               [bar]
+               en = Bar
+               es = Bar
+               uk = Bar
+
+               While in next sample all languages except 'en' should be removed (fallback_to_default = True)
+               [take_exit_number_street_verb]
+               en = NULL
+               de = NULL
+               nl = NULL
+        """
         self.sections: List[TwineSection] = []
         self.definitions_by_key: Dict[str, TwineDefinition] = {}
         self.language_codes: List[str] = []
+        self.fallback_to_default = fallback_to_default
 
     def add_language_code(self, code: str):
         """Add a language code, maintaining developer language at position 0."""
@@ -218,7 +235,7 @@ def optimize_duplicates(self):
 
     def match_fallback_lang(self, translations: dict, lang:str, key:str, value: Any) -> bool:
         # TODO: this method is invoked for each key and lang. Optimize: cache all fallback languages in a dict
-        for fallback_lang in self.fallback_languages(lang):
+        for fallback_lang in self.fallback_languages(lang, self.fallback_to_default):
             if translations.get(fallback_lang) == value:
                 print(f"Warning: key '{key}' in lang '{lang}' matches value from fallback language '{fallback_lang}'")
                 return True