v0.5.5

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.5.5
+
+* Ensure that POT content is now sorted by path when merging POTs from multiple sources (i.e., templates and content).
+* `xgettext` is used to merge POT files instead of `msgcat`, providing a better header and merging of same strings from different sources.
+* The initially generated PO files will now have a header compatible with GNOME's Translation Editor, since they will have a non-placeholder `Project-Id-Version`. Existing users hitting this problem will need to fill in the `Project-Id-Version` header manually.
+* Translations in templates now provide pgettext and ngettext methods.
+* The bug where deletion of strings from the English PO file with non-English content is resolved.
+* When updating translated PO files, the content-language PO file strings are automatically filled with the message IDs. Side effects with plurals are documented.
+
 ## 0.5.4
 
 * POT content is now sorted by path.

README.md

@@ -40,6 +40,11 @@ A `babel.cfg` must be created in your project root with the following content:
     [jinja2: **/templates/**.html]
     encoding = utf-8
 
+#### Whitespace Trimming during Extraction
+
+If you're using `{% trans %}` blocks in your template files, the `trimmed` policy is enabled for Jinja's i18n plugin, so all whitespaces would be trimmed at the beginning and end of those
+blocks.  However, in order for PyBabel's extraction to also work properly this way, one shall add `trimmed = True` to the jinja2 section of the `babel.cfg` configuration file.
+
 ### Translatable fields
 
 In order for a field to be marked as translatable, an option has to be set in the field definition. Both blocks and flowblocks fields are translatable.
@@ -96,12 +101,6 @@ For example:
 
 As with the previous example, `body` and `title` field content will be translated. However, in this example, `image` and `image_position` will not.
 
-### Non-english content
-
-Due to a limitation of `msginit`, it is difficult to translate a site when the primary language is set to anything but English.
-
-If your default content language is not English, you will have to edit the first `contents-en.po` file and remove the translations.
-
 ## Installation
 
 ### Prerequisites
@@ -166,6 +165,12 @@ All translation files (`contents-*.po`) are then compiled and merged with the or
 
 You must run `lektor build` once to generate the list of `contents-xx.po` files. After that, once a translation change is applied to a `contents-xx.po` file, the site must be built again for the changes to be applied to the associated `contents-xx.lr` file. This results in the changes being rendered on the site.
 
+### Plural Forms
+
+If you're using `{% pluralize %}` or `ngettext` or the like in your Jinja templates, make sure you fill in the plural forms in the PO headers manually, then make sure you have the correct
+number of `msgstr[x]`s.  The plugin automatically fills `msgstr`s into the PO file of your source lanaguage (which msginit only does for English), but since it doesn't parse plural forms,
+any non-English PO file will not have its plural message strings filled in.  Those must be done manually in the source-language PO file if simply singular and plural strings does not suffice .
+
 ### Project file
 
 You must modify the `.lektorproject` file to include the expected languages.

lektor_i18n.py

@@ -5,10 +5,11 @@
 import re
 import tempfile
 import time
-from os.path import exists, join, relpath
+from os.path import exists, join, relpath, basename
 from pprint import PrettyPrinter
 from textwrap import dedent
 from urllib.parse import urljoin
+import polib
 
 from lektor.context import get_ctx
 from lektor.db import Page
@@ -55,6 +56,14 @@ def ngettext(self, *x):
         self.init_translator()
         return self.translator.ngettext(*x)
 
+    def pgettext(self, *x):
+        self.init_translator()
+        return self.translator.pgettext(*x)
+
+    def npgettext(self, *x):
+        self.init_translator()
+        return self.translator.npgettext(*x)
+
 
 class Translations:
     """Memory of translations"""
@@ -136,15 +145,29 @@ def write_pot(self, pot_filename, language):
             f.write(self.as_pot(language, header))
 
     @staticmethod
-    def merge_pot(from_filenames, to_filename):
-        msgcat = locate_executable("msgcat")
-        if msgcat is None:
-            msgcat = "/usr/bin/msgcat"
-        cmdline = [msgcat, "--use-first"]
+    def merge_pot(from_filenames, to_filename, projectname):
+        # Get the POT Creation Date of the first file and inject it later.
+        pattern = r'("POT-Creation-Date:\s*)(\d{4}-\d{2}-\d{2}.*)(\\n")'
+        with open(from_filenames[0], 'r', encoding='utf-8') as f:
+            original_file1 = f.read()
+        date1 = re.search(pattern, original_file1).group(2)
+
+        xgettext = locate_executable("xgettext")
+        if xgettext is None:
+            xgettext = "/usr/bin/xgettext"
+        cmdline = [xgettext, "--sort-by-file", "--package-name=" + projectname, "--package-version=1.0"]
         cmdline.extend(from_filenames)
         cmdline.extend(("-o", to_filename))
-        reporter.report_debug_info("msgcat cmd line", cmdline)
+        reporter.report_debug_info("xgettext cmd line", cmdline)
         portable_popen(cmdline).wait()
+
+        # Inject the creation date back into the produced file
+        with open(to_filename, 'r', encoding='utf-8') as f:
+            finishedfile_orig = f.read()
+        replacement = r'\g<1>' + date1 + r'\g<3>'
+        finishedcontent = re.sub(pattern, replacement, finishedfile_orig, count=1)
+        with open(to_filename, 'w', encoding='utf-8') as f:
+            f.write(finishedcontent)
 
     @staticmethod
     def parse_templates(to_filename):
@@ -159,6 +182,35 @@ def parse_templates(to_filename):
 translations = Translations()  # let's have a singleton
 
 
+def clear_translations(po_filepath, save_path=None):
+    po = polib.pofile(po_filepath)
+    for entry in po:
+        entry.msgstr = ''
+        if entry.msgstr_plural:
+            for idx in entry.msgstr_plural:
+                entry.msgstr_plural[idx] = ''
+    po.save(save_path or po_filepath)
+
+def fill_translations(po_filepath, save_path=None):
+    po = polib.pofile(po_filepath)
+
+    for entry in po:
+        if not entry.msgstr:
+            entry.msgstr = entry.msgid
+
+        need_plural_fill = False
+        if entry.msgstr_plural:
+            for idx in entry.msgstr_plural:
+                if not entry.msgstr_plural[idx]:
+                    need_plural_fill = True
+
+        if need_plural_fill and '+en.po' in basename(po_filepath):
+            for idx in entry.msgstr_plural:
+                if not entry.msgstr_plural[idx]:
+                    entry.msgstr_plural[idx] = entry.msgid if int(idx) == 0 else entry.msgid_plural
+    po.save(save_path or po_filepath)
+
+
 class POFile:
     FILENAME_PATTERN = "contents+{}.po"
 
@@ -186,6 +238,7 @@ def _msg_init(self):
         ]
         reporter.report_debug_info("msginit cmd line", cmdline)
         portable_popen(cmdline, cwd=self.i18npath).wait()
+        clear_translations(os.path.join(self.i18npath, self.FILENAME_PATTERN.format(self.language)))
 
     def _msg_merge(self):
         """Merges an existing <language>.po file with .pot file"""
@@ -201,6 +254,11 @@ def _msg_merge(self):
         ]
         reporter.report_debug_info("msgmerge cmd line", cmdline)
         portable_popen(cmdline, cwd=self.i18npath).wait()
+
+    def reformat(self):
+        msgcat = locate_executable("msgcat")
+        cmdline = [msgcat, self.FILENAME_PATTERN.format(self.language), "-o", self.FILENAME_PATTERN.format(self.language)]
+        portable_popen(cmdline, cwd=self.i18npath).wait()
 
     def _prepare_locale_dir(self):
         """Prepares the i18n/<language>/LC_MESSAGES/ to store the .mo file;
@@ -558,7 +616,7 @@ def on_after_build_all(self, builder, **extra):
         reporter.report_generic(f"{relpath(pots[0], builder.env.root_path)} generated")
         pots = [p for p in pots if os.path.exists(p)]  # only keep existing ones
         if len(pots) > 1:
-            translations.merge_pot(pots, contents_pot_filename)
+            translations.merge_pot(pots, contents_pot_filename, self.env.project.name)
             reporter.report_generic(
                 f"Merged POT files "
                 f"{', '.join(relpath(p, builder.env.root_path) for p in pots)}"
@@ -567,3 +625,6 @@ def on_after_build_all(self, builder, **extra):
         for language in self.translations_languages:
             po_file = POFile(language, self.i18npath)
             po_file.generate()
+            if language == self.content_language:
+                fill_translations(os.path.join(po_file.i18npath, po_file.FILENAME_PATTERN.format(po_file.language)))
+            po_file.reformat()

pyproject.toml

@@ -19,6 +19,9 @@ authors = [
 maintainers = [
     {name="BeeWare Team", email="team@beeware.org"},
 ]
+dependencies = [
+    "polib",
+]
 
 [project.optional-dependencies]
 # Extras used by developers *of* briefcase are pinned to specific versions to