gh-130057: Pygettext: Support translator comments

Lib/test/test_tools/i18n_data/comments.pot

@@ -101,3 +101,10 @@ msgstr ""
 msgid "thud"
 msgstr ""
 
+#. i18n: This is a translator comment
+#. i18n: This is another translator comment
+#. i18n: This is yet another translator comment
+#: comments.py:78
+msgid "foos"
+msgstr ""
+

Lib/test/test_tools/i18n_data/comments.py

@@ -70,3 +70,9 @@
 # i18n: This is a translator comment
 _('xyzzy')
 _('thud')
+
+
+## i18n: This is a translator comment
+# # i18n: This is another translator comment
+### ###    i18n: This is yet another translator comment
+_('foos')

Lib/test/test_tools/test_i18n.py

@@ -87,7 +87,8 @@ def assert_POT_equal(self, expected, actual):
         self.maxDiff = None
         self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual))
 
-    def extract_from_str(self, module_content, *, args=(), strict=True, with_stderr=False):
+    def extract_from_str(self, module_content, *, args=(), strict=True,
+                         with_stderr=False, raw=False):
         """Return all msgids extracted from module_content."""
         filename = 'test.py'
         with temp_cwd(None):
@@ -98,10 +99,11 @@ def extract_from_str(self, module_content, *, args=(), strict=True, with_stderr=
                 self.assertEqual(res.err, b'')
             with open('messages.pot', encoding='utf-8') as fp:
                 data = fp.read()
-        msgids = self.get_msgids(data)
+        if not raw:
+            data = self.get_msgids(data)
         if not with_stderr:
-            return msgids
-        return msgids, res.err
+            return data
+        return data, res.err
 
     def extract_docstrings_from_str(self, module_content):
         """Return all docstrings extracted from module_content."""
@@ -432,6 +434,19 @@ def test_error_messages(self):
             "*** test.py:3: Variable positional arguments are not allowed in gettext calls\n"
         )
 
+    def test_extract_all_comments(self):
+        """
+        Test that the --add-comments option without an
+        explicit tag extracts all translator comments.
+        """
+        for arg in ('--add-comments', '-c'):
+            with self.subTest(arg=arg):
+                data = self.extract_from_str(dedent('''\
+                # Translator comment
+                _("foo")
+                '''), args=(arg,), raw=True)
+                self.assertIn('#. Translator comment', data)
+
 
 def update_POT_snapshots():
     for input_file in DATA_DIR.glob('*.py'):

Tools/i18n/pygettext.py

@@ -144,6 +144,7 @@
 import importlib.machinery
 import importlib.util
 import os
+import re
 import sys
 import time
 import tokenize
@@ -329,7 +330,9 @@ def get_source_comments(source):
     comments = {}
     for token in tokenize.tokenize(BytesIO(source).readline):
         if token.type == tokenize.COMMENT:
-            comments[token.start[0]] = token.string.removeprefix('#').strip()
+            # Remove any leading combination of '#' and whitespace
+            comment = re.sub(r'^[#\s]+', '', token.string)
+            comments[token.start[0]] = comment
     return comments
 
 
@@ -348,7 +351,8 @@ def visit_file(self, source, filename):
             return
 
         self.filename = filename
-        self.comments = get_source_comments(source)
+        if self.options.comment_tags:
+            self.comments = get_source_comments(source)
         self.visit(module_tree)
 
     def visit_Module(self, node):
@@ -436,7 +440,7 @@ def _extract_comments(self, node):
         return comments[first_index:]
 
     def _is_translator_comment(self, comment):
-        return comment.startswith(tuple(self.options.comment_tags))
+        return comment.startswith(self.options.comment_tags)
 
     def _add_message(
             self, lineno, msgid, msgid_plural=None, msgctxt=None, *,
@@ -546,8 +550,8 @@ def main():
     try:
         opts, args = getopt.getopt(
             sys.argv[1:],
-            'ac:d:DEhk:Kno:p:S:Vvw:x:X:',
-            ['extract-all', 'add-comments=', 'default-domain=', 'escape',
+            'ac::d:DEhk:Kno:p:S:Vvw:x:X:',
+            ['extract-all', 'add-comments=?', 'default-domain=', 'escape',
              'help', 'keyword=', 'no-default-keywords',
              'add-location', 'no-location', 'output=', 'output-dir=',
              'style=', 'verbose', 'version', 'width=', 'exclude-file=',
@@ -634,6 +638,8 @@ class Options:
             finally:
                 fp.close()
 
+    options.comment_tags = tuple(options.comment_tags)
+
     # calculate escapes
     make_escapes(not options.escape)