Fix translation of file types in CWS (#1475)

Previously, we tried to find shared-mime-info.mo and use that to
translate the descriptions obtained from xdg.Mime into the right
language. This was not the correct approach: first, the path to
shared-mime-info.mo varied (on Ubuntu it was in locale-langpack instead
of locale), and second, sometimes (e.g. on Fedora) it was not present at
all.

The right place to find these translations is in the mimetype xml
files themselves. However, this caused other issues: first, pyxdg does
not have proper support for using more than one language. I hacked
around this by setting the (global) language before each call, and
clearing pyxdg's own cache. And then adding my own cache on top that can
cache multiple languages. The second problem was that on Debian/Ubuntu,
the xml:lang tags that specify the language codes for each string use an
incorrect format: they use POSIX-style locale names (like zh_CN) instead
of XML-compliant BCP47 language codes (like zh-Hans-CN). pyxdg only
knows how to handle these incorrect names, so I had to implement a hack
to get it to recognize the correct names too.

Also deleted the undocumented and now-unused shared_mime_info_prefix
config option, and mentioned shared-mime-info as a dependency in the
installation docs.

Author: prandla

cms/conf.py

@@ -140,12 +140,6 @@ def __init__(self):
         self.max_input_length = 5_000_000  # 5 MB
         self.stl_path = "/usr/share/cppreference/doc/html/"
         self.docs_path = None
-        # Prefix of 'shared-mime-info'[1] installation. It can be found
-        # out using `pkg-config --variable=prefix shared-mime-info`, but
-        # it's almost universally the same (i.e. '/usr') so it's hardly
-        # necessary to change it.
-        # [1] http://freedesktop.org/wiki/Software/shared-mime-info
-        self.shared_mime_info_prefix = "/usr"
         self.contest_admin_token = None
 
         # AdminWebServer.

cms/locale/locale.py

@@ -33,7 +33,6 @@
 import copy
 import logging
 import math
-import os
 
 import babel.core
 import babel.dates
@@ -43,8 +42,8 @@
 import babel.units
 import importlib.resources
 
-from cms import config
 from cmscommon.datetime import utc
+from cmscommon.mimetypes import get_name_for_type
 from datetime import datetime, tzinfo, timedelta
 
 
@@ -70,9 +69,6 @@ def __init__(self, lang_code, mofile=None):
             self.translation = babel.support.Translations(mofile, domain="cms")
         else:
             self.translation = babel.support.NullTranslations()
-        self.mimetype_translation = babel.support.Translations.load(
-            os.path.join(config.shared_mime_info_prefix, "share", "locale"),
-            [self.locale], "shared-mime-info")
 
     @property
     def identifier(self) -> str:
@@ -266,7 +262,11 @@ def format_locale(self, code: str) -> str:
             return code
 
     def translate_mimetype(self, mimetype: str) -> str:
-        return self.mimetype_translation.gettext(mimetype)
+        lang_code = self.identifier
+        alt_lang_code = babel.core.get_locale_identifier(
+            (self.locale.language, self.locale.territory), sep="_"
+        )
+        return get_name_for_type(mimetype, lang_code, alt_lang_code)
 
 
 DEFAULT_TRANSLATION = Translation("en")

cms/server/contest/templates/task_description.html

@@ -177,10 +177,8 @@ <h2>{% trans %}Attachments{% endtrans %}</h2>
     {% for filename, attachment in task.attachments|dictsort(by="key") %}
         {% set mime_type = get_mimetype_for_file_name(filename) %}
         {% if mime_type is not none %}
-            {% set type_name = get_name_for_mimetype(mime_type) %}
             {% set type_icon = get_icon_for_mimetype(mime_type) %}
         {% else %}
-            {% set type_name = none %}
             {% set type_icon = none %}
         {% endif %}
         {% set file_size = handler.application.service.file_cacher.get_size(attachment.digest) %}
@@ -195,8 +193,8 @@ <h2>{% trans %}Attachments{% endtrans %}</h2>
                         <span class="name">{{ filename }}</span>
                         <span class="size">{{ file_size|format_size }}</span>
                     </span>
-            {% if type_name is not none %}
-                    <span class="type">{{ translation.translate_mimetype(type_name) }}</span>
+            {% if mime_type is not none %}
+                    <span class="type">{{ translation.translate_mimetype(mime_type) }}</span>
             {% endif %}
                 </a>
             </li>

cms/server/jinja2_toolbox.py

@@ -42,8 +42,7 @@
 from cmscommon.constants import \
     SCORE_MODE_MAX, SCORE_MODE_MAX_SUBTASK, SCORE_MODE_MAX_TOKENED_LAST
 from cmscommon.datetime import make_datetime, make_timestamp, utc, local_tz
-from cmscommon.mimetypes import get_type_for_file_name, get_name_for_type, \
-    get_icon_for_type
+from cmscommon.mimetypes import get_type_for_file_name, get_icon_for_type
 
 
 @contextfilter
@@ -211,7 +210,6 @@ def instrument_cms_toolbox(env: Environment):
     env.globals["get_score_type"] = safe_get_score_type
 
     env.globals["get_mimetype_for_file_name"] = get_type_for_file_name
-    env.globals["get_name_for_mimetype"] = get_name_for_type
     env.globals["get_icon_for_mimetype"] = get_icon_for_type
 
     env.filters["to_language"] = get_language

cmscommon/mimetypes.py

@@ -16,10 +16,12 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import functools
 import os.path
 
 import xdg.BaseDirectory
 import xdg.Mime
+import xdg.Locale
 
 
 __all__ = [
@@ -59,17 +61,36 @@ def get_icon_for_type(typename: str) -> str:
         return _icons[typename]
     return mimetype.media + "-x-generic"
 
-
-def get_name_for_type(typename: str) -> str:
+# xdg.Mime is by default memoized, but since we need to change the language, we
+# need to wipe the cache to load the correct language. So use our own caching
+# on top of it.
+@functools.cache
+def get_name_for_type(typename: str, language: str, alt_language: str) -> str:
     """Get the natural language description of the MIME type.
 
     typename: a MIME type, e.g., "application/pdf".
+    language: the BCP47 code of the language for which to return the result.
+    alt_language: underscore-separated form of the language code, to work
+        around incorrect behavior in pyxdg.
 
     return: the human-readable description (also called comment)
         of the given MIME type, e.g., "PDF document".
 
     """
+    # pyxdg expects the locale field to be provided as a posix-style locale
+    # name, e.g. zh_CN. It assumes this in both the provided language name, and
+    # in the xml:lang attribute of the mimetype xml files. Some distributions
+    # instead use BCP47 language codes, e.g. zh-Hans-CN, in the mimetype xml
+    # files (which is semantically more correct, as this is mandated by the xml
+    # spec).
+    # First parse the language from the posix format.
+    xdg.Locale.update(alt_language)
+    # Then, we make pyxdg think the BCP47 code is another variant of the
+    # current language name.
+    xdg.Locale.langs += [language]
     mimetype = xdg.Mime.lookup(typename).canonical()
+    # Force reloading the comment, because the language might have changed.
+    mimetype._comment = None
     return mimetype.get_comment()
 
 

cmstestsuite/unit_tests/cmscommon/mimetypes_test.py

@@ -42,11 +42,11 @@ def test_generic(self):
 class TestGetNameForType(unittest.TestCase):
 
     def test_basic(self):
-        self.assertEqual(get_name_for_type("application/pdf"),
+        self.assertEqual(get_name_for_type("application/pdf", "en", "en"),
                          "PDF document")
 
     def test_alias(self):
-        self.assertEqual(get_name_for_type("text/x-octave"),
+        self.assertEqual(get_name_for_type("text/x-octave", "en", "en"),
                          "MATLAB file")
 
 

cmstestsuite/unit_tests/locale/locale_test.py

@@ -53,6 +53,7 @@
 ITALIAN = Translation("it")
 DANISH = Translation("da")
 CHINESE = Translation("zh_CN")
+CHINESE_TRADITIONAL = Translation("zh_TW")
 
 
 class TestIdentifier(unittest.TestCase):
@@ -591,20 +592,15 @@ def test_localized_decimal_and_thousands_separators(self):
 
 class TestTranslateMimetype(unittest.TestCase):
 
-    @unittest.skipIf(not os.path.isfile(
-        "/usr/share/locale/it/LC_MESSAGES/shared-mime-info.mo"),
-        reason="need Italian shared-mime-info translation")
     def test_translate_mimetype(self):
-        self.assertEqual(ENGLISH.translate_mimetype("PDF document"),
+        self.assertEqual(ENGLISH.translate_mimetype("application/pdf"),
                          "PDF document")
-        self.assertEqual(ITALIAN.translate_mimetype("PDF document"),
+        self.assertEqual(ITALIAN.translate_mimetype("application/pdf"),
                          "Documento PDF")
-
-    def test_graceful_failure(self):
-        self.assertEqual(ENGLISH.translate_mimetype("Not a MIME type"),
-                         "Not a MIME type")
-        self.assertEqual(ITALIAN.translate_mimetype("Not a MIME type"),
-                         "Not a MIME type")
+        self.assertEqual(CHINESE.translate_mimetype("application/pdf"),
+                         "PDF 文档")
+        self.assertEqual(CHINESE_TRADITIONAL.translate_mimetype("application/pdf"),
+                         "PDF 文件")
 
 
 class TestFilterLanguageCodes(unittest.TestCase):

docs/Installation.rst

@@ -70,7 +70,7 @@ On Ubuntu 24.04, one will need to run the following script as root to satisfy al
         postgresql postgresql-client \
         python3.12 python3.12-dev python3-pip python3-venv \
         libpq-dev libcups2-dev libyaml-dev libffi-dev \
-        cppreference-doc-en-html zip curl
+        shared-mime-info cppreference-doc-en-html zip curl
 
     # Isolate from upstream package repository
     echo 'deb [arch=amd64 signed-by=/etc/apt/keyrings/isolate.asc] http://www.ucw.cz/isolate/debian/ noble-isolate main' >/etc/apt/sources.list.d/isolate.list
@@ -92,7 +92,8 @@ On Arch Linux, run the following commands as root to install almost all dependen
 .. sourcecode:: bash
 
     pacman -S base-devel jdk8-openjdk fpc postgresql postgresql-client \
-        python python-pip postgresql-libs libcups libyaml
+        python python-pip postgresql-libs libcups libyaml \
+        shared-mime-info
 
     # Install the following from AUR.
     # https://aur.archlinux.org/packages/cppreference/