Merge pull request #135 from openzim/add-chinese

Add support for simplified and traditional Chinese

Author: benoit74

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add support for simplified and traditional Chinese (#135)
+
 ## [2.0.2] - 2025-11-14
 
 ### Changed

offliner-definition.json

@@ -13,39 +13,47 @@
       "type": "string-enum",
       "required": true,
       "title": "Language",
-      "description": "Language of zim file and curriculum.",
+      "description": "Language of curriculum.",
       "choices": [
         {
-          "title": "ENGLISH",
+          "title": "English",
           "value": "eng"
         },
         {
-          "title": "ESPANOL",
+          "title": "Espanol",
           "value": "spa"
         },
         {
-          "title": "GERMAN",
+          "title": "German",
           "value": "deu"
         },
         {
-          "title": "ITALIAN",
+          "title": "Italian",
           "value": "ita"
         },
         {
-          "title": "JAPANESE",
+          "title": "Japanese",
           "value": "jpn"
         },
         {
-          "title": "PORTOGUESE",
+          "title": "Portuguese",
           "value": "por"
         },
         {
-          "title": "UKRAINIAN",
+          "title": "Ukrainian",
           "value": "ukr"
         },
         {
-          "title": "SWAHILI",
+          "title": "Swahili",
           "value": "swa"
+        },
+        {
+          "title": "Chinese Simplified",
+          "value": "zh-hans"
+        },
+        {
+          "title": "Chinese Traditional",
+          "value": "zh-hant"
         }
       ]
     },

scraper/src/fcc2zim/constants.py

@@ -6,12 +6,11 @@
 VERSION = __version__
 ROOT_DIR = pathlib.Path(__file__).parent
 
-# key is the language passed at CLI (and used as ZIM Language metadata)
+# key is the language passed at CLI
 # value is the name of folders used in FCC source code
 FCC_LANG_MAP = {
-    # removed until we settle on these two codes
-    # "cmn": "chinese",
-    # "lzh": "chinese-traditional",
+    "zh-hans": "chinese",
+    "zh-hant": "chinese-traditional",
     "eng": "english",
     "spa": "espanol",
     "deu": "german",
@@ -21,3 +20,16 @@
     "ukr": "ukrainian",
     "swa": "swahili",
 }
+
+ZIM_LANG_MAP = {
+    "zh-hans": "zho",
+    "zh-hant": "zho",
+    "eng": "eng",
+    "spa": "spa",
+    "deu": "deu",
+    "ita": "ita",
+    "jpn": "jpn",
+    "por": "por",
+    "ukr": "ukr",
+    "swa": "swa",
+}

scraper/src/fcc2zim/context.py

@@ -26,7 +26,7 @@ class Context:
     # list of courses to include
     course: list[str]
 
-    # ZIM language, also used to fetch proper FCC content
+    # language, used to fetch proper FCC content and compute proper ZIM metadata
     language: str
 
     # ZIM name (also used for filename if zim_file is not set) and filename

scraper/src/fcc2zim/scraper.py

@@ -9,7 +9,7 @@
 from zimscraperlib.zim import Creator, metadata
 
 from fcc2zim.build import build_command
-from fcc2zim.constants import FCC_LANG_MAP, VERSION
+from fcc2zim.constants import FCC_LANG_MAP, VERSION, ZIM_LANG_MAP
 from fcc2zim.context import Context
 from fcc2zim.fetch import fetch_command
 from fcc2zim.prebuild import prebuild_command
@@ -41,6 +41,10 @@ def __init__(self):
             raise ValueError(f"Unsupported language {context.language}")
         self.fcc_lang = FCC_LANG_MAP[context.language]
 
+        if context.language not in ZIM_LANG_MAP:
+            raise ValueError(f"Unsupported language {context.language}")
+        self.zim_lang = ZIM_LANG_MAP[context.language]
+
         context.description, context.long_description = compute_descriptions(
             context.description, context.description, context.long_description
         )
@@ -118,7 +122,7 @@ def __init__(self):
         self.creator = Creator(self.zim_path, "index.html").config_metadata(
             std_metadata=metadata.StandardMetadataList(
                 Name=metadata.NameMetadata(context.name),
-                Language=metadata.LanguageMetadata(context.language),
+                Language=metadata.LanguageMetadata(self.zim_lang),
                 Title=metadata.TitleMetadata(context.title),
                 Creator=metadata.CreatorMetadata(context.creator),
                 Publisher=metadata.PublisherMetadata(context.publisher),

scraper/tests/test_scraper.py

@@ -167,27 +167,30 @@ def test_zimui_dist_ko(self):
             self.create_scraper(zimui_dist=Path("whatever"))
 
     @pytest.mark.parametrize(
-        "language, expected_fcc_lang",
+        "language, expected_fcc_lang, expected_zim_lang",
         [
-            pytest.param("eng", "english", id="english"),
-            pytest.param("eng", "english", id="english"),
-            # pytest.param("cmn", "chinese", id="chinese"),
-            # pytest.param("lzh", "chinese-traditional", id="chinese-traditional"),
-            pytest.param("eng", "english", id="english"),
-            pytest.param("spa", "espanol", id="espanol"),
-            pytest.param("deu", "german", id="german"),
-            pytest.param("ita", "italian", id="italian"),
-            pytest.param("jpn", "japanese", id="japanese"),
-            pytest.param("por", "portuguese", id="portuguese"),
-            pytest.param("ukr", "ukrainian", id="ukrainian"),
-            pytest.param("swa", "swahili", id="swahili"),
+            pytest.param("zh-hans", "chinese", "zho", id="chinese"),
+            pytest.param(
+                "zh-hant", "chinese-traditional", "zho", id="chinese-traditional"
+            ),
+            pytest.param("eng", "english", "eng", id="english"),
+            pytest.param("spa", "espanol", "spa", id="espanol"),
+            pytest.param("deu", "german", "deu", id="german"),
+            pytest.param("ita", "italian", "ita", id="italian"),
+            pytest.param("jpn", "japanese", "jpn", id="japanese"),
+            pytest.param("por", "portuguese", "por", id="portuguese"),
+            pytest.param("ukr", "ukrainian", "ukr", id="ukrainian"),
+            pytest.param("swa", "swahili", "swa", id="swahili"),
         ],
     )
-    def test_fcc_lang_ok(self, language: str, expected_fcc_lang: str):
+    def test_fcc_lang_ok(
+        self, language: str, expected_fcc_lang: str, expected_zim_lang: str
+    ):
         scraper = self.create_scraper(language=language)
         context = Context.get()
         assert context.language == language
         assert scraper.fcc_lang == expected_fcc_lang
+        assert scraper.zim_lang == expected_zim_lang
 
     def test_language_ko(self):
         with pytest.raises(ValueError):