feat: add list of case-insentive lanuguages

This is automatically generated from CLDR.

Needed for features like WeblateOrg/weblate#9450

Author: nijel

Makefile

@@ -4,7 +4,7 @@
 
 all: weblate_language_data/languages.py weblate_language_data/plural_tags.py PLURALS_DIFF.md $(wildcard weblate_language_data/locale/*/LC_MESSAGES/django.po) $(filter-out $(patsubst modules/cldr-json/cldr-json/cldr-localenames-full/main/%/languages.json,languages-po/%.po,$(wildcard modules/cldr-json/cldr-json/cldr-localenames-full/main/*/languages.json)),languages-po/en.po)
 
-weblate_language_data/languages.py: languages.csv aliases.csv cldr.csv extraplurals.csv default_countries.csv population.csv qt.csv rtl.csv $(wildcard modules/iso-codes/data/iso_*.json) scripts/generate-language-data
+weblate_language_data/languages.py: languages.csv aliases.csv cldr.csv extraplurals.csv default_countries.csv population.csv qt.csv rtl.csv case-insensitive.csv $(wildcard modules/iso-codes/data/iso_*.json) scripts/generate-language-data
 	./scripts/generate-language-data
 
 PLURALS_DIFF.md: languages.csv cldr.csv gettext.csv l10n-guide.csv translate.csv scripts/list-diff
@@ -17,6 +17,9 @@ cldr.csv: modules/cldr-json/cldr-json/cldr-core/supplemental/plurals.json module
 rtl.csv: modules/cldr-json/cldr-json/cldr-misc-full/main/*/layout.json scripts/export-cldr-orientation languages.csv
 	./scripts/export-cldr-orientation
 
+case-insensitive.csv: modules/cldr-json/cldr-json/cldr-core/scriptMetadata.json modules/cldr-json/cldr-json/cldr-core/supplemental/languageData.json scripts/export-cldr-case languages.csv
+	./scripts/export-cldr-case
+
 qt.csv: modules/qttools/src/linguist/shared/numerus.cpp scripts/export-qt languages.csv
 	./scripts/export-qt
 

case-insensitive.csv

@@ -0,0 +1,163 @@
+code,
+aeb,
+am,
+anp,
+ar,
+arq,
+ars,
+arz,
+as,
+awa,
+az,
+bal,
+bej,
+bgc,
+bgn,
+bhb,
+bhi,
+bho,
+bjj,
+bm,
+bn,
+bo,
+bqi,
+bra,
+brh,
+brx,
+byn,
+cdo_Hans,
+cdo_Hant,
+ckb,
+cpx_Hans,
+cpx_Hant,
+cr,
+csw,
+dcc,
+doi,
+dv,
+dz,
+en_Shaw,
+fa,
+gan,
+gan_Hans,
+gan_Hant,
+gbm,
+glk,
+gon,
+gu,
+ha,
+hak,
+hak_Hans,
+hak_Hant,
+haz,
+he,
+hi,
+hnd,
+hne,
+hnj,
+hno,
+hoc,
+hoj,
+hsn,
+ii,
+iu,
+ja,
+jpr,
+jrb,
+ka,
+kfr,
+kfy,
+khn,
+kk,
+km,
+kn,
+ko,
+kok,
+kru,
+ks,
+ku,
+kxm,
+ky,
+lad,
+lki,
+lmn,
+lo,
+lrc,
+lus,
+luz,
+mag,
+mai,
+man,
+mfa,
+ml,
+mn_Mong,
+mni,
+mnw,
+mr,
+ms,
+ms_Arab,
+mtr,
+mwr,
+my,
+mzn,
+nan,
+nan_Hant,
+ne,
+new,
+nod,
+noe,
+nqo,
+oj,
+or,
+pa,
+ps,
+raj,
+rhg,
+rif,
+rkt,
+rmt,
+sat,
+sck,
+sd,
+sdh,
+shi,
+shn,
+si,
+skr,
+sou,
+swb,
+swv,
+syl,
+ta,
+tcy,
+te,
+tg,
+th,
+ti,
+tig,
+tk,
+tly,
+tsj,
+tts,
+tzm,
+ug,
+unr,
+ur,
+uz,
+vai,
+wal,
+wbq,
+wbr,
+wtm,
+wuu,
+wuu_Hans,
+wuu_Hant,
+xnr,
+yi,
+yue,
+yue_Hans,
+yue_Hant,
+zgh,
+zh,
+zh_Hans,
+zh_Hant,

scripts/export-cldr-case

@@ -0,0 +1,49 @@
+#! /usr/bin/env python3
+
+# Copyright © Michal Čihař <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+
+import json
+import csv
+
+# Read languages
+with open("languages.csv") as csvfile:
+    reader = csv.reader(csvfile, delimiter=",")
+    next(reader)
+    LANGUAGES = list(reader)
+    LANGUAGE_CODES = {lang[0] for lang in LANGUAGES}
+
+# Read
+with open("case-insensitive.csv") as csvfile:
+    reader = csv.reader(csvfile, delimiter=",")
+    next(reader)
+    CASES = list(reader)
+    CASE_INSENSITIVE_CODES = {lang[0] for lang in CASES}
+
+# Load CLDR
+with open("modules/cldr-json/cldr-json/cldr-core/scriptMetadata.json") as handle:
+    SCRIPTS = json.load(handle)["scriptMetadata"]
+with open(
+    "modules/cldr-json/cldr-json/cldr-core/supplemental/languageData.json"
+) as handle:
+    LANGUAGES = json.load(handle)["supplemental"]["languageData"]
+
+for code in LANGUAGE_CODES:
+    if "_" in code:
+        base, script = code.split("_", 1)
+    else:
+        base = code
+        script = None
+    if script is None or script not in SCRIPTS:
+        if base in LANGUAGES:
+            for script in LANGUAGES[base]["_scripts"]:
+                if SCRIPTS[script]["hasCase"] != "YES":
+                    CASE_INSENSITIVE_CODES.add(base)
+    elif SCRIPTS[script]["hasCase"] != "YES":
+        CASE_INSENSITIVE_CODES.add(code)
+
+with open("case-insensitive.csv", "w") as handle:
+    handle.write("code,\n")
+    for code in sorted(CASE_INSENSITIVE_CODES):
+        handle.write(f"{code},\n")

scripts/generate-language-data

@@ -150,6 +150,13 @@ with open("rtl.csv") as csvfile:
     RTLS = list(reader)
     RTL_CODES = {lang[0] for lang in RTLS}
 
+# Read case insentive
+with open("case-insensitive.csv") as csvfile:
+    reader = csv.reader(csvfile, delimiter=",")
+    next(reader)
+    CASES = list(reader)
+    CASE_INSENSITIVE_CODES = {lang[0] for lang in CASES}
+
 # Write language definitions
 with open("weblate_language_data/languages.py", "w") as output:
     output.write(HEADER)
@@ -207,6 +214,14 @@ with open("weblate_language_data/rtl.py", "w") as output:
         output.write(f'    "{code}",\n')
     output.write("}\n")
 
+with open("weblate_language_data/case_insensitive.py", "w") as output:
+    output.write(HEADER)
+    output.write("# List of case-insentive languages\n")
+    output.write("CASE_INSENSITIVE_LANGS: set[str] = {\n")
+    for code in sorted(CASE_INSENSITIVE_CODES):
+        output.write(f'    "{code}",\n')
+    output.write("}\n")
+
 # Generate same check blacklist
 words = set()