通过解析 IANA 语言子标签注册表增强本地化功能 (#4675)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: Glavo

.gitignore

@@ -19,6 +19,9 @@ hmcl-exported-logs-*
 /.local/
 /.cache/
 
+# IANA Language Subtag Registry
+language-subtag-registry
+
 # gradle build
 /build/
 /HMCL/build/

HMCLCore/src/main/java/org/jackhuang/hmcl/util/i18n/DefaultResourceBundleControl.java

@@ -30,7 +30,7 @@
 /// - For all Chinese locales, `zh-CN` is always added to the candidate list. If `zh-Hans` already exists in the candidate list,
 ///   `zh-CN` is inserted before `zh`; otherwise, it is inserted after `zh`.
 /// - For all Traditional Chinese locales, `zh-TW` is always added to the candidate list (before `zh`).
-/// - For all [supported][LocaleUtils#mapToISO2Language(String)] ISO 639-3 language code (such as `eng`, `zho`, `lzh`, etc.),
+/// - For all supported ISO 639-3 language code (such as `eng`, `zho`, `lzh`, etc.),
 ///  a candidate list with the language code replaced by the ISO 639-1 (Macro)language code is added to the end of the candidate list.
 ///
 /// @author Glavo

HMCLCore/src/main/java/org/jackhuang/hmcl/util/i18n/LocaleUtils.java

@@ -17,9 +17,7 @@
  */
 package org.jackhuang.hmcl.util.i18n;
 
-import org.jackhuang.hmcl.util.Lang;
 import org.jackhuang.hmcl.util.StringUtils;
-import org.jackhuang.hmcl.util.io.IOUtils;
 import org.jackhuang.hmcl.util.platform.NativeUtils;
 import org.jackhuang.hmcl.util.platform.OperatingSystem;
 import org.jackhuang.hmcl.util.platform.windows.Kernel32;
@@ -29,6 +27,8 @@
 import org.jetbrains.annotations.Unmodifiable;
 
 import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
@@ -48,66 +48,74 @@ public final class LocaleUtils {
 
     public static final Locale SYSTEM_DEFAULT = Locale.getDefault();
 
-    public static final Locale LOCALE_ZH_HANS = Locale.forLanguageTag("zh-Hans");
-    public static final Locale LOCALE_ZH_HANT = Locale.forLanguageTag("zh-Hant");
-
-    public static final String DEFAULT_LANGUAGE_KEY = "default";
+    public static final boolean IS_CHINA_MAINLAND = isChinaMainland();
 
-    private static final Map<String, String> subLanguageToParent = new HashMap<>();
-    private static final Map<String, String> iso3To2 = new HashMap<>();
-    private static final Set<String> rtl = new HashSet<>();
+    private static boolean isChinaMainland() {
+        if ("Asia/Shanghai".equals(ZoneId.systemDefault().getId()))
+            return true;
 
-    static {
-        try {
-            for (String line : Lang.toIterable(IOUtils.readFullyAsString(LocaleUtils.class.getResourceAsStream("/assets/lang/sublanguages.csv")).lines())) {
-                if (line.startsWith("#") || line.isBlank()) {
-                    continue;
-                }
+        // Check if the time zone is UTC+8
+        if (ZonedDateTime.now().getOffset().getTotalSeconds() == Duration.ofHours(8).toSeconds()) {
+            if ("CN".equals(LocaleUtils.SYSTEM_DEFAULT.getCountry()))
+                return true;
 
-                String[] languages = line.split(",");
-                if (languages.length < 2) {
-                    LOG.warning("Invalid line in sublanguages.csv: " + line);
-                    continue;
-                }
+            if (OperatingSystem.CURRENT_OS == OperatingSystem.WINDOWS && NativeUtils.USE_JNA) {
+                Kernel32 kernel32 = Kernel32.INSTANCE;
 
-                String parent = languages[0];
-                for (int i = 1; i < languages.length; i++) {
-                    subLanguageToParent.put(languages[i], parent);
-                }
+                // https://learn.microsoft.com/windows/win32/intl/table-of-geographical-locations
+                if (kernel32 != null && kernel32.GetUserGeoID(WinConstants.GEOCLASS_NATION) == 45) // China
+                    return true;
             }
-        } catch (Throwable e) {
-            LOG.warning("Failed to load sublanguages.csv", e);
         }
 
-        try {
-            // Line Format: (?<iso2>[a-z]{2}),(?<iso3>[a-z]{3})
-            for (String line : Lang.toIterable(IOUtils.readFullyAsString(LocaleUtils.class.getResourceAsStream("/assets/lang/iso_languages.csv")).lines())) {
-                if (line.startsWith("#") || line.isBlank()) {
-                    continue;
-                }
+        return false;
+    }
 
-                String[] parts = line.split(",", 3);
-                if (parts.length != 2) {
-                    LOG.warning("Invalid line in iso_languages.csv: " + line);
-                    continue;
-                }
+    public static final Locale LOCALE_ZH_HANS = Locale.forLanguageTag("zh-Hans");
+    public static final Locale LOCALE_ZH_HANT = Locale.forLanguageTag("zh-Hant");
 
-                iso3To2.put(parts[1], parts[0]);
-            }
-        } catch (Throwable e) {
-            LOG.warning("Failed to load iso_languages.csv", e);
+    public static final String DEFAULT_LANGUAGE_KEY = "default";
+
+    private static final Map<String, String> PARENT_LANGUAGE = loadCSV("sublanguages.csv");
+    private static final Map<String, String> NORMALIZED_TAG = loadCSV("language_aliases.csv");
+    private static final Map<String, String> DEFAULT_SCRIPT = loadCSV("default_script.csv");
+    private static final Map<String, String> PREFERRED_LANGUAGE = Map.of("zh", "cmn");
+    private static final Set<String> RTL_SCRIPTS = Set.of("Qabs", "Arab", "Hebr");
+    private static final Set<String> CHINESE_TRADITIONAL_REGIONS = Set.of("TW", "HK", "MO");
+
+    /// Load CSV files located in `/assets/lang/`.
+    /// Each line in these files contains at least two elements.
+    ///
+    /// For example, if a file contains `value0,value1,value2`, the return value will be `{value1=value0, value2=value0}`.
+    private static Map<String, String> loadCSV(String fileName) {
+        InputStream resource = LocaleUtils.class.getResourceAsStream("/assets/lang/" + fileName);
+        if (resource == null) {
+            LOG.warning("Can't find file: " + fileName);
+            return Map.of();
         }
 
-        try {
-            for (String line : Lang.toIterable(IOUtils.readFullyAsString(LocaleUtils.class.getResourceAsStream("/assets/lang/rtl.txt")).lines())) {
-                if (line.startsWith("#") || line.isBlank()) {
-                    continue;
+        HashMap<String, String> result = new HashMap<>();
+        try (resource) {
+            new String(resource.readAllBytes(), StandardCharsets.UTF_8).lines().forEach(line -> {
+                if (line.startsWith("#") || line.isBlank())
+                    return;
+
+                String[] items = line.split(",");
+                if (items.length < 2) {
+                    LOG.warning("Invalid line in " + fileName + ": " + line);
+                    return;
                 }
-                rtl.add(line.trim());
-            }
+
+                String parent = items[0];
+                for (int i = 1; i < items.length; i++) {
+                    result.put(items[i], parent);
+                }
+            });
         } catch (Throwable e) {
-            LOG.warning("Failed to load rtl.txt", e);
+            LOG.warning("Failed to load " + fileName, e);
         }
+
+        return Map.copyOf(result);
     }
 
     private static Locale getInstance(String language, String script, String region,
@@ -130,6 +138,31 @@ public static String toLanguageKey(Locale locale) {
                 : locale.stripExtensions().toLanguageTag();
     }
 
+    public static boolean isEnglish(Locale locale) {
+        return "en".equals(getRootLanguage(locale));
+    }
+
+    public static boolean isChinese(Locale locale) {
+        return "zh".equals(getRootLanguage(locale));
+    }
+
+    // ---
+
+    /// Normalize the language code to the code in the IANA Language Subtag Registry.
+    /// Typically, it normalizes ISO 639 alpha-3 codes to ISO 639 alpha-2 codes.
+    public static @NotNull String normalizeLanguage(String language) {
+        return language.isEmpty()
+                ? "en"
+                : NORMALIZED_TAG.getOrDefault(language, language);
+    }
+
+    /// If `language` is a sublanguage of a [macrolanguage](https://en.wikipedia.org/wiki/ISO_639_macrolanguage),
+    /// return the macrolanguage; otherwise, return `null`.
+    public static @Nullable String getParentLanguage(String language) {
+        return PARENT_LANGUAGE.get(language);
+    }
+
+    /// @see #getRootLanguage(String)
     public static @NotNull String getRootLanguage(Locale locale) {
         return getRootLanguage(locale.getLanguage());
     }
@@ -140,54 +173,54 @@ public static String toLanguageKey(Locale locale) {
     /// - If `language` is empty, return `en`;
     /// - Otherwise, return the `language`.
     public static @NotNull String getRootLanguage(String language) {
-        if (language.isEmpty()) return "en";
-        if (language.length() <= 2)
-            return language;
-
-        String iso2 = mapToISO2Language(language);
-        if (iso2 != null)
-            return iso2;
+        language = normalizeLanguage(language);
 
         String parent = getParentLanguage(language);
         return parent != null ? parent : language;
     }
 
+    /// If `language` is a macrolanguage, try to map it to the most commonly used individual language.
+    ///
+    /// For example, if `language` is `zh`, this method will return `cmn`.
+    public static @NotNull String getPreferredLanguage(String language) {
+        language = normalizeLanguage(language);
+        return PREFERRED_LANGUAGE.getOrDefault(language, language);
+    }
+
     /// Get the script of the locale. If the script is empty and the language is Chinese,
     /// the script will be inferred based on the language, the region and the variant.
     public static @NotNull String getScript(Locale locale) {
         if (locale.getScript().isEmpty()) {
-            if (isEnglish(locale)) {
-                if ("UD".equals(locale.getCountry())) {
-                    return "Qabs";
-                }
+            if (!locale.getVariant().isEmpty()) {
+                String script = DEFAULT_SCRIPT.get(locale.getVariant());
+                if (script != null)
+                    return script;
             }
 
+            if ("UD".equals(locale.getCountry())) {
+                return "Qabs";
+            }
+
+            String script = DEFAULT_SCRIPT.get(normalizeLanguage(locale.getLanguage()));
+            if (script != null)
+                return script;
+
             if (isChinese(locale)) {
-                if (CHINESE_LATN_VARIANTS.contains(locale.getVariant()))
-                    return "Latn";
-                if (locale.getLanguage().equals("lzh") || CHINESE_TRADITIONAL_REGIONS.contains(locale.getCountry()))
-                    return "Hant";
-                else
-                    return "Hans";
+                return CHINESE_TRADITIONAL_REGIONS.contains(locale.getCountry())
+                        ? "Hant"
+                        : "Hans";
             }
+
+            return "";
         }
 
         return locale.getScript();
     }
 
     public static @NotNull TextDirection getTextDirection(Locale locale) {
-        TextDirection direction = rtl.contains(getRootLanguage(locale))
+        return RTL_SCRIPTS.contains(getScript(locale))
                 ? TextDirection.RIGHT_TO_LEFT
                 : TextDirection.LEFT_TO_RIGHT;
-
-        if ("Qabs".equals(getScript(locale))) {
-            direction = switch (direction) {
-                case RIGHT_TO_LEFT -> TextDirection.LEFT_TO_RIGHT;
-                case LEFT_TO_RIGHT -> TextDirection.RIGHT_TO_LEFT;
-            };
-        }
-
-        return direction;
     }
 
     private static final ConcurrentMap<Locale, List<Locale>> CANDIDATE_LOCALES = new ConcurrentHashMap<>();
@@ -196,13 +229,8 @@ public static String toLanguageKey(Locale locale) {
         return CANDIDATE_LOCALES.computeIfAbsent(locale, LocaleUtils::createCandidateLocaleList);
     }
 
-    // -------------
-
     private static List<Locale> createCandidateLocaleList(Locale locale) {
-        String language = locale.getLanguage();
-        if (language.isEmpty())
-            return List.of(Locale.ENGLISH, Locale.ROOT);
-
+        String language = getPreferredLanguage(locale.getLanguage());
         String script = getScript(locale);
         String region = locale.getCountry();
         List<String> variants = locale.getVariant().isEmpty()
@@ -211,18 +239,7 @@ private static List<Locale> createCandidateLocaleList(Locale locale) {
 
         ArrayList<Locale> result = new ArrayList<>();
         do {
-            String currentLanguage;
-
-            if (language.length() <= 2) {
-                currentLanguage = language;
-            } else {
-                String iso2 = mapToISO2Language(language);
-                currentLanguage = iso2 != null
-                        ? iso2
-                        : language;
-            }
-
-            addCandidateLocales(result, currentLanguage, script, region, variants);
+            addCandidateLocales(result, language, script, region, variants);
         } while ((language = getParentLanguage(language)) != null);
 
         result.add(Locale.ROOT);
@@ -367,54 +384,6 @@ else if (fileName.length() > defaultFileNameLength + 1 && fileName.charAt(baseNa
         return Map.of();
     }
 
-    // ---
-
-    /// Map ISO 639 alpha-3 language codes to ISO 639 alpha-2 language codes.
-    /// Returns `null` if there is no corresponding ISO 639 alpha-2 language code.
-    public static @Nullable String mapToISO2Language(String iso3Language) {
-        return iso3To2.get(iso3Language);
-    }
-
-    /// If `language` is a sublanguage of a [macrolanguage](https://en.wikipedia.org/wiki/ISO_639_macrolanguage),
-    /// return the macrolanguage; otherwise, return `null`.
-    public static @Nullable String getParentLanguage(String language) {
-        return subLanguageToParent.get(language);
-    }
-
-    public static boolean isEnglish(Locale locale) {
-        return "en".equals(getRootLanguage(locale));
-    }
-
-    public static final Set<String> CHINESE_TRADITIONAL_REGIONS = Set.of("TW", "HK", "MO");
-    public static final Set<String> CHINESE_LATN_VARIANTS = Set.of("pinyin", "wadegile", "tongyong");
-
-    public static boolean isChinese(Locale locale) {
-        return "zh".equals(getRootLanguage(locale));
-    }
-
-    public static final boolean IS_CHINA_MAINLAND = isChinaMainland();
-
-    private static boolean isChinaMainland() {
-        if ("Asia/Shanghai".equals(ZoneId.systemDefault().getId()))
-            return true;
-
-        // Check if the time zone is UTC+8
-        if (ZonedDateTime.now().getOffset().getTotalSeconds() == Duration.ofHours(8).toSeconds()) {
-            if ("CN".equals(LocaleUtils.SYSTEM_DEFAULT.getCountry()))
-                return true;
-
-            if (OperatingSystem.CURRENT_OS == OperatingSystem.WINDOWS && NativeUtils.USE_JNA) {
-                Kernel32 kernel32 = Kernel32.INSTANCE;
-
-                // https://learn.microsoft.com/windows/win32/intl/table-of-geographical-locations
-                if (kernel32 != null && kernel32.GetUserGeoID(WinConstants.GEOCLASS_NATION) == 45) // China
-                    return true;
-            }
-        }
-
-        return false;
-    }
-
     private LocaleUtils() {
     }
 }

HMCLCore/src/main/resources/assets/lang/default_script.csv

@@ -0,0 +1,29 @@
+Arab,ar,fa,ps,ur
+Armn,hy
+Beng,as,bn
+Blis,zbl
+Cyrl,ab,be,bg,kk,mk,ru,uk
+Deva,hi,mr,ne,kok,mai
+Ethi,am,ti
+Geor,ka
+Grek,el
+Gujr,gu
+Guru,pa
+Hant,lzh
+Hebr,he,yi
+Jpan,ja
+Khmr,km
+Knda,kn
+Kore,ko
+Laoo,lo
+Latn,af,ay,bs,ca,ch,cs,cy,da,de,en,eo,es,et,eu,fi,fj,fo,fr,fy,ga,gl,gn,gv,hr,ht,hu,id,is,it,kl,la,lb,ln,lt,lv,mg,mh,ms,mt,na,nb,nd,nl,nn,no,nr,ny,om,pl,pt,qu,rm,rn,ro,rw,sg,sk,sl,sm,so,sq,ss,st,sv,sw,tl,tn,to,tr,ts,ve,vi,xh,zu,dsb,frr,frs,gsw,hsb,men,nds,niu,nso,tem,tkl,tmh,tpi,tvl,tailo,pinyin,hepburn,pehoeji,tongyong,wadegile
+Mlym,ml
+Mymr,my
+Nkoo,nqo
+Orya,or
+Sinh,si
+Taml,ta
+Telu,te
+Thaa,dv
+Thai,th
+Tibt,dz