Merge pull request #117 from sillsdev/BL-15160_anamolous_languages

BL-15160 anomalous languages (#117)

Author: StephenMcConnel

components/language-chooser/common/find-language/language-data/languageData.json

@@ -246,7 +246,7 @@ describe("Macrolanguage handling", () => {
   });
 
   // Make sure that the unusual language entries that don't behave as expected are still preserved in some form
-  // Langtags.json has anomalies/unique situations for "bnc", "aka", "nor", "hbs", "san", "zap" such that are usual code can't map them to individual languages
+  // Langtags.json has had anomalies/unique situations for "bnc", "aka", "nor", "hbs", "san", "zap" preventing our usual code from mapping them to individual languages
   // How we handle these cases may change, but make sure some result is always available for these
   it("should include results for unusual language situations", async () => {
     async function asyncExpectToFindResultByExonym(
@@ -380,7 +380,6 @@ describe("getLanguageBySubtag", () => {
     expect(getLanguageBySubtag("mg")?.iso639_3_code).toEqual("plt");
     expect(getLanguageBySubtag("zh")?.exonym).toEqual("Chinese");
     expect(getLanguageBySubtag("za")?.exonym).toEqual("Zhuang");
-    expect(getLanguageBySubtag("ak")?.iso639_3_code).toEqual("twi");
     expect(getLanguageBySubtag("bnc")?.iso639_3_code).toEqual("lbk");
     expect(getLanguageBySubtag("no")?.exonym).toEqual("Norwegian");
     expect(getLanguageBySubtag("sh")?.iso639_3_code).toEqual("hbs");
@@ -412,9 +411,6 @@ describe("getLanguageBySubtag", () => {
     expect(
       getLanguageBySubtag("za", defaultSearchResultModifier)?.exonym
     ).toEqual("Zhuang");
-    expect(
-      getLanguageBySubtag("ak", defaultSearchResultModifier)?.iso639_3_code
-    ).toEqual("twi");
     expect(
       getLanguageBySubtag("bnc", defaultSearchResultModifier)?.iso639_3_code
     ).toEqual("lbk");

components/language-chooser/common/find-language/languageSearch.spec.ts

@@ -6,6 +6,8 @@ See also
 
   <!-- - https://issues.bloomlibrary.org/youtrack/issue/BL-12657/Issues-with-macrolanguage-codes-in-the-language-picker -->
 
+- https://writingsystems.info/topics/writingsystems/language-tagging/#macrolanguages
+
 - https://iso639-3.sil.org/about/scope#Macrolanguages
 - https://github.com/silnrsi/langtags/blob/master/doc/langtags.md#macro-languages
 - https://iso639-3.sil.org/code_tables/macrolanguage_mappings/
@@ -34,8 +36,6 @@ The language data we use is primarily based on [langtags.json](https://ldml.api.
 
 For example: For the macrolanguage `pus` (Pashto), there are 4 relevant entries in langtags.json (listed below). From the first (ps-Arab-AF), we can detect that `pus`/`ps` is mapped to/equivalent to representative language `pbu` as described above. (`ps` is the ISO 639-1 equivalent of `pus`.) We therefore know the second entry (ps-Arab-PK) is also for language `pbu`. Since this language chooser delineates languages firstly by their ISO 639-3 codes, we combine the first two entries. We mark the result with `aliasMacrolanguage: pus`. The `pbt` and `pst` entries we straightforwardly handle as normal individual languages.
 
-There are a few entries in langtags.json for which we cannot straightforwardly determine the individual language. These we mark with `aliasMacrolanguage: unknown` and keep the iso639-3 code despite it being a macrolanguage code. For the react language chooser, the desired behavior for these situations should be handled in search result modifiers. As of February 2025, these entries are `bnc`, `nor`, `san`, `hbs`, and `zap`. Other unusual situations we are aware of are `aka` and `zhx`, these may also warrant special checking.
-
 ```
     {
         "full": "ps-Arab-AF",
@@ -110,3 +110,33 @@ There are a few entries in langtags.json for which we cannot straightforwardly d
 ### Stripping the "(macrolanguage)" Parentheticals
 
 Some entries in langtags.json contain "macrolanguage" in the language name, and yet contain the only data present for the representative language for that macrolanguage. For example, Dogri macrolanguage code is doi and Dogri individual language code is dgo. There are 4 entries with "iso639_3": "doi", all of which have "name": "Dogri (macrolanguage)", and dgo tags in the tags field which lists equivalent tags. The code dgo does not appear anywhere in langtags.json outside of these 4 entries. We are therefore interpreting data from these entries as applying to the individual language, and simply stripping "(macrolanguage)" wherever we find it. When we create macrolanguage search results, we set `isMacrolanguage=true`.
+
+### Anomalies and Special Situations
+
+Last updated September 2025. These should be specially checked and handled as our language data may be weird or inaccurate.
+
+- `aka` - From the ISO 639-3 site, [Akan is a macrolanguage](https://iso639-3.sil.org/code/aka) (ISO639-3: `aka`; ISO639-1: `ak`) containing individual languages Twi (ISO639-3: `twi`; ISO639-1: `tw`) and Fanti (ISO639-3: `fat`). The langtags.json entry lists `ak`, `fat`, and `tw` as equivalent and provides no names other than "Akan". In Ethnologue, [Akan has a page](https://www.ethnologue.com/language/aka/) that lists Twi and Fanti as mutually intelligible dialects, among others.
+
+  Since we don't have enough data from langtags.json to make any individual language entries, we give one result with codes "aka" and "ak" but don't mark it as a macrolanguage. For now this is somewhat consistent with Ethnologue, which treats it as a single language, and there is no point discouraging people from using it in the absence of alternatives.
+
+  See also https://unicode-org.atlassian.net/browse/CLDR-10293 and https://unicode-org.atlassian.net/browse/CLDR-17323; The current langtags.json handling of these languages might not be desired or permanent.
+
+- `hbs` - [Serbo-Croatian is a macrolanguage](https://iso639-3.sil.org/code/hbs) (ISO639-3: `hbs`; ISO639-1: `sh`) containing individual languages Bosnian (ISO639-3: `bos`; ISO639-1: `bs`), Montenegrin(ISO639-3: `cnr`), Croatian(ISO639-3: `hrv`; ISO639-1: `hr`) and Serbian(ISO639-3: `srp`; ISO639-1: `sr`). Contrary to its usual behavior, langtags.json has a separate entry for Serbo-Croatian which that is not mappable to any individual language.
+
+  We straightforwardly give an `hbs` macrolanguage result as well as all the individual language results as per usual.
+
+- `nor` - [Norwegian is a macrolanguage](https://iso639-3.sil.org/code/nor) (ISO639-3: `nor`; ISO639-1: `no`) with child languages Bokmål (`nob`) and Nynorsk (`nno`), but [Ethnologue treats it as a single language](https://www.ethnologue.com/language/nor/) and says "Norwegian has 2 written standards, both of which are assigned codes in the ISO 639-3 standard: Bokmål Norwegian (nob) and Nynorsk Norwegian (nno)."
+
+  Currently we give `nor` as a macrolanguage as well as `nob` and `nno` as individual languages.
+
+- `san` - [Sanskrit is a macrolanguage](https://iso639-3.sil.org/code/san) (ISO639-3: `san`; ISO639-1: `sa`) with child languages child languages Classical Sanskrit (`cls`) and Vedic Sanskrit (`vsn`). It has a single [Ethnologue page](https://www.ethnologue.com/language/san/) which states "Sanskrit has 2 individual historical languages, both of which are assigned codes in the ISO 639-3 standard: Classical Sanskrit (cls) and Vedic Sanskrit (vsn)."
+
+  Since we have no differentiated information on `cls` or `vsn` from langtags.json and there is only 1 Ethnologue page, for now we are giving a single entry for Sanskrit and not marking it as a macrolanguage.
+
+- `zap` - [Zapotec is a macrolanguage](https://iso639-3.sil.org/code/zap) with many child languages. Due to a known error, two of its child languages, Isthmus Zapotec (`zai`) and Las Delicias Zapotec (`zcd`) are currently being conflated in langtags.json, so both of their data shows up in a single zap entry. Langtags.json has an entry for `zap-Latn-MX` which lists `zap`, `zai`, and `zcd` as equivalent. The `zai` [Ethnologue page](https://www.ethnologue.com/language/zai/) has "Dialects - None known. 18% intelligibility of Santa María Petapa [zpe] (most similar). A member of macrolanguage Zapotec [zap]". The `zcd` [Ethnologue page](https://www.ethnologue.com/language/zcd/) has "Dialects - None known. Reportedly most similar to Rincón Zapotec [zar]. A member of macrolanguage Zapotec [zap]". CLDR lists "zai" to "zap" for macrolanguage (but doesn't mention "zcd").
+
+  For now we remove the names which obviously refer to Las Delicias Zapotec or Isthmus Zapotec from the `zap` macrolanguage card; `zai` and `zcd` do not have their own cards. We hope the bug causing the langtags.json error will be resolved soon.
+
+- `zhx` - [This is a ISO 639-5 Collective code](https://iso639-3.sil.org/code/zhx) which has an entry in langtags.json with script with script `nshu`. It does not have an ethnologue page and is the only ISO 639-5 code that we have found in langtags.json.
+
+  We do not include `zhx` in our search results.

components/language-chooser/common/find-language/macrolanguageNotes.md

@@ -79,7 +79,7 @@ function addOrCombineLangtagsEntry(
       entry.isRepresentativeForMacrolanguage;
   } else {
     const scriptCode = entry.script;
-    const scripts = {};
+    const scripts: { [key: string]: IScript } = {};
     if (scriptCode) {
       scripts[scriptCode] = {
         code: scriptCode,
@@ -106,6 +106,7 @@ function addOrCombineLangtagsEntry(
       regionNames: new Set([entry.regionname]),
       names: getAllPossibleNames(entry),
       scripts,
+      isMacrolanguage: entry.isMacrolanguage || false,
       parentMacrolanguage:
         macrolanguagesByCode[indivlangsToMacrolangs[entry.indivIsoCode]],
       isRepresentativeForMacrolanguage: entry.isRepresentativeForMacrolanguage,
@@ -132,13 +133,14 @@ function parseLangtagsJson() {
         : entry.iso639_3;
 
       if (!augmentedEntry["indivIsoCode"]) {
-        // This is a data anomaly but we do have 5 as of Feb 2025: bnc, nor, san, hbs, zap
+        // This is a data anomaly but we do have a few as of Sep 2025: nor, san, hbs, zap, ar-SA, ku-Arab-TR, and man-Latn-GN
         // See macrolanguageNotes.md. These cases should be specially handled.
         console.log(
           "No indivIsoCode found for macrolang",
           entry.iso639_3,
           entry.tag
         );
+        augmentedEntry["isMacrolanguage"] = true;
       }
     }
 
@@ -177,7 +179,7 @@ function parseLangtagsJson() {
         ].filter((name) => !!name),
         alternativeTags: [...langData.alternativeTags],
         parentMacrolanguage: langData.parentMacrolanguage,
-        isMacrolanguage: false, // we add macrolanguages separately below. See macrolanguageNotes.md
+        isMacrolanguage: langData.isMacrolanguage,
         isRepresentativeForMacrolanguage:
           langData.isRepresentativeForMacrolanguage,
         languageType: langData.languageType,

components/language-chooser/common/find-language/scripts/langtagProcessing.ts

@@ -53,6 +53,7 @@ export interface ILangtagsJsonEntryInternal {
   windows: string;
 
   // These are not in the langtags.json file but may be added in the processing
+  isMacrolanguage: boolean;
   isRepresentativeForMacrolanguage: boolean;
   indivIsoCode: string; // If iso639_3 is a macrolanguage code, this is the corresponding (representative) individual language code - see macrolanguageNotes.md
 }
@@ -158,8 +159,9 @@ for (const line of macrolangMappingFile.split("\n")) {
 // So in langtags.json, for representative languages, the iso639_3 field is often the macrolangauge code,
 // but the tags field (in some but not all entries) contains equivalent tags that use the individual language codes.
 // We want to save the individual language codes, so gather as many macrolangauge to representative individual language
-// mappings as we can. As of 2/2025, this covers all macrolanguage codes in langtags.json except for
-// bnc, nor, san, hbs, and zap which should all be handled by search result modifiers. (a fix for `man` was incorporated 8/2025)
+// mappings as we can. As of 9/2025, this covers all macrolanguage codes in langtags.json except for
+// nor, san, hbs, and zap which should all be handled by search result modifiers. (a fix for `man` was incorporated 8/2025.
+//  We had previously noted that bnc was a problem, but it seems to be working now.)
 // See macrolanguageNotes.md for more explanation.
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any

components/language-chooser/common/find-language/scripts/langtagProcessingHelpers.ts

@@ -216,4 +216,117 @@ describe("reordering entries to prioritize desired language when keywords are se
       expect(spanishResult?.names[0]).toBeTruthy();
     });
   });
+
+  // I'm not sure if this is the behavior we want to stick with, but putting it here to document it
+  // and so we notice if we accidentally change it. See macrolanguageNotes.md for more details.
+  describe("Anomalous special case handling", () => {
+    it("should handle Akan as expected", async () => {
+      // one "aka" card, is macrolanguage, has multiple names
+      const akaResults = defaultSearchResultModifier(
+        (await asyncGetAllLanguageResults("Akan")) as ILanguage[],
+        "Akan"
+      );
+      const akaResult = akaResults.find((result) =>
+        codeMatches(result.iso639_3_code, "aka")
+      );
+      expect(akaResult).toBeDefined();
+      // Not marking Akan as a macrolanguage, so we don't show the "better to pick an individual language" warning
+      // when there are no relevant individual language options
+      expect(akaResult?.isMacrolanguage).toBe(false);
+      // language subtag should be "ak"
+      expect(akaResult?.languageSubtag).toBe("ak");
+      // should have more than one script option
+      expect(akaResult?.scripts.length).toBeGreaterThan(1);
+      // Currently we don't have enough info for any of the akan individual languages
+      expect(
+        akaResults.some(
+          (result) =>
+            codeMatches(result.iso639_3_code, "twi") ||
+            codeMatches(result.iso639_3_code, "fat") ||
+            result.languageSubtag === "tw"
+        )
+      ).toBe(false);
+    });
+
+    it("should handle Serbo-Croatian as expected", async () => {
+      // one "hbs" card, is macrolanguage, has multiple names
+      const hbsResults = defaultSearchResultModifier(
+        (await asyncGetAllLanguageResults("Serbo-Croatian")) as ILanguage[],
+        "Serbo-Croatian"
+      );
+      const hbsResult = hbsResults.find((result) =>
+        codeMatches(result.iso639_3_code, "hbs")
+      );
+      expect(hbsResult).toBeDefined();
+      expect(hbsResult?.isMacrolanguage).toBe(true);
+      expect(hbsResult?.names.length).toBe(0);
+      // All of the child languages should be present
+      for (const childCode of ["bos", "cnr", "hrv", "srp"]) {
+        expect(
+          hbsResults.some(
+            (result) =>
+              codeMatches(result.iso639_3_code, childCode) &&
+              !result.isMacrolanguage
+          )
+        ).toBe(true);
+      }
+    });
+
+    it("should handle Norwegian as expected", async () => {
+      // one macrolanguage "nor" card, plus one for Bokmål and one for Nynorsk
+      const norResults = defaultSearchResultModifier(
+        (await asyncGetAllLanguageResults("Norwegian")) as ILanguage[],
+        "Norwegian"
+      );
+      const norResult = norResults.find((result) =>
+        codeMatches(result.iso639_3_code, "nor")
+      );
+      expect(norResult).toBeDefined();
+      expect(norResult?.isMacrolanguage).toBe(true);
+      // Both of the child languages should be present
+      for (const childCode of ["nob", "nno"]) {
+        expect(
+          norResults.some(
+            (result) =>
+              codeMatches(result.iso639_3_code, childCode) &&
+              !result.isMacrolanguage
+          )
+        ).toBe(true);
+      }
+    });
+
+    it("should handle Sanskrit as expected", async () => {
+      // we don't have enough info for the individual languages, so just one "san" card
+      const sanResults = defaultSearchResultModifier(
+        (await asyncGetAllLanguageResults("Sanskrit")) as ILanguage[],
+        "Sanskrit"
+      );
+      const sanResult = sanResults.find((result) =>
+        codeMatches(result.iso639_3_code, "san")
+      );
+      expect(sanResult).toBeDefined();
+      expect(sanResult?.isMacrolanguage).toBe(false);
+      // Make sure it has the list of names
+      expect(sanResult?.names.length).toBeGreaterThan(3);
+    });
+
+    it("should handle Zapotec as expected", async () => {
+      // in addition to all the individual zapotec languages, there should be one macrolanguage zap card,
+      // and its list of names should not contain "Isthmus Zapotec" nor "Las Delicias Zapotec" as that could be confusing
+      const zapResults = defaultSearchResultModifier(
+        (await asyncGetAllLanguageResults("Zapotec")) as ILanguage[],
+        "Zapotec"
+      );
+      const zapResult = zapResults.find((result) =>
+        codeMatches(result.iso639_3_code, "zap")
+      );
+      expect(zapResult).toBeDefined();
+      expect(zapResult?.isMacrolanguage).toBe(true);
+      expect(
+        zapResult?.names.some((n) =>
+          ["Isthmus Zapotec", "Las Delicias Zapotec"].includes(n)
+        )
+      ).toBe(false);
+    });
+  });
 });