impr(funbox): proper per-language handling in polyglot mode (@byseif21, @fehmer) (monkeytypegame#6666)

### Description

#### fixes some polyglot issues

* when mixing RTL (e.g. Arabic) with LTR (e.g. English), ligatures were
broken if the main `Config.Language` was a LTR lang.
* lazy mode: previously, if the main language didn’t support lazy mode,
none of the languages would, even if they did individually.

closes monkeytypegame#6665

---------

Co-authored-by: Jack <[email protected]>

Author: byseif21

frontend/src/ts/constants/layouts.ts

@@ -1,3 +1,3 @@
 import { LayoutName, LayoutNameSchema } from "@monkeytype/schemas/layouts";
 
-export const LayoutsList:LayoutName[] = LayoutNameSchema._def.values;
+export const LayoutsList: LayoutName[] = LayoutNameSchema._def.values;

frontend/src/ts/test/funbox/funbox-functions.ts

@@ -1,4 +1,3 @@
-import { Section } from "../../utils/json-data";
 import { FunboxWordsFrequency, Wordset } from "../wordset";
 import * as GetText from "../../utils/generate";
 import Config, * as UpdateConfig from "../../config";
@@ -24,17 +23,18 @@ import * as TestState from "../test-state";
 import { WordGenError } from "../../utils/word-gen-error";
 import { FunboxName, KeymapLayout, Layout } from "@monkeytype/schemas/configs";
 import { Language, LanguageObject } from "@monkeytype/schemas/languages";
+
 export type FunboxFunctions = {
   getWord?: (wordset?: Wordset, wordIndex?: number) => string;
   punctuateWord?: (word: string) => string;
-  withWords?: (words?: string[]) => Promise<Wordset>;
+  withWords?: (words?: string[]) => Promise<Wordset | PolyglotWordset>;
   alterText?: (word: string, wordIndex: number, wordsBound: number) => string;
   applyConfig?: () => void;
   applyGlobalCSS?: () => void;
   clearGlobal?: () => void;
   rememberSettings?: () => void;
   toggleScript?: (params: string[]) => void;
-  pullSection?: (language?: Language) => Promise<Section | false>;
+  pullSection?: (language?: Language) => Promise<JSONData.Section | false>;
   handleSpace?: () => void;
   handleChar?: (char: string) => string;
   isCharCorrect?: (char: string, originalChar: string) => boolean;
@@ -151,6 +151,23 @@ class PseudolangWordGenerator extends Wordset {
   }
 }
 
+export class PolyglotWordset extends Wordset {
+  public wordsWithLanguage: Map<string, Language>;
+  public languageProperties: Map<Language, JSONData.LanguageProperties>;
+
+  constructor(
+    wordsWithLanguage: Map<string, Language>,
+    languageProperties: Map<Language, JSONData.LanguageProperties>
+  ) {
+    // build and shuffle the word array
+    const wordArray = Array.from(wordsWithLanguage.keys());
+    Arrays.shuffle(wordArray);
+    super(wordArray);
+    this.wordsWithLanguage = wordsWithLanguage;
+    this.languageProperties = languageProperties;
+  }
+}
+
 const list: Partial<Record<FunboxName, FunboxFunctions>> = {
   "58008": {
     getWord(): string {
@@ -646,12 +663,12 @@ const list: Partial<Record<FunboxName, FunboxFunctions>> = {
             `Failed to load language: ${language}. It will be ignored.`,
             0
           );
-          return null; // Return null for failed languages
+          return null;
         })
       );
 
       const languages = (await Promise.all(promises)).filter(
-        (lang) => lang !== null
+        (lang): lang is LanguageObject => lang !== null
       );
 
       if (languages.length === 0) {
@@ -679,9 +696,44 @@ const list: Partial<Record<FunboxName, FunboxFunctions>> = {
         throw new WordGenError("");
       }
 
-      const wordSet = languages.flatMap((it) => it.words);
-      Arrays.shuffle(wordSet);
-      return new Wordset(wordSet);
+      // direction conflict check
+      const allRightToLeft = languages.every((lang) => lang.rightToLeft);
+      const allLeftToRight = languages.every((lang) => !lang.rightToLeft);
+      const mainLanguage = await JSONData.getLanguage(Config.language);
+      const mainLanguageIsRTL = mainLanguage?.rightToLeft ?? false;
+      if (
+        (mainLanguageIsRTL && allLeftToRight) ||
+        (!mainLanguageIsRTL && allRightToLeft)
+      ) {
+        const fallbackLanguage =
+          languages[0]?.name ?? (allRightToLeft ? "arabic" : "english");
+        UpdateConfig.setLanguage(fallbackLanguage);
+        Notifications.add(
+          `Language direction conflict: switched to ${fallbackLanguage} for consistency.`,
+          0,
+          { duration: 5 }
+        );
+        throw new WordGenError("");
+      }
+
+      // build languageProperties
+      const languageProperties = new Map(
+        languages.map((lang) => [
+          lang.name,
+          {
+            noLazyMode: lang.noLazyMode,
+            ligatures: lang.ligatures,
+            rightToLeft: lang.rightToLeft,
+            additionalAccents: lang.additionalAccents,
+          },
+        ])
+      );
+
+      const wordsWithLanguage = new Map(
+        languages.flatMap((lang) => lang.words.map((word) => [word, lang.name]))
+      );
+
+      return new PolyglotWordset(wordsWithLanguage, languageProperties);
     },
   },
 };

frontend/src/ts/test/test-logic.ts

@@ -69,6 +69,7 @@ import {
   findSingleActiveFunboxWithFunction,
   getActiveFunboxes,
   getActiveFunboxesWithFunction,
+  getActiveFunboxNames,
   isFunboxActive,
   isFunboxActiveWithProperty,
 } from "./funbox/list";
@@ -469,14 +470,54 @@ async function init(): Promise<boolean> {
   }
 
   const allowLazyMode = !language.noLazyMode || Config.mode === "custom";
-  if (Config.lazyMode && !allowLazyMode) {
-    rememberLazyMode = true;
-    Notifications.add("This language does not support lazy mode.", 0, {
-      important: true,
+
+  // polyglot mode, check to enable lazy mode if any support it
+  if (getActiveFunboxNames().includes("polyglot")) {
+    const polyglotLanguages = Config.customPolyglot;
+    const languagePromises = polyglotLanguages.map(async (langName) => {
+      const { data: lang, error } = await tryCatch(
+        JSONData.getLanguage(langName)
+      );
+      if (error) {
+        Notifications.add(
+          Misc.createErrorMessage(
+            error,
+            `Failed to load language: ${langName}`
+          ),
+          -1
+        );
+      }
+      return lang;
     });
-    UpdateConfig.setLazyMode(false, true);
-  } else if (rememberLazyMode && !language.noLazyMode) {
-    UpdateConfig.setLazyMode(true, true);
+
+    const anySupportsLazyMode = (await Promise.all(languagePromises))
+      .filter((lang) => lang !== null)
+      .some((lang) => !lang.noLazyMode);
+
+    if (Config.lazyMode && !anySupportsLazyMode) {
+      rememberLazyMode = true;
+      Notifications.add(
+        "None of the selected polyglot languages support lazy mode.",
+        0,
+        {
+          important: true,
+        }
+      );
+      UpdateConfig.setLazyMode(false, true);
+    } else if (rememberLazyMode && anySupportsLazyMode) {
+      UpdateConfig.setLazyMode(true, true);
+    }
+  } else {
+    // normal mode
+    if (Config.lazyMode && !allowLazyMode) {
+      rememberLazyMode = true;
+      Notifications.add("This language does not support lazy mode.", 0, {
+        important: true,
+      });
+      UpdateConfig.setLazyMode(false, true);
+    } else if (rememberLazyMode && !language.noLazyMode) {
+      UpdateConfig.setLazyMode(true, true);
+    }
   }
 
   if (!Config.lazyMode && !language.noLazyMode) {
@@ -502,16 +543,19 @@ async function init(): Promise<boolean> {
     currentQuote: TestWords.currentQuote,
   });
 
-  let generatedWords: string[];
-  let generatedSectionIndexes: number[];
   let wordsHaveTab = false;
   let wordsHaveNewline = false;
+  let allRightToLeft: boolean | undefined = undefined;
+  let allLigatures: boolean | undefined = undefined;
+  let generatedWords: string[] = [];
+  let generatedSectionIndexes: number[] = [];
   try {
     const gen = await WordsGenerator.generateWords(language);
     generatedWords = gen.words;
     generatedSectionIndexes = gen.sectionIndexes;
     wordsHaveTab = gen.hasTab;
     wordsHaveNewline = gen.hasNewline;
+    ({ allRightToLeft, allLigatures } = gen);
   } catch (e) {
     Loader.hide();
     if (e instanceof WordGenError || e instanceof Error) {
@@ -570,10 +614,10 @@ async function init(): Promise<boolean> {
     );
   }
   Funbox.toggleScript(TestWords.words.getCurrent());
-  TestUI.setRightToLeft(language.rightToLeft ?? false);
-  TestUI.setLigatures(language.ligatures ?? false);
+  TestUI.setLigatures(allLigatures ?? language.ligatures ?? false);
 
-  const isLanguageRTL = language.rightToLeft ?? false;
+  const isLanguageRTL = allRightToLeft ?? language.rightToLeft ?? false;
+  TestUI.setRightToLeft(isLanguageRTL);
   TestState.setIsLanguageRightToLeft(isLanguageRTL);
   TestState.setIsDirectionReversed(
     isFunboxActiveWithProperty("reverseDirection")

frontend/src/ts/test/words-generator.ts

@@ -1,6 +1,6 @@
 import Config, * as UpdateConfig from "../config";
 import * as CustomText from "./custom-text";
-import * as Wordset from "./wordset";
+import { Wordset, FunboxWordsFrequency, withWords } from "./wordset";
 import QuotesController, {
   Quote,
   QuoteWithTextSplit,
@@ -24,6 +24,7 @@ import {
 } from "./funbox/list";
 import { WordGenError } from "../utils/word-gen-error";
 import * as Loader from "../elements/loader";
+import { PolyglotWordset } from "./funbox/funbox-functions";
 import { LanguageObject } from "@monkeytype/schemas/languages";
 
 //pin implementation
@@ -311,7 +312,7 @@ async function applyEnglishPunctuationToWord(word: string): Promise<string> {
   return EnglishPunctuation.replace(word);
 }
 
-function getFunboxWordsFrequency(): Wordset.FunboxWordsFrequency | undefined {
+function getFunboxWordsFrequency(): FunboxWordsFrequency | undefined {
   const funbox = findSingleActiveFunboxWithFunction("getWordsFrequencyMode");
   if (funbox) {
     return funbox.functions.getWordsFrequencyMode();
@@ -345,7 +346,7 @@ async function getFunboxSection(): Promise<string[]> {
 function getFunboxWord(
   word: string,
   wordIndex: number,
-  wordset?: Wordset.Wordset
+  wordset?: Wordset
 ): string {
   const funbox = findSingleActiveFunboxWithFunction("getWord");
 
@@ -384,6 +385,21 @@ async function applyBritishEnglishToWord(
 }
 
 function applyLazyModeToWord(word: string, language: LanguageObject): string {
+  // polyglot mode, use the word's actual language
+  if (currentWordset && currentWordset instanceof PolyglotWordset) {
+    const langName = currentWordset.wordsWithLanguage.get(word);
+    const langProps = langName
+      ? currentWordset.languageProperties.get(langName)
+      : undefined;
+    const allowLazyMode =
+      (langProps && !langProps.noLazyMode) || Config.mode === "custom";
+    if (Config.lazyMode && allowLazyMode && langProps) {
+      word = LazyMode.replaceAccents(word, langProps.additionalAccents);
+    }
+    return word;
+  }
+
+  // normal mode
   const allowLazyMode = !language.noLazyMode || Config.mode === "custom";
   if (Config.lazyMode && allowLazyMode) {
     word = LazyMode.replaceAccents(word, language.additionalAccents);
@@ -574,7 +590,7 @@ async function getQuoteWordList(
   return TestWords.currentQuote.textSplit;
 }
 
-let currentWordset: Wordset.Wordset | null = null;
+let currentWordset: Wordset | null = null;
 let currentLanguage: LanguageObject | null = null;
 let isCurrentlyUsingFunboxSection = false;
 
@@ -583,6 +599,8 @@ type GenerateWordsReturn = {
   sectionIndexes: number[];
   hasTab: boolean;
   hasNewline: boolean;
+  allRightToLeft?: boolean;
+  allLigatures?: boolean;
 };
 
 let previousRandomQuote: QuoteWithTextSplit | null = null;
@@ -604,6 +622,8 @@ export async function generateWords(
     sectionIndexes: [],
     hasTab: false,
     hasNewline: false,
+    allRightToLeft: language.rightToLeft,
+    allLigatures: language.ligatures ?? false,
   };
 
   isCurrentlyUsingFunboxSection = isFunboxActiveWithFunction("pullSection");
@@ -634,9 +654,20 @@ export async function generateWords(
 
   const funbox = findSingleActiveFunboxWithFunction("withWords");
   if (funbox) {
-    currentWordset = await funbox.functions.withWords(wordList);
+    const result = await funbox.functions.withWords(wordList);
+    // PolyglotWordset if polyglot otherwise Wordset
+    if (result instanceof PolyglotWordset) {
+      const polyglotResult = result;
+      currentWordset = polyglotResult;
+      // set allLigatures if any language in languageProperties has ligatures true
+      ret.allLigatures = Array.from(
+        polyglotResult.languageProperties.values()
+      ).some((props) => !!props.ligatures);
+    } else {
+      currentWordset = result;
+    }
   } else {
-    currentWordset = await Wordset.withWords(wordList);
+    currentWordset = await withWords(wordList);
   }
 
   console.debug("Wordset", currentWordset);

frontend/src/ts/utils/json-data.ts

@@ -80,6 +80,12 @@ export async function getLayout(layoutName: string): Promise<LayoutObject> {
   return await cachedFetchJson<LayoutObject>(`/layouts/${layoutName}.json`);
 }
 
+// used for polyglot wordset language-specific properties
+export type LanguageProperties = Pick<
+  LanguageObject,
+  "noLazyMode" | "ligatures" | "rightToLeft" | "additionalAccents"
+>;
+
 let currentLanguage: LanguageObject;
 
 /**