added ability to translate specific languages and/or sections through command yarn scripts

Author: yihao03

i18n/controllers/recurTranslate.ts

@@ -34,6 +34,7 @@ const MAXLEN = Number(process.env.MAX_LEN) || 3000;
 // change to true to avoid calling openai api, useful for troubleshooting
 // chunking logic
 const troubleshoot = false;
+let languageNames = new Intl.DisplayNames(["en"], { type: "language" });
 
 // Centralized logging to prevent duplicate messages
 const errorMessages = new Set();
@@ -73,21 +74,29 @@ export function getFileErrors(): Record<
 const createParser = () =>
   (sax as any).createStream(false, { trim: false }, { strictEntities: true });
 
-async function translate(language: string, filePath: string): Promise<void> {
+async function translate(langCode: string, filePath: string): Promise<void> {
   const startTime = new Date().getTime();
   let assistant;
 
   try {
     // Use the provided file path directly without modification
     const input_path = filePath;
+    const language = languageNames.of(langCode);
+
+    if (language === undefined) {
+      throw new Error('Undefined language');
+    }
 
     if (!troubleshoot) assistant = await createAssistant(language, ai as any);
 
+    
     // Generate output path by replacing "/en/" with "/cn/" in the path
     const output_path = filePath.replace(
       path.sep + "en" + path.sep,
-      path.sep + "cn" + path.sep
+      path.sep + langCode + path.sep
     );
+    
+
 
     const translated: string = await recursivelyTranslate(
       language,
@@ -112,7 +121,7 @@ async function translate(language: string, filePath: string): Promise<void> {
       });
     }
     const elapsed = new Date().getTime() - startTime;
-    console.log(filePath + " took " + elapsed / 1000.0 + " seconds");
+    console.log("Took " + elapsed / 1000.0 + " seconds");
   }
 }
 
@@ -273,7 +282,7 @@ async function recursivelyTranslate(
 
   try {
     await new Promise<void>((resolve, reject) => {
-      console.log("Translating " + filePath + " at " + thread.id);
+      console.log("Translating " + filePath + " to " + language + " at " + thread.id);
       // Variables to track current depth and segments.
       let currentDepth = 0;
       let currentSegment = "";
@@ -413,7 +422,7 @@ async function recursivelyTranslate(
     try {
       await ai.beta.threads.messages.create(thread.id, {
         role: "user",
-        content: `<TRANSLATE> ${chunk} </TRANSLATE>`
+        content: `Translate to ${language}: <TRANSLATE> ${chunk} </TRANSLATE>`
       });
 
       const run = await ai.beta.threads.runs.createAndPoll(thread.id, {

i18n/index.ts

@@ -1,15 +1,23 @@
 import PathGenerator from "./controllers/path.ts";
 import translate, { getFileErrors } from "./controllers/recurTranslate.ts";
 import fs from "fs";
+import util from "util";
 import path from "path";
 import { fileURLToPath } from "url";
 import { dirname } from "path";
 import OpenAI from "openai";
+import { permission } from "process";
 
 // Get the directory name of the current module
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
+const readdir = util.promisify(fs.readdir);
+const getDirectories = async source =>
+  (await readdir(source, { withFileTypes: true }))
+    .filter(dirent => dirent.isDirectory())
+    .map(dirent => dirent.name);
+
 // Global variables for tracking translation state
 // These need to be accessible by signal handlers
 let xmlFiles: string[] = [];
@@ -52,7 +60,7 @@ async function saveSummaryLog() {
           failedDel.push(file.id);
         }
       })
-    ).then(() => console.log("successfully deleted all files"))
+    ).then(() => console.log("successfully deleted all files"));
 
     const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
     let summaryLog = `
@@ -189,11 +197,11 @@ async function findAllXmlFiles(directory: string): Promise<string[]> {
 }
 
 // Function to check if a file needs translation
-async function needsTranslation(enFilePath: string): Promise<boolean> {
+async function needsTranslation(enFilePath: string, lang: string): Promise<boolean> {
   // Generate the corresponding cn file path
   const cnFilePath = enFilePath.replace(
     path.sep + "en" + path.sep,
-    path.sep + "cn" + path.sep
+    path.sep + lang + path.sep
   );
 
   try {
@@ -212,15 +220,29 @@ async function needsTranslation(enFilePath: string): Promise<boolean> {
   }
 }
 
-export default async function fancyName(path: string) {
+export default async function fancyName(path: string, language: string) {
   const fullPath = PathGenerator(path);
-  await translate("Chinese", fullPath);
+  await translate(language, fullPath);
 }
 
 (async () => {
   await setupCleanupHandlers();
 
   try {
+    if ((process.argv[2], process.argv[3])) {
+      fancyName(process.argv[2], process.argv[3]);
+      return;
+    }
+
+    let languages: string[] = [];
+
+    if (process.argv[2] === "all") {
+      languages = await getDirectories(path.join(__dirname, "../xml"));
+      console.dir(languages);
+    } else {
+      languages.push(process.argv[2]);
+    }
+
     // Get the absolute path to the xml/en directory using proper path resolution
     const enDirPath = path.resolve(__dirname, "../xml/en");
 
@@ -230,19 +252,20 @@ export default async function fancyName(path: string) {
     xmlFiles = await findAllXmlFiles(enDirPath);
 
     console.log(`Found ${xmlFiles.length} XML files to check for translation`);
-
+   
+    for (const lang of languages) {
     // Filter files that need translation
     filesToTranslate = [];
     for (const file of xmlFiles) {
-      if (await needsTranslation(file)) {
+      if (await needsTranslation(file, lang)) {
         filesToTranslate.push(file as never);
       }
     }
 
     console.log(`${filesToTranslate.length} files need translation`);
 
     if (filesToTranslate.length === 0) {
-      console.log("No files need translation. Exiting.");
+      console.log(`No files need translation for ${lang}.`);
       return;
     }
 
@@ -263,7 +286,7 @@ export default async function fancyName(path: string) {
         batch.map(async file => {
           try {
             console.log(`Starting translation for ${file}`);
-            await translate("Chinese", file);
+            await translate(lang, file);
             return { file, success: true };
           } catch (error) {
             // Return failure with error but don't log yet
@@ -325,6 +348,7 @@ export default async function fancyName(path: string) {
 
     // Save a detailed summary to a log file
     await saveSummaryLog();
+  }
   } catch (e) {
     console.error("Error during translation process:", e);
   }