reorganised file structure and yarn prepare json to translate all languages in xml

Author: yihao03

i18n/controllers/recurTranslate.ts

@@ -19,7 +19,7 @@ const ai = new OpenAI({
   baseURL: process.env.AI_BASEURL
 });
 
-const MAXLEN = 3000;
+const MAXLEN = Number(process.env.MAX_LEN) || 3000;
 
 const createParser = () =>
   (sax as any).createStream(true, { trim: false }, { strictEntities: true });
@@ -35,7 +35,7 @@ async function translate(language: string, filePath: string): Promise<void> {
     const translated: string = await recursivelyTranslate(language, input_dir);
 
     const output_path = fileURLToPath(
-      import.meta.resolve("../../xml_cn" + filePath)
+      import.meta.resolve("../../xml/cn" + filePath)
     );
 
     // Ensure directory exists
@@ -69,7 +69,7 @@ async function recursivelyTranslate(
     await new Promise<void>((resolve, reject) => {
       const subParser = createParser();
 
-      let subCurrentDepth = 0;
+      let subCurrentDepth = 1;
       let subCurrentSegment = "";
       const subSegments: [boolean, string][] = [];
       let subIsRecording = false;
@@ -78,7 +78,7 @@ async function recursivelyTranslate(
         if (node.name === "WRAPPER") {
           return;
         }
-
+        
         subCurrentDepth++;
 
         // If we're at depth 2, this is the start of a new segment.
@@ -160,7 +160,6 @@ async function recursivelyTranslate(
           // We are closing the root element.
           subSegments.push([false, `</${tagName}>`]);
         }
-
         subCurrentDepth--;
       });
 
@@ -194,7 +193,7 @@ async function recursivelyTranslate(
   // Create a SAX parser in strict mode to split source into chunks.
   const parser = createParser();
 
-  // const assistant = await createAssistant(language, ai);
+  // const assistant = await createAssistant(language, ai as any);
   const assistant_id = "asst_BLVYfog5DpWrbu3fW3o2oD4r";
   const thread = await ai.beta.threads.create();
   let translated: String[] = [];
@@ -316,7 +315,7 @@ async function recursivelyTranslate(
       return chunk;
     }
 
-    console.log("Translating chunk of length: " + chunk.length);
+    // console.log("Translating chunk of length: " + chunk.length);
     if (chunk.length < 100) {
       console.log("\nchunk: " + chunk)
     }
@@ -333,13 +332,15 @@ async function recursivelyTranslate(
         Content to translate:
         <TRANSLATE> ${chunk} </TRANSLATE>`
       });
+      
       const run = await ai.beta.threads.runs.createAndPoll(thread.id, {
         assistant_id: assistant_id
       });
 
       const messages = await ai.beta.threads.messages.list(thread.id, {
         run_id: run.id
       });
+
       const message = messages.data.pop()!;
       const messageContent = message.content[0];
 
@@ -352,6 +353,7 @@ async function recursivelyTranslate(
       const text = messageContent.text;
 
       const safeText = escapeXML(text.value);
+      console.log(safeText);
       const textStream = Readable.from("<WRAPPER>" + safeText + "</WRAPPER>");
 
       await new Promise<void>((resolve, reject) => {
@@ -416,7 +418,7 @@ async function recursivelyTranslate(
       console.log(`Error occured while translating ${path}:\n    ` + err);
       return (
         translatedChunk +
-        `<!-- Error occured while translating this section-->\n<!-- Error: ${err.length < 50 ? err : err.subString(0, 50) + "..."}-->`
+        `<!-- Error occured while translating this section-->\n<!-- Error: ${err}-->`
       );
     }
   }

i18n/index.ts

@@ -12,11 +12,11 @@ export default async function fancyName(path: string) {
   try {
     await Promise.all([
       // fancyName("2"),
-      fancyName("1.1"),
+      // fancyName("1.1"),
       fancyName("1.1.2"),
-      fancyName("1.1.3"),
-      fancyName("1.1.4"),
-      fancyName("1.1.5"),
+      // fancyName("1.1.3"),
+      // fancyName("1.1.4"),
+      // fancyName("1.1.5"),
       // fancyName("1.1.6"),
       // fancyName("1.1.7"),
       // fancyName("1.1.8"),

i18n/initializers/initialize.ts

@@ -21,11 +21,11 @@ export default async function createAssistant(language: string, ai: OpenAI) {
   ].map(path => fs.createReadStream(path));
 
   // Create a vector store including our two files.
-  const vectorStore = await ai.beta.vectorStores.create({
+  const vectorStore = await ai.vectorStores.create({
     name: "Translation instructions"
   });
 
-  await ai.beta.vectorStores.fileBatches.uploadAndPoll(vectorStore.id, {
+  await ai.vectorStores.fileBatches.uploadAndPoll(vectorStore.id, {
     files: fileStreams
   });
 

javascript/index.js

@@ -42,6 +42,7 @@ import { writeRewritedSearchData } from "./searchRewrite";
 import { setupSnippetsJson } from "./processingFunctions/processSnippetJson";
 import { createTocJson } from "./generateTocJson";
 import { setupReferencesJson } from "./processingFunctions/processReferenceJson";
+import { SourceTextModule } from "vm";
 
 export let parseType;
 let version;
@@ -58,6 +59,11 @@ const ensureDirectoryExists = (path, cb) => {
   });
 };
 
+const getDirectories = async source =>
+  (await readdir(source, { withFileTypes: true }))
+    .filter(dirent => dirent.isDirectory())
+    .map(dirent => dirent.name);
+
 async function translateXml(filepath, filename, option) {
   const fullFilepath = path.join(inputDir, filepath, filename);
   const fileToRead = await open(fullFilepath, "r");
@@ -200,8 +206,14 @@ async function recursiveXmlToHtmlInOrder(option) {
   }
 }
 
-async function recursiveTranslateXml(filepath, option) {
+async function recursiveTranslateXml(filepath, option, lang = "en") {
   let files;
+
+  if (lang != null) {
+    filepath = path.join(filepath, lang);
+    console.log(filepath);
+  }
+
   const fullPath = path.join(inputDir, filepath);
   files = await readdir(fullPath);
   const promises = [];
@@ -222,7 +234,9 @@ async function recursiveTranslateXml(filepath, option) {
         promises.push(translateXml(filepath, file, option));
       }
     } else if (fs.lstatSync(path.join(fullPath, file)).isDirectory()) {
-      promises.push(recursiveTranslateXml(path.join(filepath, file), option));
+      promises.push(
+        recursiveTranslateXml(path.join(filepath, file), option, null)
+      );
     }
   });
   await Promise.all(promises);
@@ -349,23 +363,28 @@ async function main() {
     console.log("setup snippets done\n");
     recursiveTranslateXml("", "parseXml");
   } else if (parseType == "json") {
-    outputDir = path.join(__dirname, "../json");
+    const languages = await getDirectories(path.join(__dirname, "../xml"));
+    console.dir(languages)
 
-    createMain();
 
-    console.log("\ngenerate table of content\n");
-    await recursiveTranslateXml("", "generateTOC");
-    allFilepath = sortTOC(allFilepath);
-    createTocJson(outputDir);
+    languages.forEach(async lang => {
+      outputDir = path.join(__dirname, "../json", lang);
+      createMain();
 
-    console.log("setup snippets and references\n");
-    await recursiveXmlToHtmlInOrder("setupSnippet");
-    console.log("setup snippets and references done\n");
+      console.log("\ngenerate table of content\n");
+      await recursiveTranslateXml("", "generateTOC", lang);
+      allFilepath = sortTOC(allFilepath);
+      createTocJson(outputDir);
+
+      console.log("setup snippets and references\n");
+      await recursiveXmlToHtmlInOrder("setupSnippet");
+      console.log("setup snippets and references done\n");
 
-    await recursiveXmlToHtmlInOrder("parseXml");
-    writeRewritedSearchData();
-    // this is meant to be temp; also, will remove the original "generateSearchData" after the updation at the frontend is completed.
-    //testIndexSearch();
+      await recursiveXmlToHtmlInOrder("parseXml");
+      writeRewritedSearchData();
+      // this is meant to be temp; also, will remove the original "generateSearchData" after the updation at the frontend is completed.
+      //testIndexSearch();
+    });
   }
 }