Enhance error handling and logging in translation process, streamline file path usage

Author: yihao03

i18n/controllers/recurTranslate.ts

@@ -7,6 +7,7 @@ import sax from "sax";
 import { Readable } from "stream";
 import { fileURLToPath } from "url";
 import { isGeneratorObject } from "util/types";
+import { AssistantStream } from "openai/lib/AssistantStream.mjs";
 
 dotenv.config();
 
@@ -20,25 +21,57 @@ const ai = new OpenAI({
   baseURL: process.env.AI_BASEURL
 });
 
-const ignoredTags = ["LATEXINLINE", "LATEX", "SNIPPET", "SCHEMEINLINE", "SCHEME", "LONG_PAGE", "LABEL"];
+const ignoredTags = [
+  "LATEXINLINE",
+  "LATEX",
+  "SNIPPET",
+  "SCHEMEINLINE",
+  "SCHEME",
+  "LONG_PAGE",
+  "LABEL"
+];
 
 const MAXLEN = Number(process.env.MAX_LEN) || 3000;
 
+// Centralized logging to prevent duplicate messages
+const errorMessages = new Set();
+function logError(message: string, error?: any) {
+  // Create a unique key for this error message
+  const errorKey = message + (error ? error.toString() : "");
+  // Only log if we haven't seen this exact message before
+  if (!errorMessages.has(errorKey)) {
+    errorMessages.add(errorKey);
+    if (error) {
+      console.error(message, error);
+    } else {
+      console.error(message);
+    }
+  }
+}
+
 const createParser = () =>
   (sax as any).createStream(true, { trim: false }, { strictEntities: true });
 
 async function translate(language: string, filePath: string): Promise<void> {
   const startTime = new Date().getTime();
+  let assistant;
+
   try {
-    // Pipe the XML file into the parser.
-    const input_dir = fileURLToPath(
-      import.meta.resolve("../../xml/en" + filePath)
-    );
+    // Use the provided file path directly without modification
+    const input_path = filePath;
+
+    assistant = await createAssistant(language, ai as any);
 
-    const translated: string = await recursivelyTranslate(language, input_dir);
+    // Generate output path by replacing "/en/" with "/cn/" in the path
+    const output_path = filePath.replace(
+      path.sep + "en" + path.sep,
+      path.sep + "cn" + path.sep
+    );
 
-    const output_path = fileURLToPath(
-      import.meta.resolve("../../xml/cn" + filePath)
+    const translated: string = await recursivelyTranslate(
+      language,
+      input_path,
+      assistant.id
     );
 
     // Ensure directory exists
@@ -48,17 +81,23 @@ async function translate(language: string, filePath: string): Promise<void> {
     fs.writeFileSync(output_path, translated);
     console.log(`Translation saved to ${output_path}`);
   } catch (parseErr) {
-    console.error("Error parsing XML:", parseErr);
+    logError(`Error translating file ${filePath}:`, parseErr);
   } finally {
+    if (assistant) {
+      await ai.beta.assistants.del(assistant.id).catch(err => {
+        logError(`Error deleting assistant:`, err);
+      });
+    }
     const elapsed = new Date().getTime() - startTime;
     console.log(filePath + " took " + elapsed / 1000.0 + " seconds");
   }
 }
 
-// TODO: change the toTranslate to a file path, read the file and translate the content
+// Function to translate the content of a file
 async function recursivelyTranslate(
   language: string,
-  path: string
+  filePath: string,
+  assistant_id: string
 ): Promise<string> {
   // Recursive function to split and translate
   async function helper(ori: string): Promise<string> {
@@ -124,14 +163,12 @@ async function recursivelyTranslate(
         if (tagName === "WRAPPER") {
           return;
         }
-        
+
         subCurrentSegment += `</${tagName}>`;
 
         if (subCurrentDepth === 2) {
           // We are closing a segment element.
-          if (
-            ignoredTags.includes(tagName)
-          ) {
+          if (ignoredTags.includes(tagName)) {
             subSegments.push([false, subCurrentSegment]);
           } else if (
             subSegments.length > 0 &&
@@ -147,12 +184,12 @@ async function recursivelyTranslate(
           subCurrentSegment = "";
           subIsRecording = false;
         }
-        
+
         if (subCurrentDepth === 1) {
-          subSegments.push([false, `</${tagName}>`])
+          subSegments.push([false, `</${tagName}>`]);
           subCurrentSegment = "";
         }
-        
+
         subCurrentDepth--;
       });
 
@@ -166,16 +203,34 @@ async function recursivelyTranslate(
 
       subParser.on("end", async () => {
         for (const segment of subSegments) {
-          if (segment[0]) {
-            subTranslated.push(await helper(segment[1]));
-          } else {
-            subTranslated.push(segment[1]);
+          try {
+            if (segment[0]) {
+              subTranslated.push(await helper(segment[1]));
+            } else {
+              subTranslated.push(segment[1]);
+            }
+          } catch (error) {
+            logError(`Error translating segment in ${filePath}:`, error);
+            // Add error comment and continue with next segment
+            subTranslated.push(
+              segment[1] + `<!-- Error translating this segment -->`
+            );
           }
         }
         resolve();
       });
 
-      subParser.on("error", reject);
+      subParser.on("error", err => {
+        logError(`Error in subParser for ${filePath}:`, err);
+        // Try to recover and continue
+        try {
+          subParser._parser.error = null;
+          subParser._parser.resume();
+        } catch (resumeErr) {
+          logError(`Could not recover from parser error:`, resumeErr);
+          reject(err);
+        }
+      });
 
       Readable.from("<WRAPPER>" + ori + "</WRAPPER>").pipe(subParser);
     });
@@ -186,14 +241,12 @@ async function recursivelyTranslate(
   // Create a SAX parser in strict mode to split source into chunks.
   const parser = createParser();
 
-  // const assistant = await createAssistant(language, ai as any);
-  const assistant_id = "asst_BLVYfog5DpWrbu3fW3o2oD4r";
   const thread = await ai.beta.threads.create();
   let translated: String[] = [];
 
   try {
     await new Promise<void>((resolve, reject) => {
-      console.log("Translating " + path + " at " + thread.id);
+      console.log("Translating " + filePath + " at " + thread.id);
       // Variables to track current depth and segments.
       let currentDepth = 0;
       let currentSegment = "";
@@ -278,36 +331,51 @@ async function recursivelyTranslate(
 
       parser.on("end", async () => {
         for (const segment of segments) {
-          if (segment[0]) {
-            translated.push(await helper(segment[1]));
-          } else {
-            translated.push(segment[1]);
+          try {
+            if (segment[0]) {
+              translated.push(await helper(segment[1]));
+            } else {
+              translated.push(segment[1]);
+            }
+          } catch (error) {
+            logError(`Error translating segment in ${filePath}:`, error);
+            // Add error comment and continue with next segment
+            translated.push(
+              segment[1] + `<!-- Error translating this section -->`
+            );
           }
         }
         resolve();
       });
 
-      parser.on("error", reject);
+      parser.on("error", err => {
+        logError(`Parser error in ${filePath}:`, err);
+        // Try to recover and continue
+        try {
+          parser._parser.error = null;
+          parser._parser.resume();
+        } catch (resumeErr) {
+          logError(`Could not recover from parser error:`, resumeErr);
+          reject(err);
+        }
+      });
 
-      fs.createReadStream(path).pipe(parser);
+      // Use the file path directly without modification
+      fs.createReadStream(filePath).pipe(parser);
     });
 
     return translated.join("");
   } catch (parseErr) {
-    console.error("Error parsing XML:", parseErr);
-    return translated.join("") + "<!-- Error parsing this section -->";
+    logError(`Error parsing XML in ${filePath}:`, parseErr);
+    // Return what we have so far plus error comment
+    return translated.join("") + `<!-- Error parsing this file -->`;
   }
 
   async function translateChunk(chunk: string): Promise<string> {
     if (chunk.trim() === "" || chunk.trim() === "," || chunk.trim() === ".") {
       return chunk;
     }
 
-    // console.log("Translating chunk of length: " + chunk.length);
-    // if (chunk.length < 100) {
-    //   console.log("\nchunk: " + chunk);
-    // }
-
     let translatedChunk = "";
 
     try {
@@ -330,7 +398,7 @@ async function recursivelyTranslate(
       });
 
       const message = messages.data.pop()!;
-      const messageContent = message.content[0];
+      const messageContent = message?.content[0];
 
       if (messageContent.type !== "text") {
         throw new Error(
@@ -341,7 +409,6 @@ async function recursivelyTranslate(
       const text = messageContent.text;
 
       const safeText = escapeXML(text.value);
-      // const safeText = chunk;
       const textStream = Readable.from("<WRAPPER>" + safeText + "</WRAPPER>");
 
       await new Promise<void>((resolve, reject) => {
@@ -359,13 +426,21 @@ async function recursivelyTranslate(
 
         clean.on("opentag", node => {
           currDepth++;
-          if (node.name != "WRAPPER" && node.name != "TRANSLATE") {
+          if (
+            node.name != "WRAPPER" &&
+            node.name != "TRANSLATE" &&
+            !ignoredTags.includes(node.name)
+          ) {
             translatedChunk += `<${node.name}${formatAttributes(node.attributes)}>`;
           }
         });
 
         clean.on("closetag", tagName => {
-          if (tagName != "WRAPPER" && tagName != "TRANSLATE") {
+          if (
+            tagName != "WRAPPER" &&
+            tagName != "TRANSLATE" &&
+            !ignoredTags.includes(tagName)
+          ) {
             translatedChunk += `</${tagName}>`;
           }
           currDepth--;
@@ -380,24 +455,19 @@ async function recursivelyTranslate(
         });
 
         clean.on("error", error => {
-          console.log(
-            "error encountered when validating XML: " +
-              error +
-              "\nfile: " +
-              path +
-              "\n section: " +
-              safeText + 
-              "\n original text: " + 
-              chunk
-          );
+          // Log only once with abbreviated content
+          logError(`XML validation error in ${filePath}`, error);
 
           // Attempt to recover using the internal parser
           try {
             clean._parser.error = null;
             clean._parser.resume();
+            // Continue processing despite the error
+            resolve();
           } catch (e) {
-            console.log("Failed to resume parser:", e);
-            reject(e);
+            // Add error comment and resolve instead of rejecting
+            translatedChunk += `<!-- XML validation error -->`;
+            resolve();
           }
         });
 
@@ -408,11 +478,9 @@ async function recursivelyTranslate(
 
       return translatedChunk;
     } catch (err) {
-      console.log(`Error occured while translating ${path}:\n    ` + err);
-      return (
-        translatedChunk +
-        `<!-- Error occured while translating this section-->\n<!-- Error: ${err}-->`
-      );
+      logError(`Error occurred while translating chunk in ${filePath}:`, err);
+      // Return the original chunk with error comment rather than throwing
+      return chunk + `<!-- Error occurred while translating this section -->`;
     }
   }
 }