Merge branch 'sicp_i18n' into i18n_gh_actions

Author: coder114514

i18n/config.ts

@@ -0,0 +1,19 @@
+// maximum permissible concurrent translations
+export const max_trans_num = Number(process.env.MAX_TRANSLATION_NO) || 5;
+
+// log file configs
+export const translationSummaryPrefix: string = "translation-summary";
+export const jsonSummaryPrefix: string = "json-summary";
+export const ignoredTags: string[] = [
+    "LATEXINLINE",
+    "LATEX",
+    "SNIPPET",
+    "SCHEMEINLINE",
+    "SCHEME",
+    "LONG_PAGE",
+    "LABEL",
+    "HISTORY",
+    "REF",
+    "FIGURE",
+];
+export const max_chunk_len: Number = Number(process.env.MAX_LEN) || 3000;

i18n/controllers/fileUtilities.ts

@@ -0,0 +1,45 @@
+import path from "path";
+import fs from "fs";
+
+export async function needsTranslation(
+  enFilePath: string,
+  lang: string
+): Promise<boolean> {
+  const cnFilePath = enFilePath.replace(
+    path.sep + "en" + path.sep,
+    path.sep + lang + path.sep
+  );
+  try {
+    const cnStats = await fs.promises.stat(cnFilePath);
+    if (!cnStats.isFile()) {
+      return true;
+    }
+
+    const enStats = await fs.promises.stat(enFilePath);
+    return enStats.mtime > cnStats.mtime;
+  } catch (error) {
+    throw error;
+  }
+}
+
+// Function to recursively find all XML files in a directory
+export async function findAllXmlFiles(directory: string): Promise<string[]> {
+  const files = await fs.promises.readdir(directory);
+  const xmlFiles: string[] = [];
+
+  for (const file of files) {
+    const fullPath = path.join(directory, file);
+    const stat = await fs.promises.stat(fullPath);
+
+    if (stat.isDirectory()) {
+      // Recursively search subdirectories
+      const subDirFiles = await findAllXmlFiles(fullPath);
+      xmlFiles.push(...subDirFiles);
+    } else if (path.extname(file).toLowerCase() === ".xml") {
+      // Add XML files to the list
+      xmlFiles.push(fullPath);
+    }
+  }
+
+  return xmlFiles;
+}

i18n/controllers/loggers.ts

@@ -0,0 +1,111 @@
+import OpenAI from "openai";
+import path from "path";
+import fs from "fs";
+import { getFileErrors } from "./translator";
+import { translationSummaryPrefix } from "../config";
+
+// Function to save summary log - can be called from signal handlers
+export async function saveSummaryLog(
+    xmlFiles: string[],
+    failures: { file: string; error: any }[],
+    translateNum: number,
+    failureCount: number,
+    successCount: number
+) {
+    try {
+        const ai = new OpenAI({
+            apiKey: process.env.API_KEY,
+            baseURL: process.env.AI_BASEURL
+        });
+
+        // list and delete all assistants
+        const assistants = await ai.beta.assistants.list({ limit: 100 });
+        const failedDel: string[] = [];
+        await Promise.all(
+            assistants.data.map(async assistant => {
+                try {
+                    await ai.beta.assistants.del(assistant.id);
+                } catch (error) {
+                    failedDel.push(assistant.id);
+                }
+            })
+        ).then(() => console.log("successfully removed all assistants"));
+
+        // list and delete all uploaded files
+        const files = await ai.files.list();
+        await Promise.all(
+            files.data.map(async file => {
+                try {
+                    await ai.files.del(file.id);
+                } catch (error) {
+                    failedDel.push(file.id);
+                }
+            })
+        ).then(() => console.log("successfully deleted all files"));
+
+        const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+        let summaryLog = `
+Translation Summary (${timestamp})
+================================
+Total files scanned: ${xmlFiles.length}
+Files needing translation: ${translateNum}
+Successfully translated: ${successCount}
+Failed translations: ${failureCount}
+Success rate: ${translateNum > 0 ? ((successCount / translateNum) * 100).toFixed(2) : 0}%
+`;
+
+        if (failedDel.length > 0) {
+            summaryLog += `\nFailed to remove ${failedDel.length} assistants\n`;
+            failedDel.forEach(
+                (assistant, index) => (summaryLog += `${index + 1}. ${assistant}\n`)
+            );
+        }
+
+        // Add failed translations to the log
+        if (failures.length > 0) {
+            summaryLog += `\nFailed Translations (High-level errors):\n`;
+            failures.forEach((failure, index) => {
+                summaryLog += `${index + 1}. ${failure.file}\n   Error: ${failure.error}\n\n`;
+            });
+        }
+
+        // Add detailed errors captured during translation process
+        const fileErrors = getFileErrors();
+        if (Object.keys(fileErrors).length > 0) {
+            failureCount = Object.keys(fileErrors).length;
+            summaryLog += `\nDetailed Translation Errors:\n`;
+            summaryLog += `============================\n`;
+
+            for (const [filePath, errors] of Object.entries(fileErrors)) {
+                summaryLog += `\nFile: ${filePath}\n`;
+                errors.forEach((error, index) => {
+                    summaryLog += `  ${index + 1}. ${error.error}\n`;
+                    if (error.error) {
+                        // Format the error object/message for better readability
+                        const errorStr =
+                            typeof error.error === "object"
+                                ? JSON.stringify(error.error, null, 2).substring(0, 500) // Limit very long errors
+                                : String(error.error);
+                        summaryLog += `     Details: ${errorStr}\n`;
+                    }
+                });
+            }
+        }
+
+        const logDir = path.resolve(__dirname, "../logs");
+        if (!fs.existsSync(logDir)) {
+            fs.mkdirSync(logDir, { recursive: true });
+        }
+
+        const logPath = path.join(
+            logDir,
+            `${translationSummaryPrefix}-${timestamp}.log`
+        );
+        fs.writeFileSync(logPath, summaryLog);
+        console.log(
+            `Summary log saved to logs/translation-summary-${timestamp}.log`
+        );
+    } catch (logError) {
+        console.error("Failed to save log file:", logError);
+    }
+}