Skip to content

Commit b37b503

Browse files
yihao03yihao03
committed
added proper error logging and reporting. added ability to skip translations inside SCHEME and SCHEMEINLINE tags
1 parent 89841d6 commit b37b503

File tree

3 files changed

+377
-40
lines changed

3 files changed

+377
-40
lines changed

‎i18n/controllers/recurTranslate.ts

Lines changed: 63 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,11 @@ import createAssistant from "../initializers/initialize";
55
import dotenv from "dotenv";
66
import sax from "sax";
77
import { Readable } from "stream";
8-
import { fileURLToPath } from "url";
9-
import { isGeneratorObject } from "util/types";
10-
import { AssistantStream } from "openai/lib/AssistantStream.mjs";
118

129
dotenv.config();
1310

1411
if (process.env.AI_MODEL === undefined || process.env.API_KEY === undefined) {
15-
throw Error("Please specify AI_MODEL and API_KEY!");
12+
throw Error("Please specify AI_MODEL and API_KEY");
1613
}
1714

1815
// initialize OpenAI API
@@ -35,7 +32,10 @@ const MAXLEN = Number(process.env.MAX_LEN) || 3000;
3532

3633
// Centralized logging to prevent duplicate messages
3734
const errorMessages = new Set();
38-
function logError(message: string, error?: any) {
35+
// Track errors by file for summary reporting
36+
const fileErrors: Record<string, { message: string; error?: any }[]> = {};
37+
38+
function logError(message: string, error?: any, filePath?: string) {
3939
// Create a unique key for this error message
4040
const errorKey = message + (error ? error.toString() : "");
4141
// Only log if we haven't seen this exact message before
@@ -46,11 +46,27 @@ function logError(message: string, error?: any) {
4646
} else {
4747
console.error(message);
4848
}
49+
50+
// Store error for the summary log if filePath is provided
51+
if (filePath) {
52+
if (!fileErrors[filePath]) {
53+
fileErrors[filePath] = [];
54+
}
55+
fileErrors[filePath].push({ message, error });
56+
}
4957
}
5058
}
5159

60+
// Function to get all logged errors for summary
61+
export function getFileErrors(): Record<
62+
string,
63+
{ message: string; error?: any }[]
64+
> {
65+
return fileErrors;
66+
}
67+
5268
const createParser = () =>
53-
(sax as any).createStream(true, { trim: false }, { strictEntities: true });
69+
(sax as any).createStream(false, { trim: false }, { strictEntities: true });
5470

5571
async function translate(language: string, filePath: string): Promise<void> {
5672
const startTime = new Date().getTime();
@@ -81,11 +97,13 @@ async function translate(language: string, filePath: string): Promise<void> {
8197
fs.writeFileSync(output_path, translated);
8298
console.log(`Translation saved to ${output_path}`);
8399
} catch (parseErr) {
84-
logError(`Error translating file ${filePath}:`, parseErr);
100+
logError(`Error translating file ${filePath}:`, parseErr, filePath);
101+
// Re-throw the error to propagate it to the caller
102+
throw parseErr;
85103
} finally {
86104
if (assistant) {
87105
await ai.beta.assistants.del(assistant.id).catch(err => {
88-
logError(`Error deleting assistant:`, err);
106+
logError(`Error deleting assistant:`, err, filePath);
89107
});
90108
}
91109
const elapsed = new Date().getTime() - startTime;
@@ -210,7 +228,11 @@ async function recursivelyTranslate(
210228
subTranslated.push(segment[1]);
211229
}
212230
} catch (error) {
213-
logError(`Error translating segment in ${filePath}:`, error);
231+
logError(
232+
`Error translating segment in ${filePath}:`,
233+
error,
234+
filePath
235+
);
214236
// Add error comment and continue with next segment
215237
subTranslated.push(
216238
segment[1] + `<!-- Error translating this segment -->`
@@ -221,13 +243,13 @@ async function recursivelyTranslate(
221243
});
222244

223245
subParser.on("error", err => {
224-
logError(`Error in subParser for ${filePath}:`, err);
246+
logError(`Error in subParser for ${filePath}:`, err, filePath);
225247
// Try to recover and continue
226248
try {
227249
subParser._parser.error = null;
228250
subParser._parser.resume();
229251
} catch (resumeErr) {
230-
logError(`Could not recover from parser error:`, resumeErr);
252+
logError(`Could not recover from parser error:`, resumeErr, filePath);
231253
reject(err);
232254
}
233255
});
@@ -262,6 +284,8 @@ async function recursivelyTranslate(
262284
currentDepth++;
263285

264286
// If we're at depth 2, this is the start of a new segment.
287+
if (node.name == "SCHEME" || node.name == "SCHEMEINLINE") return;
288+
265289
if (currentDepth === 2 || isRecording) {
266290
isRecording = true;
267291
currentSegment += `<${node.name}${formatAttributes(node.attributes)}>`;
@@ -275,6 +299,10 @@ async function recursivelyTranslate(
275299

276300
parser.on("text", text => {
277301
text = strongEscapeXML(text);
302+
303+
// ignore all scheme contents
304+
if(parser._parser.tag == "SCHEME" || parser._parser.tag == "SCHEMEINLINE") return;
305+
278306
if (isRecording) {
279307
currentSegment += text;
280308
} else {
@@ -289,7 +317,7 @@ async function recursivelyTranslate(
289317
});
290318

291319
parser.on("closetag", tagName => {
292-
if (isRecording) {
320+
if (tagName !== "SCHEME" && tagName !== "SCHEMEINLINE" && isRecording) {
293321
currentSegment += `</${tagName}>`;
294322
}
295323

@@ -338,7 +366,11 @@ async function recursivelyTranslate(
338366
translated.push(segment[1]);
339367
}
340368
} catch (error) {
341-
logError(`Error translating segment in ${filePath}:`, error);
369+
logError(
370+
`Error translating segment in ${filePath}:`,
371+
error,
372+
filePath
373+
);
342374
// Add error comment and continue with next segment
343375
translated.push(
344376
segment[1] + `<!-- Error translating this section -->`
@@ -349,13 +381,13 @@ async function recursivelyTranslate(
349381
});
350382

351383
parser.on("error", err => {
352-
logError(`Parser error in ${filePath}:`, err);
384+
logError(`Parser error in ${filePath}:`, err, filePath);
353385
// Try to recover and continue
354386
try {
355387
parser._parser.error = null;
356388
parser._parser.resume();
357389
} catch (resumeErr) {
358-
logError(`Could not recover from parser error:`, resumeErr);
390+
logError(`Could not recover from parser error:`, resumeErr, filePath);
359391
reject(err);
360392
}
361393
});
@@ -366,12 +398,13 @@ async function recursivelyTranslate(
366398

367399
return translated.join("");
368400
} catch (parseErr) {
369-
logError(`Error parsing XML in ${filePath}:`, parseErr);
401+
logError(`Error parsing XML in ${filePath}:`, parseErr, filePath);
370402
// Return what we have so far plus error comment
371403
return translated.join("") + `<!-- Error parsing this file -->`;
372404
}
373405

374406
async function translateChunk(chunk: string): Promise<string> {
407+
return chunk;
375408
if (chunk.trim() === "" || chunk.trim() === "," || chunk.trim() === ".") {
376409
return chunk;
377410
}
@@ -400,6 +433,7 @@ async function recursivelyTranslate(
400433
const message = messages.data.pop()!;
401434
const messageContent = message?.content[0];
402435

436+
if (!messageContent) throw new Error(`undefined AI response`);
403437
if (messageContent.type !== "text") {
404438
throw new Error(
405439
`Unexpected message content type: ${messageContent.type}`
@@ -428,8 +462,7 @@ async function recursivelyTranslate(
428462
currDepth++;
429463
if (
430464
node.name != "WRAPPER" &&
431-
node.name != "TRANSLATE" &&
432-
!ignoredTags.includes(node.name)
465+
node.name != "TRANSLATE"
433466
) {
434467
translatedChunk += `<${node.name}${formatAttributes(node.attributes)}>`;
435468
}
@@ -438,8 +471,7 @@ async function recursivelyTranslate(
438471
clean.on("closetag", tagName => {
439472
if (
440473
tagName != "WRAPPER" &&
441-
tagName != "TRANSLATE" &&
442-
!ignoredTags.includes(tagName)
474+
tagName != "TRANSLATE"
443475
) {
444476
translatedChunk += `</${tagName}>`;
445477
}
@@ -456,7 +488,11 @@ async function recursivelyTranslate(
456488

457489
clean.on("error", error => {
458490
// Log only once with abbreviated content
459-
logError(`XML validation error in ${filePath}`, error);
491+
logError(
492+
`Error validating AI response for ${filePath}`,
493+
error,
494+
filePath
495+
);
460496

461497
// Attempt to recover using the internal parser
462498
try {
@@ -478,7 +514,11 @@ async function recursivelyTranslate(
478514

479515
return translatedChunk;
480516
} catch (err) {
481-
logError(`Error occurred while translating chunk in ${filePath}:`, err);
517+
logError(
518+
`Error occurred while translating chunk in ${filePath}:`,
519+
err,
520+
filePath
521+
);
482522
// Return the original chunk with error comment rather than throwing
483523
return chunk + `<!-- Error occurred while translating this section -->`;
484524
}
@@ -488,7 +528,7 @@ async function recursivelyTranslate(
488528
export default translate;
489529

490530
// Helper function to format attributes into a string.
491-
function formatAttributes(attrs) {
531+
function formatAttributes(attrs: string) {
492532
const attrStr = Object.entries(attrs)
493533
.map(([key, val]) => `${key}="${val}"`)
494534
.join(" ");

0 commit comments

Comments
 (0)