feat: add chunk-based translation for large files to handle TPM limits (#92)

Signed-off-by: JounQin <[email protected]>
Co-authored-by: JounQin <[email protected]>

Author: youyongsong

.changeset/selfish-worms-help.md

@@ -0,0 +1,5 @@
+---
+"@alauda/doom": minor
+---
+
+feat: add chunk-based translation for large files to handle TPM limits

src/cli/translate.ts

@@ -92,6 +92,10 @@ You are a professional technical documentation engineer, skilled in writing high
 <% if (terms) { %>
 <%- terms %>
 <% } %>
+<% if (isChunk) { %>
+## Chunk Translation Notice
+This is part of a larger document that has been split into smaller chunks for translation. Please translate this chunk as if it's part of a continuous document, maintaining consistency with the overall document style and context.
+<% } %>
 
 <% if (userPrompt || additionalPrompts) { %>
 ## Additional Requirements
@@ -117,6 +121,7 @@ export interface InternalTranslateOptions extends TranslateOptions {
   sourceContent: string
   target: Language
   additionalPrompts?: string
+  isChunk?: boolean
 }
 
 const resolveTerms = async (
@@ -220,13 +225,83 @@ function getTitleTranslation(
   return null
 }
 
+function splitContentIntoChunks(
+  content: string,
+  maxChunkSize: number,
+): string[] {
+  const lines = content.split('\n')
+  const chunks: string[] = []
+  let currentChunk: string[] = []
+  let currentSize = 0
+
+  for (const line of lines) {
+    const lineSize = Buffer.byteLength(line + '\n', 'utf8')
+
+    // If adding this line would exceed the chunk size, and we have content in current chunk
+    if (currentSize + lineSize > maxChunkSize && currentChunk.length > 0) {
+      chunks.push(currentChunk.join('\n'))
+      currentChunk = [line]
+      currentSize = lineSize
+    } else {
+      currentChunk.push(line)
+      currentSize += lineSize
+    }
+  }
+
+  // Add the last chunk if it has content
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk.join('\n'))
+  }
+
+  return chunks
+}
+
+export const translateWithChunks = async (
+  options: InternalTranslateOptions,
+): Promise<string> => {
+  const { sourceContent } = options
+  const maxChunkSize = 60 * 1024
+
+  const contentSize = Buffer.byteLength(sourceContent, 'utf8')
+  if (contentSize <= maxChunkSize) {
+    return translate(options)
+  }
+
+  logger.info(
+    `Content size (${Math.round(contentSize / 1024)}KB) exceeds limit, splitting into chunks...`,
+  )
+
+  const chunks = splitContentIntoChunks(sourceContent, maxChunkSize)
+  logger.info(`Split content into ${chunks.length} chunks`)
+
+  const translatedChunks: string[] = []
+
+  for (let i = 0; i < chunks.length; i++) {
+    logger.info(`Translating chunk ${i + 1}/${chunks.length}...`)
+
+    const translatedChunk = await translate({
+      ...options,
+      sourceContent: chunks[i],
+      isChunk: true,
+    })
+
+    translatedChunks.push(translatedChunk)
+  }
+
+  const result = translatedChunks.join('\n')
+  logger.info(`Successfully translated ${chunks.length} chunks`)
+
+  return result
+}
+
 export const translate = async ({
   source,
   sourceContent,
   target,
   systemPrompt,
   userPrompt = '',
   additionalPrompts = '',
+  isChunk = false,
 }: InternalTranslateOptions) => {
   if (!openai) {
     openai = new AzureOpenAI({
@@ -275,6 +350,7 @@ export const translate = async ({
       additionalPrompts: additionalPrompts,
       terms,
       titleTranslationPrompt,
+      isChunk,
     },
     { async: true },
   )
@@ -543,7 +619,7 @@ export const translateCommand = new Command('translate')
                 ),
               })
 
-              targetContent = await translate({
+              targetContent = await translateWithChunks({
                 ...config.translate,
                 source,
                 sourceContent: normalizedSourceContent,