add retry logic to responsesAPI client

Author: Veikkosuhonen

src/server/routes/openai.ts

@@ -10,7 +10,7 @@ import { FileSearchResultsStore } from '../services/azureFileSearch/fileSearchRe
 import { ResponsesClient } from '../util/azure/ResponsesAPI'
 import { DEFAULT_RAG_SYSTEM_PROMPT } from '../util/config'
 import logger from '../util/logger'
-import { isError } from '../util/parser'
+import { isError } from '../util/isError'
 import { pdfToText } from '../util/pdfToText'
 import getEncoding from '../util/tiktoken'
 import { getAllowedModels, getCourseModel, getMessageContext, getModelContextLimit } from '../util/util'
@@ -180,10 +180,6 @@ openaiRouter.post('/stream/v2', upload.single('file'), async (r, res) => {
     include: ragIndexId ? ['file_search_call.results'] : [],
   })
 
-  if (isError(events)) {
-    throw new ApplicationError('Error creating a response stream', 424)
-  }
-
   res.setHeader('content-type', 'text/event-stream')
 
   const result = await responsesClient.handleResponse({

src/server/util/azure/ResponsesAPI.ts

@@ -12,6 +12,8 @@ import { createFileSearchTool } from './util'
 import { FileSearchResultsStore } from '../../services/azureFileSearch/fileSearchResultsStore'
 import { getAzureOpenAIClient } from './client'
 import { RagIndex } from '../../db/models'
+import OpenAI, { APIError } from 'openai'
+import { ApplicationError } from '../ApplicationError'
 
 const client = getAzureOpenAIClient(process.env.GPT_4O_MINI ?? '')
 
@@ -68,10 +70,12 @@ export class ResponsesClient {
     input,
     prevResponseId,
     include,
+    attemptNumber = 1,
   }: {
     input: ResponseInput
     prevResponseId?: string
     include?: ResponseIncludable[]
+    attemptNumber?: number
   }): Promise<Stream<ResponseStreamEvent> | APIError> {
     try {
       const sanitizedInput = validatedInputSchema.parse(input)
@@ -102,10 +106,16 @@ export class ResponsesClient {
          */
         // background: true,
       })
-    } catch (error: any) {
-      logger.error(error)
-
-      return { error } as any as APIError
+    } catch (error: unknown) {
+      if (error instanceof OpenAI.APIError && attemptNumber < 3) {
+        const retryDelay = 2 ** attemptNumber * 200
+        logger.error(`Failed to create a response stream after ${attemptNumber} attempts. Retrying after ${retryDelay}ms`, { error })
+        // Retry the request after a delay
+        await new Promise((resolve) => setTimeout(resolve, retryDelay))
+        return await this.createResponse({ prevResponseId, input, attemptNumber: attemptNumber + 1 })
+      } else {
+        throw ApplicationError.InternalServerError(`Failed to create a response stream after ${attemptNumber} attempts, sorry`, { extra: { error } })
+      }
     }
   }