Merge branch 'update-azure-openai'

Author: sutigit

package-lock.json

@@ -92,7 +92,7 @@
     "node-cron": "^3.0.3",
     "notistack": "^3.0.1",
     "ollama": "^0.5.15",
-    "openai": "^4.0.0-beta.12",
+    "openai": "^4.102.0",
     "pdf-parse-fork": "^1.2.0",
     "pg": "^8.11.3",
     "react": "^18.2.0",

package.json

@@ -45,7 +45,7 @@
     "courseTerms": "Lukukaudet",
     "courseNameInfo": "Kurssi",
     "usageCount": "Käyttömäärä",
-    "promptCount": "Kehtotteita",
+    "promptCount": "Kehotteita",
     "programCodes": "Ohjelmat",
     "studentCount": "Opis",
     "courseCodes": "Kurssikoodit"

src/client/locales/fi.json

@@ -1,11 +1,28 @@
 import express from 'express'
 import multer from 'multer'
 
-import { CourseChatRequest, AzureOptions, RequestWithUser } from '../types'
+import {
+  CourseChatRequest,
+  AzureOptions,
+  RequestWithUser,
+  AzureOptionsV2,
+} from '../types'
 import { isError } from '../util/parser'
-import { calculateUsage, incrementUsage, checkUsage, checkCourseUsage, incrementCourseUsage } from '../services/chatInstances/usage'
-import { getCompletionEvents, streamCompletion } from '../util/azure'
-import { getMessageContext, getModelContextLimit, getCourseModel, getAllowedModels } from '../util/util'
+import {
+  calculateUsage,
+  incrementUsage,
+  checkUsage,
+  checkCourseUsage,
+  incrementCourseUsage,
+} from '../services/chatInstances/usage'
+// import { getCompletionEvents, streamCompletion } from '../util/azure'
+import { getCompletionEventsV2, streamCompletionV2 } from '../util/azureV2'
+import {
+  getMessageContext,
+  getModelContextLimit,
+  getCourseModel,
+  getAllowedModels,
+} from '../util/util'
 import getEncoding from '../util/tiktoken'
 import logger from '../util/logger'
 import { inProduction, DEFAULT_TOKEN_LIMIT, FREE_MODEL } from '../../config'
@@ -20,7 +37,14 @@ const upload = multer({ storage })
 const fileParsing = async (options: any, req: any) => {
   let fileContent = ''
 
-  const textFileTypes = ['text/plain', 'text/html', 'text/css', 'text/csv', 'text/markdown', 'text/md']
+  const textFileTypes = [
+    'text/plain',
+    'text/html',
+    'text/css',
+    'text/csv',
+    'text/markdown',
+    'text/md',
+  ]
   if (textFileTypes.includes(req.file.mimetype)) {
     const fileBuffer = req.file.buffer
     fileContent = fileBuffer.toString('utf8')
@@ -56,7 +80,9 @@ openaiRouter.post('/stream', upload.single('file'), async (r, res) => {
     return
   }
 
-  const usageAllowed = courseId ? await checkCourseUsage(user, courseId) : model === FREE_MODEL || (await checkUsage(user, model))
+  const usageAllowed = courseId
+    ? await checkCourseUsage(user, courseId)
+    : model === FREE_MODEL || (await checkUsage(user, model))
 
   if (!usageAllowed) {
     res.status(403).send('Usage limit reached')
@@ -75,14 +101,22 @@ openaiRouter.post('/stream', upload.single('file'), async (r, res) => {
     return
   }
 
-  options.messages = getMessageContext(optionsMessagesWithFile || options.messages)
+  options.messages = getMessageContext(
+    optionsMessagesWithFile || options.messages
+  )
   options.stream = true
 
   const encoding = getEncoding(model)
   let tokenCount = calculateUsage(options, encoding)
-  const tokenUsagePercentage = Math.round((tokenCount / DEFAULT_TOKEN_LIMIT) * 100)
-
-  if (model !== FREE_MODEL && tokenCount > 0.1 * DEFAULT_TOKEN_LIMIT && !userConsent) {
+  const tokenUsagePercentage = Math.round(
+    (tokenCount / DEFAULT_TOKEN_LIMIT) * 100
+  )
+
+  if (
+    model !== FREE_MODEL &&
+    tokenCount > 0.1 * DEFAULT_TOKEN_LIMIT &&
+    !userConsent
+  ) {
     res.status(201).json({
       tokenConsumtionWarning: true,
       message: `You are about to use ${tokenUsagePercentage}% of your monthly CurreChat usage`,
@@ -98,7 +132,7 @@ openaiRouter.post('/stream', upload.single('file'), async (r, res) => {
     return
   }
 
-  const events = await getCompletionEvents(options as AzureOptions)
+  const events = await getCompletionEventsV2(options)
 
   if (isError(events)) {
     res.status(424)
@@ -107,7 +141,7 @@ openaiRouter.post('/stream', upload.single('file'), async (r, res) => {
 
   res.setHeader('content-type', 'text/event-stream')
 
-  const completion = await streamCompletion(events, options as AzureOptions, encoding, res)
+  const completion = await streamCompletionV2(events, encoding, res)
 
   tokenCount += completion.tokenCount
 
@@ -135,7 +169,8 @@ openaiRouter.post('/stream', upload.single('file'), async (r, res) => {
       where: { courseId },
     }))
 
-  const consentToSave = courseId && course.saveDiscussions && options.saveConsent
+  const consentToSave =
+    courseId && course.saveDiscussions && options.saveConsent
 
   console.log('consentToSave', options.saveConsent, user.username)
 
@@ -155,79 +190,83 @@ openaiRouter.post('/stream', upload.single('file'), async (r, res) => {
   return
 })
 
-openaiRouter.post('/stream/:courseId', upload.single('file'), async (r, res) => {
-  const { courseId } = r.params
-  const req = r as CourseChatRequest
-  const { options } = JSON.parse(r.body.data)
-  const { user } = req
-
-  if (!user.id) {
-    res.status(401).send('Unauthorized')
-    return
-  }
+openaiRouter.post(
+  '/stream/:courseId',
+  upload.single('file'),
+  async (r, res) => {
+    const { courseId } = r.params
+    const req = r as CourseChatRequest
+    const { options } = JSON.parse(r.body.data)
+    const { user } = req
+
+    if (!user.id) {
+      res.status(401).send('Unauthorized')
+      return
+    }
 
-  const usageAllowed = await checkCourseUsage(user, courseId)
-  if (!usageAllowed) {
-    res.status(403).send('Usage limit reached')
-    return
-  }
+    const usageAllowed = await checkCourseUsage(user, courseId)
+    if (!usageAllowed) {
+      res.status(403).send('Usage limit reached')
+      return
+    }
 
-  options.messages = getMessageContext(options.messages)
-  options.stream = true
+    options.messages = getMessageContext(options.messages)
+    options.stream = true
 
-  const model = await getCourseModel(courseId)
+    const model = await getCourseModel(courseId)
 
-  if (options.model) {
-    const allowedModels = getAllowedModels(model)
-    if (!allowedModels.includes(options.model)) {
-      res.status(403).send('Model not allowed')
-      return
+    if (options.model) {
+      const allowedModels = getAllowedModels(model)
+      if (!allowedModels.includes(options.model)) {
+        res.status(403).send('Model not allowed')
+        return
+      }
+    } else {
+      options.model = model
     }
-  } else {
-    options.model = model
-  }
 
-  const encoding = getEncoding(options.model)
-  let tokenCount = calculateUsage(options, encoding)
+    const encoding = getEncoding(options.model)
+    let tokenCount = calculateUsage(options, encoding)
 
-  const contextLimit = getModelContextLimit(options.model)
+    const contextLimit = getModelContextLimit(options.model)
 
-  if (tokenCount > contextLimit) {
-    logger.info('Maximum context reached')
-    res.status(403).send('Model maximum context reached')
-    return
-  }
+    if (tokenCount > contextLimit) {
+      logger.info('Maximum context reached')
+      res.status(403).send('Model maximum context reached')
+      return
+    }
 
-  const events = await getCompletionEvents(options as AzureOptions)
+    const events = await getCompletionEventsV2(options)
 
-  if (isError(events)) {
-    res.status(424).send(events)
-    return
-  }
+    if (isError(events)) {
+      res.status(424).send(events)
+      return
+    }
 
-  res.setHeader('content-type', 'text/event-stream')
+    res.setHeader('content-type', 'text/event-stream')
 
-  const completion = await streamCompletion(events, options as AzureOptions, encoding, res)
+    const completion = await streamCompletionV2(events, encoding, res)
 
-  tokenCount += completion.tokenCount
+    tokenCount += completion.tokenCount
 
-  let userToCharge = user
-  if (inProduction && req.hijackedBy) {
-    userToCharge = req.hijackedBy
-  }
+    let userToCharge = user
+    if (inProduction && req.hijackedBy) {
+      userToCharge = req.hijackedBy
+    }
 
-  await incrementCourseUsage(userToCharge, courseId, tokenCount)
-  logger.info(`Stream ended. Total tokens: ${tokenCount}`, {
-    tokenCount,
-    courseId,
-    model: options.model,
-    user: user.username,
-  })
+    await incrementCourseUsage(userToCharge, courseId, tokenCount)
+    logger.info(`Stream ended. Total tokens: ${tokenCount}`, {
+      tokenCount,
+      courseId,
+      model: options.model,
+      user: user.username,
+    })
 
-  encoding.free()
+    encoding.free()
 
-  res.end()
-  return
-})
+    res.end()
+    return
+  }
+)
 
 export default openaiRouter

src/server/routes/openai.ts

@@ -1,6 +1,8 @@
 import { Request } from 'express'
 import OpenAI from 'openai'
 import { ChatRequestMessage, GetChatCompletionsOptions } from '@azure/openai'
+import { ChatCompletionMessageParam } from 'openai/resources/chat'
+import { RequestOptions } from 'openai/core'
 
 export type PartialRecord<K extends keyof any, T> = Partial<Record<K, T>>
 
@@ -34,7 +36,9 @@ export interface CourseChatRequest extends ChatRequest {
 
 export type APIError = typeof OpenAI.APIError
 
-export type OpenAIStream = ReturnType<typeof OpenAI.prototype.chat.completions.create>
+export type OpenAIStream = ReturnType<
+  typeof OpenAI.prototype.chat.completions.create
+>
 
 export type Message = OpenAI.Chat.ChatCompletionMessage
 export interface CustomMessage {
@@ -43,14 +47,21 @@ export interface CustomMessage {
 }
 export type Role = 'system' | 'assistant' | 'user'
 
-export type StreamingOptions = OpenAI.Chat.Completions.CompletionCreateParamsStreaming
+export type StreamingOptions =
+  OpenAI.Chat.Completions.CompletionCreateParamsStreaming
 
 export type AzureOptions = {
   model: string
   messages: ChatRequestMessage[]
   options: GetChatCompletionsOptions
 }
 
+export type AzureOptionsV2 = {
+  model: string
+  messages: ChatCompletionMessageParam[]
+  options: RequestOptions
+}
+
 interface RequestBody {
   id?: string
   options?: StreamingOptions