Get/Delete ChatMessage based on startDateTime / endDateTime (#3867)

* Get ChatMessage based on startDateTime / endDateTime - supplements existing startDate / endDate fields

* Proper query handling for between case

* Return 0 result rather than error

* lint fix

* update start date and end date query

---------

Co-authored-by: Henry <[email protected]>

Author: ryanhalliday

packages/server/src/controllers/chat-messages/index.ts

@@ -2,9 +2,9 @@ import { Request, Response, NextFunction } from 'express'
 import { ChatMessageRatingType, ChatType, IReactFlowObject } from '../../Interface'
 import chatflowsService from '../../services/chatflows'
 import chatMessagesService from '../../services/chat-messages'
-import { aMonthAgo, clearSessionMemory, setDateToStartOrEndOfDay } from '../../utils'
+import { aMonthAgo, clearSessionMemory } from '../../utils'
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
-import { Between, FindOptionsWhere } from 'typeorm'
+import { Between, DeleteResult, FindOptionsWhere } from 'typeorm'
 import { ChatMessage } from '../../database/entities/ChatMessage'
 import { InternalFlowiseError } from '../../errors/internalFlowiseError'
 import { StatusCodes } from 'http-status-codes'
@@ -167,21 +167,21 @@ const removeAllChatMessages = async (req: Request, res: Response, next: NextFunc
         if (!chatId) {
             const isFeedback = feedbackTypeFilters?.length ? true : false
             const hardDelete = req.query?.hardDelete as boolean | undefined
-            const messages = await utilGetChatMessage(
+            const messages = await utilGetChatMessage({
                 chatflowid,
-                _chatType as ChatType | undefined,
-                undefined,
-                undefined,
-                undefined,
-                undefined,
+                chatType: _chatType as ChatType | undefined,
                 startDate,
                 endDate,
-                undefined,
-                isFeedback,
-                feedbackTypeFilters
-            )
+                feedback: isFeedback,
+                feedbackTypes: feedbackTypeFilters
+            })
             const messageIds = messages.map((message) => message.id)
 
+            if (messages.length === 0) {
+                const result: DeleteResult = { raw: [], affected: 0 }
+                return res.json(result)
+            }
+
             // Categorize by chatId_memoryType_sessionId
             const chatIdMap = new Map<string, ChatMessage[]>()
             messages.forEach((message) => {
@@ -238,8 +238,8 @@ const removeAllChatMessages = async (req: Request, res: Response, next: NextFunc
             if (sessionId) deleteOptions.sessionId = sessionId
             if (_chatType) deleteOptions.chatType = _chatType
             if (startDate && endDate) {
-                const fromDate = setDateToStartOrEndOfDay(startDate, 'start')
-                const toDate = setDateToStartOrEndOfDay(endDate, 'end')
+                const fromDate = new Date(startDate)
+                const toDate = new Date(endDate)
                 deleteOptions.createdDate = Between(fromDate ?? aMonthAgo(), toDate ?? new Date())
             }
             const apiResponse = await chatMessagesService.removeAllChatMessages(chatId, chatflowid, deleteOptions)

packages/server/src/services/chat-messages/index.ts

@@ -39,9 +39,9 @@ const getAllChatMessages = async (
     feedbackTypes?: ChatMessageRatingType[]
 ): Promise<ChatMessage[]> => {
     try {
-        const dbResponse = await utilGetChatMessage(
-            chatflowId,
-            chatTypeFilter,
+        const dbResponse = await utilGetChatMessage({
+            chatflowid: chatflowId,
+            chatType: chatTypeFilter,
             sortOrder,
             chatId,
             memoryType,
@@ -51,7 +51,7 @@ const getAllChatMessages = async (
             messageId,
             feedback,
             feedbackTypes
-        )
+        })
         return dbResponse
     } catch (error) {
         throw new InternalFlowiseError(
@@ -76,9 +76,9 @@ const getAllInternalChatMessages = async (
     feedbackTypes?: ChatMessageRatingType[]
 ): Promise<ChatMessage[]> => {
     try {
-        const dbResponse = await utilGetChatMessage(
-            chatflowId,
-            chatTypeFilter,
+        const dbResponse = await utilGetChatMessage({
+            chatflowid: chatflowId,
+            chatType: chatTypeFilter,
             sortOrder,
             chatId,
             memoryType,
@@ -88,7 +88,7 @@ const getAllInternalChatMessages = async (
             messageId,
             feedback,
             feedbackTypes
-        )
+        })
         return dbResponse
     } catch (error) {
         throw new InternalFlowiseError(

packages/server/src/services/stats/index.ts

@@ -17,19 +17,15 @@ const getChatflowStats = async (
     feedbackTypes?: ChatMessageRatingType[]
 ): Promise<any> => {
     try {
-        const chatmessages = (await utilGetChatMessage(
+        const chatmessages = (await utilGetChatMessage({
             chatflowid,
-            chatTypeFilter,
-            undefined,
-            undefined,
-            undefined,
-            undefined,
+            chatType: chatTypeFilter,
             startDate,
             endDate,
             messageId,
             feedback,
             feedbackTypes
-        )) as Array<ChatMessage & { feedback?: ChatMessageFeedback }>
+        })) as Array<ChatMessage & { feedback?: ChatMessageFeedback }>
         const totalMessages = chatmessages.length
         const totalFeedback = chatmessages.filter((message) => message?.feedback).length
         const positiveFeedback = chatmessages.filter((message) => message?.feedback?.rating === 'THUMBS_UP').length

packages/server/src/services/upsert-history/index.ts

@@ -1,4 +1,4 @@
-import { MoreThanOrEqual, LessThanOrEqual } from 'typeorm'
+import { MoreThanOrEqual, LessThanOrEqual, Between } from 'typeorm'
 import { StatusCodes } from 'http-status-codes'
 import { getRunningExpressApp } from '../../utils/getRunningExpressApp'
 import { UpsertHistory } from '../../database/entities/UpsertHistory'
@@ -14,26 +14,20 @@ const getAllUpsertHistory = async (
     try {
         const appServer = getRunningExpressApp()
 
-        const setDateToStartOrEndOfDay = (dateTimeStr: string, setHours: 'start' | 'end') => {
-            const date = new Date(dateTimeStr)
-            if (isNaN(date.getTime())) {
-                return undefined
+        let createdDateQuery
+        if (startDate || endDate) {
+            if (startDate && endDate) {
+                createdDateQuery = Between(new Date(startDate), new Date(endDate))
+            } else if (startDate) {
+                createdDateQuery = MoreThanOrEqual(new Date(startDate))
+            } else if (endDate) {
+                createdDateQuery = LessThanOrEqual(new Date(endDate))
             }
-            setHours === 'start' ? date.setHours(0, 0, 0, 0) : date.setHours(23, 59, 59, 999)
-            return date
         }
-
-        let fromDate
-        if (startDate) fromDate = setDateToStartOrEndOfDay(startDate, 'start')
-
-        let toDate
-        if (endDate) toDate = setDateToStartOrEndOfDay(endDate, 'end')
-
         let upsertHistory = await appServer.AppDataSource.getRepository(UpsertHistory).find({
             where: {
                 chatflowid,
-                ...(fromDate && { date: MoreThanOrEqual(fromDate) }),
-                ...(toDate && { date: LessThanOrEqual(toDate) })
+                date: createdDateQuery
             },
             order: {
                 date: sortOrder === 'DESC' ? 'DESC' : 'ASC'

packages/server/src/utils/getChatMessage.ts

@@ -1,9 +1,9 @@
-import { MoreThanOrEqual, LessThanOrEqual } from 'typeorm'
+import { MoreThanOrEqual, LessThanOrEqual, Between } from 'typeorm'
 import { ChatMessageRatingType, ChatType } from '../Interface'
 import { ChatMessage } from '../database/entities/ChatMessage'
 import { ChatMessageFeedback } from '../database/entities/ChatMessageFeedback'
 import { getRunningExpressApp } from '../utils/getRunningExpressApp'
-import { aMonthAgo, setDateToStartOrEndOfDay } from '.'
+import { aMonthAgo } from '.'
 
 /**
  * Method that get chat messages.
@@ -18,26 +18,34 @@ import { aMonthAgo, setDateToStartOrEndOfDay } from '.'
  * @param {boolean} feedback
  * @param {ChatMessageRatingType[]} feedbackTypes
  */
-export const utilGetChatMessage = async (
-    chatflowid: string,
-    chatType: ChatType | undefined,
-    sortOrder: string = 'ASC',
-    chatId?: string,
-    memoryType?: string,
-    sessionId?: string,
-    startDate?: string,
-    endDate?: string,
-    messageId?: string,
-    feedback?: boolean,
+interface GetChatMessageParams {
+    chatflowid: string
+    chatType?: ChatType
+    sortOrder?: string
+    chatId?: string
+    memoryType?: string
+    sessionId?: string
+    startDate?: string
+    endDate?: string
+    messageId?: string
+    feedback?: boolean
     feedbackTypes?: ChatMessageRatingType[]
-): Promise<ChatMessage[]> => {
-    const appServer = getRunningExpressApp()
-
-    let fromDate
-    if (startDate) fromDate = setDateToStartOrEndOfDay(startDate, 'start')
+}
 
-    let toDate
-    if (endDate) toDate = setDateToStartOrEndOfDay(endDate, 'end')
+export const utilGetChatMessage = async ({
+    chatflowid,
+    chatType,
+    sortOrder = 'ASC',
+    chatId,
+    memoryType,
+    sessionId,
+    startDate,
+    endDate,
+    messageId,
+    feedback,
+    feedbackTypes
+}: GetChatMessageParams): Promise<ChatMessage[]> => {
+    const appServer = getRunningExpressApp()
 
     if (feedback) {
         const query = await appServer.AppDataSource.getRepository(ChatMessage).createQueryBuilder('chat_message')
@@ -62,10 +70,13 @@ export const utilGetChatMessage = async (
         }
 
         // set date range
-        query.andWhere('chat_message.createdDate BETWEEN :fromDate AND :toDate', {
-            fromDate: fromDate ?? aMonthAgo(),
-            toDate: toDate ?? new Date()
-        })
+        if (startDate) {
+            query.andWhere('chat_message.createdDate >= :startDateTime', { startDateTime: startDate ? new Date(startDate) : aMonthAgo() })
+        }
+        if (endDate) {
+            query.andWhere('chat_message.createdDate <= :endDateTime', { endDateTime: endDate ? new Date(endDate) : new Date() })
+        }
+
         // sort
         query.orderBy('chat_message.createdDate', sortOrder === 'DESC' ? 'DESC' : 'ASC')
 
@@ -89,15 +100,25 @@ export const utilGetChatMessage = async (
         return messages
     }
 
+    let createdDateQuery
+    if (startDate || endDate) {
+        if (startDate && endDate) {
+            createdDateQuery = Between(new Date(startDate), new Date(endDate))
+        } else if (startDate) {
+            createdDateQuery = MoreThanOrEqual(new Date(startDate))
+        } else if (endDate) {
+            createdDateQuery = LessThanOrEqual(new Date(endDate))
+        }
+    }
+
     return await appServer.AppDataSource.getRepository(ChatMessage).find({
         where: {
             chatflowid,
             chatType,
             chatId,
             memoryType: memoryType ?? undefined,
             sessionId: sessionId ?? undefined,
-            ...(fromDate && { createdDate: MoreThanOrEqual(fromDate) }),
-            ...(toDate && { createdDate: LessThanOrEqual(toDate) }),
+            createdDate: createdDateQuery,
             id: messageId ?? undefined
         },
         order: {

packages/server/src/utils/index.ts

@@ -1745,15 +1745,6 @@ export const convertToValidFilename = (word: string) => {
         .toLowerCase()
 }
 
-export const setDateToStartOrEndOfDay = (dateTimeStr: string, setHours: 'start' | 'end') => {
-    const date = new Date(dateTimeStr)
-    if (isNaN(date.getTime())) {
-        return undefined
-    }
-    setHours === 'start' ? date.setHours(0, 0, 0, 0) : date.setHours(23, 59, 59, 999)
-    return date
-}
-
 export const aMonthAgo = () => {
     const date = new Date()
     date.setMonth(new Date().getMonth() - 1)

packages/ui/src/ui-component/dialog/ViewMessagesDialog.jsx

@@ -167,28 +167,32 @@ const ViewMessagesDialog = ({ show, dialogProps, onCancel }) => {
     let storagePath = ''
 
     const onStartDateSelected = (date) => {
-        setStartDate(date)
+        const updatedDate = new Date(date)
+        updatedDate.setHours(0, 0, 0, 0)
+        setStartDate(updatedDate)
         getChatmessageApi.request(dialogProps.chatflow.id, {
-            startDate: date,
+            startDate: updatedDate,
             endDate: endDate,
             chatType: chatTypeFilter.length ? chatTypeFilter : undefined
         })
         getStatsApi.request(dialogProps.chatflow.id, {
-            startDate: date,
+            startDate: updatedDate,
             endDate: endDate,
             chatType: chatTypeFilter.length ? chatTypeFilter : undefined
         })
     }
 
     const onEndDateSelected = (date) => {
-        setEndDate(date)
+        const updatedDate = new Date(date)
+        updatedDate.setHours(23, 59, 59, 999)
+        setEndDate(updatedDate)
         getChatmessageApi.request(dialogProps.chatflow.id, {
-            endDate: date,
+            endDate: updatedDate,
             startDate: startDate,
             chatType: chatTypeFilter.length ? chatTypeFilter : undefined
         })
         getStatsApi.request(dialogProps.chatflow.id, {
-            endDate: date,
+            endDate: updatedDate,
             startDate: startDate,
             chatType: chatTypeFilter.length ? chatTypeFilter : undefined
         })

packages/ui/src/views/vectorstore/UpsertHistoryDialog.jsx

@@ -211,17 +211,21 @@ const UpsertHistoryDialog = ({ show, dialogProps, onCancel }) => {
     }
 
     const onStartDateSelected = (date) => {
-        setStartDate(date)
+        const updatedDate = new Date(date)
+        updatedDate.setHours(0, 0, 0, 0)
+        setStartDate(updatedDate)
         getUpsertHistoryApi.request(dialogProps.chatflow.id, {
-            startDate: date,
+            startDate: updatedDate,
             endDate: endDate
         })
     }
 
     const onEndDateSelected = (date) => {
-        setEndDate(date)
+        const updatedDate = new Date(date)
+        updatedDate.setHours(23, 59, 59, 999)
+        setEndDate(updatedDate)
         getUpsertHistoryApi.request(dialogProps.chatflow.id, {
-            endDate: date,
+            endDate: updatedDate,
             startDate: startDate
         })
     }