Bugfix: Citation Parsing Issue (Version 2) (#2056)

* Citation issue fix

* Citation issue fix version2

* Use startsWith

* Update comments

Author: bnodir

app/frontend/src/components/Answer/Answer.tsx

@@ -43,8 +43,7 @@ export const Answer = ({
     showSpeechOutputBrowser
 }: Props) => {
     const followupQuestions = answer.context?.followup_questions;
-    const messageContent = answer.message.content;
-    const parsedAnswer = useMemo(() => parseAnswerToHtml(messageContent, isStreaming, onCitationClicked), [answer]);
+    const parsedAnswer = useMemo(() => parseAnswerToHtml(answer, isStreaming, onCitationClicked), [answer]);
     const { t } = useTranslation();
     const sanitizedAnswerHtml = DOMPurify.sanitize(parsedAnswer.answerHtml);
 

app/frontend/src/components/Answer/AnswerParser.tsx

@@ -1,16 +1,43 @@
 import { renderToStaticMarkup } from "react-dom/server";
-import { getCitationFilePath } from "../../api";
+import { ChatAppResponse, getCitationFilePath } from "../../api";
 
 type HtmlParsedAnswer = {
     answerHtml: string;
     citations: string[];
 };
 
-export function parseAnswerToHtml(answer: string, isStreaming: boolean, onCitationClicked: (citationFilePath: string) => void): HtmlParsedAnswer {
+// Function to validate citation format and check if dataPoint starts with possible citation
+function isCitationValid(contextDataPoints: any, citationCandidate: string): boolean {
+    const regex = /^[^\s]+\.[a-zA-Z0-9]+/;
+    if (!regex.test(citationCandidate)) {
+        return false;
+    }
+
+    // Check if contextDataPoints is an object with a text property that is an array
+    let dataPointsArray: string[];
+    if (Array.isArray(contextDataPoints)) {
+        dataPointsArray = contextDataPoints;
+    } else if (contextDataPoints && Array.isArray(contextDataPoints.text)) {
+        dataPointsArray = contextDataPoints.text;
+    } else {
+        return false;
+    }
+
+    const isValidCitation = dataPointsArray.some(dataPoint => dataPoint.startsWith(citationCandidate));
+
+    if (!isValidCitation) {
+        return false;
+    }
+
+    return true;
+}
+
+export function parseAnswerToHtml(answer: ChatAppResponse, isStreaming: boolean, onCitationClicked: (citationFilePath: string) => void): HtmlParsedAnswer {
+    const contextDataPoints = answer.context.data_points;
     const citations: string[] = [];
 
-    // trim any whitespace from the end of the answer after removing follow-up questions
-    let parsedAnswer = answer.trim();
+    // Trim any whitespace from the end of the answer after removing follow-up questions
+    let parsedAnswer = answer.message.content.trim();
 
     // Omit a citation that is still being typed during streaming
     if (isStreaming) {
@@ -34,6 +61,11 @@ export function parseAnswerToHtml(answer: string, isStreaming: boolean, onCitati
             return part;
         } else {
             let citationIndex: number;
+
+            if (!isCitationValid(contextDataPoints, part)) {
+                return `[${part}]`;
+            }
+
             if (citations.indexOf(part) !== -1) {
                 citationIndex = citations.indexOf(part) + 1;
             } else {