Dev

backend/src/document_sources/wikipedia.py

@@ -11,6 +11,6 @@ def get_documents_from_Wikipedia(wiki_query:str, language:str):
   except Exception as e:
     message="Failed To Process Wikipedia Query"
     error_message = str(e)
-    logging.exception(f'Failed To Process Wikipedia Query: {file_name}, Exception Stack trace: {error_message}')
+    logging.exception(f'Failed To Process Wikipedia Query, Exception Stack trace: {error_message}')
     raise LLMGraphBuilderException(error_message+' '+message)
 

backend/test_integrationqa.py

@@ -8,8 +8,7 @@
 from dotenv import load_dotenv
 from src.main import *
 from src.QA_integration import QA_RAG
-from src.ragas_eval import get_ragas_metrics
-from datasets import Dataset
+
 # Load environment variables
 load_dotenv()
 URI = os.getenv('NEO4J_URI')
@@ -49,7 +48,7 @@ def test_graph_from_file_local(model_name):
        create_source_node_local(graph, model_name, file_name)
        result = asyncio.run(
            extract_graph_from_file_local_file(
-               URI, USERNAME, PASSWORD, DATABASE, model_name, merged_file_path, file_name, '', '', None, ''
+               URI, USERNAME, PASSWORD, DATABASE, model_name, merged_file_path, file_name, '', '',100,20,1, None,''
            )
        )
        logging.info(f"Local file test result: {result}")
@@ -67,7 +66,7 @@ def test_graph_from_wikipedia(model_name):
        create_source_node_graph_url_wikipedia(graph, model_name, wiki_query, "Wikipedia")
        result = asyncio.run(
            extract_graph_from_file_Wikipedia(
-               URI, USERNAME, PASSWORD, DATABASE, model_name, file_name, 'en', file_name, '', '', None, ''
+               URI, USERNAME, PASSWORD, DATABASE, model_name, file_name, 'en', file_name, '', '', 100,20,1,None,''
            )
        )
        logging.info(f"Wikipedia test result: {result}")
@@ -85,9 +84,9 @@ def test_graph_from_youtube_video(model_name):
        create_source_node_graph_url_youtube(graph, model_name, source_url, "youtube")
        result = asyncio.run(
            extract_graph_from_file_youtube(
-               URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, file_name, '', '', None, ''
+               URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, file_name, '', '',100,20,1, None,''
            )
-       )
+      )
        logging.info(f"YouTube video test result: {result}")
        if isinstance(result, dict) and result.get("status") == "Failed":
            return {"status": "Failed", "error": result.get("error", "Unknown error")}
@@ -104,7 +103,7 @@ def test_graph_website(model_name):
        create_source_node_graph_web_url(graph, model_name, source_url, "web-url")
        result = asyncio.run(
            extract_graph_from_web_page(
-               URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, "Google Cloud Skills Boost", '', '', None, ''
+               URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, "Google Cloud Skills Boost-www", '', '',100,20,1, None,''
            )
        )
        logging.info(f"Web URL test result: {result}")
@@ -122,7 +121,8 @@ def test_chatbot_qna(model_name, mode='vector'):
        result = QA_RAG(graph, model_name, 'Tell me about Amazon', '[]', 1, mode)
     #    assert len(result['message']) > 20
        logging.info(f"Chatbot QnA test passed for mode: {mode}")
-       return result
+       final_result = {'model_name':model_name,'mode':mode,'result':result}
+       return final_result
    except Exception as e:
        logging.error(f"Error in chatbot QnA: {e}")
        return {"status": "Failed", "error": str(e)}
@@ -170,15 +170,34 @@ def get_duplicate_nodes():
            return "Data successfully loaded"
        else:
            return "Unable to load data"
-
+
+def flatten_extract_dataframe(df: pd.DataFrame):
+   rows = []
+   for _, row in df.iterrows():
+       try:
+           col1, col2, execution_date = row[0], row[1], row[2] if len(row) > 2 else None
+           data_dict = ast.literal_eval(col1) if isinstance(col1, str) and col1.startswith("{") else {}
+           meta_dict = ast.literal_eval(col2) if isinstance(col2, str) and col2.startswith("{") else {}
+           combined_dict = {**data_dict, **meta_dict}
+           for key in combined_dict.keys():
+               if isinstance(combined_dict[key], dict) and key.startswith("processed_chunk_detail"):
+                   combined_dict[key] = str(combined_dict[key])
+           combined_dict["execution_date"] = execution_date
+           rows.append(combined_dict)
+       except (SyntaxError, ValueError, TypeError) as e:
+           print(f"Error parsing row: {row} - {e}")
+           continue
+   flat_df = pd.DataFrame(rows)
+   return flat_df
+
 def run_tests():
    """Runs all integration tests and logs results."""
    extract_list = []
    extract_error_list = []
    chatbot_list = []
    chatbot_error_list = []
    other_api_list = []
-   models = ['openai_gpt_4','openai_gpt_4o','openai_gpt_4o_mini','gemini_1.5_pro','gemini_1.5_flash','gemini_2.0_flash','bedrock_nova_micro_v1','bedrock_nova_lite_v1','bedrock_nova_pro_v1','fireworks_qwen72b_instruct']
+   models = ['openai_gpt_4o','openai_gpt_4o_mini','openai_gpt_4.1','openai_gpt_4.1_mini','gemini_2.0_flash','fireworks_llama4_maverick','bedrock_nova_pro_v1']
    chatbot_modes = [
        "vector",
        "graph+vector",
@@ -218,9 +237,7 @@ def run_tests():
 
        try:
             schema_result = test_populate_graph_schema_from_text(model_name)
-            print("KAUSTUBH : ",schema_result)
             other_api_list.append({f"{model_name}":schema_result}) 
-            print("other_api_list : ",other_api_list)
        except Exception as e:
            logging.error(f"Error in test_populate_graph_schema_from_text for {model_name}: {e}")
            other_api_list.append({f"{model_name}":str(e)}) 

frontend/package.json

@@ -32,9 +32,11 @@
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "react-icons": "^5.5.0",
-    "react-markdown": "^9.0.1",
+    "react-markdown": "^10.1.0",
     "react-router": "^6.23.1",
     "react-router-dom": "^6.23.1",
+    "rehype-raw": "^7.0.0",
+    "remark-gfm": "^4.0.1",
     "uuid": "^11.1.0"
   },
   "devDependencies": {

frontend/src/App.css

@@ -469,4 +469,5 @@
 .animate-highlight {
   animation: highlightAnimation 1.5s ease-in-out;
   border: 2px solid rgb(var(--theme-palette-primary-bg-strong));
-}
+}
+

frontend/src/components/ChatBot/Chatbot.tsx

@@ -41,6 +41,8 @@ import { downloadClickHandler, getDateTime } from '../../utils/Utils';
 import ChatModesSwitch from './ChatModesSwitch';
 import CommonActions from './CommonChatActions';
 import Loader from '../../utils/Loader';
+import remarkGfm from 'remark-gfm';
+import rehypeRaw from 'rehype-raw';
 const InfoModal = lazy(() => import('./ChatInfoModal'));
 if (typeof window !== 'undefined') {
   if (!sessionStorage.getItem('session_id')) {
@@ -467,9 +469,17 @@ const Chatbot: FC<ChatbotProps> = (props) => {
                         chat.isLoading && index === listMessages.length - 1 && chat.user === 'chatbot' ? 'loader' : ''
                       }`}
                     >
-                      <ReactMarkdown className={!isFullScreen ? 'max-w-[250px]' : ''}>
-                        {chat.modes[chat.currentMode]?.message || ''}
-                      </ReactMarkdown>
+                      <div
+                        className={
+                          !isFullScreen
+                            ? 'max-w-[250px] prose prose-sm sm:prose lg:prose-lg xl:prose-xl'
+                            : 'prose prose-sm sm:prose lg:prose-lg xl:prose-xl max-w-none'
+                        }
+                      >
+                        <ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeRaw] as any}>
+                          {chat.modes[chat.currentMode]?.message || ''}
+                        </ReactMarkdown>
+                      </div>
                     </div>
                     <div>
                       <div>

frontend/src/components/ChatBot/ChunkInfo.tsx

@@ -13,7 +13,8 @@ import { chatModeLables } from '../../utils/Constants';
 import GraphViewModal from '../Graph/GraphViewModal';
 import { handleGraphNodeClick } from './chatInfo';
 import { IconButtonWithToolTip } from '../UI/IconButtonToolTip';
-
+import remarkGfm from 'remark-gfm';
+import rehypeRaw from 'rehype-raw';
 const ChunkInfo: FC<ChunkProps> = ({ loading, chunks, mode }) => {
   const themeUtils = useContext(ThemeWrapperContext);
   const [neoNodes, setNeoNodes] = useState<any[]>([]);
@@ -257,8 +258,10 @@ const ChunkInfo: FC<ChunkProps> = ({ loading, chunks, mode }) => {
                     </div>
                   </>
                 )}
-                <div className='mt-2'>
-                  <ReactMarkdown>{chunk?.text}</ReactMarkdown>
+                <div className='mt-2 prose prose-sm sm:prose lg:prose-lg xl:prose-xl max-w-none'>
+                  <ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeRaw] as any}>
+                    {chunk?.text}
+                  </ReactMarkdown>
                 </div>
               </li>
             ))}

frontend/src/components/ChatBot/CommunitiesInfo.tsx

@@ -5,7 +5,8 @@ import { CommunitiesProps } from '../../types';
 import { chatModeLables } from '../../utils/Constants';
 import GraphViewModal from '../Graph/GraphViewModal';
 import { handleGraphNodeClick } from './chatInfo';
-
+import remarkGfm from 'remark-gfm';
+import rehypeRaw from 'rehype-raw';
 const CommunitiesInfo: FC<CommunitiesProps> = ({ loading, communities, mode }) => {
   const [neoNodes, setNeoNodes] = useState<any[]>([]);
   const [neoRels, setNeoRels] = useState<any[]>([]);
@@ -51,7 +52,11 @@ const CommunitiesInfo: FC<CommunitiesProps> = ({ loading, communities, mode }) =
                       <Typography variant='subheading-medium'>{community.score}</Typography>
                     </Flex>
                   )}
-                  <ReactMarkdown>{community.summary}</ReactMarkdown>
+                  <div className='prose prose-sm sm:prose lg:prose-lg xl:prose-xl max-w-none'>
+                    <ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeRaw] as any}>
+                      {community.summary}
+                    </ReactMarkdown>
+                  </div>
                 </div>
               </li>
             ))}

frontend/src/components/Popups/ChunkPopUp/index.tsx

@@ -5,7 +5,9 @@ import Loader from '../../../utils/Loader';
 import { useMemo } from 'react';
 import chunklogo from '../../../assets/images/chunks.svg';
 import { tokens } from '@neo4j-ndl/base';
-
+import remarkGfm from 'remark-gfm';
+import rehypeRaw from 'rehype-raw';
+import ReactMarkdown from 'react-markdown';
 const ChunkPopUp = ({
   showChunkPopup,
   chunks,
@@ -71,7 +73,12 @@ const ChunkPopUp = ({
                       <Typography variant='subheading-small'>{c.pagenumber}</Typography>
                     </Flex>
                   ) : null}
-                  <Typography variant='body-medium'>{c.text}</Typography>
+
+                  <div className='prose prose-sm sm:prose lg:prose-lg xl:prose-xl max-w-none'>
+                    <ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeRaw] as any}>
+                      {c.text}
+                    </ReactMarkdown>
+                  </div>
                 </Flex>
               </li>
             ))}

frontend/src/components/Popups/ConnectionModal/VectorIndexMisMatchAlert.tsx

@@ -21,17 +21,19 @@ export default function VectorIndexMisMatchAlert({
   return (
     <Flex>
       <Box>
-        <Markdown className='whitespace-pre-wrap'>
-          {isVectorIndexAlreadyExists
-            ? `**Vector Index Incompatibility** 
+        <div className='whitespace-pre-wrap'>
+          <Markdown>
+            {isVectorIndexAlreadyExists
+              ? `**Vector Index Incompatibility** 
 The existing Neo4j vector index dimension (${userVectorIndexDimension}) is incompatible with the supported dimension (384) for this application. 
 To proceed, please choose one of the following options: 
 1.**Recreate Vector Index:** Click "Re-Create Vector Index" to generate a compatible vector index. 
 2.**Use a Different Instance:** Connect to a Neo4j instance with a compatible vector index configuration  `
-            : chunksExists
-            ? `A vector index is essential for performing efficient similarity searches within your data. Without it, some chunks of data will be invisible to queries based on meaning and context. Creating a vector index unlocks the full potential of your data by allowing you to find related information quickly and accurately.`
-            : ''}
-        </Markdown>
+              : chunksExists
+                ? `A vector index is essential for performing efficient similarity searches within your data. Without it, some chunks of data will be invisible to queries based on meaning and context. Creating a vector index unlocks the full potential of your data by allowing you to find related information quickly and accurately.`
+                : ''}
+          </Markdown>
+        </div>
       </Box>
       <div className='n-size-full n-flex n-flex-col n-items-center n-justify-center'>
         <ButtonWithToolTip

frontend/src/index.css

@@ -16,7 +16,6 @@ body {
   margin: 0;
 }
 
-
 .ndl-progress-bar-wrapper .ndl-header .ndl-heading::after{
   animation: none !important;
 }

frontend/src/services/GetFiles.ts

@@ -3,7 +3,7 @@ import api from '../API/Index';
 export const getSourceNodes = async () => {
   try {
     const formdata = new FormData();
-    const response = await api.post<SourceListServerData>(`/sources_list?`, formdata);
+    const response = await api.post<SourceListServerData>(`/sources_list`, formdata);
     return response;
   } catch (error) {
     console.log(error);

frontend/src/utils/Constants.ts

@@ -6,7 +6,7 @@ import schemaExamples from '../assets/newSchema.json';
 export const APP_SOURCES =
   process.env.VITE_REACT_APP_SOURCES !== ''
     ? (process.env.VITE_REACT_APP_SOURCES?.split(',') as string[])
-    : ['gcs', 's3', 'local', 'wiki', 'youtube', 'web'];
+    : ['s3', 'local', 'wiki', 'youtube', 'web'];
 
 export const llms =
   process.env?.VITE_LLM_MODELS?.trim() != ''