Merge pull request #152 from neo4j-labs/DEV

DEV- to - STAGING

Author: prakriti-solankey

backend/score.py

@@ -57,16 +57,17 @@ async def create_source_knowledge_graph_url(
     uri=Form(None),
     userName=Form(None),
     password=Form(None),
-    source_url=Form(),
+    source_url=Form(None),
     database=Form(None),
     aws_access_key_id=Form(None),
     aws_secret_access_key=Form(None),
     max_limit=Form(5),
     query_source=Form(None),
+    wiki_query=Form(None),
     model=Form(None)
 ):
     return create_source_node_graph_url(
-        uri, userName, password, source_url, model, database, aws_access_key_id, aws_secret_access_key
+        uri, userName, password, model, source_url, database, wiki_query, aws_access_key_id, aws_secret_access_key
     )
 
 
@@ -126,6 +127,17 @@ async def extract_knowledge_graph_from_file(
             wiki_query=wiki_query,
             max_sources=max_sources,
         )
+    elif wiki_query:
+         return await asyncio.to_thread(
+            extract_graph_from_file,
+            uri,
+            userName,
+            password,
+            model,
+            database,
+            wiki_query=wiki_query
+        )
+            
     else:
         return {"job_status": "Failure", "error": "No file found"}
 
@@ -165,6 +177,14 @@ async def chat_bot(uri=Form(None),
     result = await asyncio.to_thread(QA_RAG,uri=uri,userName=userName,password=password,model_version=model,question=question)
     return result
 
+@app.post("/connect")
+async def connect(uri=Form(None),
+                          userName=Form(None),
+                          password=Form(None),
+                          database=Form(None)):
+    result = await asyncio.to_thread(connection_check,uri,userName,password,database)
+    return result
+
 def decode_password(pwd):
     sample_string_bytes = base64.b64decode(pwd)
     decoded_password = sample_string_bytes.decode("utf-8")

backend/src/main.py

@@ -160,7 +160,7 @@ def check_url_source(url):
       logging.error(f"Error in recognize URL: {e}")  
       raise Exception(e)
 
-def create_source_node_graph_url(uri, userName, password, source_url ,model, db_name=None,aws_access_key_id=None,aws_secret_access_key=None):
+def create_source_node_graph_url(uri, userName, password ,model, source_url=None, db_name=None,wiki_query:List[str]=None,aws_access_key_id=None,aws_secret_access_key=None):
     """
       Creates a source node in Neo4jGraph and sets properties.
       
@@ -176,60 +176,87 @@ def create_source_node_graph_url(uri, userName, password, source_url ,model, db_
         Success or Failed message of node creation
     """
     try:
-        source_type,youtube_url = check_url_source(source_url)
         graph = Neo4jGraph(url=uri, database=db_name, username=userName, password=password)
-        logging.info(f"source type URL:{source_type}")
-        if source_type == "s3 bucket":
-            lst_s3_file_name = []
-            files_info = get_s3_files_info(source_url,aws_access_key_id=aws_access_key_id,aws_secret_access_key=aws_secret_access_key)
-            if isinstance(files_info,dict):
-              return files_info
-            elif len(files_info)==0:
-              return create_api_response('Failed',success_count=0,Failed_count=0,message='No pdf files found.')  
-            logging.info(f'files info : {files_info}')
-            err_flag=0
-            success_count=0
-            Failed_count=0
-            file_type='pdf'
-            for file_info in files_info:
-                job_status = "New"
-                file_name=file_info['file_key'] 
-                file_size=file_info['file_size_bytes']
-                s3_file_path=str(source_url+file_name)
-                try:
-                  create_source_node(graph,file_name.split('/')[-1],file_size,file_type,source_type,model,s3_file_path,aws_access_key_id)
-                  success_count+=1
-                  lst_s3_file_name.append({'fileName':file_name.split('/')[-1],'fileSize':file_size,'url':s3_file_path})
-
-                except Exception as e:
-                  err_flag=1
-                  Failed_count+=1
-                  error_message = str(e)
-            if err_flag==1:
-              job_status = "Failed"
-              message="Unable to create source node for s3 bucket files"
-              return create_api_response(job_status,message=message,error=error_message,success_count=success_count,Failed_count=Failed_count,file_source='s3 bucket')  
-            return create_api_response("Success",message="Source Node created successfully",success_count=success_count,Failed_count=Failed_count,file_source='s3 bucket',file_name=lst_s3_file_name)
-        elif source_type == 'youtube':
-            source_url= youtube_url
-            match = re.search(r'(?:v=)([0-9A-Za-z_-]{11})\s*',source_url)
-            logging.info(f"match value{match}")
-            file_name = YouTube(source_url).title
-            transcript= get_youtube_transcript(match.group(1))
-            if transcript==None or len(transcript)==0:
-              file_size=''
-              job_status = "Failed"
-              message = f"Youtube transcript is not available for : {file_name}"
-              error_message = str(e)
-              logging.exception(f'Exception Stack trace:')
-              return create_api_response(job_status,message=message,error=error_message,file_source=source_type)
-            else:  
-              file_size=sys.getsizeof(transcript)
-            file_type='text'
-            aws_access_key_id=''
-            job_status = "Completed"
-            create_source_node(graph,file_name,file_size,file_type,source_type,model,source_url,aws_access_key_id)
-            return create_api_response(job_status,file_name={'fileName':file_name,'fileSize':file_size,'url':source_url})
+        if source_url:
+          source_type,youtube_url = check_url_source(source_url)
+          logging.info(f"source type URL:{source_type}")
+          if source_type == "s3 bucket":
+              lst_s3_file_name = []
+              files_info = get_s3_files_info(source_url,aws_access_key_id=aws_access_key_id,aws_secret_access_key=aws_secret_access_key)
+              if isinstance(files_info,dict):
+                return files_info
+              elif len(files_info)==0:
+                return create_api_response('Failed',success_count=0,Failed_count=0,message='No pdf files found.')  
+              logging.info(f'files info : {files_info}')
+              err_flag=0
+              success_count=0
+              Failed_count=0
+              file_type='pdf'
+              for file_info in files_info:
+                  job_status = "New"
+                  file_name=file_info['file_key'] 
+                  file_size=file_info['file_size_bytes']
+                  s3_file_path=str(source_url+file_name)
+                  try:
+                    create_source_node(graph,file_name.split('/')[-1],file_size,file_type,source_type,model,s3_file_path,aws_access_key_id)
+                    success_count+=1
+                    lst_s3_file_name.append({'fileName':file_name.split('/')[-1],'fileSize':file_size,'url':s3_file_path})
+
+                  except Exception as e:
+                    err_flag=1
+                    Failed_count+=1
+                    error_message = str(e)
+              if err_flag==1:
+                job_status = "Failed"
+                message="Unable to create source node for s3 bucket files"
+                return create_api_response(job_status,message=message,error=error_message,success_count=success_count,Failed_count=Failed_count,file_source='s3 bucket')  
+              return create_api_response("Success",message="Source Node created successfully",success_count=success_count,Failed_count=Failed_count,file_source='s3 bucket',file_name=lst_s3_file_name)
+          elif source_type == 'youtube':
+              source_url= youtube_url
+              match = re.search(r'(?:v=)([0-9A-Za-z_-]{11})\s*',source_url)
+              logging.info(f"match value{match}")
+              file_name = YouTube(source_url).title
+              transcript= get_youtube_transcript(match.group(1))
+              if transcript==None or len(transcript)==0:
+                file_size=''
+                job_status = "Failed"
+                message = f"Youtube transcript is not available for : {file_name}"
+                error_message = str(e)
+                logging.exception(f'Exception Stack trace:')
+                return create_api_response(job_status,message=message,error=error_message,file_source=source_type)
+              else:  
+                file_size=sys.getsizeof(transcript)
+              file_type='text'
+              aws_access_key_id=''
+              job_status = "Completed"
+              create_source_node(graph,file_name,file_size,file_type,source_type,model,source_url,aws_access_key_id)
+              return create_api_response(job_status,file_name={'fileName':file_name,'fileSize':file_size,'url':source_url})
+          
+        elif wiki_query:
+           success_count=0
+           Failed_count=0
+           lst_file_metadata=[]
+           queries =  wiki_query.split(',')
+           for query in queries:
+              logging.info(f"Creating source node for {query.strip()}")
+              pages = WikipediaLoader(query=query.strip(), load_max_docs=1, load_all_available_meta=True).load()
+              file_name = query.strip()
+              file_size = sys.getsizeof(pages[0].page_content)
+              file_type = 'text'
+              source_url= pages[0].metadata['source']
+              aws_access_key_id=''
+              source_type = 'Wikipedia'
+              job_status = 'Completed'
+              try:
+                create_source_node(graph,file_name,file_size,file_type,source_type,model,source_url,aws_access_key_id)
+                success_count+=1
+                lst_file_metadata.append({'fileName':file_name,'fileSize':file_size,'url':source_url})
+              except Exception as e:
+                    job_status = "Failed"
+                    Failed_count+=1
+                    error_message = str(e) 
+                    return create_api_response(job_status,message="SUnable to create source node for Wikipedia source",file_name=lst_file_metadata, success_count=success_count, Failed_count=Failed_count) 
+           return create_api_response(job_status,message="Source Node created successfully",file_name=lst_file_metadata, success_count=success_count, Failed_count=Failed_count)   
         else:
            job_status = "Failed"
            return create_api_response(job_status,message='Invalid URL')
@@ -281,25 +308,6 @@ def get_s3_pdf_content(s3_url,aws_access_key_id=None,aws_secret_access_key=None)
         logging.error(f"getting error while reading content from s3 files:{e}")
         raise Exception(e)
 
-def get_wikipedia_content(wiki_query,max_sources):
-  try:
-    searches=wiki_query.split(',')
-    if max_sources:
-      searches=searches[:int(max_sources)]
-    else:
-       searches=searches[:2] 
-    pages=[]
-    for query in searches:
-      wiki_pages = WikipediaLoader(query=query.strip(), load_max_docs=1, load_all_available_meta=False).load()
-      pages.extend(wiki_pages)
-    
-    logging.info(f"Total Pages from Wikipedia = {len(pages)}") 
-    return pages
-  except Exception as e:
-    logging.error(f"Not finding wiki content:{e}")
-    raise Exception(e)
-
-
 
 def extract_graph_from_file(uri, userName, password, model, db_name=None, file=None,source_url=None,aws_access_key_id=None,aws_secret_access_key=None,wiki_query=None,max_sources=None):
   """
@@ -329,6 +337,9 @@ def extract_graph_from_file(uri, userName, password, model, db_name=None, file=N
     if file!=None:
       file_name, file_key, pages = get_documents_from_file(file)
 
+    elif wiki_query:  
+        file_name, file_key, pages = get_documents_from_Wikipedia(wiki_query)
+      
     elif source_type =='s3 bucket':
       if(aws_access_key_id==None or aws_secret_access_key==None):
         job_status = "Failed"
@@ -339,9 +350,6 @@ def extract_graph_from_file(uri, userName, password, model, db_name=None, file=N
         logging.info(f"filename {file_name} file_key: {file_key} pages:{pages}  ")
     elif source_type =='youtube':
         file_name, file_key, pages = get_documents_from_youtube(source_url)
-        if wiki_query is not None:
-          logging.info(f"Wikipedia query source = {wiki_query}")
-          pages.extend(get_wikipedia_content(wiki_query, max_sources))
 
     else:
         job_status = "Failed"
@@ -483,6 +491,21 @@ def get_documents_from_youtube(url):
       logging.exception(f'Exception in reading transcript from youtube:{error_message}')
       raise Exception(error_message)
 
+def get_documents_from_Wikipedia(wiki_query:str):
+  try:
+    pages = WikipediaLoader(query=wiki_query.strip(), load_max_docs=1, load_all_available_meta=False).load()
+    file_name = wiki_query.strip()
+    file_key = wiki_query.strip()
+    logging.info(f"Total Pages from Wikipedia = {len(pages)}") 
+    return file_name, file_key, pages
+  except Exception as e:
+    job_status = "Failed"
+    message="Failed To Process Wikipedia Query"
+    error_message = str(e)
+    logging.error(f"Failed To Process Wikipedia Query: {file_name}")
+    logging.exception(f'Exception Stack trace: {error_message}')
+    return create_api_response(job_status,message=message,error=error_message,file_name=file_name)    
+
 def get_source_list_from_graph(uri,userName,password,db_name=None):
   """
   Args:
@@ -529,6 +552,27 @@ def update_graph(uri,userName,password,db_name):
     error_message = str(e)
     logging.exception(f'Exception in update KNN graph:{error_message}')
     raise Exception(error_message)
+  
+def connection_check(uri,userName,password,db_name):
+  """
+  Args:
+    uri: URI of the graph to extract
+    userName: Username to use for graph creation ( if None will use username from config file )
+    password: Password to use for graph creation ( if None will use password from config file )
+    db_name: db_name is database name to connect to graph db
+  Returns:
+   Returns a status of connection from NEO4j is success or failure
+ """
+  try:
+    graph = Neo4jGraph(url=uri, database=db_name, username=userName, password=password)
+    if graph:
+      return create_api_response("Success",message="Connection Successful")
+  except Exception as e:
+    job_status = "Failed"
+    message="Connection Failed"
+    error_message = str(e)
+    logging.exception(f'Exception:{error_message}')
+    return create_api_response(job_status,message=message,error=error_message)
 
 def create_api_response(status,success_count=None,Failed_count=None, data=None, error=None,message=None,file_source=None,file_name=None):
   """

frontend/src/App.css

@@ -72,7 +72,6 @@
   display: flex;
   align-items: center;
   justify-content: center;
-  height: calc(-477px + 100dvh);
   border-radius: 8px;
   outline-color: #d1d5db;
 }
@@ -122,4 +121,34 @@
   overflow: hidden;
   text-overflow: ellipsis;
   white-space: nowrap;
+}
+
+.loader {
+  width: 8px;
+  height: 8px;
+  border-radius: 50%;
+  display: block;
+  margin: 7px auto;
+  position: relative;
+  background: #FFF;
+  box-shadow: -12px 0 #FFF, 12px 0 #FFF;
+  box-sizing: border-box;
+  animation: shadowPulse 2s linear infinite;
+}
+
+@keyframes shadowPulse {
+  33% {
+    background: #FFF;
+    box-shadow: -12px 0 rgb(var(--theme-palette-primary-bg-strong)), 12px 0 #FFF;
+  }
+
+  66% {
+    background: rgb(var(--theme-palette-primary-bg-strong));
+    box-shadow: -12px 0 #FFF, 12px 0 #FFF;
+  }
+
+  100% {
+    background: #FFF;
+    box-shadow: -12px 0 #FFF, 12px 0 rgb(var(--theme-palette-primary-bg-strong));
+  }
 }

frontend/src/assets/ChatbotMessages.json

@@ -1,5 +1,5 @@
- {
-    "listMessages": [
+{
+  "listMessages": [
     {
       "id": 1,
       "message": "Hi, I need help with creating a Cypher query for Neo4j.",
@@ -8,7 +8,7 @@
     },
     {
       "id": 2,
-      "message": "Hi there! Welcome to Neo4j Chat! Your insights are just a click away from completed files",
+      "message": " Welcome to the Neo4j Knowledge Graph Chat. You can ask questions related to documents which have been completely processed.",
       "user": "chatbot",
       "datetime": "01/01/2024 00:00:00"
     },
@@ -20,8 +20,7 @@
     },
     {
       "id": 4,
-      "message":
-        "Alright, you can use the following query: `MATCH (e:Employee)-[:WORKS_IN]->(d:Department {name: 'IT'}) RETURN e.name`. This query matches nodes labeled 'Employee' related to the 'IT' department and returns their names.",
+      "message": "Alright, you can use the following query: `MATCH (e:Employee)-[:WORKS_IN]->(d:Department {name: 'IT'}) RETURN e.name`. This query matches nodes labeled 'Employee' related to the 'IT' department and returns their names.",
       "user": "chatbot",
       "datetime": "01/01/2024 00:00:00"
     },
@@ -33,10 +32,9 @@
     },
     {
       "id": 6,
-      "message":
-        "To get the count, use: `MATCH (e:Employee)-[:WORKS_IN]->(d:Department {name: 'IT'}) RETURN count(e)`. This counts all the distinct 'Employee' nodes related to the 'IT' department.",
+      "message": "To get the count, use: `MATCH (e:Employee)-[:WORKS_IN]->(d:Department {name: 'IT'}) RETURN count(e)`. This counts all the distinct 'Employee' nodes related to the 'IT' department.",
       "user": "chatbot",
       "datetime": "01/01/2024 00:00:00"
     }
-    ]
- }
+  ]
+}