custom error in extract and url_scan API as LLMGraphBuilderException.… (#989)

* custom error in extract and url_scan API as LLMGraphBuilderException. These all errors records as status='Success/Completed'

* removing allowed nodes and rels from logs

Author: praveshkumar1988

backend/score.py

@@ -155,7 +155,14 @@ async def create_source_knowledge_graph_url(
                             'gcs_project_id':gcs_project_id, 'logging_time': formatted_time(datetime.now(timezone.utc))}
         logger.log_struct(json_obj, "INFO")
         result ={'elapsed_api_time' : f'{elapsed_time:.2f}'}
-        return create_api_response("Success",message=message,success_count=success_count,failed_count=failed_count,file_name=lst_file_name,data=result)    
+        return create_api_response("Success",message=message,success_count=success_count,failed_count=failed_count,file_name=lst_file_name,data=result)
+    except LLMGraphBuilderException as e:
+        error_message = str(e)
+        message = f" Unable to create source node for source type: {source_type} and source: {source}"
+        # Set the status "Success" becuase we are treating these error already handled by application as like custom errors.
+        json_obj = {'error_message':error_message, 'status':'Success','db_url':uri, 'userName':userName, 'database':database,'success_count':1, 'source_type': source_type, 'source_url':source_url, 'wiki_query':wiki_query, 'logging_time': formatted_time(datetime.now(timezone.utc))}
+        logger.log_struct(json_obj, "INFO")
+        return create_api_response('Failed',message=message + error_message[:80],error=error_message,file_source=source_type)
     except Exception as e:
         error_message = str(e)
         message = f" Unable to create source node for source type: {source_type} and source: {source}"
@@ -261,42 +268,32 @@ async def extract_knowledge_graph_from_file(
             result['gcs_bucket_folder'] = gcs_bucket_folder
             result['gcs_blob_filename'] = gcs_blob_filename
             result['gcs_project_id'] = gcs_project_id
-            result['allowedNodes'] = allowedNodes
-            result['allowedRelationship'] = allowedRelationship
             result['language'] = language
             result['retry_condition'] = retry_condition
         logger.log_struct(result, "INFO")
         result.update(uri_latency)
         logging.info(f"extraction completed in {extract_api_time:.2f} seconds for file name {file_name}")
         return create_api_response('Success', data=result, file_source= source_type)
-    except LLMGraphBuilderException as app_exp:
-        job_status="Completed"
-        obj_source_node = sourceNode()
-        obj_source_node.file_name = file_name
-        obj_source_node.status = job_status
-        obj_source_node.error_message = str(app_exp)
-        obj_source_node.retry_condition = retry_condition
-        graphDb_data_Access.update_source_node(obj_source_node)
-        return create_api_response("Success", data={"message": str(app_exp)}, file_name=file_name)
+    except LLMGraphBuilderException as e:
+        error_message = str(e)
+        graphDb_data_Access.update_exception_db(file_name,error_message, retry_condition)
+        failed_file_process(uri,file_name, merged_file_path, source_type)
+        node_detail = graphDb_data_Access.get_current_status_document_node(file_name)
+        # Set the status "Completed" in logging becuase we are treating these error already handled by application as like custom errors.
+        json_obj = {'api_name':'extract','message':error_message,'file_created_at':node_detail[0]['created_time'],'error_message':error_message, 'file_name': file_name,'status':'Completed',
+                    'db_url':uri, 'userName':userName, 'database':database,'success_count':1, 'source_type': source_type, 'source_url':source_url, 'wiki_query':wiki_query, 'logging_time': formatted_time(datetime.now(timezone.utc))}
+        logger.log_struct(json_obj, "INFO")
+        return create_api_response("Failed", message = error_message, error=error_message, file_name=file_name)
     except Exception as e:
         message=f"Failed To Process File:{file_name} or LLM Unable To Parse Content "
         error_message = str(e)
         graphDb_data_Access.update_exception_db(file_name,error_message, retry_condition)
+        failed_file_process(uri,file_name, merged_file_path, source_type)
         node_detail = graphDb_data_Access.get_current_status_document_node(file_name)
-        gcs_file_cache = os.environ.get('GCS_FILE_CACHE')
-        if source_type == 'local file':
-            if gcs_file_cache == 'True':
-                folder_name = create_gcs_bucket_folder_name_hashed(uri,file_name)
-                copy_failed_file(BUCKET_UPLOAD, BUCKET_FAILED_FILE, folder_name, file_name)
-                time.sleep(5)
-                delete_file_from_gcs(BUCKET_UPLOAD,folder_name,file_name)
-            else:
-                logging.info(f'Deleted File Path: {merged_file_path} and Deleted File Name : {file_name}')
-                delete_uploaded_local_file(merged_file_path,file_name)
-        json_obj = {'message':message,'file_created_at':node_detail[0]['created_time'],'error_message':error_message, 'file_name': file_name,'status':'Failed',
+        
+        json_obj = {'api_name':'extract','message':message,'file_created_at':node_detail[0]['created_time'],'error_message':error_message, 'file_name': file_name,'status':'Failed',
                     'db_url':uri, 'userName':userName, 'database':database,'failed_count':1, 'source_type': source_type, 'source_url':source_url, 'wiki_query':wiki_query, 'logging_time': formatted_time(datetime.now(timezone.utc))}
         logger.log_struct(json_obj, "ERROR")
-        logging.exception(f'File Failed in extraction: {json_obj}')
         return create_api_response('Failed', message=message + error_message[:100], error=error_message, file_name = file_name)
     finally:
         gc.collect()
@@ -514,7 +511,7 @@ async def connect(uri=Form(), userName=Form(), password=Form(), database=Form())
         gcs_file_cache = os.environ.get('GCS_FILE_CACHE')
         end = time.time()
         elapsed_time = end - start
-        json_obj = {'api_name':'connect','db_url':uri, 'userName':userName, 'database':database,'status':result, 'count':1, 'logging_time': formatted_time(datetime.now(timezone.utc)), 'elapsed_api_time':f'{elapsed_time:.2f}'}
+        json_obj = {'api_name':'connect','db_url':uri, 'userName':userName, 'database':database, 'count':1, 'logging_time': formatted_time(datetime.now(timezone.utc)), 'elapsed_api_time':f'{elapsed_time:.2f}'}
         logger.log_struct(json_obj, "INFO")
         result['elapsed_api_time'] = f'{elapsed_time:.2f}'
         result['gcs_file_cache'] = gcs_file_cache

backend/src/document_sources/gcs_bucket.py

@@ -6,6 +6,7 @@
 from langchain_core.documents import Document
 from PyPDF2 import PdfReader
 import io
+from src.shared.llm_graph_builder_exception import LLMGraphBuilderException
 from google.oauth2.credentials import Credentials
 import time
 import nltk
@@ -34,12 +35,12 @@ def get_gcs_bucket_files_info(gcs_project_id, gcs_bucket_name, gcs_bucket_folder
         file_name=''
         message=f" Bucket:{gcs_bucket_name} does not exist in Project:{gcs_project_id}. Please provide valid GCS bucket name"
         logging.info(f"Bucket : {gcs_bucket_name} does not exist in project : {gcs_project_id}")
-        raise Exception(message)
+        raise LLMGraphBuilderException(message)
     except Exception as e:
       error_message = str(e)
       logging.error(f"Unable to create source node for gcs bucket file {file_name}")
       logging.exception(f'Exception Stack trace: {error_message}')
-      raise Exception(error_message)
+      raise LLMGraphBuilderException(error_message)
 
 def load_pdf(file_path):
     return PyMuPDFLoader(file_path)
@@ -66,7 +67,7 @@ def get_documents_from_gcs(gcs_project_id, gcs_bucket_name, gcs_bucket_folder, g
         loader = GCSFileLoader(project_name=gcs_project_id, bucket=gcs_bucket_name, blob=blob_name, loader_func=load_document_content)
         pages = loader.load() 
     else :
-      raise Exception('File does not exist, Please re-upload the file and try again.')
+      raise LLMGraphBuilderException('File does not exist, Please re-upload the file and try again.')
   else:
     creds= Credentials(access_token)
     storage_client = storage.Client(project=gcs_project_id, credentials=creds)
@@ -83,7 +84,7 @@ def get_documents_from_gcs(gcs_project_id, gcs_bucket_name, gcs_bucket_folder, g
             text += page.extract_text()
       pages = [Document(page_content = text)]
     else:
-      raise Exception(f'File Not Found in GCS bucket - {gcs_bucket_name}')
+      raise LLMGraphBuilderException(f'File Not Found in GCS bucket - {gcs_bucket_name}')
   return gcs_blob_filename, pages
 
 def upload_file_to_gcs(file_chunk, chunk_number, original_file_name, bucket_name, folder_name_sha1_hashed):

backend/src/document_sources/s3_bucket.py

@@ -1,5 +1,6 @@
 from langchain_community.document_loaders import S3DirectoryLoader
 import logging
+from src.shared.llm_graph_builder_exception import LLMGraphBuilderException
 import boto3
 import os
 from urllib.parse import urlparse
@@ -74,4 +75,4 @@ def get_documents_from_s3(s3_url, aws_access_key_id, aws_secret_access_key):
     except Exception as e:
       error_message = str(e)
       logging.exception(f'Exception in reading content from S3:{error_message}')
-      raise Exception(error_message)    
+      raise LLMGraphBuilderException(error_message)    

backend/src/document_sources/web_pages.py

@@ -1,4 +1,5 @@
 from langchain_community.document_loaders import WebBaseLoader
+from src.shared.llm_graph_builder_exception import LLMGraphBuilderException
 from src.shared.common_fn import last_url_segment
 
 def get_documents_from_web_page(source_url:str):
@@ -12,4 +13,4 @@ def get_documents_from_web_page(source_url:str):
       file_name = last_url_segment(source_url)
     return file_name, pages
   except Exception as e:
-    raise Exception(str(e))
+    raise LLMGraphBuilderException(str(e))

backend/src/document_sources/wikipedia.py

@@ -1,6 +1,6 @@
 import logging
 from langchain_community.document_loaders import WikipediaLoader
-from src.api_response import create_api_response
+from src.shared.llm_graph_builder_exception import LLMGraphBuilderException
 
 def get_documents_from_Wikipedia(wiki_query:str, language:str):
   try:
@@ -9,9 +9,8 @@ def get_documents_from_Wikipedia(wiki_query:str, language:str):
     logging.info(f"Total Pages from Wikipedia = {len(pages)}") 
     return file_name, pages
   except Exception as e:
-    job_status = "Failed"
     message="Failed To Process Wikipedia Query"
     error_message = str(e)
-    logging.error(f"Failed To Process Wikipedia Query: {file_name}")
-    logging.exception(f'Exception Stack trace: {error_message}')
-    return create_api_response(job_status,message=message,error=error_message,file_name=file_name) 
+    logging.exception(f'Failed To Process Wikipedia Query: {file_name}, Exception Stack trace: {error_message}')
+    raise LLMGraphBuilderException(error_message+' '+message)
+  

backend/src/document_sources/youtube.py

@@ -1,4 +1,5 @@
 from langchain.docstore.document import Document
+from src.shared.llm_graph_builder_exception import LLMGraphBuilderException
 from youtube_transcript_api import YouTubeTranscriptApi 
 import logging
 from urllib.parse import urlparse,parse_qs
@@ -16,7 +17,7 @@ def get_youtube_transcript(youtube_id):
     return transcript_pieces
   except Exception as e:
     message = f"Youtube transcript is not available for youtube Id: {youtube_id}"
-    raise Exception(message)
+    raise LLMGraphBuilderException(message)
 
 def get_youtube_combined_transcript(youtube_id):
   try:
@@ -27,7 +28,7 @@ def get_youtube_combined_transcript(youtube_id):
     return transcript
   except Exception as e:
     message = f"Youtube transcript is not available for youtube Id: {youtube_id}"
-    raise Exception(message)
+    raise LLMGraphBuilderException(message)
 
 
 def create_youtube_url(url):
@@ -63,7 +64,7 @@ def get_documents_from_youtube(url):
     except Exception as e:
       error_message = str(e)
       logging.exception(f'Exception in reading transcript from youtube:{error_message}')
-      raise Exception(error_message)  
+      raise LLMGraphBuilderException(error_message)  
 
 def get_calculated_timestamps(chunks, youtube_id):
   logging.info('Calculating timestamps for chunks')