diff --git a/README.md b/README.md index e965ed4ae..6eb02aaea 100644 --- a/README.md +++ b/README.md @@ -31,32 +31,14 @@ If you are using Neo4j Desktop, you will not be able to use the docker-compose b ### Local deployment #### Running through docker-compose By default only OpenAI and Diffbot are enabled since Gemini requires extra GCP configurations. -Accoroding to enviornment we are configuring the models which is indicated by VITE_LLM_MODELS_PROD variable we can configure model based on our need. +According to enviornment we are configuring the models which is indicated by VITE_LLM_MODELS_PROD variable we can configure model based on our need. EX: ```env VITE_LLM_MODELS_PROD="openai_gpt_4o,openai_gpt_4o_mini,diffbot,gemini_1.5_flash" ``` - -if you only want OpenAI: -```env -VITE_LLM_MODELS_PROD="diffbot,openai-gpt-3.5,openai-gpt-4o" -``` - -Backend ENV -```env OPENAI_API_KEY="your-openai-key" ``` -if you only want Diffbot: -```env -VITE_LLM_MODELS_PROD="diffbot" -``` - -Backend ENV -```env -DIFFBOT_API_KEY="your-diffbot-key" -``` - You can then run Docker Compose to build and start all components: ```bash docker-compose up --build @@ -89,6 +71,7 @@ VITE_CHAT_MODES="" If however you want to specify the only vector mode or only graph mode you can do that by specifying the mode in the env: ```env VITE_CHAT_MODES="vector,graph" +VITE_CHAT_MODES="vector,graph" ``` #### Running Backend and Frontend separately (dev environment) diff --git a/backend/example.env b/backend/example.env index 85b8d1054..6bef36f78 100644 --- a/backend/example.env +++ b/backend/example.env @@ -25,8 +25,9 @@ NEO4J_USER_AGENT="" ENABLE_USER_AGENT = "" LLM_MODEL_CONFIG_model_version="" ENTITY_EMBEDDING="" True or False -DUPLICATE_SCORE_VALUE = "" -DUPLICATE_TEXT_DISTANCE = "" +DUPLICATE_SCORE_VALUE =0.97 +DUPLICATE_TEXT_DISTANCE =3 +DEFAULT_DIFFBOT_CHAT_MODEL="openai_gpt_4o" #whichever model specified here , need to add config for that model in below format) #examples LLM_MODEL_CONFIG_openai_gpt_3.5="gpt-3.5-turbo-0125,openai_api_key" LLM_MODEL_CONFIG_openai_gpt_4o_mini="gpt-4o-mini-2024-07-18,openai_api_key" diff --git a/backend/score.py b/backend/score.py index 737a82afe..a5da52aab 100644 --- a/backend/score.py +++ b/backend/score.py @@ -18,24 +18,19 @@ from src.communities import create_communities from src.neighbours import get_neighbour_nodes import json -from typing import List, Mapping, Union +from typing import List from starlette.middleware.sessions import SessionMiddleware -import google_auth_oauthlib.flow from google.oauth2.credentials import Credentials import os from src.logger import CustomLogger from datetime import datetime, timezone import time import gc -from Secweb import SecWeb -from Secweb.StrictTransportSecurity import HSTS -from Secweb.ContentSecurityPolicy import ContentSecurityPolicy from Secweb.XContentTypeOptions import XContentTypeOptions from Secweb.XFrameOptions import XFrame from fastapi.middleware.gzip import GZipMiddleware from src.ragas_eval import * from starlette.types import ASGIApp, Message, Receive, Scope, Send -import gzip from langchain_neo4j import Neo4jGraph logger = CustomLogger() @@ -493,11 +488,13 @@ async def connect(uri=Form(), userName=Form(), password=Form(), database=Form()) start = time.time() graph = create_graph_database_connection(uri, userName, password, database) result = await asyncio.to_thread(connection_check_and_get_vector_dimensions, graph, database) + gcs_file_cache = os.environ.get('GCS_FILE_CACHE') end = time.time() elapsed_time = end - start json_obj = {'api_name':'connect','db_url':uri, 'userName':userName, 'database':database,'status':result, 'count':1, 'logging_time': formatted_time(datetime.now(timezone.utc)), 'elapsed_api_time':f'{elapsed_time:.2f}'} logger.log_struct(json_obj, "INFO") result['elapsed_api_time'] = f'{elapsed_time:.2f}' + result['gcs_file_cache'] = gcs_file_cache return create_api_response('Success',data=result) except Exception as e: job_status = "Failed" @@ -571,6 +568,8 @@ async def generate(): uri = url if " " in url: uri= url.replace(" ","+") + graph = create_graph_database_connection(uri, userName, decoded_password, database) + graphDb_data_Access = graphDBdataAccess(graph) while True: try: if await request.is_disconnected(): @@ -579,8 +578,6 @@ async def generate(): # get the current status of document node else: - graph = create_graph_database_connection(uri, userName, decoded_password, database) - graphDb_data_Access = graphDBdataAccess(graph) result = graphDb_data_Access.get_current_status_document_node(file_name) if len(result) > 0: status = json.dumps({'fileName':file_name, @@ -968,22 +965,30 @@ async def fetch_chunktext( gc.collect() -@app.post("/backend_connection_configuation") -async def backend_connection_configuation(): +@app.post("/backend_connection_configuration") +async def backend_connection_configuration(): try: - graph = Neo4jGraph() - logging.info(f'login connection status of object: {graph}') - if graph is not None: - graph_connection = True - isURI = os.getenv('NEO4J_URI') - isUsername= os.getenv('NEO4J_USERNAME') - isDatabase= os.getenv('NEO4J_DATABASE') - isPassword= os.getenv('NEO4J_PASSWORD') - encoded_password = encode_password(isPassword) - graphDb_data_Access = graphDBdataAccess(graph) - gds_status = graphDb_data_Access.check_gds_version() - write_access = graphDb_data_Access.check_account_access(database=isDatabase) - return create_api_response('Success',message=f"Backend connection successful",data={'graph_connection':graph_connection,'uri':isURI,'user_name':isUsername,'database':isDatabase,'password':encoded_password,'gds_status':gds_status,'write_access':write_access}) + uri = os.getenv('NEO4J_URI') + username= os.getenv('NEO4J_USERNAME') + database= os.getenv('NEO4J_DATABASE') + password= os.getenv('NEO4J_PASSWORD') + gcs_file_cache = os.environ.get('GCS_FILE_CACHE') + if all([uri, username, database, password]): + print(f'uri:{uri}, usrName:{username}, database :{database}, password: {password}') + graph = Neo4jGraph() + logging.info(f'login connection status of object: {graph}') + if graph is not None: + graph_connection = True + encoded_password = encode_password(password) + graphDb_data_Access = graphDBdataAccess(graph) + result = graphDb_data_Access.connection_check_and_get_vector_dimensions(database) + result["graph_connection"] = graph_connection + result["uri"] = uri + result["user_name"] = username + result["database"] = database + result["password"] = encoded_password + result['gcs_file_cache'] = gcs_file_cache + return create_api_response('Success',message=f"Backend connection successful",data=result) else: graph_connection = False return create_api_response('Success',message=f"Backend connection is not successful",data=graph_connection) diff --git a/backend/src/entities/source_node.py b/backend/src/entities/source_node.py index a162b6d04..9e52497e3 100644 --- a/backend/src/entities/source_node.py +++ b/backend/src/entities/source_node.py @@ -11,6 +11,12 @@ class sourceNode: gcsBucketFolder:str=None gcsProjectId:str=None awsAccessKeyId:str=None + chunkNodeCount:int=None + chunkRelCount:int=None + entityNodeCount:int=None + entityEntityRelCount:int=None + communityNodeCount:int=None + communityRelCount:int=None node_count:int=None relationship_count:str=None model:str=None diff --git a/backend/src/graphDB_dataAccess.py b/backend/src/graphDB_dataAccess.py index 1f387a2d2..1780b6203 100644 --- a/backend/src/graphDB_dataAccess.py +++ b/backend/src/graphDB_dataAccess.py @@ -46,14 +46,24 @@ def create_source_node(self, obj_source_node:sourceNode): d.relationshipCount = $r_count, d.model= $model, d.gcsBucket=$gcs_bucket, d.gcsBucketFolder= $gcs_bucket_folder, d.language= $language,d.gcsProjectId= $gcs_project_id, d.is_cancelled=False, d.total_chunks=0, d.processed_chunk=0, - d.access_token=$access_token""", + d.access_token=$access_token, + d.chunkNodeCount=$chunkNodeCount,d.chunkRelCount=$chunkRelCount, + d.entityNodeCount=$entityNodeCount,d.entityEntityRelCount=$entityEntityRelCount, + d.communityNodeCount=$communityNodeCount,d.communityRelCount=$communityRelCount""", {"fn":obj_source_node.file_name, "fs":obj_source_node.file_size, "ft":obj_source_node.file_type, "st":job_status, "url":obj_source_node.url, "awsacc_key_id":obj_source_node.awsAccessKeyId, "f_source":obj_source_node.file_source, "c_at":obj_source_node.created_at, "u_at":obj_source_node.created_at, "pt":0, "e_message":'', "n_count":0, "r_count":0, "model":obj_source_node.model, "gcs_bucket": obj_source_node.gcsBucket, "gcs_bucket_folder": obj_source_node.gcsBucketFolder, "language":obj_source_node.language, "gcs_project_id":obj_source_node.gcsProjectId, - "access_token":obj_source_node.access_token}) + "access_token":obj_source_node.access_token, + "chunkNodeCount":obj_source_node.chunkNodeCount, + "chunkRelCount":obj_source_node.chunkRelCount, + "entityNodeCount":obj_source_node.entityNodeCount, + "entityEntityRelCount":obj_source_node.entityEntityRelCount, + "communityNodeCount":obj_source_node.communityNodeCount, + "communityRelCount":obj_source_node.communityRelCount + }) except Exception as e: error_message = str(e) logging.info(f"error_message = {error_message}") @@ -108,7 +118,7 @@ def update_source_node(self, obj_source_node:sourceNode): self.graph.query(query,param) except Exception as e: error_message = str(e) - self.update_exception_db(self.file_name,error_message) + self.update_exception_db(self,self.file_name,error_message) raise Exception(error_message) def get_source_list(self): @@ -463,51 +473,52 @@ def update_node_relationship_count(self,document_name): param = {"document_name": document_name} result = self.execute_query(NODEREL_COUNT_QUERY_WITHOUT_COMMUNITY, param) response = {} - for record in result: - filename = record["filename"] - chunkNodeCount = record["chunkNodeCount"] - chunkRelCount = record["chunkRelCount"] - entityNodeCount = record["entityNodeCount"] - entityEntityRelCount = record["entityEntityRelCount"] - if (not document_name) and (community_flag): - communityNodeCount = record["communityNodeCount"] - communityRelCount = record["communityRelCount"] - else: - communityNodeCount = 0 - communityRelCount = 0 - nodeCount = int(chunkNodeCount) + int(entityNodeCount) + int(communityNodeCount) - relationshipCount = int(chunkRelCount) + int(entityEntityRelCount) + int(communityRelCount) - update_query = """ - MATCH (d:Document {fileName: $filename}) - SET d.chunkNodeCount = $chunkNodeCount, - d.chunkRelCount = $chunkRelCount, - d.entityNodeCount = $entityNodeCount, - d.entityEntityRelCount = $entityEntityRelCount, - d.communityNodeCount = $communityNodeCount, - d.communityRelCount = $communityRelCount, - d.nodeCount = $nodeCount, - d.relationshipCount = $relationshipCount - """ - self.execute_query(update_query,{ - "filename": filename, - "chunkNodeCount": chunkNodeCount, - "chunkRelCount": chunkRelCount, - "entityNodeCount": entityNodeCount, - "entityEntityRelCount": entityEntityRelCount, - "communityNodeCount": communityNodeCount, - "communityRelCount": communityRelCount, - "nodeCount" : nodeCount, - "relationshipCount" : relationshipCount - }) - - response[filename] = {"chunkNodeCount": chunkNodeCount, - "chunkRelCount": chunkRelCount, - "entityNodeCount": entityNodeCount, - "entityEntityRelCount": entityEntityRelCount, - "communityNodeCount": communityNodeCount, - "communityRelCount": communityRelCount, - "nodeCount" : nodeCount, - "relationshipCount" : relationshipCount - } - + if result: + for record in result: + filename = record.get("filename",None) + chunkNodeCount = int(record.get("chunkNodeCount",0)) + chunkRelCount = int(record.get("chunkRelCount",0)) + entityNodeCount = int(record.get("entityNodeCount",0)) + entityEntityRelCount = int(record.get("entityEntityRelCount",0)) + if (not document_name) and (community_flag): + communityNodeCount = int(record.get("communityNodeCount",0)) + communityRelCount = int(record.get("communityRelCount",0)) + else: + communityNodeCount = 0 + communityRelCount = 0 + nodeCount = int(chunkNodeCount) + int(entityNodeCount) + int(communityNodeCount) + relationshipCount = int(chunkRelCount) + int(entityEntityRelCount) + int(communityRelCount) + update_query = """ + MATCH (d:Document {fileName: $filename}) + SET d.chunkNodeCount = $chunkNodeCount, + d.chunkRelCount = $chunkRelCount, + d.entityNodeCount = $entityNodeCount, + d.entityEntityRelCount = $entityEntityRelCount, + d.communityNodeCount = $communityNodeCount, + d.communityRelCount = $communityRelCount, + d.nodeCount = $nodeCount, + d.relationshipCount = $relationshipCount + """ + self.execute_query(update_query,{ + "filename": filename, + "chunkNodeCount": chunkNodeCount, + "chunkRelCount": chunkRelCount, + "entityNodeCount": entityNodeCount, + "entityEntityRelCount": entityEntityRelCount, + "communityNodeCount": communityNodeCount, + "communityRelCount": communityRelCount, + "nodeCount" : nodeCount, + "relationshipCount" : relationshipCount + }) + + response[filename] = {"chunkNodeCount": chunkNodeCount, + "chunkRelCount": chunkRelCount, + "entityNodeCount": entityNodeCount, + "entityEntityRelCount": entityEntityRelCount, + "communityNodeCount": communityNodeCount, + "communityRelCount": communityRelCount, + "nodeCount" : nodeCount, + "relationshipCount" : relationshipCount + } + return response \ No newline at end of file diff --git a/backend/src/graph_query.py b/backend/src/graph_query.py index dc5a64a2c..f9af6102b 100644 --- a/backend/src/graph_query.py +++ b/backend/src/graph_query.py @@ -226,6 +226,7 @@ def get_graph_results(uri, username, password,database,document_names): def get_chunktext_results(uri, username, password, database, document_name, page_no): """Retrieves chunk text, position, and page number from graph data with pagination.""" + driver = None try: logging.info("Starting chunk text query process") offset = 10 @@ -254,4 +255,5 @@ def get_chunktext_results(uri, username, password, database, document_name, page logging.error(f"An error occurred in get_chunktext_results. Error: {str(e)}") raise Exception("An error occurred in get_chunktext_results. Please check the logs for more details.") from e finally: - driver.close() \ No newline at end of file + if driver: + driver.close() \ No newline at end of file diff --git a/backend/src/llm.py b/backend/src/llm.py index f19648ed6..381a38a68 100644 --- a/backend/src/llm.py +++ b/backend/src/llm.py @@ -16,95 +16,106 @@ def get_llm(model: str): """Retrieve the specified language model based on the model name.""" - env_key = "LLM_MODEL_CONFIG_" + model + model = model.lower().strip() + env_key = f"LLM_MODEL_CONFIG_{model}" env_value = os.environ.get(env_key) - logging.info("Model: {}".format(env_key)) + + if not env_value: + err = f"Environment variable '{env_key}' is not defined as per format or missing" + logging.error(err) + raise Exception(err) - if "gemini" in model: - model_name = env_value - credentials, project_id = google.auth.default() - llm = ChatVertexAI( - model_name=model_name, - #convert_system_message_to_human=True, - credentials=credentials, - project=project_id, - temperature=0, - safety_settings={ - HarmCategory.HARM_CATEGORY_UNSPECIFIED: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE, - }, - ) - elif "openai" in model: - model_name, api_key = env_value.split(",") - llm = ChatOpenAI( - api_key=api_key, - model=model_name, - temperature=0, - ) + logging.info("Model: {}".format(env_key)) + try: + if "gemini" in model: + model_name = env_value + credentials, project_id = google.auth.default() + llm = ChatVertexAI( + model_name=model_name, + #convert_system_message_to_human=True, + credentials=credentials, + project=project_id, + temperature=0, + safety_settings={ + HarmCategory.HARM_CATEGORY_UNSPECIFIED: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE, + }, + ) + elif "openai" in model: + model_name, api_key = env_value.split(",") + llm = ChatOpenAI( + api_key=api_key, + model=model_name, + temperature=0, + ) - elif "azure" in model: - model_name, api_endpoint, api_key, api_version = env_value.split(",") - llm = AzureChatOpenAI( - api_key=api_key, - azure_endpoint=api_endpoint, - azure_deployment=model_name, # takes precedence over model parameter - api_version=api_version, - temperature=0, - max_tokens=None, - timeout=None, - ) + elif "azure" in model: + model_name, api_endpoint, api_key, api_version = env_value.split(",") + llm = AzureChatOpenAI( + api_key=api_key, + azure_endpoint=api_endpoint, + azure_deployment=model_name, # takes precedence over model parameter + api_version=api_version, + temperature=0, + max_tokens=None, + timeout=None, + ) - elif "anthropic" in model: - model_name, api_key = env_value.split(",") - llm = ChatAnthropic( - api_key=api_key, model=model_name, temperature=0, timeout=None - ) + elif "anthropic" in model: + model_name, api_key = env_value.split(",") + llm = ChatAnthropic( + api_key=api_key, model=model_name, temperature=0, timeout=None + ) - elif "fireworks" in model: - model_name, api_key = env_value.split(",") - llm = ChatFireworks(api_key=api_key, model=model_name) - - elif "groq" in model: - model_name, base_url, api_key = env_value.split(",") - llm = ChatGroq(api_key=api_key, model_name=model_name, temperature=0) - - elif "bedrock" in model: - model_name, aws_access_key, aws_secret_key, region_name = env_value.split(",") - bedrock_client = boto3.client( - service_name="bedrock-runtime", - region_name=region_name, - aws_access_key_id=aws_access_key, - aws_secret_access_key=aws_secret_key, - ) + elif "fireworks" in model: + model_name, api_key = env_value.split(",") + llm = ChatFireworks(api_key=api_key, model=model_name) + + elif "groq" in model: + model_name, base_url, api_key = env_value.split(",") + llm = ChatGroq(api_key=api_key, model_name=model_name, temperature=0) + + elif "bedrock" in model: + model_name, aws_access_key, aws_secret_key, region_name = env_value.split(",") + bedrock_client = boto3.client( + service_name="bedrock-runtime", + region_name=region_name, + aws_access_key_id=aws_access_key, + aws_secret_access_key=aws_secret_key, + ) - llm = ChatBedrock( - client=bedrock_client, model_id=model_name, model_kwargs=dict(temperature=0) - ) + llm = ChatBedrock( + client=bedrock_client, model_id=model_name, model_kwargs=dict(temperature=0) + ) - elif "ollama" in model: - model_name, base_url = env_value.split(",") - llm = ChatOllama(base_url=base_url, model=model_name) + elif "ollama" in model: + model_name, base_url = env_value.split(",") + llm = ChatOllama(base_url=base_url, model=model_name) - elif "diffbot" in model: - #model_name = "diffbot" - model_name, api_key = env_value.split(",") - llm = DiffbotGraphTransformer( - diffbot_api_key=api_key, - extract_types=["entities", "facts"], - ) - - else: - model_name, api_endpoint, api_key = env_value.split(",") - llm = ChatOpenAI( - api_key=api_key, - base_url=api_endpoint, - model=model_name, - temperature=0, - ) - + elif "diffbot" in model: + #model_name = "diffbot" + model_name, api_key = env_value.split(",") + llm = DiffbotGraphTransformer( + diffbot_api_key=api_key, + extract_types=["entities", "facts"], + ) + + else: + model_name, api_endpoint, api_key = env_value.split(",") + llm = ChatOpenAI( + api_key=api_key, + base_url=api_endpoint, + model=model_name, + temperature=0, + ) + except Exception as e: + err = f"Error while creating LLM '{model}': {str(e)}" + logging.error(err) + raise Exception(err) + logging.info(f"Model created - Model Version: {model}") return llm, model_name @@ -179,21 +190,24 @@ async def get_graph_document_list( async def get_graph_from_llm(model, chunkId_chunkDoc_list, allowedNodes, allowedRelationship): - - llm, model_name = get_llm(model) - combined_chunk_document_list = get_combined_chunks(chunkId_chunkDoc_list) - #combined_chunk_document_list = get_chunk_id_as_doc_metadata(chunkId_chunkDoc_list) - - if allowedNodes is None or allowedNodes=="": - allowedNodes =[] - else: - allowedNodes = allowedNodes.split(',') - if allowedRelationship is None or allowedRelationship=="": - allowedRelationship=[] - else: - allowedRelationship = allowedRelationship.split(',') + try: + llm, model_name = get_llm(model) + combined_chunk_document_list = get_combined_chunks(chunkId_chunkDoc_list) - graph_document_list = await get_graph_document_list( - llm, combined_chunk_document_list, allowedNodes, allowedRelationship - ) - return graph_document_list + if allowedNodes is None or allowedNodes=="": + allowedNodes =[] + else: + allowedNodes = allowedNodes.split(',') + if allowedRelationship is None or allowedRelationship=="": + allowedRelationship=[] + else: + allowedRelationship = allowedRelationship.split(',') + + graph_document_list = await get_graph_document_list( + llm, combined_chunk_document_list, allowedNodes, allowedRelationship + ) + return graph_document_list + except Exception as e: + err = f"Error during extracting graph with llm: {e}" + logging.error(err) + raise diff --git a/backend/src/main.py b/backend/src/main.py index 5ef1e4354..d47061c46 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -56,6 +56,12 @@ def create_source_node_graph_url_s3(graph, model, source_url, aws_access_key_id, obj_source_node.url = str(source_url+file_name) obj_source_node.awsAccessKeyId = aws_access_key_id obj_source_node.created_at = datetime.now() + obj_source_node.chunkNodeCount=0 + obj_source_node.chunkRelCount=0 + obj_source_node.entityNodeCount=0 + obj_source_node.entityEntityRelCount=0 + obj_source_node.communityNodeCount=0 + obj_source_node.communityRelCount=0 try: graphDb_data_Access = graphDBdataAccess(graph) graphDb_data_Access.create_source_node(obj_source_node) @@ -88,6 +94,12 @@ def create_source_node_graph_url_gcs(graph, model, gcs_project_id, gcs_bucket_na obj_source_node.gcsProjectId = file_metadata['gcsProjectId'] obj_source_node.created_at = datetime.now() obj_source_node.access_token = credentials.token + obj_source_node.chunkNodeCount=0 + obj_source_node.chunkRelCount=0 + obj_source_node.entityNodeCount=0 + obj_source_node.entityEntityRelCount=0 + obj_source_node.communityNodeCount=0 + obj_source_node.communityRelCount=0 try: graphDb_data_Access = graphDBdataAccess(graph) @@ -119,7 +131,12 @@ def create_source_node_graph_web_url(graph, model, source_url, source_type): obj_source_node.file_name = pages[0].metadata['title'] obj_source_node.language = pages[0].metadata['language'] obj_source_node.file_size = sys.getsizeof(pages[0].page_content) - + obj_source_node.chunkNodeCount=0 + obj_source_node.chunkRelCount=0 + obj_source_node.entityNodeCount=0 + obj_source_node.entityEntityRelCount=0 + obj_source_node.communityNodeCount=0 + obj_source_node.communityRelCount=0 graphDb_data_Access = graphDBdataAccess(graph) graphDb_data_Access.create_source_node(obj_source_node) lst_file_name.append({'fileName':obj_source_node.file_name,'fileSize':obj_source_node.file_size,'url':obj_source_node.url,'status':'Success'}) @@ -138,6 +155,12 @@ def create_source_node_graph_url_youtube(graph, model, source_url, source_type): obj_source_node.model = model obj_source_node.url = youtube_url obj_source_node.created_at = datetime.now() + obj_source_node.chunkNodeCount=0 + obj_source_node.chunkRelCount=0 + obj_source_node.entityNodeCount=0 + obj_source_node.entityEntityRelCount=0 + obj_source_node.communityNodeCount=0 + obj_source_node.communityRelCount=0 match = re.search(r'(?:v=)([0-9A-Za-z_-]{11})\s*',obj_source_node.url) logging.info(f"match value: {match}") video_id = parse_qs(urlparse(youtube_url).query).get('v') @@ -180,6 +203,12 @@ def create_source_node_graph_url_wikipedia(graph, model, wiki_query, source_type obj_source_node.url = urllib.parse.unquote(pages[0].metadata['source']) obj_source_node.created_at = datetime.now() obj_source_node.language = language + obj_source_node.chunkNodeCount=0 + obj_source_node.chunkRelCount=0 + obj_source_node.entityNodeCount=0 + obj_source_node.entityEntityRelCount=0 + obj_source_node.communityNodeCount=0 + obj_source_node.communityRelCount=0 graphDb_data_Access = graphDBdataAccess(graph) graphDb_data_Access.create_source_node(obj_source_node) success_count+=1 @@ -279,6 +308,7 @@ async def processing_source(uri, userName, password, database, model, file_name, logging.info(f'Time taken database connection: {elapsed_create_connection:.2f} seconds') uri_latency["create_connection"] = f'{elapsed_create_connection:.2f}' graphDb_data_Access = graphDBdataAccess(graph) + create_chunk_vector_index(graph) start_get_chunkId_chunkDoc_list = time.time() total_chunks, chunkId_chunkDoc_list = get_chunkId_chunkDoc_list(graph, file_name, pages, retry_condition) end_get_chunkId_chunkDoc_list = time.time() @@ -428,7 +458,7 @@ async def processing_chunks(chunkId_chunkDoc_list,graph,uri, userName, password, graph = create_graph_database_connection(uri, userName, password, database) start_update_embedding = time.time() - update_embedding_create_vector_index( graph, chunkId_chunkDoc_list, file_name) + create_chunk_embeddings( graph, chunkId_chunkDoc_list, file_name) end_update_embedding = time.time() elapsed_update_embedding = end_update_embedding - start_update_embedding logging.info(f'Time taken to update embedding in chunk node: {elapsed_update_embedding:.2f} seconds') @@ -618,6 +648,12 @@ def upload_file(graph, model, chunk, chunk_number:int, total_chunks:int, origina obj_source_node.file_source = 'local file' obj_source_node.model = model obj_source_node.created_at = datetime.now() + obj_source_node.chunkNodeCount=0 + obj_source_node.chunkRelCount=0 + obj_source_node.entityNodeCount=0 + obj_source_node.entityEntityRelCount=0 + obj_source_node.communityNodeCount=0 + obj_source_node.communityRelCount=0 graphDb_data_Access = graphDBdataAccess(graph) graphDb_data_Access.create_source_node(obj_source_node) diff --git a/backend/src/make_relationships.py b/backend/src/make_relationships.py index 20e95913f..410a383fd 100644 --- a/backend/src/make_relationships.py +++ b/backend/src/make_relationships.py @@ -6,6 +6,7 @@ import os import hashlib import time +from langchain_neo4j import Neo4jVector logging.basicConfig(format='%(asctime)s - %(message)s',level='INFO') @@ -39,7 +40,7 @@ def merge_relationship_between_chunk_and_entites(graph: Neo4jGraph, graph_docume graph.query(unwind_query, params={"batch_data": batch_data}) -def update_embedding_create_vector_index(graph, chunkId_chunkDoc_list, file_name): +def create_chunk_embeddings(graph, chunkId_chunkDoc_list, file_name): #create embedding isEmbedding = os.getenv('IS_EMBEDDING') # embedding_model = os.getenv('EMBEDDING_MODEL') @@ -58,35 +59,6 @@ def update_embedding_create_vector_index(graph, chunkId_chunkDoc_list, file_name "chunkId": row['chunk_id'], "embeddings": embeddings_arr }) - # graph.query("""MATCH (d:Document {fileName : $fileName}) - # MERGE (c:Chunk {id:$chunkId}) SET c.embedding = $embeddings - # MERGE (c)-[:PART_OF]->(d) - # """, - # { - # "fileName" : file_name, - # "chunkId": row['chunk_id'], - # "embeddings" : embeddings_arr - # } - # ) - logging.info('create vector index on chunk embedding') - # result = graph.query("SHOW INDEXES YIELD * WHERE labelsOrTypes = ['Chunk'] and name = 'vector'") - vector_index = graph.query("SHOW INDEXES YIELD * WHERE labelsOrTypes = ['Chunk'] and type = 'VECTOR' AND name = 'vector' return options") - # if result: - # logging.info(f"vector index dropped for 'Chunk'") - # graph.query("DROP INDEX vector IF EXISTS;") - - if len(vector_index) == 0: - logging.info(f'vector index is not exist, will create in next query') - graph.query("""CREATE VECTOR INDEX `vector` if not exists for (c:Chunk) on (c.embedding) - OPTIONS {indexConfig: { - `vector.dimensions`: $dimensions, - `vector.similarity_function`: 'cosine' - }} - """, - { - "dimensions" : dimension - } - ) query_to_create_embedding = """ UNWIND $data AS row @@ -185,4 +157,27 @@ def create_relation_between_chunks(graph, file_name, chunks: List[Document])->li """ graph.query(query_to_create_NEXT_CHUNK_relation, params={"relationships": relationships}) - return lst_chunks_including_hash \ No newline at end of file + return lst_chunks_including_hash + + +def create_chunk_vector_index(graph): + start_time = time.time() + try: + vector_index = graph.query("SHOW INDEXES YIELD * WHERE labelsOrTypes = ['Chunk'] and type = 'VECTOR' AND name = 'vector' return options") + + if not vector_index: + vector_store = Neo4jVector(embedding=EMBEDDING_FUNCTION, + graph=graph, + node_label="Chunk", + embedding_node_property="embedding", + index_name="vector" + ) + vector_store.create_new_index() + logging.info(f"Index created successfully. Time taken: {time.time() - start_time:.2f} seconds") + else: + logging.info(f"Index already exist,Skipping creation. Time taken: {time.time() - start_time:.2f} seconds") + except Exception as e: + if "EquivalentSchemaRuleAlreadyExists" in str(e): + logging.info("Vector index already exists, skipping creation.") + else: + raise \ No newline at end of file diff --git a/backend/src/ragas_eval.py b/backend/src/ragas_eval.py index 29f7e026c..251ab71c0 100644 --- a/backend/src/ragas_eval.py +++ b/backend/src/ragas_eval.py @@ -17,6 +17,7 @@ load_dotenv() EMBEDDING_MODEL = os.getenv("RAGAS_EMBEDDING_MODEL") +logging.info(f"Loading embedding model '{EMBEDDING_MODEL}' for ragas evaluation") EMBEDDING_FUNCTION, _ = load_embedding_model(EMBEDDING_MODEL) def get_ragas_metrics(question: str, context: list, answer: list, model: str): diff --git a/backend/test_integrationqa.py b/backend/test_integrationqa.py index ede8077f7..03d0d470b 100644 --- a/backend/test_integrationqa.py +++ b/backend/test_integrationqa.py @@ -13,7 +13,8 @@ from src.ragas_eval import get_ragas_metrics from datasets import Dataset from ragas import evaluate -from ragas.metrics import answer_relevancy, context_utilization, faithfulness +# from ragas.metrics import answer_relevancy, context_utilization, faithfulness +# from ragas.dataset_schema import SingleTurnSample # Load environment variables if needed load_dotenv() import os @@ -53,19 +54,18 @@ def delete_extracted_files(file_path): logging.error(f"Failed to delete file {file_path}. Error: {e}") def test_graph_from_file_local(model_name): - """Test graph creation from a local file.""" - file_name = 'About Amazon.pdf' - shutil.copyfile('/workspaces/llm-graph-builder/backend/files/About Amazon.pdf', - os.path.join(MERGED_DIR, file_name)) - - create_source_node_local(graph, model_name, file_name) - merged_file_path = os.path.join(MERGED_DIR, file_name) - - local_file_result = asyncio.run(extract_graph_from_file_local_file( - URI, USERNAME, PASSWORD, DATABASE, model_name, merged_file_path, file_name, '', '',None)) - logging.info("Local file processing complete") - print(local_file_result) - return local_file_result + """Test graph creation from a local file.""" + try: + file_name = 'About Amazon.pdf' + shutil.copyfile('/workspaces/llm-graph-builder/backend/files/About Amazon.pdf',os.path.join(MERGED_DIR, file_name)) + create_source_node_local(graph, model_name, file_name) + merged_file_path = os.path.join(MERGED_DIR, file_name) + local_file_result = asyncio.run(extract_graph_from_file_local_file(URI, USERNAME, PASSWORD, DATABASE, model_name, merged_file_path, file_name, '', '',None)) + logging.info("Local file processing complete") + print(local_file_result) + return local_file_result + except Exception as e: + logging.error(f"Failed to delete file. Error: {e}") # try: # assert local_file_result['status'] == 'Completed' @@ -81,88 +81,68 @@ def test_graph_from_file_local(model_name): #return local_file_result def test_graph_from_wikipedia(model_name): - # try: + try: """Test graph creation from a Wikipedia page.""" - wiki_query = 'https://en.wikipedia.org/wiki/Berkshire_Hathaway' + wiki_query = 'https://en.wikipedia.org/wiki/Apollo_program' source_type = 'Wikipedia' - file_name = "Berkshire_Hathaway" + file_name = "Apollo_program" create_source_node_graph_url_wikipedia(graph, model_name, wiki_query, source_type) - wiki_result = asyncio.run(extract_graph_from_file_Wikipedia(URI, USERNAME, PASSWORD, DATABASE, model_name, file_name, 'en',file_name, '', '',None)) + wiki_result = asyncio.run(extract_graph_from_file_Wikipedia(URI, USERNAME, PASSWORD, DATABASE, model_name, wiki_query, 'en',file_name, '', '',None)) logging.info("Wikipedia test done") print(wiki_result) - try: - assert wiki_result['status'] == 'Completed' - assert wiki_result['nodeCount'] > 0 - assert wiki_result['relationshipCount'] > 0 - print("Success") - except AssertionError as e: - print("Fail: ", e) + # try: + # assert wiki_result['status'] == 'Completed' + # assert wiki_result['nodeCount'] > 0 + # assert wiki_result['relationshipCount'] > 0 + # print("Success") + # except AssertionError as e: + # print("Fail: ", e) return wiki_result - # except Exception as ex: - # print(ex) - -def test_graph_website(model_name): - """Test graph creation from a Website page.""" - #graph, model, source_url, source_type - source_url = 'https://www.amazon.com/' - source_type = 'web-url' - file_name = [] - create_source_node_graph_web_url(graph, model_name, source_url, source_type) - - weburl_result = asyncio.run(extract_graph_from_web_page(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url,file_name, '', '',None)) - logging.info("WebUrl test done") - print(weburl_result) - - try: - assert weburl_result['status'] == 'Completed' - assert weburl_result['nodeCount'] > 0 - assert weburl_result['relationshipCount'] > 0 - print("Success") - except AssertionError as e: - print("Fail: ", e) - return weburl_result + except Exception as ex: + print('Hello error herte') + print(ex) def test_graph_website(model_name): """Test graph creation from a Website page.""" #graph, model, source_url, source_type - source_url = 'https://www.amazon.com/' + source_url = 'https://www.cloudskillsboost.google/' source_type = 'web-url' + file_name = 'Google Cloud Skills Boost' + # file_name = [] create_source_node_graph_web_url(graph, model_name, source_url, source_type) - weburl_result = extract_graph_from_web_page(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, '', '') + weburl_result = asyncio.run(extract_graph_from_web_page(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url,file_name, '', '',None)) logging.info("WebUrl test done") print(weburl_result) - try: - assert weburl_result['status'] == 'Completed' - assert weburl_result['nodeCount'] > 0 - assert weburl_result['relationshipCount'] > 0 - print("Success") - except AssertionError as e: - print("Fail: ", e) + # try: + # assert weburl_result['status'] == 'Completed' + # assert weburl_result['nodeCount'] > 0 + # assert weburl_result['relationshipCount'] > 0 + # print("Success") + # except AssertionError as e: + # print("Fail: ", e) return weburl_result - def test_graph_from_youtube_video(model_name): """Test graph creation from a YouTube video.""" source_url = 'https://www.youtube.com/watch?v=T-qy-zPWgqA' + file_name = 'NKc8Tr5_L3w' source_type = 'youtube' create_source_node_graph_url_youtube(graph, model_name, source_url, source_type) - youtube_result = extract_graph_from_file_youtube( - URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, '', '' - ) + youtube_result = asyncio.run(extract_graph_from_file_youtube(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url,file_name,'','',None)) logging.info("YouTube Video test done") print(youtube_result) - try: - assert youtube_result['status'] == 'Completed' - assert youtube_result['nodeCount'] > 1 - assert youtube_result['relationshipCount'] > 1 - print("Success") - except AssertionError as e: - print("Failed: ", e) +# try: +# assert youtube_result['status'] == 'Completed' +# assert youtube_result['nodeCount'] > 1 +# assert youtube_result['relationshipCount'] > 1 +# print("Success") +# except AssertionError as e: +# print("Failed: ", e) return youtube_result @@ -218,7 +198,7 @@ def get_duplicate_nodes(): #Test populate_graph_schema def test_populate_graph_schema_from_text(model_name): - schema_text =('Amazon was founded on July 5, 1994, by Jeff Bezos in Bellevue, Washington.The company originally started as an online marketplace for books but gradually expanded its offerings to include a wide range of product categories. This diversification led to it being referred.') + schema_text =('Amazon was founded on July 5, 1994, by Jeff Bezos in Bellevue, Washington.The company originally started as an online marketplace for books but gradually expanded its offerings to include a wide range of product categories. This diversification led to it being referred.') #result_schema='' try: result_schema = populate_graph_schema_from_text(schema_text, model_name, True) @@ -232,7 +212,7 @@ def run_tests(): final_list = [] error_list = [] - models = ['openai_gpt_3_5','openai_gpt_4o','openai_gpt_4o_mini','azure-ai-gpt-35','azure-ai-gpt-4o','gemini_1_5_pro','gemini_1_5_flash','anthropic-claude-3-5-sonnet','bedrock-claude-3-5-sonnet','groq-llama3-70b','fireworks-llama-v3-70b'] + models = ['openai_gpt_4','openai_gpt_4o','openai_gpt_4o_mini','gemini_1.5_pro','gemini_1.5_flash'] for model_name in models: try: @@ -256,7 +236,7 @@ def run_tests(): dis_elementid, dis_status = disconected_nodes() lst_element_id = [dis_elementid] delt = delete_disconected_nodes(lst_element_id) -# dup = get_duplicate_nodes() + dup = get_duplicate_nodes() print(final_list) schma = test_populate_graph_schema_from_text(model_name) # Save final results to CSV @@ -265,7 +245,7 @@ def run_tests(): df['execution_date'] = dt.today().strftime('%Y-%m-%d') #diconnected nodes df['disconnected_nodes']=dis_status -# df['get_duplicate_nodes']=dup + df['get_duplicate_nodes']=dup df['delete_disconected_nodes']=delt df['test_populate_graph_schema_from_text'] = schma diff --git a/frontend/src/App.css b/frontend/src/App.css index a20f7d351..bbc0fc44f 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -1,84 +1,7 @@ .filetable { - /* width: calc(-360px + 100dvw); */ - height: calc(-350px + 100dvh); border: 1px solid #d1d5db; + width: 95% !important; } - -.fileTableWithExpansion { - width: calc(-640px + 100dvw) !important; - height: calc(-350px + 100dvh); - border: 1px solid #d1d5db; -} - -.fileTableWithBothDrawers { - width: calc(-650px + 100dvw) !important; - height: calc(-350px + 100dvh); - border: 1px solid #d1d5db; -} - -.fileTableWithExpansion .ndl-div-table { - width: max-content !important; -} - -.filetable .ndl-div-table { - width: max-content !important; -} - -.contentWithExpansion { - width: calc(-807px + 100dvw); - height: calc(100dvh - 58px); - padding: 3px; - display: flex; - flex-direction: column; - align-items: center; - gap: 5px; - position: relative; -} - -.contentWithOneExpansion { - width: calc(-550px + 100dvw); - height: calc(100dvh - 58px); - padding: 3px; - display: flex; - flex-direction: column; - align-items: center; - gap: 5px; - position: relative; -} - -.contentWithBothDrawers { - width: calc(-786px + 100dvw); - height: calc(100dvh - 58px); - padding: 3px; - display: flex; - flex-direction: column; - align-items: center; - gap: 5px; - position: relative; -} - -.contentWithChatBot { - width: calc(-512px + 100dvw); - height: calc(100dvh - 58px); - padding: 3px; - display: flex; - flex-direction: column; - align-items: center; - gap: 5px; - position: relative; -} - -.contentWithDropzoneExpansion { - width: calc(100% - 422px); - height: calc(100dvh - 58px); - padding: 3px; - display: flex; - flex-direction: column; - align-items: center; - gap: 5px; - position: relative; -} - .s3Container { display: flex; align-items: center; @@ -409,3 +332,27 @@ opacity: 0.8; background-color: rgb(201 201 201 / 40%); } +.layout-wrapper{ + display: grid; + grid-template-rows: auto; + grid-template-columns: 64px 1fr minmax(min-content,4fr) 1.4fr 64px; + max-height: calc(100vh - 58px); + max-width: 100%; +} +.layout-wrapper.drawerdropzoneclosed{ + grid-template-columns: 64px 4fr 1.1fr 64px; + +} +.layout-wrapper.drawerchatbotclosed{ + grid-template-columns: 64px 0.6fr minmax(min-content,4fr) 64px; +} +.layout-wrapper.drawerclosed{ + grid-template-columns: 64px minmax(min-content,4fr) 64px; +} +.main-content-wrapper{ + display: flex; + flex-direction: column; + max-width: 100%; + position: relative; + height: 100%; +} diff --git a/frontend/src/components/BreakDownPopOver.tsx b/frontend/src/components/BreakDownPopOver.tsx index f6798bf97..46654a913 100644 --- a/frontend/src/components/BreakDownPopOver.tsx +++ b/frontend/src/components/BreakDownPopOver.tsx @@ -1,5 +1,4 @@ import CustomPopOver from './UI/CustomPopOver'; -import { IconButton } from '@neo4j-ndl/react'; import { InformationCircleIconOutline } from '@neo4j-ndl/react/icons'; import { CustomFileBase } from '../types'; import { useCredentials } from '../context/UserCredentials'; @@ -8,13 +7,7 @@ export default function BreakDownPopOver({ file, isNodeCount = true }: { file: C const { isGdsActive } = useCredentials(); return ( - - - - } - > + }> {isNodeCount ? (
  • Chunk Nodes: {file.chunkNodeCount}
    • diff --git a/frontend/src/components/ChatBot/ChatInfoModal.tsx b/frontend/src/components/ChatBot/ChatInfoModal.tsx index ab66a52f1..97a244b32 100644 --- a/frontend/src/components/ChatBot/ChatInfoModal.tsx +++ b/frontend/src/components/ChatBot/ChatInfoModal.tsx @@ -393,7 +393,7 @@ const ChatInfoModal: React.FC = ({ Currently ragas evaluation works on{' '} {supportedLLmsForRagas.map((s, idx) => ( - + {capitalizeWithUnderscore(s) + (idx != supportedLLmsForRagas.length - 1 ? ',' : '')} ))} diff --git a/frontend/src/components/ChatBot/EntitiesInfo.tsx b/frontend/src/components/ChatBot/EntitiesInfo.tsx index ed6029d68..3c1626cef 100644 --- a/frontend/src/components/ChatBot/EntitiesInfo.tsx +++ b/frontend/src/components/ChatBot/EntitiesInfo.tsx @@ -93,11 +93,11 @@ const EntitiesInfo: FC = ({ loading, mode, graphonly_entities, in
)) - : sortedLabels.map((label, index) => { + : sortedLabels.map((label) => { const entity = groupedEntities[label == 'undefined' ? 'Entity' : label]; return (
  • diff --git a/frontend/src/components/Content.tsx b/frontend/src/components/Content.tsx index 7fcd12b03..cbd7875f7 100644 --- a/frontend/src/components/Content.tsx +++ b/frontend/src/components/Content.tsx @@ -4,7 +4,15 @@ import { Button, Typography, Flex, StatusIndicator, useMediaQuery } from '@neo4j import { useCredentials } from '../context/UserCredentials'; import { useFileContext } from '../context/UsersFiles'; import { extractAPI } from '../utils/FileAPI'; -import { BannerAlertProps, ChildRef, ContentProps, CustomFile, OptionType, UserCredentials, chunkdata } from '../types'; +import { + BannerAlertProps, + ContentProps, + CustomFile, + OptionType, + UserCredentials, + chunkdata, + FileTableHandle, +} from '../types'; import deleteAPI from '../services/DeleteFiles'; import { postProcessing } from '../services/PostProcessing'; import { triggerStatusUpdateAPI } from '../services/ServerSideStatusUpdateAPI'; @@ -35,14 +43,13 @@ import { useMessageContext } from '../context/UserMessages'; import PostProcessingToast from './Popups/GraphEnhancementDialog/PostProcessingCheckList/PostProcessingToast'; import { getChunkText } from '../services/getChunkText'; import ChunkPopUp from './Popups/ChunkPopUp'; +import { isExpired, isFileReadyToProcess } from '../utils/Utils'; const ConfirmationDialog = lazy(() => import('./Popups/LargeFilePopUp/ConfirmationDialog')); let afterFirstRender = false; const Content: React.FC = ({ - isLeftExpanded, - isRightExpanded, isSchema, setIsSchema, showEnhancementDialog, @@ -57,8 +64,10 @@ const Content: React.FC = ({ const [openGraphView, setOpenGraphView] = useState(false); const [inspectedName, setInspectedName] = useState(''); const [documentName, setDocumentName] = useState(''); - const { setUserCredentials, userCredentials, setConnectionStatus, isGdsActive, isReadOnlyUser } = useCredentials(); + const { setUserCredentials, userCredentials, setConnectionStatus, isGdsActive, isReadOnlyUser, isGCSActive } = + useCredentials(); const [showConfirmationModal, setshowConfirmationModal] = useState(false); + const [showExpirationModal, setshowExpirationModal] = useState(false); const [extractLoading, setextractLoading] = useState(false); const [retryFile, setRetryFile] = useState(''); const [retryLoading, setRetryLoading] = useState(false); @@ -106,7 +115,7 @@ const Content: React.FC = ({ showErrorToast(`${fileName} Failed to process`); } ); - const childRef = useRef(null); + const childRef = useRef(null); const incrementPage = async () => { setCurrentPage((prev) => prev + 1); @@ -267,7 +276,7 @@ const Content: React.FC = ({ fileItem.retryOption ?? '', fileItem.sourceUrl, localStorage.getItem('accesskey'), - localStorage.getItem('secretkey'), + atob(localStorage.getItem('secretkey') ?? ''), fileItem.name ?? '', fileItem.gcsBucket ?? '', fileItem.gcsBucketFolder ?? '', @@ -520,15 +529,6 @@ const Content: React.FC = ({ window.open(replacedUrl, '_blank'); }; - const classNameCheck = - isLeftExpanded && isRightExpanded - ? 'contentWithExpansion' - : isRightExpanded - ? 'contentWithChatBot' - : !isLeftExpanded && !isRightExpanded - ? 'w-[calc(100%-128px)]' - : 'contentWithDropzoneExpansion'; - const handleGraphView = () => { setOpenGraphView(true); setViewPoint('showGraphView'); @@ -671,35 +671,25 @@ const Content: React.FC = ({ const onClickHandler = () => { const selectedRows = childRef.current?.getSelectedRows(); if (selectedRows?.length) { - let selectedLargeFiles: CustomFile[] = []; - for (let index = 0; index < selectedRows.length; index++) { - const parsedData: CustomFile = selectedRows[index]; - if ( - parsedData.fileSource === 'local file' && - typeof parsedData.size === 'number' && - (parsedData.status === 'New' || parsedData.status == 'Ready to Reprocess') && - parsedData.size > largeFileSize - ) { - selectedLargeFiles.push(parsedData); - } - } - if (selectedLargeFiles.length) { + const expiredFilesExists = selectedRows.some( + (c) => isFileReadyToProcess(c, true) && isExpired(c?.createdAt as Date) + ); + const largeFileExists = selectedRows.some( + (c) => isFileReadyToProcess(c, true) && typeof c.size === 'number' && c.size > largeFileSize + ); + if (expiredFilesExists) { setshowConfirmationModal(true); + } else if (largeFileExists && isGCSActive) { + setshowExpirationModal(true); } else { - handleGenerateGraph(selectedRows.filter((f) => f.status === 'New' || f.status === 'Ready to Reprocess')); + handleGenerateGraph(selectedRows.filter((f) => isFileReadyToProcess(f, false))); } } else if (filesData.length) { - const largefiles = filesData.filter((f) => { - if ( - typeof f.size === 'number' && - (f.status === 'New' || f.status == 'Ready to Reprocess') && - f.size > largeFileSize - ) { - return true; - } - return false; - }); - const selectAllNewFiles = filesData.filter((f) => f.status === 'New' || f.status === 'Ready to Reprocess'); + const expiredFileExists = filesData.some((c) => isFileReadyToProcess(c, true) && isExpired(c.createdAt as Date)); + const largeFileExists = filesData.some( + (c) => isFileReadyToProcess(c, true) && typeof c.size === 'number' && c.size > largeFileSize + ); + const selectAllNewFiles = filesData.filter((f) => isFileReadyToProcess(f, false)); const stringified = selectAllNewFiles.reduce((accu, f) => { const key = f.id; // @ts-ignore @@ -707,10 +697,12 @@ const Content: React.FC = ({ return accu; }, {}); setRowSelection(stringified); - if (largefiles.length) { + if (largeFileExists) { setshowConfirmationModal(true); + } else if (expiredFileExists && isGCSActive) { + setshowExpirationModal(true); } else { - handleGenerateGraph(filesData.filter((f) => f.status === 'New' || f.status === 'Ready to Reprocess')); + handleGenerateGraph(filesData.filter((f) => isFileReadyToProcess(f, false))); } } }; @@ -757,6 +749,19 @@ const Content: React.FC = ({ > )} + {showExpirationModal && filesForProcessing.length && ( + }> + setshowExpirationModal(false)} + loading={extractLoading} + selectedRows={childRef.current?.getSelectedRows() as CustomFile[]} + isLargeDocumentAlert={false} + > + + )} {showDeletePopUp && ( = ({ {showEnhancementDialog && ( )} -
    - + +
    +
    Neo4j connection {isReadOnlyUser ? '(Read only Mode)' : ''} @@ -847,7 +865,6 @@ const Content: React.FC = ({
    { @@ -874,21 +891,21 @@ const Content: React.FC = ({ handleGenerateGraph={processWaitingFilesOnRefresh} > - - +
    + +
    + = ({
    - ); }; diff --git a/frontend/src/components/DataSources/AWS/S3Modal.tsx b/frontend/src/components/DataSources/AWS/S3Modal.tsx index 0e9b2f49f..6ccf03c60 100644 --- a/frontend/src/components/DataSources/AWS/S3Modal.tsx +++ b/frontend/src/components/DataSources/AWS/S3Modal.tsx @@ -54,7 +54,7 @@ const S3Modal: React.FC = ({ hideModal, open }) => { localStorage.setItem('accesskey', accessKey); } if (accessKey.length) { - localStorage.setItem('secretkey', secretKey); + localStorage.setItem('secretkey', btoa(secretKey)); } if (isValid && accessKey.trim() != '' && secretKey.trim() != '') { try { diff --git a/frontend/src/components/DataSources/Local/DropZone.tsx b/frontend/src/components/DataSources/Local/DropZone.tsx index 209573eaf..4a5d92efe 100644 --- a/frontend/src/components/DataSources/Local/DropZone.tsx +++ b/frontend/src/components/DataSources/Local/DropZone.tsx @@ -40,6 +40,7 @@ const DropZone: FunctionComponent = () => { entityEntityRelCount: 0, communityNodeCount: 0, communityRelCount: 0, + createdAt: new Date(), }; const copiedFilesData: CustomFile[] = [...filesData]; @@ -186,6 +187,7 @@ const DropZone: FunctionComponent = () => { ...curfile, status: 'New', uploadProgress: 100, + createdAt: new Date(), }; } return curfile; @@ -217,7 +219,7 @@ const DropZone: FunctionComponent = () => { text={ - Microsoft Office (.docx, .pptx, .xls) + Microsoft Office (.docx, .pptx, .xls, .xlsx) PDF (.pdf) Images (.jpeg, .jpg, .png, .svg) Text (.html, .txt , .md) @@ -241,6 +243,7 @@ const DropZone: FunctionComponent = () => { 'application/vnd.ms-powerpoint': ['.pptx'], 'application/vnd.ms-excel': ['.xls'], 'text/markdown': ['.md'], + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'], }, onDrop: (f: Partial[]) => { onDropHandler(f); diff --git a/frontend/src/components/DataSources/Local/DropZoneForSmallLayouts.tsx b/frontend/src/components/DataSources/Local/DropZoneForSmallLayouts.tsx index f3fbff852..2305a614e 100644 --- a/frontend/src/components/DataSources/Local/DropZoneForSmallLayouts.tsx +++ b/frontend/src/components/DataSources/Local/DropZoneForSmallLayouts.tsx @@ -118,6 +118,7 @@ export default function DropZoneForSmallLayouts() { ...curfile, status: 'New', uploadprogess: 100, + createdAt: new Date(), }; } return curfile; @@ -141,6 +142,7 @@ export default function DropZoneForSmallLayouts() { 'application/vnd.ms-powerpoint': ['.pptx'], 'application/vnd.ms-excel': ['.xls'], 'text/markdown': ['.md'], + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'], }, onDrop: (f: Partial[]) => { onDropHandler(f); @@ -175,6 +177,7 @@ export default function DropZoneForSmallLayouts() { entityEntityRelCount: 0, communityNodeCount: 0, communityRelCount: 0, + createdAt: new Date(), }; const copiedFilesData: CustomFile[] = [...filesData]; diff --git a/frontend/src/components/FileTable.tsx b/frontend/src/components/FileTable.tsx index 01fc0b0be..7c3e74bd0 100644 --- a/frontend/src/components/FileTable.tsx +++ b/frontend/src/components/FileTable.tsx @@ -9,8 +9,18 @@ import { Typography, useCopyToClipboard, Checkbox, + useMediaQuery, } from '@neo4j-ndl/react'; -import { forwardRef, useContext, useEffect, useImperativeHandle, useMemo, useRef, useState } from 'react'; +import { + forwardRef, + useContext, + useEffect, + useImperativeHandle, + useMemo, + useRef, + useState, + ForwardRefRenderFunction, +} from 'react'; import { useReactTable, getCoreRowModel, @@ -33,6 +43,7 @@ import { getFileSourceStatus, isProcessingFileValid, capitalizeWithUnderscore, + getParsedDate, } from '../utils/Utils'; import { SourceNode, CustomFile, FileTableProps, UserCredentials, statusupdate, ChildRef } from '../types'; import { useCredentials } from '../context/UserCredentials'; @@ -57,8 +68,8 @@ import BreakDownPopOver from './BreakDownPopOver'; let onlyfortheFirstRender = true; -const FileTable = forwardRef((props, ref) => { - const { isExpanded, connectionStatus, setConnectionStatus, onInspect, onRetry, onChunkView } = props; +const FileTable: ForwardRefRenderFunction = (props, ref) => { + const { connectionStatus, setConnectionStatus, onInspect, onRetry, onChunkView } = props; const { filesData, setFilesData, model, rowSelection, setRowSelection, setSelectedRows, setProcessedCount, queue } = useFileContext(); const { userCredentials, isReadOnlyUser } = useCredentials(); @@ -73,6 +84,7 @@ const FileTable = forwardRef((props, ref) => { const [_, copy] = useCopyToClipboard(); const { colorMode } = useContext(ThemeWrapperContext); const [copyRow, setCopyRow] = useState(false); + const largedesktops = useMediaQuery(`(min-width:1440px )`); const tableRef = useRef(null); @@ -761,6 +773,7 @@ const FileTable = forwardRef((props, ref) => { entityEntityRelCount: item.entityEntityRelCount ?? 0, communityNodeCount: item.communityNodeCount ?? 0, communityRelCount: item.communityRelCount ?? 0, + createdAt: item.createdAt != undefined ? getParsedDate(item?.createdAt) : undefined, }); } }); @@ -781,6 +794,7 @@ const FileTable = forwardRef((props, ref) => { } setIsLoading(false); } catch (error: any) { + console.log(error); if (error instanceof Error) { showErrorToast(error.message); } @@ -976,8 +990,6 @@ const FileTable = forwardRef((props, ref) => { [table] ); - const classNameCheck = isExpanded ? 'fileTableWithExpansion' : `filetable`; - useEffect(() => { setSelectedRows(table.getSelectedRowModel().rows.map((i) => i.id)); }, [table.getSelectedRowModel()]); @@ -986,53 +998,51 @@ const FileTable = forwardRef((props, ref) => { <> {filesData ? ( <> -
    - ( - ( + + ), + PaginationNumericButton: ({ isSelected, innerProps, ...restProps }) => { + return ( + - ), - PaginationNumericButton: ({ isSelected, innerProps, ...restProps }) => { - return ( - - ); - }, - }} - isKeyboardNavigable={false} - /> -
    + ); + }, + }} + isKeyboardNavigable={false} + /> ) : null} ); -}); +}; -export default FileTable; +export default forwardRef(FileTable); diff --git a/frontend/src/components/Graph/ResultOverview.tsx b/frontend/src/components/Graph/ResultOverview.tsx index 9fd691b0f..17fdbbc78 100644 --- a/frontend/src/components/Graph/ResultOverview.tsx +++ b/frontend/src/components/Graph/ResultOverview.tsx @@ -144,10 +144,10 @@ const ResultOverview: React.FunctionComponent = ({
    - {nodeCheck.map((nodeLabel, index) => ( + {nodeCheck.map((nodeLabel) => ( = ({ isExpanded, clearHistoryD return messages.some((msg) => msg.isTyping || msg.isLoading); }; return ( -
    +
    = ({ }) => { const { closeAlert, alertState } = useAlertContext(); const { isReadOnlyUser, isBackendConnected } = useCredentials(); + const largedesktops = useMediaQuery(`(min-width:1440px )`); const isYoutubeOnlyCheck = useMemo( () => APP_SOURCES?.includes('youtube') && !APP_SOURCES.includes('wiki') && !APP_SOURCES.includes('web'), @@ -40,7 +41,7 @@ const DrawerDropzone: React.FC = ({ ); return ( -
    +
    { setErrorMessage, setShowDisconnectButton, showDisconnectButton, + setIsGCSActive, } = useCredentials(); const { cancel } = useSpeechSynthesis(); @@ -97,6 +98,7 @@ const PageLayout: React.FC = () => { }); setGdsActive(parsedConnection.isgdsActive); setIsReadOnlyUser(parsedConnection.isReadOnlyUser); + setIsGCSActive(parsedConnection.isGCSActive); } else { console.error('Invalid parsed session data:', parsedConnection); } @@ -105,7 +107,7 @@ const PageLayout: React.FC = () => { } }; // To update credentials if environment values differ - const updateSessionIfNeeded = (envCredentials: UserCredentials, storedSession: string) => { + const updateSessionIfNeeded = (envCredentials: any, storedSession: string) => { try { const storedCredentials = JSON.parse(storedSession); const isDiffCreds = @@ -115,6 +117,7 @@ const PageLayout: React.FC = () => { envCredentials.database !== storedCredentials.database; if (isDiffCreds) { setUserCredentials(envCredentials); + setIsGCSActive(envCredentials.isGCSActive ?? false); localStorage.setItem( 'neo4j.connection', JSON.stringify({ @@ -147,7 +150,9 @@ const PageLayout: React.FC = () => { database: connectionData.data.database, isReadonlyUser: !connectionData.data.write_access, isgdsActive: connectionData.data.gds_status, + isGCSActive: connectionData?.data?.gcs_file_cache === 'True', }; + setIsGCSActive(connectionData?.data?.gcs_file_cache === 'True'); if (session) { const updated = updateSessionIfNeeded(envCredentials, session); if (!updated) { @@ -168,6 +173,7 @@ const PageLayout: React.FC = () => { userDbVectorIndex: 384, isReadOnlyUser: envCredentials.isReadonlyUser, isgdsActive: envCredentials.isgdsActive, + isGCSActive: envCredentials.isGCSActive, }) ); setConnectionStatus(true); @@ -226,7 +232,7 @@ const PageLayout: React.FC = () => { }; return ( -
    + <> }> { chunksExistsWithDifferentEmbedding={openConnection.chunksExistsWithDifferentDimension} /> - - { @@ -267,46 +256,132 @@ const PageLayout: React.FC = () => { } }} > - setShowChatBot(true)} - isLeftExpanded={isLeftExpanded} - isRightExpanded={isRightExpanded} - showChatBot={showChatBot} - openTextSchema={() => { - setShowTextFromSchemaDialog({ triggeredFrom: 'schemadialog', show: true }); - }} - isSchema={isSchema} - setIsSchema={setIsSchema} - showEnhancementDialog={showEnhancementDialog} - toggleEnhancementDialog={toggleEnhancementDialog} - setOpenConnection={setOpenConnection} - showDisconnectButton={showDisconnectButton} - connectionStatus={connectionStatus} - /> - {showDrawerChatbot && ( - + {largedesktops ? ( +
    + + {isLeftExpanded && ( + + )} + setShowChatBot(true)} + showChatBot={showChatBot} + openTextSchema={() => { + setShowTextFromSchemaDialog({ triggeredFrom: 'schemadialog', show: true }); + }} + isSchema={isSchema} + setIsSchema={setIsSchema} + showEnhancementDialog={showEnhancementDialog} + toggleEnhancementDialog={toggleEnhancementDialog} + setOpenConnection={setOpenConnection} + showDisconnectButton={showDisconnectButton} + connectionStatus={connectionStatus} + /> + {isRightExpanded && ( + + )} + +
    + ) : ( + <> + + +
    + + + setShowChatBot(true)} + showChatBot={showChatBot} + openTextSchema={() => { + setShowTextFromSchemaDialog({ triggeredFrom: 'schemadialog', show: true }); + }} + isSchema={isSchema} + setIsSchema={setIsSchema} + showEnhancementDialog={showEnhancementDialog} + toggleEnhancementDialog={toggleEnhancementDialog} + setOpenConnection={setOpenConnection} + showDisconnectButton={showDisconnectButton} + connectionStatus={connectionStatus} + /> + {isRightExpanded && ( + + )} + +
    + )} - -
    + ); }; diff --git a/frontend/src/components/Popups/ChunkPopUp/index.tsx b/frontend/src/components/Popups/ChunkPopUp/index.tsx index c0e2a016b..054d77dca 100644 --- a/frontend/src/components/Popups/ChunkPopUp/index.tsx +++ b/frontend/src/components/Popups/ChunkPopUp/index.tsx @@ -58,8 +58,8 @@ const ChunkPopUp = ({ ) : (
      - {sortedChunksData.map((c, idx) => ( -
    1. + {sortedChunksData.map((c) => ( +
    2. Position : diff --git a/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx b/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx index 683d1ddb8..2bc82735e 100644 --- a/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx +++ b/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx @@ -41,7 +41,8 @@ export default function ConnectionModal({ const [username, setUsername] = useState(initialusername ?? 'neo4j'); const [password, setPassword] = useState(''); const [connectionMessage, setMessage] = useState({ type: 'unknown', content: '' }); - const { setUserCredentials, userCredentials, setGdsActive, setIsReadOnlyUser, errorMessage } = useCredentials(); + const { setUserCredentials, userCredentials, setGdsActive, setIsReadOnlyUser, errorMessage, setIsGCSActive } = + useCredentials(); const [isLoading, setIsLoading] = useState(false); const [searchParams, setSearchParams] = useSearchParams(); const [userDbVectorIndex, setUserDbVectorIndex] = useState(initialuserdbvectorindex ?? undefined); @@ -215,8 +216,11 @@ export default function ConnectionModal({ } else { const isgdsActive = response.data.data.gds_status; const isReadOnlyUser = !response.data.data.write_access; + const isGCSActive = response.data.data.gcs_file_cache === 'True'; + setIsGCSActive(isGCSActive); setGdsActive(isgdsActive); setIsReadOnlyUser(isReadOnlyUser); + localStorage.setItem( 'neo4j.connection', JSON.stringify({ @@ -227,6 +231,7 @@ export default function ConnectionModal({ userDbVectorIndex, isgdsActive, isReadOnlyUser, + isGCSActive, }) ); setUserDbVectorIndex(response.data.data.db_vector_dimension); diff --git a/frontend/src/components/Popups/ExpirationModal/ExpiredFilesAlert.tsx b/frontend/src/components/Popups/ExpirationModal/ExpiredFilesAlert.tsx new file mode 100644 index 000000000..23738820c --- /dev/null +++ b/frontend/src/components/Popups/ExpirationModal/ExpiredFilesAlert.tsx @@ -0,0 +1,74 @@ +import { Checkbox, Flex, Typography } from '@neo4j-ndl/react'; +import { DocumentTextIconOutline } from '@neo4j-ndl/react/icons'; +import { LargefilesProps } from '../../../types'; +import { List, ListItem, ListItemAvatar, ListItemButton, ListItemIcon, ListItemText } from '@mui/material'; +import { FC } from 'react'; +import BellImage from '../../../assets/images/Stopwatch-blue.svg'; +import AlertIcon from '../../Layout/AlertIcon'; +import { isExpired } from '../../../utils/Utils'; +import { EXPIRATION_DAYS } from '../../../utils/Constants'; + +const ExpiredFilesAlert: FC = ({ Files, handleToggle, checked }) => { + return ( +
      +
      + alert icon +
      + Document Expiration Notice + + One or more of your selected documents are expired as per the expiration policy we are storing documents for + only {EXPIRATION_DAYS} days . Please delete and reupload the documents. + + + {Files.map((f) => { + return ( + + + + { + if (e.target.checked) { + handleToggle(true, f.id); + } else { + handleToggle(false, f.id); + } + }} + isChecked={checked.indexOf(f.id) !== -1} + htmlAttributes={{ tabIndex: -1 }} + /> + + + + + + + {f.name} - {Math.floor((f?.size as number) / 1000)?.toFixed(2)}KB + + {f.createdAt != undefined && isExpired(f.createdAt) ? ( + + + + ) : ( + <> + )} + + } + /> + + + ); + })} + +
      +
      +
      + ); +}; +export default ExpiredFilesAlert; diff --git a/frontend/src/components/Popups/GraphEnhancementDialog/DeleteTabForOrphanNodes/index.tsx b/frontend/src/components/Popups/GraphEnhancementDialog/DeleteTabForOrphanNodes/index.tsx index de8cdc186..1fae795b1 100644 --- a/frontend/src/components/Popups/GraphEnhancementDialog/DeleteTabForOrphanNodes/index.tsx +++ b/frontend/src/components/Popups/GraphEnhancementDialog/DeleteTabForOrphanNodes/index.tsx @@ -141,8 +141,8 @@ export default function DeletePopUpForOrphanNodes({ cell: (info) => { return ( - {info.getValue().map((l, index) => ( - + {info.getValue().map((l) => ( + ))} ); diff --git a/frontend/src/components/Popups/LargeFilePopUp/ConfirmationDialog.tsx b/frontend/src/components/Popups/LargeFilePopUp/ConfirmationDialog.tsx index c0159be65..7762d0d77 100644 --- a/frontend/src/components/Popups/LargeFilePopUp/ConfirmationDialog.tsx +++ b/frontend/src/components/Popups/LargeFilePopUp/ConfirmationDialog.tsx @@ -3,6 +3,8 @@ import { CustomFile } from '../../../types'; import LargeFilesAlert from './LargeFilesAlert'; import { memo, useEffect, useState } from 'react'; import { useFileContext } from '../../../context/UsersFiles'; +import ExpiredFilesAlert from '../ExpirationModal/ExpiredFilesAlert'; +import { isExpired } from '../../../utils/Utils'; function ConfirmationDialog({ largeFiles, @@ -11,6 +13,7 @@ function ConfirmationDialog({ loading, extractHandler, selectedRows, + isLargeDocumentAlert = true, }: { largeFiles: CustomFile[]; open: boolean; @@ -18,6 +21,7 @@ function ConfirmationDialog({ loading: boolean; extractHandler: (selectedFilesFromAllfiles: CustomFile[]) => void; selectedRows: CustomFile[]; + isLargeDocumentAlert?: boolean; }) { const { setSelectedRows, filesData, setRowSelection } = useFileContext(); const [checked, setChecked] = useState([...largeFiles.map((f) => f.id)]); @@ -82,8 +86,10 @@ function ConfirmationDialog({ {largeFiles.length === 0 && loading ? ( Files are under processing + ) : isLargeDocumentAlert ? ( + ) : ( - + )} @@ -106,6 +112,9 @@ function ConfirmationDialog({ onClose(); }} size='large' + isDisabled={largeFiles.some( + (f) => f.createdAt != undefined && checked.includes(f.id) && isExpired(f?.createdAt as Date) + )} > Continue diff --git a/frontend/src/components/Popups/LargeFilePopUp/LargeFilesAlert.tsx b/frontend/src/components/Popups/LargeFilePopUp/LargeFilesAlert.tsx index b8dfa0251..a107dcb2f 100644 --- a/frontend/src/components/Popups/LargeFilePopUp/LargeFilesAlert.tsx +++ b/frontend/src/components/Popups/LargeFilePopUp/LargeFilesAlert.tsx @@ -16,7 +16,7 @@ import s3logo from '../../../assets/images/s3logo.png'; import { calculateProcessingTime } from '../../../utils/Utils'; import { ThemeWrapperContext } from '../../../context/ThemeWrapper'; -const LargeFilesAlert: FC = ({ largeFiles, handleToggle, checked }) => { +const LargeFilesAlert: FC = ({ Files, handleToggle, checked }) => { const { colorMode } = useContext(ThemeWrapperContext); const imageIcon: Record = useMemo( @@ -44,7 +44,7 @@ const LargeFilesAlert: FC = ({ largeFiles, handleToggle, checke estimated times below - {largeFiles.map((f, i) => { + {Files.map((f, i) => { const { minutes, seconds } = calculateProcessingTime(f.size as number, 0.2); return ( diff --git a/frontend/src/components/UI/CustomMenu.tsx b/frontend/src/components/UI/CustomMenu.tsx index fa33c368d..65aa58801 100644 --- a/frontend/src/components/UI/CustomMenu.tsx +++ b/frontend/src/components/UI/CustomMenu.tsx @@ -21,9 +21,9 @@ export default function CustomMenu({ }) { return ( - {items?.map((i, idx) => ( + {items?.map((i) => ( { } return ( - + {children} diff --git a/frontend/src/context/UserCredentials.tsx b/frontend/src/context/UserCredentials.tsx index 29eee0c14..d466c109f 100644 --- a/frontend/src/context/UserCredentials.tsx +++ b/frontend/src/context/UserCredentials.tsx @@ -20,6 +20,8 @@ export const UserConnection = createContext({ setErrorMessage: () => null, showDisconnectButton: false, setShowDisconnectButton: () => null, + isGCSActive: false, + setIsGCSActive: () => null, }); export const useCredentials = () => { const userCredentials = useContext(UserConnection); @@ -33,6 +35,7 @@ const UserCredentialsWrapper: FunctionComponent = (props) => { const [isBackendConnected, setIsBackendConnected] = useState(false); const [errorMessage, setErrorMessage] = useState(''); const [showDisconnectButton, setShowDisconnectButton] = useState(false); + const [isGCSActive, setIsGCSActive] = useState(false); const value = { userCredentials, setUserCredentials, @@ -48,6 +51,8 @@ const UserCredentialsWrapper: FunctionComponent = (props) => { setErrorMessage, showDisconnectButton, setShowDisconnectButton, + isGCSActive, + setIsGCSActive, }; return {props.children}; diff --git a/frontend/src/services/ConnectAPI.ts b/frontend/src/services/ConnectAPI.ts index c802a5946..7432dd83a 100644 --- a/frontend/src/services/ConnectAPI.ts +++ b/frontend/src/services/ConnectAPI.ts @@ -21,7 +21,7 @@ const connectAPI = async (connectionURI: string, username: string, password: str const envConnectionAPI = async () => { try { - const conectionUrl = `/backend_connection_configuation`; + const conectionUrl = `/backend_connection_configuration`; const response = await api.post(conectionUrl); return response; } catch (error) { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 1c782103e..367739f9e 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -6,6 +6,7 @@ import type { Node, Relationship } from '@neo4j-nvl/base'; import { NonOAuthError } from '@react-oauth/google'; import { BannerType } from '@neo4j-ndl/react'; import Queue from './utils/Queue'; +import FileTable from './components/FileTable'; export interface CustomFileBase extends Partial { processingTotalTime: number | string; @@ -34,6 +35,7 @@ export interface CustomFileBase extends Partial { entityEntityRelCount: number; communityNodeCount: number; communityRelCount: number; + createdAt?: Date; } export interface CustomFile extends CustomFileBase { id: string; @@ -51,7 +53,7 @@ export type UserCredentials = { database: string; } & { [key: string]: any }; -export interface SourceNode extends Omit { +export interface SourceNode extends Omit { fileName: string; fileSize: number; fileType: string; @@ -65,6 +67,7 @@ export interface SourceNode extends Omit { processed_chunk?: number; total_chunks?: number; retry_condition?: string; + createdAt: filedate; } export type ExtractParams = Pick & { @@ -143,8 +146,6 @@ export interface DrawerProps { } export interface ContentProps { - isLeftExpanded: boolean; - isRightExpanded: boolean; showChatBot: boolean; openChatBot: () => void; openTextSchema: () => void; @@ -158,7 +159,6 @@ export interface ContentProps { } export interface FileTableProps { - isExpanded: boolean; connectionStatus: boolean; setConnectionStatus: Dispatch>; onInspect: (id: string) => void; @@ -580,7 +580,7 @@ export interface UploadResponse extends Partial { data: uploadData; } export interface LargefilesProps { - largeFiles: CustomFile[]; + Files: CustomFile[]; handleToggle: (ischecked: boolean, id: string) => void; checked: string[]; } @@ -758,6 +758,8 @@ export interface ContextProps { setErrorMessage: Dispatch>; showDisconnectButton: boolean; setShowDisconnectButton: Dispatch>; + isGCSActive: boolean; + setIsGCSActive: Dispatch>; } export interface MessageContextType { messages: Messages[] | []; @@ -773,11 +775,6 @@ export interface GraphContextType { setLoadingGraph: Dispatch>; } -export interface GraphContextType { - loadingGraph: boolean; - setLoadingGraph: Dispatch>; -} - export interface DatabaseStatusProps { isConnected: boolean; isGdsActive: boolean; @@ -923,3 +920,21 @@ export interface GraphViewHandlerProps { export interface ChatProps { chatMessages: Messages[]; } + +export interface filedate { + _DateTime__date: { + _Date__ordinal: number; + _Date__year: number; + _Date__month: number; + _Date__day: number; + }; + _DateTime__time: { + _Time__ticks: number; + _Time__hour: number; + _Time__minute: number; + _Time__second: number; + _Time__nanosecond: number; + _Time__tzinfo: null; + }; +} +export type FileTableHandle = React.ElementRef; diff --git a/frontend/src/utils/Constants.ts b/frontend/src/utils/Constants.ts index 52ea23813..3a5702b91 100644 --- a/frontend/src/utils/Constants.ts +++ b/frontend/src/utils/Constants.ts @@ -368,3 +368,4 @@ export const metricsinfo: Record = { semantic_score: 'Determines How well the generated answer understands the meaning of the reference answer.', context_entity_recall: 'Determines the recall of entities present in both generated answer and retrieved contexts', }; +export const EXPIRATION_DAYS = 3; diff --git a/frontend/src/utils/Utils.ts b/frontend/src/utils/Utils.ts index 7314c55e7..206a2d097 100644 --- a/frontend/src/utils/Utils.ts +++ b/frontend/src/utils/Utils.ts @@ -5,6 +5,7 @@ import { Entity, ExtendedNode, ExtendedRelationship, + filedate, GraphType, Messages, Scheme, @@ -19,7 +20,7 @@ import youtubedarklogo from '../assets/images/youtube-darkmode.svg'; import youtubelightlogo from '../assets/images/youtube-lightmode.svg'; import s3logo from '../assets/images/s3logo.png'; import gcslogo from '../assets/images/gcs.webp'; -import { chatModeLables } from './Constants'; +import { chatModeLables, EXPIRATION_DAYS } from './Constants'; // Get the Url export const url = () => { @@ -522,3 +523,24 @@ export function getNodes(nodesData: Array EXPIRATION_DAYS; +} + +export function isFileReadyToProcess(file: CustomFile, withLocalCheck: boolean) { + if (withLocalCheck) { + return file.fileSource === 'local file' && (file.status === 'New' || file.status == 'Ready to Reprocess'); + } + return file.status === 'New' || file.status == 'Ready to Reprocess'; +}