diff --git a/backend/example.env b/backend/example.env index 85b8d1054..6bef36f78 100644 --- a/backend/example.env +++ b/backend/example.env @@ -25,8 +25,9 @@ NEO4J_USER_AGENT="" ENABLE_USER_AGENT = "" LLM_MODEL_CONFIG_model_version="" ENTITY_EMBEDDING="" True or False -DUPLICATE_SCORE_VALUE = "" -DUPLICATE_TEXT_DISTANCE = "" +DUPLICATE_SCORE_VALUE =0.97 +DUPLICATE_TEXT_DISTANCE =3 +DEFAULT_DIFFBOT_CHAT_MODEL="openai_gpt_4o" #whichever model specified here , need to add config for that model in below format) #examples LLM_MODEL_CONFIG_openai_gpt_3.5="gpt-3.5-turbo-0125,openai_api_key" LLM_MODEL_CONFIG_openai_gpt_4o_mini="gpt-4o-mini-2024-07-18,openai_api_key" diff --git a/backend/requirements.txt b/backend/requirements.txt index 1df0ff4de..270e27f4b 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -30,7 +30,7 @@ nltk==3.9.1 openai==1.55.1 opencv-python==4.10.0.84 psutil==6.1.0 -pydantic==2.9.0 +pydantic==2.9.2 python-dotenv==1.0.1 python-magic==0.4.27 PyPDF2==3.0.1 @@ -39,9 +39,9 @@ starlette==0.41.3 sse-starlette==2.1.3 starlette-session==0.4.3 tqdm==4.67.1 -unstructured[all-docs]==0.16.6 -unstructured==0.16.6 -unstructured-client==0.26.2 +unstructured[all-docs] +unstructured==0.16.11 +unstructured-client==0.28.1 unstructured-inference==0.8.1 urllib3==2.2.2 uvicorn==0.32.1 diff --git a/backend/score.py b/backend/score.py index 737a82afe..23c3d14a8 100644 --- a/backend/score.py +++ b/backend/score.py @@ -18,24 +18,19 @@ from src.communities import create_communities from src.neighbours import get_neighbour_nodes import json -from typing import List, Mapping, Union +from typing import List from starlette.middleware.sessions import SessionMiddleware -import google_auth_oauthlib.flow from google.oauth2.credentials import Credentials import os from src.logger import CustomLogger from datetime import datetime, timezone import time import gc -from Secweb import SecWeb -from Secweb.StrictTransportSecurity import HSTS -from Secweb.ContentSecurityPolicy import ContentSecurityPolicy from Secweb.XContentTypeOptions import XContentTypeOptions from Secweb.XFrameOptions import XFrame from fastapi.middleware.gzip import GZipMiddleware from src.ragas_eval import * -from starlette.types import ASGIApp, Message, Receive, Scope, Send -import gzip +from starlette.types import ASGIApp, Receive, Scope, Send from langchain_neo4j import Neo4jGraph logger = CustomLogger() @@ -493,11 +488,13 @@ async def connect(uri=Form(), userName=Form(), password=Form(), database=Form()) start = time.time() graph = create_graph_database_connection(uri, userName, password, database) result = await asyncio.to_thread(connection_check_and_get_vector_dimensions, graph, database) + gcs_file_cache = os.environ.get('GCS_FILE_CACHE') end = time.time() elapsed_time = end - start json_obj = {'api_name':'connect','db_url':uri, 'userName':userName, 'database':database,'status':result, 'count':1, 'logging_time': formatted_time(datetime.now(timezone.utc)), 'elapsed_api_time':f'{elapsed_time:.2f}'} logger.log_struct(json_obj, "INFO") result['elapsed_api_time'] = f'{elapsed_time:.2f}' + result['gcs_file_cache'] = gcs_file_cache return create_api_response('Success',data=result) except Exception as e: job_status = "Failed" @@ -571,6 +568,8 @@ async def generate(): uri = url if " " in url: uri= url.replace(" ","+") + graph = create_graph_database_connection(uri, userName, decoded_password, database) + graphDb_data_Access = graphDBdataAccess(graph) while True: try: if await request.is_disconnected(): @@ -579,8 +578,6 @@ async def generate(): # get the current status of document node else: - graph = create_graph_database_connection(uri, userName, decoded_password, database) - graphDb_data_Access = graphDBdataAccess(graph) result = graphDb_data_Access.get_current_status_document_node(file_name) if len(result) > 0: status = json.dumps({'fileName':file_name, @@ -968,22 +965,30 @@ async def fetch_chunktext( gc.collect() -@app.post("/backend_connection_configuation") -async def backend_connection_configuation(): +@app.post("/backend_connection_configuration") +async def backend_connection_configuration(): try: - graph = Neo4jGraph() - logging.info(f'login connection status of object: {graph}') - if graph is not None: - graph_connection = True - isURI = os.getenv('NEO4J_URI') - isUsername= os.getenv('NEO4J_USERNAME') - isDatabase= os.getenv('NEO4J_DATABASE') - isPassword= os.getenv('NEO4J_PASSWORD') - encoded_password = encode_password(isPassword) - graphDb_data_Access = graphDBdataAccess(graph) - gds_status = graphDb_data_Access.check_gds_version() - write_access = graphDb_data_Access.check_account_access(database=isDatabase) - return create_api_response('Success',message=f"Backend connection successful",data={'graph_connection':graph_connection,'uri':isURI,'user_name':isUsername,'database':isDatabase,'password':encoded_password,'gds_status':gds_status,'write_access':write_access}) + uri = os.getenv('NEO4J_URI') + username= os.getenv('NEO4J_USERNAME') + database= os.getenv('NEO4J_DATABASE') + password= os.getenv('NEO4J_PASSWORD') + gcs_file_cache = os.environ.get('GCS_FILE_CACHE') + if all([uri, username, database, password]): + print(f'uri:{uri}, usrName:{username}, database :{database}, password: {password}') + graph = Neo4jGraph() + logging.info(f'login connection status of object: {graph}') + if graph is not None: + graph_connection = True + encoded_password = encode_password(password) + graphDb_data_Access = graphDBdataAccess(graph) + result = graphDb_data_Access.connection_check_and_get_vector_dimensions(database) + result["graph_connection"] = graph_connection + result["uri"] = uri + result["user_name"] = username + result["database"] = database + result["password"] = encoded_password + result['gcs_file_cache'] = gcs_file_cache + return create_api_response('Success',message=f"Backend connection successful",data=result) else: graph_connection = False return create_api_response('Success',message=f"Backend connection is not successful",data=graph_connection) diff --git a/backend/src/entities/source_node.py b/backend/src/entities/source_node.py index a162b6d04..9e52497e3 100644 --- a/backend/src/entities/source_node.py +++ b/backend/src/entities/source_node.py @@ -11,6 +11,12 @@ class sourceNode: gcsBucketFolder:str=None gcsProjectId:str=None awsAccessKeyId:str=None + chunkNodeCount:int=None + chunkRelCount:int=None + entityNodeCount:int=None + entityEntityRelCount:int=None + communityNodeCount:int=None + communityRelCount:int=None node_count:int=None relationship_count:str=None model:str=None diff --git a/backend/src/graphDB_dataAccess.py b/backend/src/graphDB_dataAccess.py index 1f387a2d2..ddb270862 100644 --- a/backend/src/graphDB_dataAccess.py +++ b/backend/src/graphDB_dataAccess.py @@ -46,14 +46,24 @@ def create_source_node(self, obj_source_node:sourceNode): d.relationshipCount = $r_count, d.model= $model, d.gcsBucket=$gcs_bucket, d.gcsBucketFolder= $gcs_bucket_folder, d.language= $language,d.gcsProjectId= $gcs_project_id, d.is_cancelled=False, d.total_chunks=0, d.processed_chunk=0, - d.access_token=$access_token""", + d.access_token=$access_token, + d.chunkNodeCount=$chunkNodeCount,d.chunkRelCount=$chunkRelCount, + d.entityNodeCount=$entityNodeCount,d.entityEntityRelCount=$entityEntityRelCount, + d.communityNodeCount=$communityNodeCount,d.communityRelCount=$communityRelCount""", {"fn":obj_source_node.file_name, "fs":obj_source_node.file_size, "ft":obj_source_node.file_type, "st":job_status, "url":obj_source_node.url, "awsacc_key_id":obj_source_node.awsAccessKeyId, "f_source":obj_source_node.file_source, "c_at":obj_source_node.created_at, "u_at":obj_source_node.created_at, "pt":0, "e_message":'', "n_count":0, "r_count":0, "model":obj_source_node.model, "gcs_bucket": obj_source_node.gcsBucket, "gcs_bucket_folder": obj_source_node.gcsBucketFolder, "language":obj_source_node.language, "gcs_project_id":obj_source_node.gcsProjectId, - "access_token":obj_source_node.access_token}) + "access_token":obj_source_node.access_token, + "chunkNodeCount":obj_source_node.chunkNodeCount, + "chunkRelCount":obj_source_node.chunkRelCount, + "entityNodeCount":obj_source_node.entityNodeCount, + "entityEntityRelCount":obj_source_node.entityEntityRelCount, + "communityNodeCount":obj_source_node.communityNodeCount, + "communityRelCount":obj_source_node.communityRelCount + }) except Exception as e: error_message = str(e) logging.info(f"error_message = {error_message}") @@ -108,7 +118,7 @@ def update_source_node(self, obj_source_node:sourceNode): self.graph.query(query,param) except Exception as e: error_message = str(e) - self.update_exception_db(self.file_name,error_message) + self.update_exception_db(self,self.file_name,error_message) raise Exception(error_message) def get_source_list(self): @@ -463,51 +473,52 @@ def update_node_relationship_count(self,document_name): param = {"document_name": document_name} result = self.execute_query(NODEREL_COUNT_QUERY_WITHOUT_COMMUNITY, param) response = {} - for record in result: - filename = record["filename"] - chunkNodeCount = record["chunkNodeCount"] - chunkRelCount = record["chunkRelCount"] - entityNodeCount = record["entityNodeCount"] - entityEntityRelCount = record["entityEntityRelCount"] - if (not document_name) and (community_flag): - communityNodeCount = record["communityNodeCount"] - communityRelCount = record["communityRelCount"] - else: - communityNodeCount = 0 - communityRelCount = 0 - nodeCount = int(chunkNodeCount) + int(entityNodeCount) + int(communityNodeCount) - relationshipCount = int(chunkRelCount) + int(entityEntityRelCount) + int(communityRelCount) - update_query = """ - MATCH (d:Document {fileName: $filename}) - SET d.chunkNodeCount = $chunkNodeCount, - d.chunkRelCount = $chunkRelCount, - d.entityNodeCount = $entityNodeCount, - d.entityEntityRelCount = $entityEntityRelCount, - d.communityNodeCount = $communityNodeCount, - d.communityRelCount = $communityRelCount, - d.nodeCount = $nodeCount, - d.relationshipCount = $relationshipCount - """ - self.execute_query(update_query,{ - "filename": filename, - "chunkNodeCount": chunkNodeCount, - "chunkRelCount": chunkRelCount, - "entityNodeCount": entityNodeCount, - "entityEntityRelCount": entityEntityRelCount, - "communityNodeCount": communityNodeCount, - "communityRelCount": communityRelCount, - "nodeCount" : nodeCount, - "relationshipCount" : relationshipCount - }) - - response[filename] = {"chunkNodeCount": chunkNodeCount, - "chunkRelCount": chunkRelCount, - "entityNodeCount": entityNodeCount, - "entityEntityRelCount": entityEntityRelCount, - "communityNodeCount": communityNodeCount, - "communityRelCount": communityRelCount, - "nodeCount" : nodeCount, - "relationshipCount" : relationshipCount - } + if result: + for record in result: + filename = record.get("filename",None) + chunkNodeCount = int(record.get("chunkNodeCount",0)) + chunkRelCount = int(record.get("chunkRelCount",0)) + entityNodeCount = int(record.get("entityNodeCount",0)) + entityEntityRelCount = int(record.get("entityEntityRelCount",0)) + if (not document_name) and (community_flag): + communityNodeCount = int(record.get("communityNodeCount",0)) + communityRelCount = int(record.get("communityRelCount",0)) + else: + communityNodeCount = 0 + communityRelCount = 0 + nodeCount = int(chunkNodeCount) + int(entityNodeCount) + int(communityNodeCount) + relationshipCount = int(chunkRelCount) + int(entityEntityRelCount) + int(communityRelCount) + update_query = """ + MATCH (d:Document {fileName: $filename}) + SET d.chunkNodeCount = $chunkNodeCount, + d.chunkRelCount = $chunkRelCount, + d.entityNodeCount = $entityNodeCount, + d.entityEntityRelCount = $entityEntityRelCount, + d.communityNodeCount = $communityNodeCount, + d.communityRelCount = $communityRelCount, + d.nodeCount = $nodeCount, + d.relationshipCount = $relationshipCount + """ + self.execute_query(update_query,{ + "filename": filename, + "chunkNodeCount": chunkNodeCount, + "chunkRelCount": chunkRelCount, + "entityNodeCount": entityNodeCount, + "entityEntityRelCount": entityEntityRelCount, + "communityNodeCount": communityNodeCount, + "communityRelCount": communityRelCount, + "nodeCount" : nodeCount, + "relationshipCount" : relationshipCount + }) + + response[filename] = {"chunkNodeCount": chunkNodeCount, + "chunkRelCount": chunkRelCount, + "entityNodeCount": entityNodeCount, + "entityEntityRelCount": entityEntityRelCount, + "communityNodeCount": communityNodeCount, + "communityRelCount": communityRelCount, + "nodeCount" : nodeCount, + "relationshipCount" : relationshipCount + } return response \ No newline at end of file diff --git a/backend/src/graph_query.py b/backend/src/graph_query.py index dc5a64a2c..f9af6102b 100644 --- a/backend/src/graph_query.py +++ b/backend/src/graph_query.py @@ -226,6 +226,7 @@ def get_graph_results(uri, username, password,database,document_names): def get_chunktext_results(uri, username, password, database, document_name, page_no): """Retrieves chunk text, position, and page number from graph data with pagination.""" + driver = None try: logging.info("Starting chunk text query process") offset = 10 @@ -254,4 +255,5 @@ def get_chunktext_results(uri, username, password, database, document_name, page logging.error(f"An error occurred in get_chunktext_results. Error: {str(e)}") raise Exception("An error occurred in get_chunktext_results. Please check the logs for more details.") from e finally: - driver.close() \ No newline at end of file + if driver: + driver.close() \ No newline at end of file diff --git a/backend/src/llm.py b/backend/src/llm.py index f19648ed6..381a38a68 100644 --- a/backend/src/llm.py +++ b/backend/src/llm.py @@ -16,95 +16,106 @@ def get_llm(model: str): """Retrieve the specified language model based on the model name.""" - env_key = "LLM_MODEL_CONFIG_" + model + model = model.lower().strip() + env_key = f"LLM_MODEL_CONFIG_{model}" env_value = os.environ.get(env_key) - logging.info("Model: {}".format(env_key)) + + if not env_value: + err = f"Environment variable '{env_key}' is not defined as per format or missing" + logging.error(err) + raise Exception(err) - if "gemini" in model: - model_name = env_value - credentials, project_id = google.auth.default() - llm = ChatVertexAI( - model_name=model_name, - #convert_system_message_to_human=True, - credentials=credentials, - project=project_id, - temperature=0, - safety_settings={ - HarmCategory.HARM_CATEGORY_UNSPECIFIED: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE, - HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE, - }, - ) - elif "openai" in model: - model_name, api_key = env_value.split(",") - llm = ChatOpenAI( - api_key=api_key, - model=model_name, - temperature=0, - ) + logging.info("Model: {}".format(env_key)) + try: + if "gemini" in model: + model_name = env_value + credentials, project_id = google.auth.default() + llm = ChatVertexAI( + model_name=model_name, + #convert_system_message_to_human=True, + credentials=credentials, + project=project_id, + temperature=0, + safety_settings={ + HarmCategory.HARM_CATEGORY_UNSPECIFIED: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE, + HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE, + }, + ) + elif "openai" in model: + model_name, api_key = env_value.split(",") + llm = ChatOpenAI( + api_key=api_key, + model=model_name, + temperature=0, + ) - elif "azure" in model: - model_name, api_endpoint, api_key, api_version = env_value.split(",") - llm = AzureChatOpenAI( - api_key=api_key, - azure_endpoint=api_endpoint, - azure_deployment=model_name, # takes precedence over model parameter - api_version=api_version, - temperature=0, - max_tokens=None, - timeout=None, - ) + elif "azure" in model: + model_name, api_endpoint, api_key, api_version = env_value.split(",") + llm = AzureChatOpenAI( + api_key=api_key, + azure_endpoint=api_endpoint, + azure_deployment=model_name, # takes precedence over model parameter + api_version=api_version, + temperature=0, + max_tokens=None, + timeout=None, + ) - elif "anthropic" in model: - model_name, api_key = env_value.split(",") - llm = ChatAnthropic( - api_key=api_key, model=model_name, temperature=0, timeout=None - ) + elif "anthropic" in model: + model_name, api_key = env_value.split(",") + llm = ChatAnthropic( + api_key=api_key, model=model_name, temperature=0, timeout=None + ) - elif "fireworks" in model: - model_name, api_key = env_value.split(",") - llm = ChatFireworks(api_key=api_key, model=model_name) - - elif "groq" in model: - model_name, base_url, api_key = env_value.split(",") - llm = ChatGroq(api_key=api_key, model_name=model_name, temperature=0) - - elif "bedrock" in model: - model_name, aws_access_key, aws_secret_key, region_name = env_value.split(",") - bedrock_client = boto3.client( - service_name="bedrock-runtime", - region_name=region_name, - aws_access_key_id=aws_access_key, - aws_secret_access_key=aws_secret_key, - ) + elif "fireworks" in model: + model_name, api_key = env_value.split(",") + llm = ChatFireworks(api_key=api_key, model=model_name) + + elif "groq" in model: + model_name, base_url, api_key = env_value.split(",") + llm = ChatGroq(api_key=api_key, model_name=model_name, temperature=0) + + elif "bedrock" in model: + model_name, aws_access_key, aws_secret_key, region_name = env_value.split(",") + bedrock_client = boto3.client( + service_name="bedrock-runtime", + region_name=region_name, + aws_access_key_id=aws_access_key, + aws_secret_access_key=aws_secret_key, + ) - llm = ChatBedrock( - client=bedrock_client, model_id=model_name, model_kwargs=dict(temperature=0) - ) + llm = ChatBedrock( + client=bedrock_client, model_id=model_name, model_kwargs=dict(temperature=0) + ) - elif "ollama" in model: - model_name, base_url = env_value.split(",") - llm = ChatOllama(base_url=base_url, model=model_name) + elif "ollama" in model: + model_name, base_url = env_value.split(",") + llm = ChatOllama(base_url=base_url, model=model_name) - elif "diffbot" in model: - #model_name = "diffbot" - model_name, api_key = env_value.split(",") - llm = DiffbotGraphTransformer( - diffbot_api_key=api_key, - extract_types=["entities", "facts"], - ) - - else: - model_name, api_endpoint, api_key = env_value.split(",") - llm = ChatOpenAI( - api_key=api_key, - base_url=api_endpoint, - model=model_name, - temperature=0, - ) - + elif "diffbot" in model: + #model_name = "diffbot" + model_name, api_key = env_value.split(",") + llm = DiffbotGraphTransformer( + diffbot_api_key=api_key, + extract_types=["entities", "facts"], + ) + + else: + model_name, api_endpoint, api_key = env_value.split(",") + llm = ChatOpenAI( + api_key=api_key, + base_url=api_endpoint, + model=model_name, + temperature=0, + ) + except Exception as e: + err = f"Error while creating LLM '{model}': {str(e)}" + logging.error(err) + raise Exception(err) + logging.info(f"Model created - Model Version: {model}") return llm, model_name @@ -179,21 +190,24 @@ async def get_graph_document_list( async def get_graph_from_llm(model, chunkId_chunkDoc_list, allowedNodes, allowedRelationship): - - llm, model_name = get_llm(model) - combined_chunk_document_list = get_combined_chunks(chunkId_chunkDoc_list) - #combined_chunk_document_list = get_chunk_id_as_doc_metadata(chunkId_chunkDoc_list) - - if allowedNodes is None or allowedNodes=="": - allowedNodes =[] - else: - allowedNodes = allowedNodes.split(',') - if allowedRelationship is None or allowedRelationship=="": - allowedRelationship=[] - else: - allowedRelationship = allowedRelationship.split(',') + try: + llm, model_name = get_llm(model) + combined_chunk_document_list = get_combined_chunks(chunkId_chunkDoc_list) - graph_document_list = await get_graph_document_list( - llm, combined_chunk_document_list, allowedNodes, allowedRelationship - ) - return graph_document_list + if allowedNodes is None or allowedNodes=="": + allowedNodes =[] + else: + allowedNodes = allowedNodes.split(',') + if allowedRelationship is None or allowedRelationship=="": + allowedRelationship=[] + else: + allowedRelationship = allowedRelationship.split(',') + + graph_document_list = await get_graph_document_list( + llm, combined_chunk_document_list, allowedNodes, allowedRelationship + ) + return graph_document_list + except Exception as e: + err = f"Error during extracting graph with llm: {e}" + logging.error(err) + raise diff --git a/backend/src/main.py b/backend/src/main.py index 5ef1e4354..d47061c46 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -56,6 +56,12 @@ def create_source_node_graph_url_s3(graph, model, source_url, aws_access_key_id, obj_source_node.url = str(source_url+file_name) obj_source_node.awsAccessKeyId = aws_access_key_id obj_source_node.created_at = datetime.now() + obj_source_node.chunkNodeCount=0 + obj_source_node.chunkRelCount=0 + obj_source_node.entityNodeCount=0 + obj_source_node.entityEntityRelCount=0 + obj_source_node.communityNodeCount=0 + obj_source_node.communityRelCount=0 try: graphDb_data_Access = graphDBdataAccess(graph) graphDb_data_Access.create_source_node(obj_source_node) @@ -88,6 +94,12 @@ def create_source_node_graph_url_gcs(graph, model, gcs_project_id, gcs_bucket_na obj_source_node.gcsProjectId = file_metadata['gcsProjectId'] obj_source_node.created_at = datetime.now() obj_source_node.access_token = credentials.token + obj_source_node.chunkNodeCount=0 + obj_source_node.chunkRelCount=0 + obj_source_node.entityNodeCount=0 + obj_source_node.entityEntityRelCount=0 + obj_source_node.communityNodeCount=0 + obj_source_node.communityRelCount=0 try: graphDb_data_Access = graphDBdataAccess(graph) @@ -119,7 +131,12 @@ def create_source_node_graph_web_url(graph, model, source_url, source_type): obj_source_node.file_name = pages[0].metadata['title'] obj_source_node.language = pages[0].metadata['language'] obj_source_node.file_size = sys.getsizeof(pages[0].page_content) - + obj_source_node.chunkNodeCount=0 + obj_source_node.chunkRelCount=0 + obj_source_node.entityNodeCount=0 + obj_source_node.entityEntityRelCount=0 + obj_source_node.communityNodeCount=0 + obj_source_node.communityRelCount=0 graphDb_data_Access = graphDBdataAccess(graph) graphDb_data_Access.create_source_node(obj_source_node) lst_file_name.append({'fileName':obj_source_node.file_name,'fileSize':obj_source_node.file_size,'url':obj_source_node.url,'status':'Success'}) @@ -138,6 +155,12 @@ def create_source_node_graph_url_youtube(graph, model, source_url, source_type): obj_source_node.model = model obj_source_node.url = youtube_url obj_source_node.created_at = datetime.now() + obj_source_node.chunkNodeCount=0 + obj_source_node.chunkRelCount=0 + obj_source_node.entityNodeCount=0 + obj_source_node.entityEntityRelCount=0 + obj_source_node.communityNodeCount=0 + obj_source_node.communityRelCount=0 match = re.search(r'(?:v=)([0-9A-Za-z_-]{11})\s*',obj_source_node.url) logging.info(f"match value: {match}") video_id = parse_qs(urlparse(youtube_url).query).get('v') @@ -180,6 +203,12 @@ def create_source_node_graph_url_wikipedia(graph, model, wiki_query, source_type obj_source_node.url = urllib.parse.unquote(pages[0].metadata['source']) obj_source_node.created_at = datetime.now() obj_source_node.language = language + obj_source_node.chunkNodeCount=0 + obj_source_node.chunkRelCount=0 + obj_source_node.entityNodeCount=0 + obj_source_node.entityEntityRelCount=0 + obj_source_node.communityNodeCount=0 + obj_source_node.communityRelCount=0 graphDb_data_Access = graphDBdataAccess(graph) graphDb_data_Access.create_source_node(obj_source_node) success_count+=1 @@ -279,6 +308,7 @@ async def processing_source(uri, userName, password, database, model, file_name, logging.info(f'Time taken database connection: {elapsed_create_connection:.2f} seconds') uri_latency["create_connection"] = f'{elapsed_create_connection:.2f}' graphDb_data_Access = graphDBdataAccess(graph) + create_chunk_vector_index(graph) start_get_chunkId_chunkDoc_list = time.time() total_chunks, chunkId_chunkDoc_list = get_chunkId_chunkDoc_list(graph, file_name, pages, retry_condition) end_get_chunkId_chunkDoc_list = time.time() @@ -428,7 +458,7 @@ async def processing_chunks(chunkId_chunkDoc_list,graph,uri, userName, password, graph = create_graph_database_connection(uri, userName, password, database) start_update_embedding = time.time() - update_embedding_create_vector_index( graph, chunkId_chunkDoc_list, file_name) + create_chunk_embeddings( graph, chunkId_chunkDoc_list, file_name) end_update_embedding = time.time() elapsed_update_embedding = end_update_embedding - start_update_embedding logging.info(f'Time taken to update embedding in chunk node: {elapsed_update_embedding:.2f} seconds') @@ -618,6 +648,12 @@ def upload_file(graph, model, chunk, chunk_number:int, total_chunks:int, origina obj_source_node.file_source = 'local file' obj_source_node.model = model obj_source_node.created_at = datetime.now() + obj_source_node.chunkNodeCount=0 + obj_source_node.chunkRelCount=0 + obj_source_node.entityNodeCount=0 + obj_source_node.entityEntityRelCount=0 + obj_source_node.communityNodeCount=0 + obj_source_node.communityRelCount=0 graphDb_data_Access = graphDBdataAccess(graph) graphDb_data_Access.create_source_node(obj_source_node) diff --git a/backend/src/make_relationships.py b/backend/src/make_relationships.py index 20e95913f..410a383fd 100644 --- a/backend/src/make_relationships.py +++ b/backend/src/make_relationships.py @@ -6,6 +6,7 @@ import os import hashlib import time +from langchain_neo4j import Neo4jVector logging.basicConfig(format='%(asctime)s - %(message)s',level='INFO') @@ -39,7 +40,7 @@ def merge_relationship_between_chunk_and_entites(graph: Neo4jGraph, graph_docume graph.query(unwind_query, params={"batch_data": batch_data}) -def update_embedding_create_vector_index(graph, chunkId_chunkDoc_list, file_name): +def create_chunk_embeddings(graph, chunkId_chunkDoc_list, file_name): #create embedding isEmbedding = os.getenv('IS_EMBEDDING') # embedding_model = os.getenv('EMBEDDING_MODEL') @@ -58,35 +59,6 @@ def update_embedding_create_vector_index(graph, chunkId_chunkDoc_list, file_name "chunkId": row['chunk_id'], "embeddings": embeddings_arr }) - # graph.query("""MATCH (d:Document {fileName : $fileName}) - # MERGE (c:Chunk {id:$chunkId}) SET c.embedding = $embeddings - # MERGE (c)-[:PART_OF]->(d) - # """, - # { - # "fileName" : file_name, - # "chunkId": row['chunk_id'], - # "embeddings" : embeddings_arr - # } - # ) - logging.info('create vector index on chunk embedding') - # result = graph.query("SHOW INDEXES YIELD * WHERE labelsOrTypes = ['Chunk'] and name = 'vector'") - vector_index = graph.query("SHOW INDEXES YIELD * WHERE labelsOrTypes = ['Chunk'] and type = 'VECTOR' AND name = 'vector' return options") - # if result: - # logging.info(f"vector index dropped for 'Chunk'") - # graph.query("DROP INDEX vector IF EXISTS;") - - if len(vector_index) == 0: - logging.info(f'vector index is not exist, will create in next query') - graph.query("""CREATE VECTOR INDEX `vector` if not exists for (c:Chunk) on (c.embedding) - OPTIONS {indexConfig: { - `vector.dimensions`: $dimensions, - `vector.similarity_function`: 'cosine' - }} - """, - { - "dimensions" : dimension - } - ) query_to_create_embedding = """ UNWIND $data AS row @@ -185,4 +157,27 @@ def create_relation_between_chunks(graph, file_name, chunks: List[Document])->li """ graph.query(query_to_create_NEXT_CHUNK_relation, params={"relationships": relationships}) - return lst_chunks_including_hash \ No newline at end of file + return lst_chunks_including_hash + + +def create_chunk_vector_index(graph): + start_time = time.time() + try: + vector_index = graph.query("SHOW INDEXES YIELD * WHERE labelsOrTypes = ['Chunk'] and type = 'VECTOR' AND name = 'vector' return options") + + if not vector_index: + vector_store = Neo4jVector(embedding=EMBEDDING_FUNCTION, + graph=graph, + node_label="Chunk", + embedding_node_property="embedding", + index_name="vector" + ) + vector_store.create_new_index() + logging.info(f"Index created successfully. Time taken: {time.time() - start_time:.2f} seconds") + else: + logging.info(f"Index already exist,Skipping creation. Time taken: {time.time() - start_time:.2f} seconds") + except Exception as e: + if "EquivalentSchemaRuleAlreadyExists" in str(e): + logging.info("Vector index already exists, skipping creation.") + else: + raise \ No newline at end of file diff --git a/backend/src/ragas_eval.py b/backend/src/ragas_eval.py index 29f7e026c..251ab71c0 100644 --- a/backend/src/ragas_eval.py +++ b/backend/src/ragas_eval.py @@ -17,6 +17,7 @@ load_dotenv() EMBEDDING_MODEL = os.getenv("RAGAS_EMBEDDING_MODEL") +logging.info(f"Loading embedding model '{EMBEDDING_MODEL}' for ragas evaluation") EMBEDDING_FUNCTION, _ = load_embedding_model(EMBEDDING_MODEL) def get_ragas_metrics(question: str, context: list, answer: list, model: str): diff --git a/backend/test_integrationqa.py b/backend/test_integrationqa.py index ede8077f7..03d0d470b 100644 --- a/backend/test_integrationqa.py +++ b/backend/test_integrationqa.py @@ -13,7 +13,8 @@ from src.ragas_eval import get_ragas_metrics from datasets import Dataset from ragas import evaluate -from ragas.metrics import answer_relevancy, context_utilization, faithfulness +# from ragas.metrics import answer_relevancy, context_utilization, faithfulness +# from ragas.dataset_schema import SingleTurnSample # Load environment variables if needed load_dotenv() import os @@ -53,19 +54,18 @@ def delete_extracted_files(file_path): logging.error(f"Failed to delete file {file_path}. Error: {e}") def test_graph_from_file_local(model_name): - """Test graph creation from a local file.""" - file_name = 'About Amazon.pdf' - shutil.copyfile('/workspaces/llm-graph-builder/backend/files/About Amazon.pdf', - os.path.join(MERGED_DIR, file_name)) - - create_source_node_local(graph, model_name, file_name) - merged_file_path = os.path.join(MERGED_DIR, file_name) - - local_file_result = asyncio.run(extract_graph_from_file_local_file( - URI, USERNAME, PASSWORD, DATABASE, model_name, merged_file_path, file_name, '', '',None)) - logging.info("Local file processing complete") - print(local_file_result) - return local_file_result + """Test graph creation from a local file.""" + try: + file_name = 'About Amazon.pdf' + shutil.copyfile('/workspaces/llm-graph-builder/backend/files/About Amazon.pdf',os.path.join(MERGED_DIR, file_name)) + create_source_node_local(graph, model_name, file_name) + merged_file_path = os.path.join(MERGED_DIR, file_name) + local_file_result = asyncio.run(extract_graph_from_file_local_file(URI, USERNAME, PASSWORD, DATABASE, model_name, merged_file_path, file_name, '', '',None)) + logging.info("Local file processing complete") + print(local_file_result) + return local_file_result + except Exception as e: + logging.error(f"Failed to delete file. Error: {e}") # try: # assert local_file_result['status'] == 'Completed' @@ -81,88 +81,68 @@ def test_graph_from_file_local(model_name): #return local_file_result def test_graph_from_wikipedia(model_name): - # try: + try: """Test graph creation from a Wikipedia page.""" - wiki_query = 'https://en.wikipedia.org/wiki/Berkshire_Hathaway' + wiki_query = 'https://en.wikipedia.org/wiki/Apollo_program' source_type = 'Wikipedia' - file_name = "Berkshire_Hathaway" + file_name = "Apollo_program" create_source_node_graph_url_wikipedia(graph, model_name, wiki_query, source_type) - wiki_result = asyncio.run(extract_graph_from_file_Wikipedia(URI, USERNAME, PASSWORD, DATABASE, model_name, file_name, 'en',file_name, '', '',None)) + wiki_result = asyncio.run(extract_graph_from_file_Wikipedia(URI, USERNAME, PASSWORD, DATABASE, model_name, wiki_query, 'en',file_name, '', '',None)) logging.info("Wikipedia test done") print(wiki_result) - try: - assert wiki_result['status'] == 'Completed' - assert wiki_result['nodeCount'] > 0 - assert wiki_result['relationshipCount'] > 0 - print("Success") - except AssertionError as e: - print("Fail: ", e) + # try: + # assert wiki_result['status'] == 'Completed' + # assert wiki_result['nodeCount'] > 0 + # assert wiki_result['relationshipCount'] > 0 + # print("Success") + # except AssertionError as e: + # print("Fail: ", e) return wiki_result - # except Exception as ex: - # print(ex) - -def test_graph_website(model_name): - """Test graph creation from a Website page.""" - #graph, model, source_url, source_type - source_url = 'https://www.amazon.com/' - source_type = 'web-url' - file_name = [] - create_source_node_graph_web_url(graph, model_name, source_url, source_type) - - weburl_result = asyncio.run(extract_graph_from_web_page(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url,file_name, '', '',None)) - logging.info("WebUrl test done") - print(weburl_result) - - try: - assert weburl_result['status'] == 'Completed' - assert weburl_result['nodeCount'] > 0 - assert weburl_result['relationshipCount'] > 0 - print("Success") - except AssertionError as e: - print("Fail: ", e) - return weburl_result + except Exception as ex: + print('Hello error herte') + print(ex) def test_graph_website(model_name): """Test graph creation from a Website page.""" #graph, model, source_url, source_type - source_url = 'https://www.amazon.com/' + source_url = 'https://www.cloudskillsboost.google/' source_type = 'web-url' + file_name = 'Google Cloud Skills Boost' + # file_name = [] create_source_node_graph_web_url(graph, model_name, source_url, source_type) - weburl_result = extract_graph_from_web_page(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, '', '') + weburl_result = asyncio.run(extract_graph_from_web_page(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url,file_name, '', '',None)) logging.info("WebUrl test done") print(weburl_result) - try: - assert weburl_result['status'] == 'Completed' - assert weburl_result['nodeCount'] > 0 - assert weburl_result['relationshipCount'] > 0 - print("Success") - except AssertionError as e: - print("Fail: ", e) + # try: + # assert weburl_result['status'] == 'Completed' + # assert weburl_result['nodeCount'] > 0 + # assert weburl_result['relationshipCount'] > 0 + # print("Success") + # except AssertionError as e: + # print("Fail: ", e) return weburl_result - def test_graph_from_youtube_video(model_name): """Test graph creation from a YouTube video.""" source_url = 'https://www.youtube.com/watch?v=T-qy-zPWgqA' + file_name = 'NKc8Tr5_L3w' source_type = 'youtube' create_source_node_graph_url_youtube(graph, model_name, source_url, source_type) - youtube_result = extract_graph_from_file_youtube( - URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, '', '' - ) + youtube_result = asyncio.run(extract_graph_from_file_youtube(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url,file_name,'','',None)) logging.info("YouTube Video test done") print(youtube_result) - try: - assert youtube_result['status'] == 'Completed' - assert youtube_result['nodeCount'] > 1 - assert youtube_result['relationshipCount'] > 1 - print("Success") - except AssertionError as e: - print("Failed: ", e) +# try: +# assert youtube_result['status'] == 'Completed' +# assert youtube_result['nodeCount'] > 1 +# assert youtube_result['relationshipCount'] > 1 +# print("Success") +# except AssertionError as e: +# print("Failed: ", e) return youtube_result @@ -218,7 +198,7 @@ def get_duplicate_nodes(): #Test populate_graph_schema def test_populate_graph_schema_from_text(model_name): - schema_text =('Amazon was founded on July 5, 1994, by Jeff Bezos in Bellevue, Washington.The company originally started as an online marketplace for books but gradually expanded its offerings to include a wide range of product categories. This diversification led to it being referred.') + schema_text =('Amazon was founded on July 5, 1994, by Jeff Bezos in Bellevue, Washington.The company originally started as an online marketplace for books but gradually expanded its offerings to include a wide range of product categories. This diversification led to it being referred.') #result_schema='' try: result_schema = populate_graph_schema_from_text(schema_text, model_name, True) @@ -232,7 +212,7 @@ def run_tests(): final_list = [] error_list = [] - models = ['openai_gpt_3_5','openai_gpt_4o','openai_gpt_4o_mini','azure-ai-gpt-35','azure-ai-gpt-4o','gemini_1_5_pro','gemini_1_5_flash','anthropic-claude-3-5-sonnet','bedrock-claude-3-5-sonnet','groq-llama3-70b','fireworks-llama-v3-70b'] + models = ['openai_gpt_4','openai_gpt_4o','openai_gpt_4o_mini','gemini_1.5_pro','gemini_1.5_flash'] for model_name in models: try: @@ -256,7 +236,7 @@ def run_tests(): dis_elementid, dis_status = disconected_nodes() lst_element_id = [dis_elementid] delt = delete_disconected_nodes(lst_element_id) -# dup = get_duplicate_nodes() + dup = get_duplicate_nodes() print(final_list) schma = test_populate_graph_schema_from_text(model_name) # Save final results to CSV @@ -265,7 +245,7 @@ def run_tests(): df['execution_date'] = dt.today().strftime('%Y-%m-%d') #diconnected nodes df['disconnected_nodes']=dis_status -# df['get_duplicate_nodes']=dup + df['get_duplicate_nodes']=dup df['delete_disconected_nodes']=delt df['test_populate_graph_schema_from_text'] = schma diff --git a/frontend/src/App.css b/frontend/src/App.css index a20f7d351..bbc0fc44f 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -1,84 +1,7 @@ .filetable { - /* width: calc(-360px + 100dvw); */ - height: calc(-350px + 100dvh); border: 1px solid #d1d5db; + width: 95% !important; } - -.fileTableWithExpansion { - width: calc(-640px + 100dvw) !important; - height: calc(-350px + 100dvh); - border: 1px solid #d1d5db; -} - -.fileTableWithBothDrawers { - width: calc(-650px + 100dvw) !important; - height: calc(-350px + 100dvh); - border: 1px solid #d1d5db; -} - -.fileTableWithExpansion .ndl-div-table { - width: max-content !important; -} - -.filetable .ndl-div-table { - width: max-content !important; -} - -.contentWithExpansion { - width: calc(-807px + 100dvw); - height: calc(100dvh - 58px); - padding: 3px; - display: flex; - flex-direction: column; - align-items: center; - gap: 5px; - position: relative; -} - -.contentWithOneExpansion { - width: calc(-550px + 100dvw); - height: calc(100dvh - 58px); - padding: 3px; - display: flex; - flex-direction: column; - align-items: center; - gap: 5px; - position: relative; -} - -.contentWithBothDrawers { - width: calc(-786px + 100dvw); - height: calc(100dvh - 58px); - padding: 3px; - display: flex; - flex-direction: column; - align-items: center; - gap: 5px; - position: relative; -} - -.contentWithChatBot { - width: calc(-512px + 100dvw); - height: calc(100dvh - 58px); - padding: 3px; - display: flex; - flex-direction: column; - align-items: center; - gap: 5px; - position: relative; -} - -.contentWithDropzoneExpansion { - width: calc(100% - 422px); - height: calc(100dvh - 58px); - padding: 3px; - display: flex; - flex-direction: column; - align-items: center; - gap: 5px; - position: relative; -} - .s3Container { display: flex; align-items: center; @@ -409,3 +332,27 @@ opacity: 0.8; background-color: rgb(201 201 201 / 40%); } +.layout-wrapper{ + display: grid; + grid-template-rows: auto; + grid-template-columns: 64px 1fr minmax(min-content,4fr) 1.4fr 64px; + max-height: calc(100vh - 58px); + max-width: 100%; +} +.layout-wrapper.drawerdropzoneclosed{ + grid-template-columns: 64px 4fr 1.1fr 64px; + +} +.layout-wrapper.drawerchatbotclosed{ + grid-template-columns: 64px 0.6fr minmax(min-content,4fr) 64px; +} +.layout-wrapper.drawerclosed{ + grid-template-columns: 64px minmax(min-content,4fr) 64px; +} +.main-content-wrapper{ + display: flex; + flex-direction: column; + max-width: 100%; + position: relative; + height: 100%; +} diff --git a/frontend/src/components/BreakDownPopOver.tsx b/frontend/src/components/BreakDownPopOver.tsx index f6798bf97..46654a913 100644 --- a/frontend/src/components/BreakDownPopOver.tsx +++ b/frontend/src/components/BreakDownPopOver.tsx @@ -1,5 +1,4 @@ import CustomPopOver from './UI/CustomPopOver'; -import { IconButton } from '@neo4j-ndl/react'; import { InformationCircleIconOutline } from '@neo4j-ndl/react/icons'; import { CustomFileBase } from '../types'; import { useCredentials } from '../context/UserCredentials'; @@ -8,13 +7,7 @@ export default function BreakDownPopOver({ file, isNodeCount = true }: { file: C const { isGdsActive } = useCredentials(); return ( - - - - } - > + }> {isNodeCount ? (
  • Chunk Nodes: {file.chunkNodeCount}
    • diff --git a/frontend/src/components/ChatBot/ChatInfoModal.tsx b/frontend/src/components/ChatBot/ChatInfoModal.tsx index ab66a52f1..97a244b32 100644 --- a/frontend/src/components/ChatBot/ChatInfoModal.tsx +++ b/frontend/src/components/ChatBot/ChatInfoModal.tsx @@ -393,7 +393,7 @@ const ChatInfoModal: React.FC = ({ Currently ragas evaluation works on{' '} {supportedLLmsForRagas.map((s, idx) => ( - + {capitalizeWithUnderscore(s) + (idx != supportedLLmsForRagas.length - 1 ? ',' : '')} ))} diff --git a/frontend/src/components/ChatBot/EntitiesInfo.tsx b/frontend/src/components/ChatBot/EntitiesInfo.tsx index ed6029d68..3c1626cef 100644 --- a/frontend/src/components/ChatBot/EntitiesInfo.tsx +++ b/frontend/src/components/ChatBot/EntitiesInfo.tsx @@ -93,11 +93,11 @@ const EntitiesInfo: FC = ({ loading, mode, graphonly_entities, in
)) - : sortedLabels.map((label, index) => { + : sortedLabels.map((label) => { const entity = groupedEntities[label == 'undefined' ? 'Entity' : label]; return (
  • diff --git a/frontend/src/components/Content.tsx b/frontend/src/components/Content.tsx index 7fcd12b03..cbd7875f7 100644 --- a/frontend/src/components/Content.tsx +++ b/frontend/src/components/Content.tsx @@ -4,7 +4,15 @@ import { Button, Typography, Flex, StatusIndicator, useMediaQuery } from '@neo4j import { useCredentials } from '../context/UserCredentials'; import { useFileContext } from '../context/UsersFiles'; import { extractAPI } from '../utils/FileAPI'; -import { BannerAlertProps, ChildRef, ContentProps, CustomFile, OptionType, UserCredentials, chunkdata } from '../types'; +import { + BannerAlertProps, + ContentProps, + CustomFile, + OptionType, + UserCredentials, + chunkdata, + FileTableHandle, +} from '../types'; import deleteAPI from '../services/DeleteFiles'; import { postProcessing } from '../services/PostProcessing'; import { triggerStatusUpdateAPI } from '../services/ServerSideStatusUpdateAPI'; @@ -35,14 +43,13 @@ import { useMessageContext } from '../context/UserMessages'; import PostProcessingToast from './Popups/GraphEnhancementDialog/PostProcessingCheckList/PostProcessingToast'; import { getChunkText } from '../services/getChunkText'; import ChunkPopUp from './Popups/ChunkPopUp'; +import { isExpired, isFileReadyToProcess } from '../utils/Utils'; const ConfirmationDialog = lazy(() => import('./Popups/LargeFilePopUp/ConfirmationDialog')); let afterFirstRender = false; const Content: React.FC = ({ - isLeftExpanded, - isRightExpanded, isSchema, setIsSchema, showEnhancementDialog, @@ -57,8 +64,10 @@ const Content: React.FC = ({ const [openGraphView, setOpenGraphView] = useState(false); const [inspectedName, setInspectedName] = useState(''); const [documentName, setDocumentName] = useState(''); - const { setUserCredentials, userCredentials, setConnectionStatus, isGdsActive, isReadOnlyUser } = useCredentials(); + const { setUserCredentials, userCredentials, setConnectionStatus, isGdsActive, isReadOnlyUser, isGCSActive } = + useCredentials(); const [showConfirmationModal, setshowConfirmationModal] = useState(false); + const [showExpirationModal, setshowExpirationModal] = useState(false); const [extractLoading, setextractLoading] = useState(false); const [retryFile, setRetryFile] = useState(''); const [retryLoading, setRetryLoading] = useState(false); @@ -106,7 +115,7 @@ const Content: React.FC = ({ showErrorToast(`${fileName} Failed to process`); } ); - const childRef = useRef(null); + const childRef = useRef(null); const incrementPage = async () => { setCurrentPage((prev) => prev + 1); @@ -267,7 +276,7 @@ const Content: React.FC = ({ fileItem.retryOption ?? '', fileItem.sourceUrl, localStorage.getItem('accesskey'), - localStorage.getItem('secretkey'), + atob(localStorage.getItem('secretkey') ?? ''), fileItem.name ?? '', fileItem.gcsBucket ?? '', fileItem.gcsBucketFolder ?? '', @@ -520,15 +529,6 @@ const Content: React.FC = ({ window.open(replacedUrl, '_blank'); }; - const classNameCheck = - isLeftExpanded && isRightExpanded - ? 'contentWithExpansion' - : isRightExpanded - ? 'contentWithChatBot' - : !isLeftExpanded && !isRightExpanded - ? 'w-[calc(100%-128px)]' - : 'contentWithDropzoneExpansion'; - const handleGraphView = () => { setOpenGraphView(true); setViewPoint('showGraphView'); @@ -671,35 +671,25 @@ const Content: React.FC = ({ const onClickHandler = () => { const selectedRows = childRef.current?.getSelectedRows(); if (selectedRows?.length) { - let selectedLargeFiles: CustomFile[] = []; - for (let index = 0; index < selectedRows.length; index++) { - const parsedData: CustomFile = selectedRows[index]; - if ( - parsedData.fileSource === 'local file' && - typeof parsedData.size === 'number' && - (parsedData.status === 'New' || parsedData.status == 'Ready to Reprocess') && - parsedData.size > largeFileSize - ) { - selectedLargeFiles.push(parsedData); - } - } - if (selectedLargeFiles.length) { + const expiredFilesExists = selectedRows.some( + (c) => isFileReadyToProcess(c, true) && isExpired(c?.createdAt as Date) + ); + const largeFileExists = selectedRows.some( + (c) => isFileReadyToProcess(c, true) && typeof c.size === 'number' && c.size > largeFileSize + ); + if (expiredFilesExists) { setshowConfirmationModal(true); + } else if (largeFileExists && isGCSActive) { + setshowExpirationModal(true); } else { - handleGenerateGraph(selectedRows.filter((f) => f.status === 'New' || f.status === 'Ready to Reprocess')); + handleGenerateGraph(selectedRows.filter((f) => isFileReadyToProcess(f, false))); } } else if (filesData.length) { - const largefiles = filesData.filter((f) => { - if ( - typeof f.size === 'number' && - (f.status === 'New' || f.status == 'Ready to Reprocess') && - f.size > largeFileSize - ) { - return true; - } - return false; - }); - const selectAllNewFiles = filesData.filter((f) => f.status === 'New' || f.status === 'Ready to Reprocess'); + const expiredFileExists = filesData.some((c) => isFileReadyToProcess(c, true) && isExpired(c.createdAt as Date)); + const largeFileExists = filesData.some( + (c) => isFileReadyToProcess(c, true) && typeof c.size === 'number' && c.size > largeFileSize + ); + const selectAllNewFiles = filesData.filter((f) => isFileReadyToProcess(f, false)); const stringified = selectAllNewFiles.reduce((accu, f) => { const key = f.id; // @ts-ignore @@ -707,10 +697,12 @@ const Content: React.FC = ({ return accu; }, {}); setRowSelection(stringified); - if (largefiles.length) { + if (largeFileExists) { setshowConfirmationModal(true); + } else if (expiredFileExists && isGCSActive) { + setshowExpirationModal(true); } else { - handleGenerateGraph(filesData.filter((f) => f.status === 'New' || f.status === 'Ready to Reprocess')); + handleGenerateGraph(filesData.filter((f) => isFileReadyToProcess(f, false))); } } }; @@ -757,6 +749,19 @@ const Content: React.FC = ({ > )} + {showExpirationModal && filesForProcessing.length && ( + }> + setshowExpirationModal(false)} + loading={extractLoading} + selectedRows={childRef.current?.getSelectedRows() as CustomFile[]} + isLargeDocumentAlert={false} + > + + )} {showDeletePopUp && ( = ({ {showEnhancementDialog && ( )} -
    - + +
    +
    Neo4j connection {isReadOnlyUser ? '(Read only Mode)' : ''} @@ -847,7 +865,6 @@ const Content: React.FC = ({
    { @@ -874,21 +891,21 @@ const Content: React.FC = ({ handleGenerateGraph={processWaitingFilesOnRefresh} > - - +
    + +
    + = ({
    - ); }; diff --git a/frontend/src/components/DataSources/AWS/S3Modal.tsx b/frontend/src/components/DataSources/AWS/S3Modal.tsx index 0e9b2f49f..6ccf03c60 100644 --- a/frontend/src/components/DataSources/AWS/S3Modal.tsx +++ b/frontend/src/components/DataSources/AWS/S3Modal.tsx @@ -54,7 +54,7 @@ const S3Modal: React.FC = ({ hideModal, open }) => { localStorage.setItem('accesskey', accessKey); } if (accessKey.length) { - localStorage.setItem('secretkey', secretKey); + localStorage.setItem('secretkey', btoa(secretKey)); } if (isValid && accessKey.trim() != '' && secretKey.trim() != '') { try { diff --git a/frontend/src/components/DataSources/Local/DropZone.tsx b/frontend/src/components/DataSources/Local/DropZone.tsx index 209573eaf..4a5d92efe 100644 --- a/frontend/src/components/DataSources/Local/DropZone.tsx +++ b/frontend/src/components/DataSources/Local/DropZone.tsx @@ -40,6 +40,7 @@ const DropZone: FunctionComponent = () => { entityEntityRelCount: 0, communityNodeCount: 0, communityRelCount: 0, + createdAt: new Date(), }; const copiedFilesData: CustomFile[] = [...filesData]; @@ -186,6 +187,7 @@ const DropZone: FunctionComponent = () => { ...curfile, status: 'New', uploadProgress: 100, + createdAt: new Date(), }; } return curfile; @@ -217,7 +219,7 @@ const DropZone: FunctionComponent = () => { text={ - Microsoft Office (.docx, .pptx, .xls) + Microsoft Office (.docx, .pptx, .xls, .xlsx) PDF (.pdf) Images (.jpeg, .jpg, .png, .svg) Text (.html, .txt , .md) @@ -241,6 +243,7 @@ const DropZone: FunctionComponent = () => { 'application/vnd.ms-powerpoint': ['.pptx'], 'application/vnd.ms-excel': ['.xls'], 'text/markdown': ['.md'], + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'], }, onDrop: (f: Partial[]) => { onDropHandler(f); diff --git a/frontend/src/components/DataSources/Local/DropZoneForSmallLayouts.tsx b/frontend/src/components/DataSources/Local/DropZoneForSmallLayouts.tsx index f3fbff852..2305a614e 100644 --- a/frontend/src/components/DataSources/Local/DropZoneForSmallLayouts.tsx +++ b/frontend/src/components/DataSources/Local/DropZoneForSmallLayouts.tsx @@ -118,6 +118,7 @@ export default function DropZoneForSmallLayouts() { ...curfile, status: 'New', uploadprogess: 100, + createdAt: new Date(), }; } return curfile; @@ -141,6 +142,7 @@ export default function DropZoneForSmallLayouts() { 'application/vnd.ms-powerpoint': ['.pptx'], 'application/vnd.ms-excel': ['.xls'], 'text/markdown': ['.md'], + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'], }, onDrop: (f: Partial[]) => { onDropHandler(f); @@ -175,6 +177,7 @@ export default function DropZoneForSmallLayouts() { entityEntityRelCount: 0, communityNodeCount: 0, communityRelCount: 0, + createdAt: new Date(), }; const copiedFilesData: CustomFile[] = [...filesData]; diff --git a/frontend/src/components/FileTable.tsx b/frontend/src/components/FileTable.tsx index 01fc0b0be..7c3e74bd0 100644 --- a/frontend/src/components/FileTable.tsx +++ b/frontend/src/components/FileTable.tsx @@ -9,8 +9,18 @@ import { Typography, useCopyToClipboard, Checkbox, + useMediaQuery, } from '@neo4j-ndl/react'; -import { forwardRef, useContext, useEffect, useImperativeHandle, useMemo, useRef, useState } from 'react'; +import { + forwardRef, + useContext, + useEffect, + useImperativeHandle, + useMemo, + useRef, + useState, + ForwardRefRenderFunction, +} from 'react'; import { useReactTable, getCoreRowModel, @@ -33,6 +43,7 @@ import { getFileSourceStatus, isProcessingFileValid, capitalizeWithUnderscore, + getParsedDate, } from '../utils/Utils'; import { SourceNode, CustomFile, FileTableProps, UserCredentials, statusupdate, ChildRef } from '../types'; import { useCredentials } from '../context/UserCredentials'; @@ -57,8 +68,8 @@ import BreakDownPopOver from './BreakDownPopOver'; let onlyfortheFirstRender = true; -const FileTable = forwardRef((props, ref) => { - const { isExpanded, connectionStatus, setConnectionStatus, onInspect, onRetry, onChunkView } = props; +const FileTable: ForwardRefRenderFunction = (props, ref) => { + const { connectionStatus, setConnectionStatus, onInspect, onRetry, onChunkView } = props; const { filesData, setFilesData, model, rowSelection, setRowSelection, setSelectedRows, setProcessedCount, queue } = useFileContext(); const { userCredentials, isReadOnlyUser } = useCredentials(); @@ -73,6 +84,7 @@ const FileTable = forwardRef((props, ref) => { const [_, copy] = useCopyToClipboard(); const { colorMode } = useContext(ThemeWrapperContext); const [copyRow, setCopyRow] = useState(false); + const largedesktops = useMediaQuery(`(min-width:1440px )`); const tableRef = useRef(null); @@ -761,6 +773,7 @@ const FileTable = forwardRef((props, ref) => { entityEntityRelCount: item.entityEntityRelCount ?? 0, communityNodeCount: item.communityNodeCount ?? 0, communityRelCount: item.communityRelCount ?? 0, + createdAt: item.createdAt != undefined ? getParsedDate(item?.createdAt) : undefined, }); } }); @@ -781,6 +794,7 @@ const FileTable = forwardRef((props, ref) => { } setIsLoading(false); } catch (error: any) { + console.log(error); if (error instanceof Error) { showErrorToast(error.message); } @@ -976,8 +990,6 @@ const FileTable = forwardRef((props, ref) => { [table] ); - const classNameCheck = isExpanded ? 'fileTableWithExpansion' : `filetable`; - useEffect(() => { setSelectedRows(table.getSelectedRowModel().rows.map((i) => i.id)); }, [table.getSelectedRowModel()]); @@ -986,53 +998,51 @@ const FileTable = forwardRef((props, ref) => { <> {filesData ? ( <> -
    - ( - ( + + ), + PaginationNumericButton: ({ isSelected, innerProps, ...restProps }) => { + return ( + - ), - PaginationNumericButton: ({ isSelected, innerProps, ...restProps }) => { - return ( - - ); - }, - }} - isKeyboardNavigable={false} - /> -
    + ); + }, + }} + isKeyboardNavigable={false} + /> ) : null} ); -}); +}; -export default FileTable; +export default forwardRef(FileTable); diff --git a/frontend/src/components/Graph/ResultOverview.tsx b/frontend/src/components/Graph/ResultOverview.tsx index 9fd691b0f..17fdbbc78 100644 --- a/frontend/src/components/Graph/ResultOverview.tsx +++ b/frontend/src/components/Graph/ResultOverview.tsx @@ -144,10 +144,10 @@ const ResultOverview: React.FunctionComponent = ({
    - {nodeCheck.map((nodeLabel, index) => ( + {nodeCheck.map((nodeLabel) => ( = ({ isExpanded, clearHistoryD return messages.some((msg) => msg.isTyping || msg.isLoading); }; return ( -
    +
    = ({ }) => { const { closeAlert, alertState } = useAlertContext(); const { isReadOnlyUser, isBackendConnected } = useCredentials(); + const largedesktops = useMediaQuery(`(min-width:1440px )`); const isYoutubeOnlyCheck = useMemo( () => APP_SOURCES?.includes('youtube') && !APP_SOURCES.includes('wiki') && !APP_SOURCES.includes('web'), @@ -40,7 +41,7 @@ const DrawerDropzone: React.FC = ({ ); return ( -
    +
    { setErrorMessage, setShowDisconnectButton, showDisconnectButton, + setIsGCSActive, } = useCredentials(); const { cancel } = useSpeechSynthesis(); @@ -97,6 +98,7 @@ const PageLayout: React.FC = () => { }); setGdsActive(parsedConnection.isgdsActive); setIsReadOnlyUser(parsedConnection.isReadOnlyUser); + setIsGCSActive(parsedConnection.isGCSActive); } else { console.error('Invalid parsed session data:', parsedConnection); } @@ -105,7 +107,8 @@ const PageLayout: React.FC = () => { } }; // To update credentials if environment values differ - const updateSessionIfNeeded = (envCredentials: UserCredentials, storedSession: string) => { + + const updateSessionIfNeeded = (envCredentials: any, storedSession: string) => { try { const storedCredentials = JSON.parse(storedSession); const isDiffCreds = @@ -115,6 +118,7 @@ const PageLayout: React.FC = () => { envCredentials.database !== storedCredentials.database; if (isDiffCreds) { setUserCredentials(envCredentials); + setIsGCSActive(envCredentials.isGCSActive ?? false); localStorage.setItem( 'neo4j.connection', JSON.stringify({ @@ -147,7 +151,9 @@ const PageLayout: React.FC = () => { database: connectionData.data.database, isReadonlyUser: !connectionData.data.write_access, isgdsActive: connectionData.data.gds_status, + isGCSActive: connectionData?.data?.gcs_file_cache === 'True', }; + setIsGCSActive(connectionData?.data?.gcs_file_cache === 'True'); if (session) { const updated = updateSessionIfNeeded(envCredentials, session); if (!updated) { @@ -168,6 +174,7 @@ const PageLayout: React.FC = () => { userDbVectorIndex: 384, isReadOnlyUser: envCredentials.isReadonlyUser, isgdsActive: envCredentials.isgdsActive, + isGCSActive: envCredentials.isGCSActive, }) ); setConnectionStatus(true); @@ -226,7 +233,7 @@ const PageLayout: React.FC = () => { }; return ( -
    + <> }> { chunksExistsWithDifferentEmbedding={openConnection.chunksExistsWithDifferentDimension} /> - - { @@ -267,46 +257,132 @@ const PageLayout: React.FC = () => { } }} > - setShowChatBot(true)} - isLeftExpanded={isLeftExpanded} - isRightExpanded={isRightExpanded} - showChatBot={showChatBot} - openTextSchema={() => { - setShowTextFromSchemaDialog({ triggeredFrom: 'schemadialog', show: true }); - }} - isSchema={isSchema} - setIsSchema={setIsSchema} - showEnhancementDialog={showEnhancementDialog} - toggleEnhancementDialog={toggleEnhancementDialog} - setOpenConnection={setOpenConnection} - showDisconnectButton={showDisconnectButton} - connectionStatus={connectionStatus} - /> - {showDrawerChatbot && ( - + {largedesktops ? ( +
    + + {isLeftExpanded && ( + + )} + setShowChatBot(true)} + showChatBot={showChatBot} + openTextSchema={() => { + setShowTextFromSchemaDialog({ triggeredFrom: 'schemadialog', show: true }); + }} + isSchema={isSchema} + setIsSchema={setIsSchema} + showEnhancementDialog={showEnhancementDialog} + toggleEnhancementDialog={toggleEnhancementDialog} + setOpenConnection={setOpenConnection} + showDisconnectButton={showDisconnectButton} + connectionStatus={connectionStatus} + /> + {isRightExpanded && ( + + )} + +
    + ) : ( + <> + + +
    + + + setShowChatBot(true)} + showChatBot={showChatBot} + openTextSchema={() => { + setShowTextFromSchemaDialog({ triggeredFrom: 'schemadialog', show: true }); + }} + isSchema={isSchema} + setIsSchema={setIsSchema} + showEnhancementDialog={showEnhancementDialog} + toggleEnhancementDialog={toggleEnhancementDialog} + setOpenConnection={setOpenConnection} + showDisconnectButton={showDisconnectButton} + connectionStatus={connectionStatus} + /> + {isRightExpanded && ( + + )} + +
    + )} - -
    + ); }; diff --git a/frontend/src/components/Popups/ChunkPopUp/index.tsx b/frontend/src/components/Popups/ChunkPopUp/index.tsx index c0e2a016b..054d77dca 100644 --- a/frontend/src/components/Popups/ChunkPopUp/index.tsx +++ b/frontend/src/components/Popups/ChunkPopUp/index.tsx @@ -58,8 +58,8 @@ const ChunkPopUp = ({ ) : (
      - {sortedChunksData.map((c, idx) => ( -
    1. + {sortedChunksData.map((c) => ( +
    2. Position : diff --git a/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx b/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx index 683d1ddb8..2bc82735e 100644 --- a/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx +++ b/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx @@ -41,7 +41,8 @@ export default function ConnectionModal({ const [username, setUsername] = useState(initialusername ?? 'neo4j'); const [password, setPassword] = useState(''); const [connectionMessage, setMessage] = useState({ type: 'unknown', content: '' }); - const { setUserCredentials, userCredentials, setGdsActive, setIsReadOnlyUser, errorMessage } = useCredentials(); + const { setUserCredentials, userCredentials, setGdsActive, setIsReadOnlyUser, errorMessage, setIsGCSActive } = + useCredentials(); const [isLoading, setIsLoading] = useState(false); const [searchParams, setSearchParams] = useSearchParams(); const [userDbVectorIndex, setUserDbVectorIndex] = useState(initialuserdbvectorindex ?? undefined); @@ -215,8 +216,11 @@ export default function ConnectionModal({ } else { const isgdsActive = response.data.data.gds_status; const isReadOnlyUser = !response.data.data.write_access; + const isGCSActive = response.data.data.gcs_file_cache === 'True'; + setIsGCSActive(isGCSActive); setGdsActive(isgdsActive); setIsReadOnlyUser(isReadOnlyUser); + localStorage.setItem( 'neo4j.connection', JSON.stringify({ @@ -227,6 +231,7 @@ export default function ConnectionModal({ userDbVectorIndex, isgdsActive, isReadOnlyUser, + isGCSActive, }) ); setUserDbVectorIndex(response.data.data.db_vector_dimension); diff --git a/frontend/src/components/Popups/ExpirationModal/ExpiredFilesAlert.tsx b/frontend/src/components/Popups/ExpirationModal/ExpiredFilesAlert.tsx new file mode 100644 index 000000000..23738820c --- /dev/null +++ b/frontend/src/components/Popups/ExpirationModal/ExpiredFilesAlert.tsx @@ -0,0 +1,74 @@ +import { Checkbox, Flex, Typography } from '@neo4j-ndl/react'; +import { DocumentTextIconOutline } from '@neo4j-ndl/react/icons'; +import { LargefilesProps } from '../../../types'; +import { List, ListItem, ListItemAvatar, ListItemButton, ListItemIcon, ListItemText } from '@mui/material'; +import { FC } from 'react'; +import BellImage from '../../../assets/images/Stopwatch-blue.svg'; +import AlertIcon from '../../Layout/AlertIcon'; +import { isExpired } from '../../../utils/Utils'; +import { EXPIRATION_DAYS } from '../../../utils/Constants'; + +const ExpiredFilesAlert: FC = ({ Files, handleToggle, checked }) => { + return ( +
      +
      + alert icon +
      + Document Expiration Notice + + One or more of your selected documents are expired as per the expiration policy we are storing documents for + only {EXPIRATION_DAYS} days . Please delete and reupload the documents. + + + {Files.map((f) => { + return ( + + + + { + if (e.target.checked) { + handleToggle(true, f.id); + } else { + handleToggle(false, f.id); + } + }} + isChecked={checked.indexOf(f.id) !== -1} + htmlAttributes={{ tabIndex: -1 }} + /> + + + + + + + {f.name} - {Math.floor((f?.size as number) / 1000)?.toFixed(2)}KB + + {f.createdAt != undefined && isExpired(f.createdAt) ? ( + + + + ) : ( + <> + )} + + } + /> + + + ); + })} + +
      +
      +
      + ); +}; +export default ExpiredFilesAlert; diff --git a/frontend/src/components/Popups/GraphEnhancementDialog/DeleteTabForOrphanNodes/index.tsx b/frontend/src/components/Popups/GraphEnhancementDialog/DeleteTabForOrphanNodes/index.tsx index de8cdc186..1fae795b1 100644 --- a/frontend/src/components/Popups/GraphEnhancementDialog/DeleteTabForOrphanNodes/index.tsx +++ b/frontend/src/components/Popups/GraphEnhancementDialog/DeleteTabForOrphanNodes/index.tsx @@ -141,8 +141,8 @@ export default function DeletePopUpForOrphanNodes({ cell: (info) => { return ( - {info.getValue().map((l, index) => ( - + {info.getValue().map((l) => ( + ))} ); diff --git a/frontend/src/components/Popups/LargeFilePopUp/ConfirmationDialog.tsx b/frontend/src/components/Popups/LargeFilePopUp/ConfirmationDialog.tsx index c0159be65..7762d0d77 100644 --- a/frontend/src/components/Popups/LargeFilePopUp/ConfirmationDialog.tsx +++ b/frontend/src/components/Popups/LargeFilePopUp/ConfirmationDialog.tsx @@ -3,6 +3,8 @@ import { CustomFile } from '../../../types'; import LargeFilesAlert from './LargeFilesAlert'; import { memo, useEffect, useState } from 'react'; import { useFileContext } from '../../../context/UsersFiles'; +import ExpiredFilesAlert from '../ExpirationModal/ExpiredFilesAlert'; +import { isExpired } from '../../../utils/Utils'; function ConfirmationDialog({ largeFiles, @@ -11,6 +13,7 @@ function ConfirmationDialog({ loading, extractHandler, selectedRows, + isLargeDocumentAlert = true, }: { largeFiles: CustomFile[]; open: boolean; @@ -18,6 +21,7 @@ function ConfirmationDialog({ loading: boolean; extractHandler: (selectedFilesFromAllfiles: CustomFile[]) => void; selectedRows: CustomFile[]; + isLargeDocumentAlert?: boolean; }) { const { setSelectedRows, filesData, setRowSelection } = useFileContext(); const [checked, setChecked] = useState([...largeFiles.map((f) => f.id)]); @@ -82,8 +86,10 @@ function ConfirmationDialog({ {largeFiles.length === 0 && loading ? ( Files are under processing + ) : isLargeDocumentAlert ? ( + ) : ( - + )} @@ -106,6 +112,9 @@ function ConfirmationDialog({ onClose(); }} size='large' + isDisabled={largeFiles.some( + (f) => f.createdAt != undefined && checked.includes(f.id) && isExpired(f?.createdAt as Date) + )} > Continue diff --git a/frontend/src/components/Popups/LargeFilePopUp/LargeFilesAlert.tsx b/frontend/src/components/Popups/LargeFilePopUp/LargeFilesAlert.tsx index b8dfa0251..a107dcb2f 100644 --- a/frontend/src/components/Popups/LargeFilePopUp/LargeFilesAlert.tsx +++ b/frontend/src/components/Popups/LargeFilePopUp/LargeFilesAlert.tsx @@ -16,7 +16,7 @@ import s3logo from '../../../assets/images/s3logo.png'; import { calculateProcessingTime } from '../../../utils/Utils'; import { ThemeWrapperContext } from '../../../context/ThemeWrapper'; -const LargeFilesAlert: FC = ({ largeFiles, handleToggle, checked }) => { +const LargeFilesAlert: FC = ({ Files, handleToggle, checked }) => { const { colorMode } = useContext(ThemeWrapperContext); const imageIcon: Record = useMemo( @@ -44,7 +44,7 @@ const LargeFilesAlert: FC = ({ largeFiles, handleToggle, checke estimated times below - {largeFiles.map((f, i) => { + {Files.map((f, i) => { const { minutes, seconds } = calculateProcessingTime(f.size as number, 0.2); return ( diff --git a/frontend/src/components/UI/CustomMenu.tsx b/frontend/src/components/UI/CustomMenu.tsx index fa33c368d..65aa58801 100644 --- a/frontend/src/components/UI/CustomMenu.tsx +++ b/frontend/src/components/UI/CustomMenu.tsx @@ -21,9 +21,9 @@ export default function CustomMenu({ }) { return ( - {items?.map((i, idx) => ( + {items?.map((i) => ( { } return ( - + {children} diff --git a/frontend/src/context/UserCredentials.tsx b/frontend/src/context/UserCredentials.tsx index 29eee0c14..530b5a865 100644 --- a/frontend/src/context/UserCredentials.tsx +++ b/frontend/src/context/UserCredentials.tsx @@ -20,6 +20,8 @@ export const UserConnection = createContext({ setErrorMessage: () => null, showDisconnectButton: false, setShowDisconnectButton: () => null, + isGCSActive: false, + setIsGCSActive: () => null, }); export const useCredentials = () => { const userCredentials = useContext(UserConnection); @@ -33,6 +35,8 @@ const UserCredentialsWrapper: FunctionComponent = (props) => { const [isBackendConnected, setIsBackendConnected] = useState(false); const [errorMessage, setErrorMessage] = useState(''); const [showDisconnectButton, setShowDisconnectButton] = useState(false); + const [isGCSActive, setIsGCSActive] = useState(false); + const value = { userCredentials, setUserCredentials, @@ -48,6 +52,8 @@ const UserCredentialsWrapper: FunctionComponent = (props) => { setErrorMessage, showDisconnectButton, setShowDisconnectButton, + isGCSActive, + setIsGCSActive, }; return {props.children}; diff --git a/frontend/src/services/ConnectAPI.ts b/frontend/src/services/ConnectAPI.ts index c802a5946..7432dd83a 100644 --- a/frontend/src/services/ConnectAPI.ts +++ b/frontend/src/services/ConnectAPI.ts @@ -21,7 +21,7 @@ const connectAPI = async (connectionURI: string, username: string, password: str const envConnectionAPI = async () => { try { - const conectionUrl = `/backend_connection_configuation`; + const conectionUrl = `/backend_connection_configuration`; const response = await api.post(conectionUrl); return response; } catch (error) { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 1c782103e..367739f9e 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -6,6 +6,7 @@ import type { Node, Relationship } from '@neo4j-nvl/base'; import { NonOAuthError } from '@react-oauth/google'; import { BannerType } from '@neo4j-ndl/react'; import Queue from './utils/Queue'; +import FileTable from './components/FileTable'; export interface CustomFileBase extends Partial { processingTotalTime: number | string; @@ -34,6 +35,7 @@ export interface CustomFileBase extends Partial { entityEntityRelCount: number; communityNodeCount: number; communityRelCount: number; + createdAt?: Date; } export interface CustomFile extends CustomFileBase { id: string; @@ -51,7 +53,7 @@ export type UserCredentials = { database: string; } & { [key: string]: any }; -export interface SourceNode extends Omit { +export interface SourceNode extends Omit { fileName: string; fileSize: number; fileType: string; @@ -65,6 +67,7 @@ export interface SourceNode extends Omit { processed_chunk?: number; total_chunks?: number; retry_condition?: string; + createdAt: filedate; } export type ExtractParams = Pick & { @@ -143,8 +146,6 @@ export interface DrawerProps { } export interface ContentProps { - isLeftExpanded: boolean; - isRightExpanded: boolean; showChatBot: boolean; openChatBot: () => void; openTextSchema: () => void; @@ -158,7 +159,6 @@ export interface ContentProps { } export interface FileTableProps { - isExpanded: boolean; connectionStatus: boolean; setConnectionStatus: Dispatch>; onInspect: (id: string) => void; @@ -580,7 +580,7 @@ export interface UploadResponse extends Partial { data: uploadData; } export interface LargefilesProps { - largeFiles: CustomFile[]; + Files: CustomFile[]; handleToggle: (ischecked: boolean, id: string) => void; checked: string[]; } @@ -758,6 +758,8 @@ export interface ContextProps { setErrorMessage: Dispatch>; showDisconnectButton: boolean; setShowDisconnectButton: Dispatch>; + isGCSActive: boolean; + setIsGCSActive: Dispatch>; } export interface MessageContextType { messages: Messages[] | []; @@ -773,11 +775,6 @@ export interface GraphContextType { setLoadingGraph: Dispatch>; } -export interface GraphContextType { - loadingGraph: boolean; - setLoadingGraph: Dispatch>; -} - export interface DatabaseStatusProps { isConnected: boolean; isGdsActive: boolean; @@ -923,3 +920,21 @@ export interface GraphViewHandlerProps { export interface ChatProps { chatMessages: Messages[]; } + +export interface filedate { + _DateTime__date: { + _Date__ordinal: number; + _Date__year: number; + _Date__month: number; + _Date__day: number; + }; + _DateTime__time: { + _Time__ticks: number; + _Time__hour: number; + _Time__minute: number; + _Time__second: number; + _Time__nanosecond: number; + _Time__tzinfo: null; + }; +} +export type FileTableHandle = React.ElementRef; diff --git a/frontend/src/utils/Constants.ts b/frontend/src/utils/Constants.ts index 52ea23813..3a5702b91 100644 --- a/frontend/src/utils/Constants.ts +++ b/frontend/src/utils/Constants.ts @@ -368,3 +368,4 @@ export const metricsinfo: Record = { semantic_score: 'Determines How well the generated answer understands the meaning of the reference answer.', context_entity_recall: 'Determines the recall of entities present in both generated answer and retrieved contexts', }; +export const EXPIRATION_DAYS = 3; diff --git a/frontend/src/utils/Utils.ts b/frontend/src/utils/Utils.ts index 7314c55e7..206a2d097 100644 --- a/frontend/src/utils/Utils.ts +++ b/frontend/src/utils/Utils.ts @@ -5,6 +5,7 @@ import { Entity, ExtendedNode, ExtendedRelationship, + filedate, GraphType, Messages, Scheme, @@ -19,7 +20,7 @@ import youtubedarklogo from '../assets/images/youtube-darkmode.svg'; import youtubelightlogo from '../assets/images/youtube-lightmode.svg'; import s3logo from '../assets/images/s3logo.png'; import gcslogo from '../assets/images/gcs.webp'; -import { chatModeLables } from './Constants'; +import { chatModeLables, EXPIRATION_DAYS } from './Constants'; // Get the Url export const url = () => { @@ -522,3 +523,24 @@ export function getNodes(nodesData: Array EXPIRATION_DAYS; +} + +export function isFileReadyToProcess(file: CustomFile, withLocalCheck: boolean) { + if (withLocalCheck) { + return file.fileSource === 'local file' && (file.status === 'New' || file.status == 'Ready to Reprocess'); + } + return file.status === 'New' || file.status == 'Ready to Reprocess'; +} diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 82c021a0e..b5886d06f 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -613,6 +613,19 @@ resolved "https://registry.yarnpkg.com/@internationalized/string/-/string-3.2.5.tgz#2f387b256e79596a2e62ddd5e15c619fe241189c" integrity sha512-rKs71Zvl2OKOHM+mzAFMIyqR5hI1d1O6BBkMK2/lkfg3fkmVh9Eeg0awcA8W2WqYqDOv6a86DIOlFpggwLtbuw== dependencies: + "@swc/helpers" "^0.5.0" + +"@isaacs/cliui@^8.0.2": + version "8.0.2" + resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" + integrity sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA== + dependencies: + string-width "^5.1.2" + string-width-cjs "npm:string-width@^4.2.0" + strip-ansi "^7.0.1" + strip-ansi-cjs "npm:strip-ansi@^6.0.1" + wrap-ansi "^8.1.0" + wrap-ansi-cjs "npm:wrap-ansi@^7.0.0" "@jridgewell/gen-mapping@^0.3.2", "@jridgewell/gen-mapping@^0.3.5": version "0.3.5" @@ -2706,11 +2719,6 @@ comma-separated-tokens@^2.0.0: resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz#4e89c9458acb61bc8fef19f4529973b2392839ee" integrity sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg== -commander@7: - version "7.2.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" - integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== - commander@^4.0.0: version "4.1.1" resolved "https://registry.yarnpkg.com/commander/-/commander-4.1.1.tgz#9fd602bd936294e9e9ef46a3f4d6964044b18068" @@ -2817,90 +2825,12 @@ cytoscape@^3.23.0: resolved "https://registry.yarnpkg.com/cytoscape/-/cytoscape-3.30.2.tgz#94149707fb6547a55e3b44f03ffe232706212161" integrity sha512-oICxQsjW8uSaRmn4UK/jkczKOqTrVqt5/1WL0POiJUT2EKNc9STM4hYFHv917yu55aTBMFNRzymlJhVAiWPCxw== -"d3-array@2 - 3", "d3-array@2.10.0 - 3", "d3-array@2.5.0 - 3", d3-array@3, d3-array@^3.2.0: - version "3.2.4" - resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-3.2.4.tgz#15fec33b237f97ac5d7c986dc77da273a8ed0bb5" - integrity sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg== - dependencies: - internmap "1 - 2" - -d3-axis@3: - version "3.0.0" - resolved "https://registry.yarnpkg.com/d3-axis/-/d3-axis-3.0.0.tgz#c42a4a13e8131d637b745fc2973824cfeaf93322" - integrity sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw== - -d3-brush@3: - version "3.0.0" - resolved "https://registry.yarnpkg.com/d3-brush/-/d3-brush-3.0.0.tgz#6f767c4ed8dcb79de7ede3e1c0f89e63ef64d31c" - integrity sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ== - dependencies: - d3-dispatch "1 - 3" - d3-drag "2 - 3" - d3-interpolate "1 - 3" - d3-selection "3" - d3-transition "3" - -d3-chord@3: - version "3.0.1" - resolved "https://registry.yarnpkg.com/d3-chord/-/d3-chord-3.0.1.tgz#d156d61f485fce8327e6abf339cb41d8cbba6966" - integrity sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g== - dependencies: - d3-path "1 - 3" - -"d3-color@1 - 3", d3-color@3: - version "3.1.0" - resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-3.1.0.tgz#395b2833dfac71507f12ac2f7af23bf819de24e2" - integrity sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA== - -d3-contour@4: - version "4.0.2" - resolved "https://registry.yarnpkg.com/d3-contour/-/d3-contour-4.0.2.tgz#bb92063bc8c5663acb2422f99c73cbb6c6ae3bcc" - integrity sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA== - dependencies: - d3-array "^3.2.0" - -d3-delaunay@6: - version "6.0.4" - resolved "https://registry.yarnpkg.com/d3-delaunay/-/d3-delaunay-6.0.4.tgz#98169038733a0a5babbeda55054f795bb9e4a58b" - integrity sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A== - dependencies: - delaunator "5" - -"d3-dispatch@1 - 3", d3-dispatch@3: +"d3-dispatch@1 - 3": version "3.0.1" resolved "https://registry.yarnpkg.com/d3-dispatch/-/d3-dispatch-3.0.1.tgz#5fc75284e9c2375c36c839411a0cf550cbfc4d5e" integrity sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg== -"d3-drag@2 - 3", d3-drag@3: - version "3.0.0" - resolved "https://registry.yarnpkg.com/d3-drag/-/d3-drag-3.0.0.tgz#994aae9cd23c719f53b5e10e3a0a6108c69607ba" - integrity sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg== - dependencies: - d3-dispatch "1 - 3" - d3-selection "3" - -"d3-dsv@1 - 3", d3-dsv@3: - version "3.0.1" - resolved "https://registry.yarnpkg.com/d3-dsv/-/d3-dsv-3.0.1.tgz#c63af978f4d6a0d084a52a673922be2160789b73" - integrity sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q== - dependencies: - commander "7" - iconv-lite "0.6" - rw "1" - -"d3-ease@1 - 3", d3-ease@3: - version "3.0.1" - resolved "https://registry.yarnpkg.com/d3-ease/-/d3-ease-3.0.1.tgz#9658ac38a2140d59d346160f1f6c30fda0bd12f4" - integrity sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w== - -d3-fetch@3: - version "3.0.1" - resolved "https://registry.yarnpkg.com/d3-fetch/-/d3-fetch-3.0.1.tgz#83141bff9856a0edb5e38de89cdcfe63d0a60a22" - integrity sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw== - dependencies: - d3-dsv "1 - 3" - -d3-force@3, d3-force@^3.0.0: +d3-force@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/d3-force/-/d3-force-3.0.0.tgz#3e2ba1a61e70888fe3d9194e30d6d14eece155c4" integrity sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg== @@ -2909,158 +2839,16 @@ d3-force@3, d3-force@^3.0.0: d3-quadtree "1 - 3" d3-timer "1 - 3" -"d3-format@1 - 3", d3-format@3: - version "3.1.0" - resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-3.1.0.tgz#9260e23a28ea5cb109e93b21a06e24e2ebd55641" - integrity sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA== - -d3-geo@3: - version "3.1.1" - resolved "https://registry.yarnpkg.com/d3-geo/-/d3-geo-3.1.1.tgz#6027cf51246f9b2ebd64f99e01dc7c3364033a4d" - integrity sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q== - dependencies: - d3-array "2.5.0 - 3" - -d3-hierarchy@3: - version "3.1.2" - resolved "https://registry.yarnpkg.com/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz#b01cd42c1eed3d46db77a5966cf726f8c09160c6" - integrity sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA== - -"d3-interpolate@1 - 3", "d3-interpolate@1.2.0 - 3", d3-interpolate@3: - version "3.0.1" - resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-3.0.1.tgz#3c47aa5b32c5b3dfb56ef3fd4342078a632b400d" - integrity sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g== - dependencies: - d3-color "1 - 3" - -"d3-path@1 - 3", d3-path@3, d3-path@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/d3-path/-/d3-path-3.1.0.tgz#22df939032fb5a71ae8b1800d61ddb7851c42526" - integrity sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ== - -d3-polygon@3: - version "3.0.1" - resolved "https://registry.yarnpkg.com/d3-polygon/-/d3-polygon-3.0.1.tgz#0b45d3dd1c48a29c8e057e6135693ec80bf16398" - integrity sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg== - -"d3-quadtree@1 - 3", d3-quadtree@3: +"d3-quadtree@1 - 3": version "3.0.1" resolved "https://registry.yarnpkg.com/d3-quadtree/-/d3-quadtree-3.0.1.tgz#6dca3e8be2b393c9a9d514dabbd80a92deef1a4f" integrity sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw== -d3-random@3: - version "3.0.1" - resolved "https://registry.yarnpkg.com/d3-random/-/d3-random-3.0.1.tgz#d4926378d333d9c0bfd1e6fa0194d30aebaa20f4" - integrity sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ== - -d3-scale-chromatic@3: - version "3.1.0" - resolved "https://registry.yarnpkg.com/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz#34c39da298b23c20e02f1a4b239bd0f22e7f1314" - integrity sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ== - dependencies: - d3-color "1 - 3" - d3-interpolate "1 - 3" - -d3-scale@4: - version "4.0.2" - resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-4.0.2.tgz#82b38e8e8ff7080764f8dcec77bd4be393689396" - integrity sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ== - dependencies: - d3-array "2.10.0 - 3" - d3-format "1 - 3" - d3-interpolate "1.2.0 - 3" - d3-time "2.1.1 - 3" - d3-time-format "2 - 4" - -"d3-selection@2 - 3", d3-selection@3: - version "3.0.0" - resolved "https://registry.yarnpkg.com/d3-selection/-/d3-selection-3.0.0.tgz#c25338207efa72cc5b9bd1458a1a41901f1e1b31" - integrity sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ== - -d3-shape@3, d3-shape@3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-3.2.0.tgz#a1a839cbd9ba45f28674c69d7f855bcf91dfc6a5" - integrity sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA== - dependencies: - d3-path "^3.1.0" - -"d3-time-format@2 - 4", d3-time-format@4: - version "4.1.0" - resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-4.1.0.tgz#7ab5257a5041d11ecb4fe70a5c7d16a195bb408a" - integrity sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg== - dependencies: - d3-time "1 - 3" - -"d3-time@1 - 3", "d3-time@2.1.1 - 3", d3-time@3: - version "3.1.0" - resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-3.1.0.tgz#9310db56e992e3c0175e1ef385e545e48a9bb5c7" - integrity sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q== - dependencies: - d3-array "2 - 3" - -"d3-timer@1 - 3", d3-timer@3: +"d3-timer@1 - 3": version "3.0.1" resolved "https://registry.yarnpkg.com/d3-timer/-/d3-timer-3.0.1.tgz#6284d2a2708285b1abb7e201eda4380af35e63b0" integrity sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA== -"d3-transition@2 - 3", d3-transition@3: - version "3.0.1" - resolved "https://registry.yarnpkg.com/d3-transition/-/d3-transition-3.0.1.tgz#6869fdde1448868077fdd5989200cb61b2a1645f" - integrity sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w== - dependencies: - d3-color "1 - 3" - d3-dispatch "1 - 3" - d3-ease "1 - 3" - d3-interpolate "1 - 3" - d3-timer "1 - 3" - -d3-zoom@3: - version "3.0.0" - resolved "https://registry.yarnpkg.com/d3-zoom/-/d3-zoom-3.0.0.tgz#d13f4165c73217ffeaa54295cd6969b3e7aee8f3" - integrity sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw== - dependencies: - d3-dispatch "1 - 3" - d3-drag "2 - 3" - d3-interpolate "1 - 3" - d3-selection "2 - 3" - d3-transition "2 - 3" - -d3@7.9.0: - version "7.9.0" - resolved "https://registry.yarnpkg.com/d3/-/d3-7.9.0.tgz#579e7acb3d749caf8860bd1741ae8d371070cd5d" - integrity sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA== - dependencies: - d3-array "3" - d3-axis "3" - d3-brush "3" - d3-chord "3" - d3-color "3" - d3-contour "4" - d3-delaunay "6" - d3-dispatch "3" - d3-drag "3" - d3-dsv "3" - d3-ease "3" - d3-fetch "3" - d3-force "3" - d3-format "3" - d3-geo "3" - d3-hierarchy "3" - d3-interpolate "3" - d3-path "3" - d3-polygon "3" - d3-quadtree "3" - d3-random "3" - d3-scale "4" - d3-scale-chromatic "3" - d3-selection "3" - d3-shape "3" - d3-time "3" - d3-time-format "4" - d3-timer "3" - d3-transition "3" - d3-zoom "3" - data-view-buffer@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/data-view-buffer/-/data-view-buffer-1.0.1.tgz#8ea6326efec17a2e42620696e671d7d5a8bc66b2" @@ -3135,13 +2923,6 @@ define-properties@^1.1.3, define-properties@^1.2.0, define-properties@^1.2.1: has-property-descriptors "^1.0.0" object-keys "^1.1.1" -delaunator@5: - version "5.0.1" - resolved "https://registry.yarnpkg.com/delaunator/-/delaunator-5.0.1.tgz#39032b08053923e924d6094fe2cde1a99cc51278" - integrity sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw== - dependencies: - robust-predicates "^3.0.2" - delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" @@ -3981,13 +3762,6 @@ hyphenate-style-name@^1.0.3: resolved "https://registry.yarnpkg.com/hyphenate-style-name/-/hyphenate-style-name-1.1.0.tgz#1797bf50369588b47b72ca6d5e65374607cf4436" integrity sha512-WDC/ui2VVRrz3jOVi+XtjqkDjiVjTtFaAGiW37k6b+ohyQ5wYDOGkvCZa8+H0nx3gyvv0+BST9xuOgIyGQ00gw== -iconv-lite@0.6: - version "0.6.3" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" - integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== - dependencies: - safer-buffer ">= 2.1.2 < 3.0.0" - ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -4045,11 +3819,6 @@ internal-slot@^1.0.7: hasown "^2.0.0" side-channel "^1.0.4" -"internmap@1 - 2": - version "2.0.3" - resolved "https://registry.yarnpkg.com/internmap/-/internmap-2.0.3.tgz#6685f23755e43c524e251d29cbc97248e3061009" - integrity sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg== - intl-messageformat@^10.1.0: version "10.5.14" resolved "https://registry.yarnpkg.com/intl-messageformat/-/intl-messageformat-10.5.14.tgz#e5bb373f8a37b88fbe647d7b941f3ab2a37ed00a" @@ -5863,11 +5632,6 @@ robust-predicates@^2.0.4: resolved "https://registry.yarnpkg.com/robust-predicates/-/robust-predicates-2.0.4.tgz#0a2367a93abd99676d075981707f29cfb402248b" integrity sha512-l4NwboJM74Ilm4VKfbAtFeGq7aEjWL+5kVFcmgFA2MrdnQWx9iE/tUGvxY5HyMI7o/WpSIUFLbC5fbeaHgSCYg== -robust-predicates@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/robust-predicates/-/robust-predicates-3.0.2.tgz#d5b28528c4824d20fc48df1928d41d9efa1ad771" - integrity sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg== - rollup@^3.27.1: version "3.29.4" resolved "https://registry.yarnpkg.com/rollup/-/rollup-3.29.4.tgz#4d70c0f9834146df8705bfb69a9a19c9e1109981" @@ -5889,11 +5653,6 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -rw@1: - version "1.3.3" - resolved "https://registry.yarnpkg.com/rw/-/rw-1.3.3.tgz#3f862dfa91ab766b14885ef4d01124bfda074fb4" - integrity sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ== - rxjs@^7.8.1: version "7.8.1" resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.8.1.tgz#6f6f3d99ea8044291efd92e7c7fcf562c4057543" @@ -5925,11 +5684,6 @@ safe-regex-test@^1.0.3: es-errors "^1.3.0" is-regex "^1.1.4" -"safer-buffer@>= 2.1.2 < 3.0.0": - version "2.1.2" - resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" - integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== - scheduler@^0.23.2: version "0.23.2" resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.23.2.tgz#414ba64a3b282892e944cf2108ecc078d115cdc3"