diff --git a/README.md b/README.md index bf56e54b8..d1074064c 100644 --- a/README.md +++ b/README.md @@ -1,118 +1,206 @@ -# Knowledge Graph Builder App - -Creating knowledge graphs from unstructured data - - -# LLM Graph Builder +# Knowledge Graph Builder ![Python](https://img.shields.io/badge/Python-yellow) ![FastAPI](https://img.shields.io/badge/FastAPI-green) ![React](https://img.shields.io/badge/React-blue) -## Overview -This application is designed to turn Unstructured data (pdfs,docs,txt,youtube video,web pages,etc.) into a knowledge graph stored in Neo4j. It utilizes the power of Large language models (OpenAI,Gemini,etc.) to extract nodes, relationships and their properties from the text and create a structured knowledge graph using Langchain framework. +Transform unstructured data (PDFs, DOCs, TXT, YouTube videos, web pages, etc.) into a structured Knowledge Graph stored in Neo4j using the power of Large Language Models (LLMs) and the LangChain framework. + +This application allows you to upload files from various sources (local machine, GCS, S3 bucket, or web sources), choose your preferred LLM model, and generate a Knowledge Graph. -Upload your files from local machine, GCS or S3 bucket or from web sources, choose your LLM model and generate knowledge graph. +--- ## Key Features -- **Knowledge Graph Creation**: Transform unstructured data into structured knowledge graphs using LLMs. -- **Providing Schema**: Provide your own custom schema or use existing schema in settings to generate graph. -- **View Graph**: View graph for a particular source or multiple sources at a time in Bloom. -- **Chat with Data**: Interact with your data in a Neo4j database through conversational queries, also retrieve metadata about the source of response to your queries.For a dedicated chat interface, access the standalone chat application at: [Chat-Only](https://dev-frontend-dcavk67s4a-uc.a.run.app/chat-only). This link provides a focused chat experience for querying your data. -## Getting started +### **Knowledge Graph Creation** +- Seamlessly transform unstructured data into structured Knowledge Graphs using advanced LLMs. +- Extract nodes, relationships, and their properties to create structured graphs. -:warning: You will need to have a Neo4j Database V5.15 or later with [APOC installed](https://neo4j.com/docs/apoc/current/installation/) to use this Knowledge Graph Builder. -You can use any [Neo4j Aura database](https://neo4j.com/aura/) (including the free database) -If you are using Neo4j Desktop, you will not be able to use the docker-compose but will have to follow the [separate deployment of backend and frontend section](#running-backend-and-frontend-separately-dev-environment). :warning: +### **Schema Support** +- Use a custom schema or existing schemas configured in the settings to generate graphs. +### **Graph Visualization** +- View graphs for specific or multiple data sources simultaneously in **Neo4j Bloom**. -## Deployment -### Local deployment -#### Running through docker-compose -By default only OpenAI and Diffbot are enabled since Gemini requires extra GCP configurations. -According to enviornment we are configuring the models which is indicated by VITE_LLM_MODELS_PROD variable we can configure model based on our need. +### **Chat with Data** +- Interact with your data in the Neo4j database through conversational queries. +- Retrieve metadata about the source of responses to your queries. +- For a dedicated chat interface, use the standalone chat application with **[/chat-only](/chat-only) route.** -EX: -```env -VITE_LLM_MODELS_PROD="openai_gpt_4o,openai_gpt_4o_mini,diffbot,gemini_1.5_flash" +### **LLMs Supported** +1. OpenAI +2. Gemini +3. Diffbot +4. Azure OpenAI(dev deployed version) +5. Anthropic(dev deployed version) +6. Fireworks(dev deployed version) +7. Groq(dev deployed version) +8. Amazon Bedrock(dev deployed version) +9. Ollama(dev deployed version) +10. Deepseek(dev deployed version) +11. Other OpenAI Compatible baseurl models(dev deployed version) + +--- + +## Getting Started + +### **Prerequisites** +- Neo4j Database **5.23 or later** with APOC installed. + - **Neo4j Aura** databases (including the free tier) are supported. + - If using **Neo4j Desktop**, you will need to deploy the backend and frontend separately (docker-compose is not supported). + +--- + +## Deployment Options + +### **Local Deployment** + +#### Using Docker-Compose +Run the application using the default `docker-compose` configuration. + +1. **Supported LLM Models**: + - By default, only OpenAI and Diffbot are enabled. Gemini requires additional GCP configurations. + - Use the `VITE_LLM_MODELS_PROD` variable to configure the models you need. Example: + ```bash + VITE_LLM_MODELS_PROD="openai_gpt_4o,openai_gpt_4o_mini,diffbot,gemini_1.5_flash" + ``` + +2. **Input Sources**: + - By default, the following sources are enabled: `local`, `YouTube`, `Wikipedia`, `AWS S3`, and `web`. + - To add Google Cloud Storage (GCS) integration, include `gcs` and your Google client ID: + ```bash + VITE_REACT_APP_SOURCES="local,youtube,wiki,s3,gcs,web" + VITE_GOOGLE_CLIENT_ID="your-google-client-id" + ``` + +#### Chat Modes +Configure chat modes using the `VITE_CHAT_MODES` variable: +- By default, all modes are enabled: `vector`, `graph_vector`, `graph`, `fulltext`, `graph_vector_fulltext`, `entity_vector`, and `global_vector`. +- To specify specific modes, update the variable. For example: + ```bash + VITE_CHAT_MODES="vector,graph" + ``` + +--- + +### **Running Backend and Frontend Separately** + +For development, you can run the backend and frontend independently. + +#### **Frontend Setup** +1. Create the `.env` file in the `frontend` folder by copying `frontend/example.env`. +2. Update environment variables as needed. +3. Run: + ```bash + cd frontend + yarn + yarn run dev + ``` + +#### **Backend Setup** +1. Create the `.env` file in the `backend` folder by copying `backend/example.env`. +2. Preconfigure user credentials in the `.env` file to bypass the login dialog: + ```bash + NEO4J_URI= + NEO4J_USERNAME= + NEO4J_PASSWORD= + NEO4J_DATABASE= + ``` +3. Run: + ```bash + cd backend + python -m venv envName + source envName/bin/activate + pip install -r requirements.txt + uvicorn score:app --reload + ``` + +--- + +### **Cloud Deployment** + +Deploy the application on **Google Cloud Platform** using the following commands: + +#### **Frontend Deployment** +```bash +gcloud run deploy dev-frontend \ + --source . \ + --region us-central1 \ + --allow-unauthenticated ``` -#### Additional configs - -By default, the input sources will be: Local files, Youtube, Wikipedia ,AWS S3 and Webpages. As this default config is applied: -```env -VITE_REACT_APP_SOURCES="local,youtube,wiki,s3,web" +#### **Backend Deployment** +```bash +gcloud run deploy dev-backend \ + --set-env-vars "OPENAI_API_KEY=" \ + --set-env-vars "DIFFBOT_API_KEY=" \ + --set-env-vars "NEO4J_URI=" \ + --set-env-vars "NEO4J_USERNAME=" \ + --set-env-vars "NEO4J_PASSWORD=" \ + --source . \ + --region us-central1 \ + --allow-unauthenticated ``` -If however you want the Google GCS integration, add `gcs` and your Google client ID: -```env -VITE_REACT_APP_SOURCES="local,youtube,wiki,s3,gcs,web" -VITE_GOOGLE_CLIENT_ID="xxxx" +--- +## For local llms (Ollama) +1. Pull the docker imgage of ollama +```bash +docker pull ollama/ollama ``` - -You can of course combine all (local, youtube, wikipedia, s3 and gcs) or remove any you don't want/need. - -### Chat Modes - -By default,all of the chat modes will be available: vector, graph_vector, graph, fulltext, graph_vector_fulltext , entity_vector and global_vector. - -If none of the mode is mentioned in the chat modes variable all modes will be available: +2. Run the ollama docker image +```bash +docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama +``` +3. Execute any llm model ex llama3 +```bash +docker exec -it ollama ollama run llama3 +``` +4. Configure env variable in docker compose. ```env -VITE_CHAT_MODES="" +LLM_MODEL_CONFIG_ollama_ +#example +LLM_MODEL_CONFIG_ollama_llama3=${LLM_MODEL_CONFIG_ollama_llama3-llama3, +http://host.docker.internal:11434} ``` - -If however you want to specify the only vector mode or only graph mode you can do that by specifying the mode in the env: +5. Configure the backend API url ```env -VITE_CHAT_MODES="vector,graph" -VITE_CHAT_MODES="vector,graph" +VITE_BACKEND_API_URL=${VITE_BACKEND_API_URL-backendurl} ``` +6. Open the application in browser and select the ollama model for the extraction. +7. Enjoy Graph Building. +--- + +## Additional Configuration -#### Running Backend and Frontend separately (dev environment) -Alternatively, you can run the backend and frontend separately: - -- For the frontend: -1. Create the frontend/.env file by copy/pasting the frontend/example.env. -2. Change values as needed -3. - ```bash - cd frontend - yarn - yarn run dev - ``` - -- For the backend: -1. Create the backend/.env file by copy/pasting the backend/example.env. To streamline the initial setup and testing of the application, you can preconfigure user credentials directly within the backend .env file. This bypasses the login dialog and allows you to immediately connect with a predefined user. - - **NEO4J_URI**: - - **NEO4J_USERNAME**: - - **NEO4J_PASSWORD**: - - **NEO4J_DATABASE**: -3. Change values as needed -4. - ```bash - cd backend - python -m venv envName - source envName/bin/activate - pip install -r requirements.txt - uvicorn score:app --reload - ``` -### Deploy in Cloud -To deploy the app and packages on Google Cloud Platform, run the following command on google cloud run: +### **LLM Models** +Configure LLM models using the `VITE_LLM_MODELS_PROD` variable. Example: ```bash -# Frontend deploy -gcloud run deploy dev-frontend -source location current directory > Frontend -region : 32 [us-central 1] -Allow unauthenticated request : Yes +VITE_LLM_MODELS_PROD="openai_gpt_4o,openai_gpt_4o_mini,diffbot,gemini_1.5_flash" ``` + +### **Input Sources** +The default input sources are: `local`, `YouTube`, `Wikipedia`, `AWS S3`, and `web`. + +To enable GCS integration, include `gcs` and your Google client ID: ```bash -# Backend deploy -gcloud run deploy --set-env-vars "OPENAI_API_KEY = " --set-env-vars "DIFFBOT_API_KEY = " --set-env-vars "NEO4J_URI = " --set-env-vars "NEO4J_PASSWORD = " --set-env-vars "NEO4J_USERNAME = " -source location current directory > Backend -region : 32 [us-central 1] -Allow unauthenticated request : Yes +VITE_REACT_APP_SOURCES="local,youtube,wiki,s3,gcs,web" +VITE_GOOGLE_CLIENT_ID="your-google-client-id" ``` +## Usage +1. Connect to Neo4j Aura Instance which can be both AURA DS or AURA DB by passing URI and password through Backend env, fill using login dialog or drag and drop the Neo4j credentials file. +2. To differntiate we have added different icons. For AURA DB we have a database icon and for AURA DS we have scientific molecule icon right under Neo4j Connection details label. +3. Choose your source from a list of Unstructured sources to create graph. +4. Change the LLM (if required) from drop down, which will be used to generate graph. +5. Optionally, define schema(nodes and relationship labels) in entity graph extraction settings. +6. Either select multiple files to 'Generate Graph' or all the files in 'New' status will be processed for graph creation. +7. Have a look at the graph for individual files using 'View' in grid or select one or more files and 'Preview Graph' +8. Ask questions related to the processed/completed sources to chat-bot, Also get detailed information about your answers generated by LLM. + +--- + + ## ENV | Env Variable Name | Mandatory/Optional | Default Value | Description | |-------------------------|--------------------|---------------|--------------------------------------------------------------------------------------------------| @@ -166,60 +254,6 @@ Allow unauthenticated request : Yes | VITE_TOKENS_PER_CHUNK | Optional | 100 | variable to configure tokens count per chunk.This gives flexibility for users who may require different chunk sizes for various tokenization tasks, especially when working with large datasets or specific language models. | VITE_CHUNK_TO_COMBINE | Optional | 1 | variable to configure number of chunks to combine for parllel processing. -## LLMs Supported -1. OpenAI -2. Gemini -3. Diffbot -4. Azure OpenAI(dev deployed version) -5. Anthropic(dev deployed version) -6. Fireworks(dev deployed version) -7. Groq(dev deployed version) -8. Amazon Bedrock(dev deployed version) -9. Ollama(dev deployed version) -10. Deepseek(dev deployed version) -11. Other OpenAI compabtile baseurl models(dev deployed version) - -## For local llms (Ollama) -1. Pull the docker imgage of ollama -```bash -docker pull ollama/ollama -``` -2. Run the ollama docker image -```bash -docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama -``` -3. Pull specific ollama model. -```bash -ollama pull llama3 -``` -4. Execute any llm model ex🦙3 -```bash -docker exec -it ollama ollama run llama3 -``` -5. Configure env variable in docker compose. -```env -LLM_MODEL_CONFIG_ollama_ -#example -LLM_MODEL_CONFIG_ollama_llama3=${LLM_MODEL_CONFIG_ollama_llama3-llama3, -http://host.docker.internal:11434} -``` -6. Configure the backend API url -```env -VITE_BACKEND_API_URL=${VITE_BACKEND_API_URL-backendurl} -``` -7. Open the application in browser and select the ollama model for the extraction. -8. Enjoy Graph Building. - - -## Usage -1. Connect to Neo4j Aura Instance which can be both AURA DS or AURA DB by passing URI and password through Backend env, fill using login dialog or drag and drop the Neo4j credentials file. -2. To differntiate we have added different icons. For AURA DB we have a database icon and for AURA DS we have scientific molecule icon right under Neo4j Connection details label. -3. Choose your source from a list of Unstructured sources to create graph. -4. Change the LLM (if required) from drop down, which will be used to generate graph. -5. Optionally, define schema(nodes and relationship labels) in entity graph extraction settings. -6. Either select multiple files to 'Generate Graph' or all the files in 'New' status will be processed for graph creation. -7. Have a look at the graph for individual files using 'View' in grid or select one or more files and 'Preview Graph' -8. Ask questions related to the processed/completed sources to chat-bot, Also get detailed information about your answers generated by LLM. ## Links diff --git a/backend/example.env b/backend/example.env index 4a66791a9..2822c92e7 100644 --- a/backend/example.env +++ b/backend/example.env @@ -31,16 +31,19 @@ DEFAULT_DIFFBOT_CHAT_MODEL="openai_gpt_4o" #whichever model specified here , ne LLM_MODEL_CONFIG_openai_gpt_3.5="gpt-3.5-turbo-0125,openai_api_key" LLM_MODEL_CONFIG_openai_gpt_4o_mini="gpt-4o-mini-2024-07-18,openai_api_key" LLM_MODEL_CONFIG_openai_gpt_4o="gpt-4o-2024-11-20,openai_api_key" -LLM_MODEL_CONFIG_openai-gpt-o3-mini="o3-mini-2025-01-31,openai_api_key" +LLM_MODEL_CONFIG_openai_gpt_4.1_mini="gpt-4.1-mini,openai_api_key" +LLM_MODEL_CONFIG_openai_gpt_4.1="gpt-4.1,openai_api_key" +LLM_MODEL_CONFIG_openai_gpt_o3_mini="o3-mini-2025-01-31,openai_api_key" LLM_MODEL_CONFIG_gemini_1.5_pro="gemini-1.5-pro-002" LLM_MODEL_CONFIG_gemini_1.5_flash="gemini-1.5-flash-002" LLM_MODEL_CONFIG_gemini_2.0_flash="gemini-2.0-flash-001" +LLM_MODEL_CONFIG_gemini_2.5_pro="gemini-2.5-pro-exp-03-25" LLM_MODEL_CONFIG_diffbot="diffbot,diffbot_api_key" LLM_MODEL_CONFIG_azure_ai_gpt_35="azure_deployment_name,azure_endpoint or base_url,azure_api_key,api_version" LLM_MODEL_CONFIG_azure_ai_gpt_4o="gpt-4o,https://YOUR-ENDPOINT.openai.azure.com/,azure_api_key,api_version" LLM_MODEL_CONFIG_groq_llama3_70b="model_name,base_url,groq_api_key" LLM_MODEL_CONFIG_anthropic_claude_3_5_sonnet="model_name,anthropic_api_key" -LLM_MODEL_CONFIG_fireworks_llama_v3_70b="model_name,fireworks_api_key" +LLM_MODEL_CONFIG_fireworks_llama4_maverick="model_name,fireworks_api_key" LLM_MODEL_CONFIG_bedrock_claude_3_5_sonnet="model_name,aws_access_key_id,aws_secret__access_key,region_name" LLM_MODEL_CONFIG_ollama_llama3="model_name,model_local_url" YOUTUBE_TRANSCRIPT_PROXY="https://user:pass@domain:port" diff --git a/backend/requirements.txt b/backend/requirements.txt index d1b801805..7ced40181 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,6 +1,7 @@ +accelerate==1.6.0 asyncio==3.4.3 -boto3==1.37.11 -botocore==1.37.11 +boto3==1.37.29 +botocore==1.37.29 certifi==2025.1.31 fastapi==0.115.11 fastapi-health==0.4.0 @@ -8,48 +9,48 @@ google-api-core==2.24.2 google-auth==2.38.0 google_auth_oauthlib==1.2.1 google-cloud-core==2.4.3 -json-repair==0.30.3 +json-repair==0.39.1 pip-install==1.3.5 -langchain==0.3.20 -langchain-aws==0.2.15 +langchain==0.3.23 +langchain-aws==0.2.18 langchain-anthropic==0.3.9 -langchain-fireworks==0.2.7 +langchain-fireworks==0.2.9 langchain-community==0.3.19 -langchain-core==0.3.45 +langchain-core==0.3.51 langchain-experimental==0.3.4 -langchain-google-vertexai==2.0.15 +langchain-google-vertexai==2.0.19 langchain-groq==0.2.5 -langchain-openai==0.3.8 -langchain-text-splitters==0.3.6 +langchain-openai==0.3.12 +langchain-text-splitters==0.3.8 langchain-huggingface==0.1.2 langdetect==1.0.9 -langsmith==0.3.13 +langsmith==0.3.26 langserve==0.3.1 neo4j-rust-ext nltk==3.9.1 -openai==1.66.2 +openai==1.71.0 opencv-python==4.11.0.86 psutil==7.0.0 pydantic==2.10.6 python-dotenv==1.0.1 python-magic==0.4.27 PyPDF2==3.0.1 -PyMuPDF==1.25.3 +PyMuPDF==1.25.5 starlette==0.46.1 sse-starlette==2.2.1 starlette-session==0.4.3 tqdm==4.67.1 unstructured[all-docs] -unstructured==0.16.25 -unstructured-client==0.31.1 -unstructured-inference==0.8.9 +unstructured==0.17.2 +unstructured-client==0.32.3 +unstructured-inference==0.8.10 urllib3==2.3.0 uvicorn==0.34.0 gunicorn==23.0.0 wikipedia==1.4.0 wrapt==1.17.2 yarl==1.18.3 -youtube-transcript-api==1.0.0 +youtube-transcript-api==1.0.3 zipp==3.21.0 sentence-transformers==4.0.2 google-cloud-logging==3.11.4 diff --git a/backend/score.py b/backend/score.py index 75181e096..2318c22e8 100644 --- a/backend/score.py +++ b/backend/score.py @@ -585,8 +585,11 @@ async def upload_large_file_into_chunks(file:UploadFile = File(...), chunkNumber else: return create_api_response('Success', message=result) except Exception as e: - message="Unable to upload large file into chunks. " + message="Unable to upload file in chunks" error_message = str(e) + graph = create_graph_database_connection(uri, userName, password, database) + graphDb_data_Access = graphDBdataAccess(graph) + graphDb_data_Access.update_exception_db(originalname,error_message) logging.info(message) logging.exception(f'Exception:{error_message}') return create_api_response('Failed', message=message + error_message[:100], error=error_message, file_name = originalname) @@ -597,8 +600,7 @@ async def upload_large_file_into_chunks(file:UploadFile = File(...), chunkNumber async def get_structured_schema(uri=Form(None), userName=Form(None), password=Form(None), database=Form(None),email=Form(None)): try: start = time.time() - graph = create_graph_database_connection(uri, userName, password, database) - result = await asyncio.to_thread(get_labels_and_relationtypes, graph) + result = await asyncio.to_thread(get_labels_and_relationtypes, uri, userName, password, database) end = time.time() elapsed_time = end - start logging.info(f'Schema result from DB: {result}') @@ -765,10 +767,10 @@ async def cancelled_job(uri=Form(None), userName=Form(None), password=Form(None) gc.collect() @app.post("/populate_graph_schema") -async def populate_graph_schema(input_text=Form(None), model=Form(None), is_schema_description_checked=Form(None),email=Form(None)): +async def populate_graph_schema(input_text=Form(None), model=Form(None), is_schema_description_checked=Form(None),is_local_storage=Form(None),email=Form(None)): try: start = time.time() - result = populate_graph_schema_from_text(input_text, model, is_schema_description_checked) + result = populate_graph_schema_from_text(input_text, model, is_schema_description_checked, is_local_storage) end = time.time() elapsed_time = end - start json_obj = {'api_name':'populate_graph_schema', 'model':model, 'is_schema_description_checked':is_schema_description_checked, 'input_text':input_text, 'logging_time': formatted_time(datetime.now(timezone.utc)), 'elapsed_api_time':f'{elapsed_time:.2f}','email':email} diff --git a/backend/src/graphDB_dataAccess.py b/backend/src/graphDB_dataAccess.py index 397227a9a..e3142a10b 100644 --- a/backend/src/graphDB_dataAccess.py +++ b/backend/src/graphDB_dataAccess.py @@ -41,7 +41,7 @@ def update_exception_db(self, file_name, exp_msg, retry_condition=None): def create_source_node(self, obj_source_node:sourceNode): try: job_status = "New" - logging.info("creating source node if does not exist") + logging.info(f"creating source node if does not exist in database {self.graph._database}") self.graph.query("""MERGE(d:Document {fileName :$fn}) SET d.fileSize = $fs, d.fileType = $ft , d.status = $st, d.url = $url, d.awsAccessKeyId = $awsacc_key_id, d.fileSource = $f_source, d.createdAt = $c_at, d.updatedAt = $u_at, diff --git a/backend/src/llm.py b/backend/src/llm.py index f40f5492c..b83773f42 100644 --- a/backend/src/llm.py +++ b/backend/src/llm.py @@ -15,6 +15,7 @@ import google.auth from src.shared.constants import ADDITIONAL_INSTRUCTIONS import re +import json def get_llm(model: str): """Retrieve the specified language model based on the model name.""" @@ -200,15 +201,13 @@ async def get_graph_from_llm(model, chunkId_chunkDoc_list, allowedNodes, allowed llm, model_name = get_llm(model) combined_chunk_document_list = get_combined_chunks(chunkId_chunkDoc_list, chunks_to_combine) - if allowedNodes is None or allowedNodes=="": - allowedNodes =[] - else: - allowedNodes = allowedNodes.split(',') - if allowedRelationship is None or allowedRelationship=="": - allowedRelationship=[] + allowedNodes = allowedNodes.split(',') if allowedNodes else [] + + if not allowedRelationship: + allowedRelationship = [] else: - allowedRelationship = allowedRelationship.split(',') - + items = allowedRelationship.split(',') + allowedRelationship = [tuple(items[i:i+3]) for i in range(0, len(items), 3)] graph_document_list = await get_graph_document_list( llm, combined_chunk_document_list, allowedNodes, allowedRelationship, additional_instructions ) diff --git a/backend/src/main.py b/backend/src/main.py index 41e69e6f4..c2f2a80f3 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -23,6 +23,7 @@ from src.shared.common_fn import * from src.make_relationships import * from src.document_sources.web_pages import * +from src.graph_query import get_graphDB_driver import re from langchain_community.document_loaders import WikipediaLoader, WebBaseLoader import warnings @@ -663,22 +664,40 @@ def upload_file(graph, model, chunk, chunk_number:int, total_chunks:int, origina return {'file_size': file_size, 'file_name': originalname, 'file_extension':file_extension, 'message':f"Chunk {chunk_number}/{total_chunks} saved"} return f"Chunk {chunk_number}/{total_chunks} saved" -def get_labels_and_relationtypes(graph): - query = """ - RETURN collect { - CALL db.labels() yield label - WHERE NOT label IN ['Document','Chunk','_Bloom_Perspective_', '__Community__', '__Entity__', 'Session', 'Message'] - return label order by label limit 100 } as labels, - collect { - CALL db.relationshipTypes() yield relationshipType as type - WHERE NOT type IN ['PART_OF', 'NEXT_CHUNK', 'HAS_ENTITY', '_Bloom_Perspective_','FIRST_CHUNK','SIMILAR','IN_COMMUNITY','PARENT_COMMUNITY', 'NEXT', 'LAST_MESSAGE'] - return type order by type LIMIT 100 } as relationshipTypes - """ - graphDb_data_Access = graphDBdataAccess(graph) - result = graphDb_data_Access.execute_query(query) - if result is None: - result=[] - return result +def get_labels_and_relationtypes(uri, userName, password, database): + excluded_labels = {'Document', 'Chunk', '_Bloom_Perspective_', '__Community__', '__Entity__', 'Session', 'Message'} + excluded_relationships = { + 'PART_OF', 'NEXT_CHUNK', 'HAS_ENTITY', '_Bloom_Perspective_', 'FIRST_CHUNK', + 'SIMILAR', 'IN_COMMUNITY', 'PARENT_COMMUNITY', 'NEXT', 'LAST_MESSAGE'} + driver = get_graphDB_driver(uri, userName, password,database) + with driver.session(database=database) as session: + result = session.run("CALL db.schema.visualization() YIELD nodes, relationships RETURN nodes, relationships") + if not result: + return [] + record = result.single() + nodes = record["nodes"] + relationships = record["relationships"] + node_map = {} + for node in nodes: + node_id = node.element_id + labels = list(node.labels) + if labels: + node_map[node_id] = ":".join(labels) + triples = [] + for rel in relationships: + start_id = rel.start_node.element_id + end_id = rel.end_node.element_id + rel_type = rel.type + start_label = node_map.get(start_id) + end_label = node_map.get(end_id) + if start_label and end_label: + if ( + start_label not in excluded_labels and + end_label not in excluded_labels and + rel_type not in excluded_relationships + ): + triples.append(f"{start_label}-{rel_type}->{end_label}") + return {"triplets" : list(set(triples))} def manually_cancelled_job(graph, filenames, source_types, merged_dir, uri): @@ -705,7 +724,7 @@ def manually_cancelled_job(graph, filenames, source_types, merged_dir, uri): delete_uploaded_local_file(merged_file_path,file_name) return "Cancelled the processing job successfully" -def populate_graph_schema_from_text(text, model, is_schema_description_cheked): +def populate_graph_schema_from_text(text, model, is_schema_description_checked, is_local_storage): """_summary_ Args: @@ -716,8 +735,8 @@ def populate_graph_schema_from_text(text, model, is_schema_description_cheked): Returns: data (list): list of lebels and relationTypes """ - result = schema_extraction_from_text(text, model, is_schema_description_cheked) - return {"labels": result.labels, "relationshipTypes": result.relationshipTypes} + result = schema_extraction_from_text(text, model, is_schema_description_checked, is_local_storage) + return result def set_status_retry(graph, file_name, retry_condition): graphDb_data_Access = graphDBdataAccess(graph) diff --git a/backend/src/shared/schema_extraction.py b/backend/src/shared/schema_extraction.py index d57703e37..952ba2095 100644 --- a/backend/src/shared/schema_extraction.py +++ b/backend/src/shared/schema_extraction.py @@ -2,12 +2,12 @@ from pydantic.v1 import BaseModel, Field from src.llm import get_llm from langchain_core.prompts import ChatPromptTemplate +import logging class Schema(BaseModel): """Knowledge Graph Schema.""" - labels: List[str] = Field(description="list of node labels or types in a graph schema") - relationshipTypes: List[str] = Field(description="list of relationship types in a graph schema") + triplets: List[str] = Field(description="list of node labels and relationship types in a graph schema in --> format") PROMPT_TEMPLATE_WITH_SCHEMA = ( "You are an expert in schema extraction, especially for extracting graph schema information from various formats." @@ -17,23 +17,35 @@ class Schema(BaseModel): "Only return the string types for nodes and relationships. Don't return attributes." ) -PROMPT_TEMPLATE_WITHOUT_SCHEMA = ( - "You are an expert in schema extraction, especially for deriving graph schema information from example texts." - "Analyze the following text and extract only the types of entities and relationships from the example prose." - "Don't return any actual entities like people's names or instances of organizations." - "Only return the string types for nodes and relationships, don't return attributes." +PROMPT_TEMPLATE_WITHOUT_SCHEMA = ( """ +You are an expert in schema extraction, especially in identifying node and relationship types from example texts. +Analyze the following text and extract only the types of entities (node types) and their relationship types. +Do not return specific instances or attributes — only abstract schema information. +Return the result in the following format: +{{"triplets": ["-->"]}} +For example, if the text says “John works at Microsoft”, the output should be: +{{"triplets": ["Person-WORKS_AT->Company"]}}" +""" ) -def schema_extraction_from_text(input_text:str, model:str, is_schema_description_cheked:bool): - - llm, model_name = get_llm(model) - if is_schema_description_cheked: - schema_prompt = PROMPT_TEMPLATE_WITH_SCHEMA - else: - schema_prompt = PROMPT_TEMPLATE_WITHOUT_SCHEMA - +PROMPT_TEMPLATE_FOR_LOCAL_STORAGE = (""" +You are an expert in knowledge graph modeling. +The user will provide a JSON input with two keys: +- "nodes": a list of objects with "label" and "value" representing node types in the schema. +- "rels": a list of objects with "label" and "value" representing relationship types in the schema. +Your task: +1. Understand the meaning of each node and relationship label. +2. Use them to generate logical triplets in the format: +--> +3. Only return a JSON list of strings like: +["User-ANSWERED->Question", "Question-ACCEPTED->Answer"] +Make sure each triplet is semantically meaningful. +""" +) + +def get_schema_local_storage(input_text,llm): prompt = ChatPromptTemplate.from_messages( - [("system", schema_prompt), ("user", "{text}")] + [("system", PROMPT_TEMPLATE_FOR_LOCAL_STORAGE), ("user", "{text}")] ) runnable = prompt | llm.with_structured_output( @@ -43,4 +55,34 @@ def schema_extraction_from_text(input_text:str, model:str, is_schema_description ) raw_schema = runnable.invoke({"text": input_text}) - return raw_schema \ No newline at end of file + return raw_schema + + +def schema_extraction_from_text(input_text:str, model:str, is_schema_description_checked:bool,is_local_storage:bool): + try: + llm, model_name = get_llm(model) + if str(is_local_storage).lower().strip() == "true": + raw_schema = get_schema_local_storage(input_text,llm) + return raw_schema + if str(is_schema_description_checked).lower().strip() == "true": + schema_prompt = PROMPT_TEMPLATE_WITH_SCHEMA + else: + schema_prompt = PROMPT_TEMPLATE_WITHOUT_SCHEMA + prompt = ChatPromptTemplate.from_messages( + [("system", schema_prompt), ("user", "{text}")] + ) + + runnable = prompt | llm.with_structured_output( + schema=Schema, + method="function_calling", + include_raw=False, + ) + + raw_schema = runnable.invoke({"text": input_text}) + if raw_schema: + return raw_schema + else: + raise Exception("Unable to get schema from text for given model") + except Exception as e: + logging.info(str(e)) + raise Exception(str(e)) \ No newline at end of file diff --git a/docs/backend/backend_docs.adoc b/docs/backend/backend_docs.adoc index d82f12870..aaf524053 100644 --- a/docs/backend/backend_docs.adoc +++ b/docs/backend/backend_docs.adoc @@ -8,15 +8,16 @@ POST /connect ----- -Neo4j database connection on frontend is done with this API. +This API is used to authenticate and connect frontend to backend using NEO4j database credential. +Based on the response from this API, the frontend UI displays the connection status and the database type icon, such as GDS DB or Aura DB. **API Parameters :** * `uri`= Neo4j uri, * `userName`= Neo4j db username, * `password`= Neo4j db password, -* `database`= Neo4j database name - +* `database`= Neo4j database name, +* `email`= Logged in User Email **Response :** [source,json,indent=0] @@ -29,8 +30,10 @@ Neo4j database connection on frontend is done with this API. "message": "Connection Successful", "gds_status": true, "write_access": true, - "elapsed_api_time": "5.48" + "elapsed_api_time": "1.52", + "gcs_file_cache": "True" } +} ---- @@ -39,7 +42,8 @@ Neo4j database connection on frontend is done with this API. POST /upload ---- -The upload endpoint is designed to handle the uploading of large files by breaking them into smaller chunks. This method ensures that large files can be uploaded efficiently without overloading the server. +This API handles the uploading of large files by breaking them into smaller chunks. +This method ensures that large files can be uploaded efficiently without overloading the server. ***API Parameters*** @@ -51,14 +55,21 @@ The upload endpoint is designed to handle the uploading of large files by breaki * `uri`=Neo4j uri, * `userName`= Neo4j db username, * `password`= Neo4j db password, -* `database`= Neo4j database name +* `database`= Neo4j database name, +* `email`= Logged in User Email **Response :** [source,json,indent=0] .... { "status": "Success", - "message": "File uploaded and chunks merged successfully." + "data": { + "file_size": 393322, + "file_name": "Untitled Diagram.png", + "file_extension": "png", + "message": "Chunk 1/1 saved" + }, + "message": "Source Node Created Successfully" } .... @@ -68,42 +79,41 @@ The upload endpoint is designed to handle the uploading of large files by breaki POST /schema ---- -User can set schema for graph generation (i.e. Nodes and relationship labels) in settings panel or get existing db schema through this API. +This API gets the labels and relationships from existing Neo4j database data. Users can set the schema for graph generation (i.e., nodes and relationship labels) in the settings panel. **API Parameters :** * `uri`=Neo4j uri, * `userName`= Neo4j db username, * `password`= Neo4j db password, -* `database`= Neo4j database name - +* `database`= Neo4j database name, +* `email`= Logged in User Email **Response :** [source,json,indent=0] .... { "status": "Success", - "data": [ - { - "labels": [ - "Access_token", - "Activity", - "Ai chatbot", - "Book", - "Metric", - "Mode", - "Mountain" - ], - "relationshipTypes": [ - "ACCELERATE", - "ACCEPTS", - "CONVERT", - "CORRELATE", - "ESTABLISHED", - "EXAMPLE_OF" - ] - } - ] + "data": { + "triplets": [ + "Location-HAS_DIPLOMATIC_MISSIONS->Entity", + "Location-FORMED->Alliance", + "Company-HAS_TECH_ROLE_COMPOSITION->Demographic", + "Objective-INCLUDES->Feature", + "Location-IS_LARGEST_URBAN_AREA_OF->Location", + "Country-DECLARED_AS->Government Type", + "Ecoregion-LOCATED_IN->Region", + "Organization-SUFFERED_CASUALTIES->Number", + "Company-HAS->Judicial System", + "Organization-ELECTED_BY->Legislative Body", + "Location-TOOK_IN->Population Group", + "Location-NUMBER_OF->Count", + "Organization-PROVIDES->Infrastructure", + "Location-HAS->Team", + "Country-FOCUSES_ON->Category" + ] + }, + "message": "Total elapsed API time 1.56" } .... @@ -119,6 +129,8 @@ The API is used to populate a graph schema based on the provided input text, mod * `input_text`=The input text used to populate the graph schema. * `model`=The model to be used for populating the graph schema. * `is_schema_description_checked`=A flag indicating whether the schema description should be considered. +* `is_local_storage`= Generate the generalized graph schema based on input text if value is false +* `email`= Logged in User Email **Response :** @@ -126,26 +138,13 @@ The API is used to populate a graph schema based on the provided input text, mod .... { "status": "Success", - "data": [ - { - "labels": [ - "Technology", - "Company", - "Person", - "Location", - "Organization", - "Concept" - ], - "relationshipTypes": [ - "LOCATED_AT", - "SUBSIDARY_OF", - "BORN_IN", - "LAST_MESSAGE", - "ATTENDED", - "PARTNERED_WITH" - ] - } - ] + "data": { + "triplets": [ + "User-PURCHASES->Product", + "Product-SOLD_BY->Store", + "Product-HAS->Warranty" + ] + } } .... @@ -155,7 +154,7 @@ The API is used to populate a graph schema based on the provided input text, mod POST /url/scan ---- -Create Document node for other sources - s3 bucket, gcs bucket, wikipedia, youtube url and web pages. +This API creates Document source nodes for all supported sources, including S3 buckets, GCS buckets, Wikipedia, web pages, YouTube videos, and local files **API Parameters :** @@ -173,22 +172,26 @@ Create Document node for other sources - s3 bucket, gcs bucket, wikipedia, youtu * `gcs_bucket_folder`= GCS bucket folder, * `source_type`= s3 bucket/ gcs bucket/ youtube/Wikipedia as source type * `gcs_project_id`=Form(None), -* `access_token`=Form(None) - +* `access_token`=Form(None), +* `email`= Logged in User Email **Response :** [source,json,indent=0] .... { "status": "Success", - "success_count": 2, + "data": { + "elapsed_api_time": "3.22" + }, + "success_count": 1, "failed_count": 0, - "message": "Source Node created successfully for source type: Wikipedia and source: Albert Einstein, neo4j", + "message": "Source Node created successfully for source type: Wikipedia and source: ", "file_name": [ { - "fileName": "Albert Einstein", + "fileName": "Google_DeepMind", "fileSize": 8074, - "url": "https://en.wikipedia.org/wiki/Albert_Einstein", + "url": "https://en.wikipedia.org/wiki/Google_DeepMind", + "language": "en", "status": "Success" } ] @@ -258,9 +261,15 @@ This API is responsible for - * `gcs_bucket_folder`= GCS bucket folder, * `gcs_blob_filename` = GCS file name, * `source_type`= local file/ s3 bucket/ gcs bucket/ youtube/ Wikipedia as source, -allowedNodes=Node labels passed from settings panel, +* `allowedNodes=Node labels passed from settings panel, * `allowedRelationship`=Relationship labels passed from settings panel, -* `language`=Language in which wikipedia content will be extracted +* `token_chunk_size`= chunk split size, +* `chunk_overlap`= numric value of chunk overlap, +* `chunks_to_combine`= value of combine chunks to process for extraction, +* `language`=Language in which wikipedia content will be extracted, +* `retry_condition`= re-process the file based on selection, +* `additional_instructions`= additional instruction for LLM while extraction, +* `email`= Logged in User Email **Response :** [source,json,indent=0] @@ -268,20 +277,59 @@ allowedNodes=Node labels passed from settings panel, { "status": "Success", "data": { - "fileName": , - "nodeCount": , - "relationshipCount": , - "processingTime": , + "fileName": "Untitled Diagram.png", + "nodeCount": 19, + "relationshipCount": 33, + "total_processing_time": 15.91, "status": "Completed", - "model": - } + "model": "openai_gpt_4.5", + "success_count": 1, + "chunkNodeCount": 5, + "chunkRelCount": 23, + "entityNodeCount": 14, + "entityEntityRelCount": 10, + "communityNodeCount": 0, + "communityRelCount": 0, + "db_url": "neo4j+s://demo.neo4jlabs.com:7687", + "api_name": "extract", + "source_url": null, + "wiki_query": null, + "source_type": "local file", + "logging_time": "2025-04-10 17:06:17 UTC", + "elapsed_api_time": "30.65", + "userName": "persistent", + "database": "persistent1", + "aws_access_key_id": null, + "gcs_bucket_name": null, + "gcs_bucket_folder": null, + "gcs_blob_filename": null, + "gcs_project_id": null, + "language": null, + "retry_condition": "", + "email": null, + "create_connection": "0.29", + "create_list_chunk_and_document": "1.75", + "total_chunks": 5, + "get_status_document_node": "0.06", + "update_source_node": "0.50", + "processed_combine_chunk_0-5": "12.85", + "processed_chunk_detail_0-5": { + "update_embedding": "0.74", + "entity_extraction": "6.54", + "save_graphDocuments": "4.81", + "relationship_between_chunk_entity": "0.56" + }, + "Processed_source": "16.40", + "Per_entity_latency": "0.8421052631578947/s" + }, + "file_source": "local file" } .... === Get list of sources ---- -GET /sources_list +POST /sources_list ---- List all sources (Document nodes) present in Neo4j graph database. @@ -291,7 +339,8 @@ List all sources (Document nodes) present in Neo4j graph database. * `uri`=Neo4j uri, * `userName`= Neo4j db username, * `password`= Neo4j db password, -* `database`= Neo4j database name +* `database`= Neo4j database name, +* `email`= Logged in User Email **Response :** [source,json,indent=0] @@ -327,7 +376,8 @@ List all sources (Document nodes) present in Neo4j graph database. } } } - ] + ], + "message": "Total elapsed API time 3.20" } .... @@ -337,7 +387,7 @@ List all sources (Document nodes) present in Neo4j graph database. POST /post_processing : ---- -This API is called at the end of processing of whole document to get create k-nearest neighbor relations between similar chunks of document based on KNN_MIN_SCORE which is 0.8 by default and to drop and create a full text index on db labels. +This API is called at the end of document processing to create k-nearest neighbor relationships between similar chunks of documents based on KNN_MIN_SCORE, which is 0.8 by default, compute community clusters, generate community summaries, and recreate a full-text index on all labels in the database so Neo4j Bloom can make use of it. **API Parameters :** @@ -345,15 +395,39 @@ This API is called at the end of processing of whole document to get create k-ne * `userName`= Neo4j db username, * `password`= Neo4j db password, * `database`= Neo4j database name -* `tasks`= List of tasks to perform - +* `tasks`= List of tasks to perform, +* `email`= Logged in User Email **Response :** [source,json,indent=0] .... { - "status":"Success", - "message":"All tasks completed successfully" + "status": "Success", + "data": [ + { + "filename": "Google", + "chunkNodeCount": 100, + "chunkRelCount": 1310, + "entityNodeCount": 670, + "entityEntityRelCount": 775, + "communityNodeCount": 289, + "communityRelCount": 883, + "nodeCount": 1059, + "relationshipCount": 2968 + }, + { + "filename": "Germany", + "chunkNodeCount": 100, + "chunkRelCount": 1402, + "entityNodeCount": 780, + "entityEntityRelCount": 813, + "communityNodeCount": 422, + "communityRelCount": 1079, + "nodeCount": 1302, + "relationshipCount": 3294 + } + ], + "message": "All tasks completed successfully" } .... @@ -363,7 +437,7 @@ This API is called at the end of processing of whole document to get create k-ne POST /chat_bot ---- -The API responsible for a chatbot system designed to leverage multiple AI models and a Neo4j graph database, providing answers to user queries. It interacts with AI models from OpenAI and Google's Vertex AI and utilizes embedding models to enhance the retrieval of relevant information. +The API responsible for a chatbot system designed to leverage multiple AI models and a Neo4j graph database, providing answers to user queries. It interacts with AI models from OpenAI and Google's Vertex AI etc and utilizes embedding models to enhance the retrieval of relevant information. It utilises different retrievers (Retrieval Detail) to extract relevant information to the user query and uses LLM to formulate the answer. If no relevant information found the chatbot gracefully conveys to user. **Components :** @@ -383,7 +457,8 @@ The API responsible for a chatbot system designed to leverage multiple AI models * `question`= User query for the chatbot * `session_id`= Session ID used to maintain the history of chats during the user's connection * `mode` = chat mode to use -* `document_names` = the names of documents to be filtered works for vector mode and vector+Graph mode +* `document_names` = the names of documents to be filtered works for vector mode and vector+Graph mode, +* `email`= Logged in User Email **Response :** [source,json,indent=0] @@ -391,45 +466,56 @@ The API responsible for a chatbot system designed to leverage multiple AI models { "status": "Success", "data": { - - "session_id": "0cbd04a8-abc3-4776-b393-6a9a2cea36b3", - "message": "response generated by the chat", + "session_id": "f4e352bf-f57a-4a15-819e-68d2ffca82a2", + "message": "Germany has sixteen constituent states, collectively referred to as Länder.", "info": { "sources": [ - "About Amazon.pdf" + "https://en.wikipedia.org/wiki/Germany" ], - "model": "gpt-4o-2024-08-06", + "model": "gpt-4.5-preview", "nodedetails": { "chunkdetails": [ { - "id": "73bc9c9170bcd807d2fa87d87a0eeb3d82f95160", + "id": "0c92f93e837e6b31f8d2429dd76c3db4ab37ce14", "score": 1.0 }, { - "id": "de5486776978353c9f8ac530bcff33eeecbdbbad", - "score": 0.9425 + "id": "ac8c9c1e05c718cc612160d6580caf1af97dfb1f", + "score": 0.9455 + }, + { + "id": "b91415a3bbfb99d64b3a2aa8b1413bebd5b5650e", + "score": 0.9307 } ], "entitydetails": [], "communitydetails": [] }, - "total_tokens": 4575, - "response_time": 17.19, + "total_tokens": 2493, + "response_time": 5.53, "mode": "graph_vector_fulltext", "entities": { "entityids": [ - "4:98e5e9bb-8095-440d-9462-03985fed2fa2:307", - "4:98e5e9bb-8095-440d-9462-03985fed2fa2:1877", + "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:8329", + "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7783", + "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:8327", + "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7780", + "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:8512", + "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:8157", + "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:8549", + "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7823" ], "relationshipids": [ - "5:98e5e9bb-8095-440d-9462-03985fed2fa2:8072566611095062357", - "5:98e5e9bb-8095-440d-9462-03985fed2fa2:8072566508015847224" + "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:146149", + "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:145082", + "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:146554", + "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:140377" ] }, "metric_details": { - "question": "tell me about amazon ", - "contexts": "context sent to LLM" - "answer": "response generated by the LLM" + "question": "how many states in germany", + "contexts": "Document start\nThis Document belongs to the source https://en.wikipedia.org/wiki/Germany\nContent: Text Content:\nGermany, officially the Federal Republic of Germany, is a country in Central Europe. It lies between the Baltic Sea and the North Sea to the north and the Alps to the south. Its sixteen constituent states have a total population of over 82 million in an area of 357,596 km2 (138,069 sq mi), making it the most populous member state of the European Union. It borders Denmark to the north, Poland and the Czech Republic to the east, Austria and Switzerland to the south,\n----\n sixteen constituent states which are collectively referred to as Länder. Each state (Land) has its own constitution, and is largely autonomous in regard to its internal organisation. As of 2017, Germany is divided into 401 districts (Kreise) at a municipal level; these consist of 294 rural districts and 107 urban districts. === Law === Germany has a civil law system based on Roman law with some references to Germanic law. The Bundesverfassungsgericht (Federal\n----\n resort. == Demographics == With a population of 84.7 million according to the 2023 German census, Germany is the most populous member state of the European Union, the second-most populous country in Europe after Russia, and the nineteenth-most populous country in the world. Its population density stands at 236 inhabitants per square kilometre (610 inhabitants/sq mi). The fertility rate of 1.57 children born per woman (2022 estimates) is below the replacement rate of 2\n----\nEntities:\nAdministrative Division:107 urban districts\nAdministrative Division:294 rural districts\nAdministrative Division:401 districts\nAdministrative Division:sixteen constituent states\nArea:357,596 km2\nBody of Water:Baltic Sea\nBrand:Volkswagen\nCompany:Deutsche Telekom\nConcept:defence\nCountry:Austria\nCountry:Czech Republic\nCountry:Denmark\nCountry:Federal Republic of Germany\nCountry:Germany\nCountry:Hungary\nCountry:Poland\nCountry:Switzerland\nCountry:Ukraine\nCountry:United States\nCountry:West Germany\nEnergy Source:40% renewable sources\nGeographical Feature:Alps\nGeographical Feature:North Sea\nGroup:East Germans\nInitiative:Energiewende\nKingdom:East Francia\nLaw System:Germanic law\nLaw System:Roman law\nLaw System:civil law system\nLegal Domain:constitutional matters\nLegal Power:judicial review\nLocation:Berlin\nName:Länder\nOrganization Membership:founding member of the European Economic Community\nOrganization:Bundesverfassungsgericht\nOrganization:European Economic Community\nOrganization:European Union\nOrganization:Federal Constitutional Court\nOrganization:German Supreme Court\nOrganization:North German Confederation\nOrganization:Population Division of the United Nations Department of Economic and Social Affairs\nOrganization:coalition\nPercentage:11% between 1990 and 2015\nPercentage:65%\nPerson:Bismarck\nPopulation:over 82 million\nRank:14th highest emitting nation of greenhouse gases\nRanking:fourth globally in number of science and engineering research papers published\nRanking:fourth in research and development expenditure\nRanking:third in quality-adjusted Nature Index\nRegion:Central Europe\nResearch Institution:Fraunhofer Society\nResearch Institution:Helmholtz Association\nResearch Institution:Leibniz Association\nResearch Institution:Max Planck Society\nStatistic:percentage of migrants in population\nTerritory:Western sectors\n----\nRelationships:\nAdministrative Division:401 districts CONSISTS_OF Administrative Division:107 urban districts\nAdministrative Division:401 districts CONSISTS_OF Administrative Division:294 rural districts\nAdministrative Division:sixteen constituent states REFERRED_AS Name:Länder\nAdministrative Division:sixteen constituent states REFERRED_TO_AS Name:Länder\nCountry:Austria BORDERS Country:Germany\nCountry:Czech Republic BORDERS Country:Germany\nCountry:Federal Republic of Germany ALSO_KNOWN_AS Country:West Germany\nCountry:Federal Republic of Germany FOUNDING_MEMBER_OF Organization:European Economic Community\nCountry:Federal Republic of Germany FOUNDING_MEMBER_OF Organization:European Union\nCountry:Federal Republic of Germany HAS_STATUS Organization Membership:founding member of the European Economic Community\nCountry:Germany ANNEXED Country:Austria\nCountry:Germany BORDERED_BY Body of Water:Baltic Sea\nCountry:Germany BORDERED_BY Geographical Feature:North Sea\nCountry:Germany BORDERS Body of Water:Baltic Sea\nCountry:Germany BORDERS Country:Austria\nCountry:Germany BORDERS Country:Czech Republic\nCountry:Germany BORDERS Country:Denmark\nCountry:Germany BORDERS Country:Poland\nCountry:Germany BORDERS Country:Switzerland\nCountry:Germany BORDERS Geographical Feature:Alps\nCountry:Germany BORDERS Geographical Feature:North Sea\nCountry:Germany COMPRISES Administrative Division:sixteen constituent states\nCountry:Germany CONQUERED Country:Denmark\nCountry:Germany DIVIDED_INTO Administrative Division:401 districts\nCountry:Germany HAS_ADMINISTRATIVE_DIVISION Administrative Division:sixteen constituent states\nCountry:Germany HAS_AREA Area:357,596 km2\nCountry:Germany HAS_BRANDS Brand:Volkswagen\nCountry:Germany HAS_BRANDS Company:Deutsche Telekom\nCountry:Germany HAS_ENERGY_TRANSITION Initiative:Energiewende\nCountry:Germany HAS_FEATURE Geographical Feature:Alps\nCountry:Germany HAS_FEATURE Geographical Feature:North Sea\nCountry:Germany HAS_INSTITUTION Organization:Bundesverfassungsgericht\nCountry:Germany HAS_LAW_SYSTEM Law System:civil law system\nCountry:Germany HAS_PART Administrative Division:sixteen constituent states\nCountry:Germany HAS_POPULATION Population:over 82 million\nCountry:Germany HAS_RECYCLING_RATE Percentage:65%\nCountry:Germany HAS_RESEARCH_INSTITUTION Research Institution:Fraunhofer Society\nCountry:Germany HAS_RESEARCH_INSTITUTION Research Institution:Helmholtz Association\nCountry:Germany HAS_RESEARCH_INSTITUTION Research Institution:Leibniz Association\nCountry:Germany HAS_RESEARCH_INSTITUTION Research Institution:Max Planck Society\nCountry:Germany HAS_ROLE Organization:European Union\nCountry:Germany HAS_TERRITORY Area:357,596 km2\nCountry:Germany INVADED Country:Poland\nCountry:Germany LOCATED_IN Region:Central Europe\nCountry:Germany MEETS_POWER_DEMAND Energy Source:40% renewable sources\nCountry:Germany MEMBER_OF Organization:European Union\nCountry:Germany OFFICIALLY_KNOWN_AS Country:Federal Republic of Germany\nCountry:Germany ORGANIZED_INTO Country:Federal Republic of Germany\nCountry:Germany PLAYS_ROLE_IN Organization:European Union\nCountry:Germany RANKS_IN Ranking:fourth globally in number of science and engineering research papers published\nCountry:Germany RANKS_IN Ranking:fourth in research and development expenditure\nCountry:Germany RANKS_IN Ranking:third in quality-adjusted Nature Index\nCountry:Germany RANKS_SECOND_AFTER Country:United States\nCountry:Germany RANKS_SEVENTH_IN Statistic:percentage of migrants in population\nCountry:Germany RECEIVED_REFUGEES_FROM Country:Ukraine\nCountry:Germany REDUCED_ENERGY_CONSUMPTION Percentage:11% between 1990 and 2015\nCountry:Germany WAS_EMITTING_NATION Rank:14th highest emitting nation of greenhouse gases\nCountry:Hungary OPENED_BORDER_WITH Country:Austria\nCountry:Poland BORDERS Country:Germany\nCountry:Switzerland BORDERS Country:Germany\nGroup:East Germans EMIGRATED_VIA Country:Austria\nKingdom:East Francia STRETCHED_FROM Geographical Feature:North Sea\nKingdom:East Francia STRETCHED_TO Geographical Feature:Alps\nLaw System:civil law system BASED_ON Law System:Roman law\nLaw System:civil law system REFERENCES Law System:Germanic law\nLocation:Berlin IS_HUB Country:Germany\nOrganization:Bundesverfassungsgericht HAS_POWER Legal Power:judicial review\nOrganization:Bundesverfassungsgericht IS Organization:German Supreme Court\nOrganization:Bundesverfassungsgericht REFERRED_AS Organization:Federal Constitutional Court\nOrganization:Bundesverfassungsgericht RESPONSIBLE_FOR Legal Domain:constitutional matters\nOrganization:Federal Constitutional Court DEFINED_TERM Concept:defence\nOrganization:North German Confederation EXCLUDED Country:Austria\nOrganization:Population Division of the United Nations Department of Economic and Social Affairs LISTED_AS_HOST_TO Country:Germany\nOrganization:coalition OPERATES_IN Country:Switzerland\nPerson:Bismarck CONCLUDED_WAR Country:Denmark\nTerritory:Western sectors MERGED_TO_FORM Country:Federal Republic of Germany\nDocument end\n", + "answer": "Germany has sixteen constituent states, collectively referred to as Länder." } }, "user": "chatbot" @@ -449,10 +535,10 @@ This API is used to get the entities and relations associated with a particular * `uri`=Neo4j uri, * `userName`= Neo4j db username, * `password`= Neo4j db password, -* `database`= Neo4j database name -* `nodedetails` = Node element id's to get information(chunks,entities,communities) -* `entities` = entities received from the retriver for graph based modes - +* `database`= Neo4j database name, +* `nodedetails` = Node element id's to get information(chunks,entities,communities), +* `entities` = entities received from the retriver for graph based modes, +* `email`= Logged in User Email **Response :** [source,json,indent=0] @@ -462,51 +548,106 @@ This API is used to get the entities and relations associated with a particular "data": { "nodes": [ { - "element_id": "4:98e5e9bb-8095-440d-9462-03985fed2fa2:307", + "element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7787", + "labels": [ + "Country", + "Location" + ], + "properties": { + "id": "Germany", + "description": null + } + }, + { + "element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7779", "labels": [ - "Company" + "Organization" ], "properties": { - "id": "Amazon", - "description": "Initially an online bookstore, Amazon has transformed into a $48 billion retail giant, offering products in over forty categories, from books and electronics to groceries. Today, it operates as a logistics platform, a search engine, an Internet advertising platform, an e-commerce platform, and an IT platform." + "id": "European Union", + "description": null + } + }, + { + "element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:5977", + "labels": [ + "Organization" + ], + "properties": { + "id": "coalition", + "description": null } } ], "relationships": [ { - "element_id": "5:98e5e9bb-8095-440d-9462-03985fed2fa2:6917952339617775946", - "type": "OFFERS", - "start_node_element_id": "4:98e5e9bb-8095-440d-9462-03985fed2fa2:307", - "end_node_element_id": "4:98e5e9bb-8095-440d-9462-03985fed2fa2:330" + "element_id": "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:146579", + "type": "RANKS_SECOND_AFTER", + "start_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7787", + "end_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:5973" + }, + { + "element_id": "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:145089", + "type": "BORDERS", + "start_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7782", + "end_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7787" + }, + { + "element_id": "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:146509", + "type": "HAS_BRANDS", + "start_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7787", + "end_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:8457" + }, + { + "element_id": "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:146121", + "type": "HAS_POWER", + "start_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:8326", + "end_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:8345" } ], "chunk_data": [ { - "element_id": "4:98e5e9bb-8095-440d-9462-03985fed2fa2:14", - "id": "d1e92be81a0872d621242cee9fed69d14b0cd68d", - "position": 13, - "text": " 6 eBay, operating as the biggest online auction house and focusing as a service provider, employs cost leadership strategy by solely operating e-commerce as an intermediary without holding any inventories or physical infrastructures. It also applies a differentiation strategy by providing a ....", - "content_offset": 9886, - "fileName": "About Amazon.pdf", - "page_number": 7, - "length": 1024, - "fileSource": "local file", - "embedding": null - } - ], - "community_data": [ + "element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7678", + "id": "0c92f93e837e6b31f8d2429dd76c3db4ab37ce14", + "position": 1, + "text": "Germany, officially the Federal Republic of Germany, is a country in Central Europe. It lies between the Baltic Sea and the North Sea to the north and the Alps to the south. Its sixteen constituent states have a total population of over 82 million in an area of 357,596 km2 (138,069 sq mi), making it the most populous member state of the European Union. It borders Denmark to the north, Poland and the Czech Republic to the east, Austria and Switzerland to the south,", + "content_offset": 0, + "fileName": "Germany", + "length": 468, + "embedding": null, + "fileSource": "Wikipedia", + "fileType": "text", + "url": "https://en.wikipedia.org/wiki/Germany" + }, { - "element_id": "4:98e5e9bb-8095-440d-9462-03985fed2fa2:1026", - "summary": "Google, led by CEO Sundar Pichai, is actively involved in various business and product initiatives.", - "id": "0-311", - "level": 0, - "weight": 7, + "element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7747", + "id": "ac8c9c1e05c718cc612160d6580caf1af97dfb1f", + "position": 70, + "text": " sixteen constituent states which are collectively referred to as Länder. Each state (Land) has its own constitution, and is largely autonomous in regard to its internal organisation. As of 2017, Germany is divided into 401 districts (Kreise) at a municipal level; these consist of 294 rural districts and 107 urban districts. === Law === Germany has a civil law system based on Roman law with some references to Germanic law. The Bundesverfassungsgericht (Federal", + "content_offset": 33460, + "fileName": "Germany", + "length": 467, + "embedding": null, + "fileSource": "Wikipedia", + "fileType": "text", + "url": "https://en.wikipedia.org/wiki/Germany" + }, + { + "element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7773", + "id": "b91415a3bbfb99d64b3a2aa8b1413bebd5b5650e", + "position": 96, + "text": " resort. == Demographics == With a population of 84.7 million according to the 2023 German census, Germany is the most populous member state of the European Union, the second-most populous country in Europe after Russia, and the nineteenth-most populous country in the world. Its population density stands at 236 inhabitants per square kilometre (610 inhabitants/sq mi). The fertility rate of 1.57 children born per woman (2022 estimates) is below the replacement rate of 2", + "content_offset": 46388, + "fileName": "Germany", + "length": 476, "embedding": null, - "community_rank": 1 + "fileSource": "Wikipedia", + "fileType": "text", + "url": "https://en.wikipedia.org/wiki/Germany" } ] }, - "message": "Total elapsed API time 3.75" + "message": "Total elapsed API time 0.55" } .... @@ -515,16 +656,17 @@ This API is used to get the entities and relations associated with a particular POST /graph_query ---- -This API is used to view graph for a particular file. +This API is used to visualize graphs for a particular document or list of multiple documents; +it will return documents, chunks, entities, relationships and communities to the front-end to be shown in a graph visualization. **API Parameters :** * `uri`=Neo4j uri, * `userName`= Neo4j db username, * `password`= Neo4j db password, -* `query_type`= Neo4j database name -* `document_names` = File name for which user wants to view graph - +* `database`= Neo4j database name, +* `document_names` = File name for which user wants to view graph, +* `email`= Logged in User Email **Response :** [source,json,indent=0] @@ -534,33 +676,105 @@ This API is used to view graph for a particular file. "data": { "nodes": [ { - "element_id": "4:98e5e9bb-8095-440d-9462-03985fed2fa2:9972", + "element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:10497", "labels": [ - "Person" + "Document" ], "properties": { - "id": "Jeff" + "fileName": "Untitled Diagram.png", + "communityNodeCount": 12, + "errorMessage": "", + "chunkRelCount": 28, + "fileSource": "local file", + "communityRelCount": 22, + "total_chunks": 5, + "processingTime": 15.91, + "entityNodeCount": 14, + "chunkNodeCount": 5, + "createdAt": "2025-04-10T16:33:22.331776000", + "entityEntityRelCount": 10, + "fileSize": 393322, + "model": "openai_gpt_4.5", + "nodeCount": 31, + "processed_chunk": 5, + "is_cancelled": false, + "relationshipCount": 60, + "fileType": "png", + "status": "Completed", + "updatedAt": "2025-04-10T17:06:15.896962000" } }, { - "element_id": "4:98e5e9bb-8095-440d-9462-03985fed2fa2:9973", + "element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:10501", "labels": [ - "Team" + "Chunk" ], "properties": { - "id": "Miami" + "fileName": "Untitled Diagram.png", + "content_offset": 9, + "page_number": 1, + "length": 21, + "id": "e6200cc319ae833a42f3ea85bd3f48fe57f528ac", + "position": 3 + } + }, + { + "element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:10499", + "labels": [ + "Chunk" + ], + "properties": { + "fileName": "Untitled Diagram.png", + "content_offset": 0, + "page_number": 1, + "length": 1, + "id": "091385be99b45f459a231582d583ec9f3fa3d194", + "position": 1 + } + }, + { + "element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:10503", + "labels": [ + "Chunk" + ], + "properties": { + "fileName": "Untitled Diagram.png", + "content_offset": 113, + "page_number": 1, + "length": 14, + "id": "2fe558452be341af4450be97b79ecdd8ea64b188", + "position": 5 } } ], "relationships": [ { - "element_id": "5:98e5e9bb-8095-440d-9462-03985fed2fa2:1153200780560312052", - "type": "PLAYER", - "start_node_element_id": "4:98e5e9bb-8095-440d-9462-03985fed2fa2:9972", - "end_node_element_id": "4:98e5e9bb-8095-440d-9462-03985fed2fa2:9973" - } + "element_id": "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:155207", + "type": "IN_COMMUNITY", + "start_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:10515", + "end_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:10907" + }, + { + "element_id": "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:155845", + "type": "PARENT_COMMUNITY", + "start_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:10907", + "end_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:11378" + }, + { + "element_id": "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:155846", + "type": "PARENT_COMMUNITY", + "start_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:11378", + "end_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:11379" + }, + { + "element_id": "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:153551", + "type": "HAS_ENTITY", + "start_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:10503", + "end_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:10516" + } ] - } + }, + "message": "Total elapsed API time 0.79" } .... @@ -569,7 +783,7 @@ This API is used to view graph for a particular file. POST /get_neighbours ---- -This API is used to retrive the neighbor nodes of the given element id of the node. +This API is used to get the nearby nodes and relationships based on the element id of the node for graph visualization of details of specific nodes. **API Parameters :** @@ -577,7 +791,8 @@ This API is used to retrive the neighbor nodes of the given element id of the no * `userName`= Neo4j db username, * `password`= Neo4j db password, * `database`= Neo4j database name, -* `elementId` = Element id of the node to retrive its neighbours +* `elementId` = Element id of the node to retrive its neighbours, +* `email`= Logged in User Email **Response :** @@ -589,33 +804,57 @@ This API is used to retrive the neighbor nodes of the given element id of the no "nodes": [ { "summary": null, - "element_id": "4:98e5e9bb-8095-440d-9462-03985fed2fa2:3", - "id": "73bc9c9170bcd807d2fa87d87a0eeb3d82f95160", - "position": 2, + "element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:11925", + "id": "0-554", "text": null, - "content_offset": 186, + "title": "Western Sectors Control,", + "weight": 2, + "level": 0, "labels": [ - "Chunk" + "__Community__" + ], + "properties": { + "id": "0-554", + "title": "Western Sectors Control," + }, + "embedding": null + }, + { + "summary": null, + "element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:5978", + "id": "United Kingdom", + "text": null, + "communities": [ + 554, + 246, + 16 + ], + "labels": [ + "Country" ], - "page_number": 2, - "fileName": "About Amazon.pdf", - "length": 904, "properties": { - "id": "73bc9c9170bcd807d2fa87d87a0eeb3d82f95160" + "id": "United Kingdom", + "title": " " }, "embedding": null } ], "relationships": [ { - "element_id": "5:98e5e9bb-8095-440d-9462-03985fed2fa2:1175445000301838339", - "end_node_element_id": "4:98e5e9bb-8095-440d-9462-03985fed2fa2:18", - "start_node_element_id": "4:98e5e9bb-8095-440d-9462-03985fed2fa2:3", - "type": "HAS_ENTITY" + "element_id": "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:145729", + "end_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:7792", + "start_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:8154", + "type": "CONTROLLED_BY" }, + { + "element_id": "5:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:145730", + "end_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:5978", + "start_node_element_id": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:8154", + "type": "CONTROLLED_BY" + } ] }, - "message": "Total elapsed API time 0.24" + "message": "Total elapsed API time 0.43" } .... @@ -634,8 +873,8 @@ This API is used to clear the chat history which is saved in Neo4j DB. * `userName`= Neo4j db username, * `password`= Neo4j db password, * `database`= Neo4j database name, -* `session_id` = User session id for QA chat - +* `session_id` = User session id for QA chat, +* `email`= Logged in User Email **Response :** [source,json,indent=0] @@ -643,8 +882,8 @@ This API is used to clear the chat history which is saved in Neo4j DB. { "status": "Success", "data": { - "session_id": "99c1a808-377f-448f-9ea6-4b4a8de46b14", - "message": "The chat History is cleared", + "session_id": "f4e352bf-f57a-4a15-819e-68d2ffca82a2", + "message": "The chat history has been cleared.", "user": "chatbot" } } @@ -663,8 +902,7 @@ The API provides a continuous update on the extraction status of a specified fil * `uri`=Neo4j uri, * `userName`= Neo4j db username, * `password`= Neo4j db password, -* `database`= Neo4j database name - +* `database`= Neo4j database name, **Response :** [source,json,indent=0] @@ -699,15 +937,13 @@ Deleteion of nodes and relations for multiple files is done through this API. Us * `database`= Neo4j database name, * `filenames`= List of files to be deleted, * `source_types`= Document sources(Wikipedia, youtube, etc.), -* `deleteEntities`= Boolean value to check entities deletion is requested or not +* `deleteEntities`= Boolean value to check entities deletion is requested or not, +* `email`= Logged in User Email **Response :** [source,json,indent=0] .... -{ - "status": "Success", - "message": "Deleted 1 documents with 68 entities from database" -} +{"status":"Success","message":"Deleted 1 documents with entities from database"} .... === Cancel processing job @@ -724,8 +960,8 @@ This API is responsible for cancelling an in process job. * `password`= Neo4j db password, * `database`= Neo4j database name, * `filenames`= Name of the file whose processing need to be stopped, -* `source_types`= Source of the file - +* `source_types`= Source of the file, +* `email`= Logged in User Email **Response :** [source,json,indent=0] @@ -741,31 +977,51 @@ This API is responsible for cancelling an in process job. POST /get_unconnected_nodes_list ---- -The API retrieves a list of nodes in the graph database that are not connected to any other nodes. +The API retrieves a list of nodes in the graph database that are not connected to any other entity nodes, +and only to chunks that they were extracted from. So to say orphan nodes from an domain graph perspective. **API Parameters :** * `uri`=Neo4j uri, * `userName`= Neo4j db username, * `password`= Neo4j db password, -* `database`= Neo4j database name - +* `database`= Neo4j database name, +* `email`= Logged in User Email **Response :** [source,json,indent=0] .... -{ "status": "Success", +{ + "status": "Success", "data": [ - "e": - { - "id": "Leela Chess Zero", - "elementId": "4:abf6f691-928d-4b1c-80fc-2914ae517b4c:336", - "labels": ["Technology"], - "embedding": null - }, - "documents": ["AlphaZero - Wikipedia.pdf"], - "chunkConnections": 7 - ] + { + "e": { + "id": "46c949fb-b451-4b69-b3bd-87f26ac8f9e6", + "elementId": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:5853", + "labels": [ + "Entity" + ], + "embedding": null + }, + "documents": [], + "chunkConnections": 0 + }, + { + "e": { + "id": "f4e352bf-f57a-4a15-819e-68d2ffca82a2", + "elementId": "4:8b7ad735-1828-4d80-b8c3-798dcbfdd95d:11380", + "labels": [ + "Entity" + ], + "embedding": null + }, + "documents": [], + "chunkConnections": 0 + } + ], + "message": { + "total": 2 + } } .... @@ -775,7 +1031,7 @@ The API retrieves a list of nodes in the graph database that are not connected t POST /delete_unconnected_nodes ---- -The API is used to delete unconnected entities from database. +The API is used to delete unconnected entities from the neo4j database with the input provided as selection from the user. **API Parameters :** @@ -783,31 +1039,20 @@ The API is used to delete unconnected entities from database. * `userName`= Neo4j db username, * `password`= Neo4j db password, * `database`= Neo4j database name, -* `unconnected_entities_list`=selected entities list to delete of unconnected entities. +* `unconnected_entities_list`=selected entities list to delete of unconnected entities, +* `email`= Logged in User Email **Response :** [source,json,indent=0] .... -{ +{ "status": "Success", - "message: "Unconnected entities delete successfully" + "data": [], + "message": "Unconnected entities delete successfully" } .... - -==== Decisions - -* Process only 1st page of Wikipedia -* Split document content into chunks of size 200 and overlap of 20 -* Configurable elements - -** Number of chunks to combine -** Generate Embedding or not -** Embedding model -** minimum score for KNN graph -** Uploaded file storage location (GCS bucket or container) - - === Get duplicate nodes ---- POST /get_duplicate_nodes @@ -821,7 +1066,7 @@ The API is used to fetch duplicate entities from database. * `userName`= Neo4j db username, * `password`= Neo4j db password, * `database`= Neo4j database name, - +* `email`= Logged in User Email **Response :** [source,json,indent=0] @@ -879,7 +1124,8 @@ The API is used to merge duplicate entities from database selected by user. * `userName`= Neo4j db username, * `password`= Neo4j db password, * `database`= Neo4j database name, -* `duplicate_nodes_list`= selected entities list to merge of with similar entities. +* `duplicate_nodes_list`= selected entities list to merge of with similar entities, +* `email`= Logged in User Email **Response :** [source,json,indent=0] @@ -908,6 +1154,7 @@ The API is used to drop and create the vector index when vector index dimesion a * `password`= Neo4j db password, * `database`= Neo4j database name, * `isVectorIndexExist`= True or False based on whether vector index exist in database, +* `email`= Logged in User Email **Response :** [source,json,indent=0] @@ -923,14 +1170,12 @@ The API is used to drop and create the vector index when vector index dimesion a POST /retry_processing ---- -This API is used to Ready to Reprocess cancelled, completed or failed file sources. -Users have 3 options to Ready to Reprocess files: - -* Start from begnning - In this condition file will be processed from the begnning i.e. 1st chunk again. -* Delete entities and start from begnning - If the file source is already processed and have any existing nodes and relations then those will be deleted and file will be reprocessed from the 1st chunk. -* Start from last processed postion - Cancelled or failed files will be processed from the last successfully processed chunk position. This option is not available for completed files. - -Ones the status is set to 'Ready to Reprocess', user can again click on Generate graph to process the file for knowledge graph creation. +This API is used to reprocess canceled, completed or failed file sources. +Users have 3 options to reprocess files: +* Start from beginning - In this condition file will be processed from the beginning i.e. 1st chunk again. +* Delete entities and start from beginning - If the file source is already processed and has any existing nodes and relationships then those will be deleted and the file will be reprocessed from the 1st chunk. +* Start from the last processed position - Canceled or failed files will be processed from the last successfully processed chunk position. This option is not available for completed files. +* Once the status is set to 'Reprocess', users can again click on Generate Graph to process the file for knowledge graph creation. **API Parameters :** @@ -940,7 +1185,8 @@ Ones the status is set to 'Ready to Reprocess', user can again click on Generate * `database`= Neo4j database name, * `file_name`= Name of the file which user want to Ready to Reprocess. * `retry_condition` = One of the above 3 conditions which is selected for reprocessing. - +* `email`= Logged in User Email, + **Response :** [source,json,indent=0] @@ -956,7 +1202,8 @@ Ones the status is set to 'Ready to Reprocess', user can again click on Generate POST /metric ---- -The API responsible for a evaluating chatbot responses on the basis of different metrics such as faithfulness and answer relevancy. This utilises RAGAS library to calculate these metrics. +The API responsible for evaluating the chatbot response for the different retrievers on the basis of different metrics +such as faithfulness and answer relevancy. This utilises the RAGAS library to calculate these metrics. **API Parameters :** @@ -972,9 +1219,10 @@ The API responsible for a evaluating chatbot responses on the basis of different { "status": "Success", "data": { - "graph+vector+fulltext": { + "graph_vector_fulltext": { "faithfulness": 1.0, - "answer_relevancy": 0.9699 + "answer_relevancy": 0.9118, + "context_entity_recall": 0.6667 } } } @@ -1004,8 +1252,7 @@ The API responsible for a evaluating chatbot responses on the basis of different "data": { "graph_vector_fulltext": { "rouge_score": 1.0, - "semantic_score": 0.9842, - "context_entity_recall_score": 0.5 + "semantic_score": 0.9842 } } } @@ -1027,6 +1274,7 @@ The API responsible for a fetching text associated with a particular chunk and c * `database`= Neo4j database name * `document_name` = Name of document for which chunks needs to be fetched. * `page no` = page number for multipage +* `email`= Logged in User Email **Response :** [source,json,indent=0] @@ -1069,14 +1317,6 @@ The API responsible for create the connection obj from Neo4j DB based on environ } .... -.... -{ - "status": "Failed", - "error": "Could not connect to Neo4j database. Please ensure that the username and password are correct", - "message": "Unable to connect backend DB" -} -.... - === Visualize graph DB schema ---- POST /schema_visualization @@ -1089,8 +1329,8 @@ User can visualize schema of the db through this API. * `uri`=Neo4j uri, * `userName`= Neo4j db username, * `password`= Neo4j db password, -* `database`= Neo4j database name - +* `database`= Neo4j database name, +* `email`= Logged in User Email **Response :** [source,json,indent=0] diff --git a/frontend/package.json b/frontend/package.json index 311560023..68ac3260b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -16,7 +16,7 @@ "@auth0/auth0-react": "^2.2.4", "@emotion/styled": "^11.11.0", "@mui/material": "^5.15.10", - "@mui/styled-engine": "^5.15.9", + "@mui/styled-engine": "^7.0.1", "@neo4j-devtools/word-color": "^0.0.8", "@neo4j-ndl/base": "^3.2.9", "@neo4j-ndl/react": "^3.2.18", @@ -25,7 +25,7 @@ "@react-oauth/google": "^0.12.1", "@tanstack/react-table": "^8.20.5", "@types/uuid": "^9.0.7", - "axios": "^1.8.3", + "axios": "^1.8.4", "clsx": "^2.1.1", "eslint-plugin-react": "^7.37.4", "re-resizable": "^6.11.2", @@ -35,7 +35,7 @@ "react-markdown": "^9.0.1", "react-router": "^6.23.1", "react-router-dom": "^6.23.1", - "uuid": "^9.0.1" + "uuid": "^11.1.0" }, "devDependencies": { "@tailwindcss/postcss": "^4.0.12", @@ -46,11 +46,11 @@ "@typescript-eslint/parser": "^6.0.0", "@vitejs/plugin-react": "^4.0.3", "eslint": "^8.45.0", - "eslint-config-prettier": "^10.0.2", + "eslint-config-prettier": "^10.1.1", "eslint-plugin-react-hooks": "^5.1.0", "eslint-plugin-react-refresh": "^0.4.19", "husky": "^9.1.7", - "lint-staged": "^15.4.3", + "lint-staged": "^15.5.0", "postcss": "^8.5.3", "prettier": "^3.5.3", "react-dropzone": "^14.3.8", diff --git a/frontend/src/App.css b/frontend/src/App.css index 9d3ba9601..a5efdd056 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -90,27 +90,33 @@ box-shadow: -12px 0 #FFF, 12px 0 rgb(var(--theme-palette-primary-bg-strong)); } } -.dropdownbtn{ + +.dropdownbtn { background-color: #014063; color: white !important; border-radius: unset !important; width: 35px; padding: 0.5rem; } -.dropdownbtn:hover{ + +.dropdownbtn:hover { background-color: #02293d !important; } -.graphbtn{ + +.graphbtn { border-top-right-radius: 0px !important; border-bottom-right-radius: 0px !important; } -.dropdownbtn.darktheme{ + +.dropdownbtn.darktheme { background-color: #51A6B1; } -.dropdownbtn.small{ + +.dropdownbtn.small { height: 24px; } -.darktheme:hover{ + +.darktheme:hover { background-color: #44929c !important; } @@ -334,21 +340,25 @@ min-width: 250px; max-width: 305px; } -.ndl-modal-root{ + +.ndl-modal-root { z-index: 39 !important; } + .tbody-dark .ndl-data-grid-tr:hover { --cell-background: rgb(60 63 68) !important; } + .tbody-light .ndl-data-grid-tr:hover { --cell-background: rgb(226 227 229) !important; } -.chatbot-deleteLoader{ - position:absolute; - top:0; - bottom:0; - width:100%; - height:100%; + +.chatbot-deleteLoader { + position: absolute; + top: 0; + bottom: 0; + width: 100%; + height: 100%; display: flex; align-items: center; justify-content: center; @@ -356,24 +366,29 @@ opacity: 0.8; background-color: rgb(201 201 201 / 40%); } -.layout-wrapper{ + +.layout-wrapper { display: grid; grid-template-rows: auto; grid-template-columns: 64px 1fr minmax(min-content,4fr) 1.1fr 64px; max-height: calc(100vh - 58px); max-width: 100%; } -.layout-wrapper.drawerdropzoneclosed{ + +.layout-wrapper.drawerdropzoneclosed { grid-template-columns: 64px 4fr 1.1fr 64px; } -.layout-wrapper.drawerchatbotclosed{ - grid-template-columns: 64px 0.6fr minmax(min-content,4fr) 64px; + +.layout-wrapper.drawerchatbotclosed { + grid-template-columns: 64px 0.6fr minmax(min-content, 4fr) 64px; } -.layout-wrapper.drawerclosed{ - grid-template-columns: 64px minmax(min-content,4fr) 64px; + +.layout-wrapper.drawerclosed { + grid-template-columns: 64px minmax(min-content, 4fr) 64px; } -.main-content-wrapper{ + +.main-content-wrapper { display: flex; flex-direction: column; max-width: 100%; @@ -384,37 +399,74 @@ @media screen and (min-width:1025px) and (max-width:1440px){ .layout-wrapper{ grid-template-columns: 64px 1fr minmax(min-content,4fr) minmax(min-content,2fr) 64px; - } } -@media screen and (min-width:1536px) and (max-width:2560px){ - .layout-wrapper{ - grid-template-columns: 64px 1fr minmax(min-content,6.5fr) minmax(max-content,1fr) 64px; + +@media screen and (min-width:1536px) and (max-width:2560px) { + .layout-wrapper { + grid-template-columns: 64px 1fr minmax(min-content, 6.5fr) minmax(max-content, 1fr) 64px; } } -.sidenav-container{ + +.sidenav-container { height: calc(100vh - 58px); min-height: 200px; display: flex } - .resource-sections.blur-sm { +.resource-sections.blur-sm { filter: blur(2px); - } -.profile-container{ - display:flex; - gap:4px; - align-items:center; +} + +.profile-container { + display: flex; + gap: 4px; + align-items: center; border: 1px solid rgb(var(--theme-palette-neutral-border-strong)); border-radius: 12px; } -.websource-btn-container{ + +.websource-btn-container { display: flex; gap: 10px; } -.enhancement-btn__wrapper{ +.enhancement-btn__wrapper { padding-right: 12px; display: flex; +} + +.patternContainer { + border: 1px solid rgb(var(--theme-palette-neutral-border-strong)); + overflow-y: scroll; + max-height: fit-content; + height: 150px; + padding: 10px; + width:100%; +} + +@keyframes highlightAnimation { + 0% { + background-color:rgb(var(--theme-palette-neutral-bg-weak)); + border-color: rgb(var(--theme-palette-primary-bg-weak)); + transform: scale(1); + } + + 50% { + background-color:rgb(var(--theme-palette-neutral-bg-weak)); + border-color: rgb(var(--theme-palette-primary-bg-strong)); + transform: scale(1.05); + } + + 100% { + background-color: rgb(var(--theme-palette-neutral-bg-weak)); + border-color: rgb(var(--theme-palette-primary-bg-strong)); + transform: scale(1); + } +} + +.animate-highlight { + animation: highlightAnimation 1.5s ease-in-out; + border: 2px solid rgb(var(--theme-palette-primary-bg-strong)); } \ No newline at end of file diff --git a/frontend/src/assets/newSchema.json b/frontend/src/assets/newSchema.json new file mode 100644 index 000000000..1f27435f9 --- /dev/null +++ b/frontend/src/assets/newSchema.json @@ -0,0 +1,167 @@ +[ + { + "schema": "crime", + "triplet": [ + "PhoneCall-CALLED->Phone", + "Phone-CALLER->PhoneCall", + "Person-CURRENT_ADDRESS->Location", + "Person-FAMILY_REL->Person", + "Person-HAS_EMAIL->Email", + "Person-HAS_PHONE->Phone", + "Location-HAS_POSTCODE->PostCode", + "Crime-INVESTIGATED_BY->Officer", + "Person-INVOLVED_IN->Crime", + "Person-KNOWS->Person", + "Person-KNOWS_LW->Person", + "Person-KNOWS_PHONE->Phone", + "Person-KNOWS_SN->Person", + "Location-LOCATION_IN_AREA->Area", + "Crime-OCCURRED_AT->Location", + "Person-PARTY_TO->Crime", + "PostCode-POSTCODE_IN_AREA->Area", + "Object-INVOLVED_IN->Crime", + "Vehicle-INVOLVED_IN->Crime" + ] + }, + { + "schema": "flights", + "triplet": [ + "Airport-HAS_ROUTE->Airport", + "Airport-IN_CITY->City", + "City-IN_COUNTRY->Country", + "Country-IN_REGION->Region", + "City-ON_CONTINENT->Continent", + "Country-ON_CONTINENT->Continent", + "Region-ON_CONTINENT->Continent" + ] + }, + { + "schema": "healthcare", + "triplet": [ + "Case-FALLS_UNDER->AgeGroup", + "Case-HAS_REACTION->Reaction", + "Case-RESULTED_IN->Outcome", + "Case-REPORTED_BY->ReportSource", + "Case-REGISTERED->Manufacturer", + "Drug-IS_PRIMARY_SUSPECT->Case", + "Drug-IS_SECONDARY_SUSPECT->Case", + "Drug-IS_CONCOMITANT->Case", + "Drug-IS_INTERACTING->Drug", + "Drug-PRESCRIBED->Therapy", + "Drug-RECEIVED->Manufacturer" + ] + }, + { + "schema": "movies", + "triplet": ["Actor-ACTED_IN->Movie", "Director-DIRECTED->Movie", "Movie-IN_GENRE->Genre", "User-RATED->Movie"] + }, + { + "schema": "network", + "triplet": [ + "Application-CONNECTS->Network", + "Application-DEPENDS_ON->Service", + "Application-RUNS->Software", + "DataCenter-CONTAINS->Rack", + "DataCenter-CONTAINS->Zone", + "Egress-ROUTES->Network", + "Interface-CONNECTS->Router", + "Interface-CONNECTS->Switch", + "Machine-CONTAINS->Interface", + "Machine-RUNS->OS", + "Machine-RUNS->Process", + "Network-CONNECTS->Router", + "Network-CONNECTS->Switch", + "Network-EXPOSES->Port", + "OS-VERSION->Version", + "Port-EXPOSES->Service", + "Process-LISTENS->Port", + "Rack-CONTAINS->Machine", + "Rack-CONTAINS->Router", + "Rack-CONTAINS->Switch", + "Router-CONNECTS->Network", + "Router-CONNECTS->Switch", + "Service-DEPENDS_ON->Application", + "Service-EXPOSES->Port", + "Software-VERSION->Version", + "Switch-CONNECTS->Network", + "Switch-CONNECTS->Router", + "Type-HOLDS->Application", + "Type-HOLDS->Machine", + "Type-HOLDS->Service", + "Version-PREVIOUS->Version", + "Zone-CONTAINS->DataCenter" + ] + }, + { + "schema": "payments", + "triplet": [ + "Client-HAS_EMAIL->Email", + "Client-HAS_PHONE->Phone", + "Client-HAS_SSN->SSN", + "Transaction-NEXT->Transaction", + "Client-PERFORMED->Transaction", + "Transaction-SENT_TO->Merchant", + "Transaction-RECEIVED_FROM->Client", + "Transaction-RECEIVED_FROM->Bank", + "Transaction-SENT_TO->Bank", + "Transaction-SENT_TO->Client", + "Transaction-RECEIVED_FROM->Merchant", + "Transaction-SENT_TO->CashOut", + "Transaction-RECEIVED_FROM->CashIn", + "Transaction-SENT_TO->CashIn", + "Transaction-RECEIVED_FROM->CashOut", + "Transaction-SENT_TO->Debit", + "Transaction-RECEIVED_FROM->Debit", + "Transaction-SENT_TO->Transfer", + "Transaction-RECEIVED_FROM->Transfer" + ] + }, + { + "schema": "retail", + "triplet": [ + "Customer-ORDERS->Order", + "Order-PART_OF->Category", + "Product-PART_OF->Category", + "Customer-PURCHASED->Product", + "Supplier-SUPPLIES->Product" + ] + }, + { + "schema": "reviews", + "triplet": [ + "User-FRIENDS->User", + "Business-HAS_LABEL->Label", + "Business-HAS_PHOTO->Photo", + "Business-IN_CATEGORY->Category", + "Business-IN_CITY->City", + "User-REVIEWS->Business", + "Business-SIMILAR->Business", + "User-WROTE->Review", + "User-WROTE_TIP->Review" + ] + }, + { + "schema": "social", + "triplet": [ + "User-FOLLOWS->User", + "User-POSTS->Post", + "Post-CONTAINS->Link", + "Post-TAGS->Tag", + "Post-MENTIONS->User", + "User-REPLY_TO->Post", + "User-REPOST->Post" + ] + }, + { + "schema": "stackoverflow", + "triplet": [ + "User-ANSWERED->Question", + "User-POSTED->Question", + "User-POSTED->Answer", + "User-ACCEPTED->Answer", + "Question-SIMILAR->Question", + "Question-TAGGED->Tag", + "Answer-TAGGED->Tag" + ] + } +] diff --git a/frontend/src/components/Auth/Auth.tsx b/frontend/src/components/Auth/Auth.tsx index 44c67ab02..df74aa3f0 100644 --- a/frontend/src/components/Auth/Auth.tsx +++ b/frontend/src/components/Auth/Auth.tsx @@ -1,4 +1,3 @@ - import React, { useEffect } from 'react'; import { AppState, Auth0Provider, useAuth0 } from '@auth0/auth0-react'; import { useNavigate } from 'react-router'; @@ -26,7 +25,6 @@ const Auth0ProviderWithHistory: React.FC<{ children: React.ReactNode }> = ({ chi }; export const AuthenticationGuard: React.FC<{ component: React.ComponentType }> = ({ component }) => { - const { isAuthenticated, isLoading } = useAuth0(); const Component = component; const navigate = useNavigate(); @@ -37,7 +35,6 @@ export const AuthenticationGuard: React.FC<{ component: React.ComponentType; }; diff --git a/frontend/src/components/Content.tsx b/frontend/src/components/Content.tsx index 4f8904fd9..5a69ef911 100644 --- a/frontend/src/components/Content.tsx +++ b/frontend/src/components/Content.tsx @@ -103,6 +103,7 @@ const Content: React.FC = ({ selectedChunk_overlap, selectedChunks_to_combine, setSelectedNodes, + setAllPatterns, setRowSelection, setSelectedRels, setSelectedTokenChunkSize, @@ -586,6 +587,7 @@ const Content: React.FC = ({ setUserCredentials({ uri: '', password: '', userName: '', database: '', email: '' }); setSelectedNodes([]); setSelectedRels([]); + setAllPatterns([]); localStorage.removeItem('selectedTokenChunkSize'); setSelectedTokenChunkSize(tokenchunkSize); localStorage.removeItem('selectedChunk_overlap'); diff --git a/frontend/src/components/Graph/GraphViewModal.tsx b/frontend/src/components/Graph/GraphViewModal.tsx index 28aede125..7738bfb3e 100644 --- a/frontend/src/components/Graph/GraphViewModal.tsx +++ b/frontend/src/components/Graph/GraphViewModal.tsx @@ -8,6 +8,7 @@ import { ExtendedRelationship, GraphType, GraphViewModalProps, + OptionType, Scheme, } from '../../types'; import { InteractiveNvlWrapper } from '@neo4j-nvl/react'; @@ -19,6 +20,7 @@ import { InformationCircleIconOutline, MagnifyingGlassMinusIconOutline, MagnifyingGlassPlusIconOutline, + ExploreIcon } from '@neo4j-ndl/react/icons'; import { IconButtonWithToolTip } from '../UI/IconButtonToolTip'; import { filterData, getCheckboxConditions, graphTypeFromNodes, processGraphData } from '../../utils/Utils'; @@ -31,6 +33,8 @@ import CheckboxSelection from './CheckboxSelection'; import ResultOverview from './ResultOverview'; import { ResizePanelDetails } from './ResizePanel'; import GraphPropertiesPanel from './GraphPropertiesPanel'; +import SchemaViz from '../Graph/SchemaViz'; +import { extractGraphSchemaFromRawData } from '../../utils/Utils'; const GraphViewModal: React.FunctionComponent = ({ open, @@ -42,8 +46,8 @@ const GraphViewModal: React.FunctionComponent = ({ selectedRows, }) => { const nvlRef = useRef(null); - const [nodes, setNodes] = useState([]); - const [relationships, setRelationships] = useState([]); + const [node, setNode] = useState([]); + const [relationship, setRelationship] = useState([]); const [allNodes, setAllNodes] = useState([]); const [allRelationships, setAllRelationships] = useState([]); const [loading, setLoading] = useState(false); @@ -59,15 +63,19 @@ const GraphViewModal: React.FunctionComponent = ({ const [selected, setSelected] = useState<{ type: EntityType; id: string } | undefined>(undefined); const [mode, setMode] = useState(false); const graphQueryAbortControllerRef = useRef(); + const [openGraphView, setOpenGraphView] = useState(false); + const [schemaNodes, setSchemaNodes] = useState([]); + const [schemaRels, setSchemaRels] = useState([]); + const [viewCheck, setViewcheck] = useState('enhancement'); const graphQuery: string = graphType.includes('DocumentChunk') && graphType.includes('Entities') ? queryMap.DocChunkEntities : graphType.includes('DocumentChunk') - ? queryMap.DocChunks - : graphType.includes('Entities') - ? queryMap.Entities - : ''; + ? queryMap.DocChunks + : graphType.includes('Entities') + ? queryMap.Entities + : ''; // fit graph to original position const handleZoomToFit = () => { @@ -88,8 +96,8 @@ const GraphViewModal: React.FunctionComponent = ({ setGraphType([]); clearTimeout(timeoutId); setScheme({}); - setNodes([]); - setRelationships([]); + setNode([]); + setRelationship([]); setAllNodes([]); setAllRelationships([]); setSearchQuery(''); @@ -100,7 +108,7 @@ const GraphViewModal: React.FunctionComponent = ({ useEffect(() => { let updateGraphType; if (mode) { - updateGraphType = graphTypeFromNodes(nodes); + updateGraphType = graphTypeFromNodes(node); } else { updateGraphType = graphTypeFromNodes(allNodes); } @@ -149,8 +157,8 @@ const GraphViewModal: React.FunctionComponent = ({ if (mode === 'refreshMode') { initGraph(graphType, finalNodes, finalRels, schemeVal); } else { - setNodes(finalNodes); - setRelationships(finalRels); + setNode(finalNodes); + setRelationship(finalRels); setNewScheme(schemeVal); setLoading(false); } @@ -181,8 +189,8 @@ const GraphViewModal: React.FunctionComponent = ({ setAllNodes(finalNodes); setAllRelationships(finalRels); setScheme(schemeVal); - setNodes(finalNodes); - setRelationships(finalRels); + setNode(finalNodes); + setRelationship(finalRels); setNewScheme(schemeVal); setLoading(false); } @@ -195,6 +203,27 @@ const GraphViewModal: React.FunctionComponent = ({ } }, [debouncedQuery]); + const mouseEventCallbacks = useMemo(() => ({ + onNodeClick: (clickedNode: Node) => { + if (selected?.id !== clickedNode.id || selected?.type !== 'node') { + setSelected({ type: 'node', id: clickedNode.id }); + } + }, + onRelationshipClick: (clickedRelationship: Relationship) => { + if (selected?.id !== clickedRelationship.id || selected?.type !== 'relationship') { + setSelected({ type: 'relationship', id: clickedRelationship.id }); + } + }, + onCanvasClick: () => { + if (selected !== undefined) { + setSelected(undefined); + } + }, + onPan: true, + onZoom: true, + onDrag: true, + }), [selected]); + const initGraph = ( graphType: GraphType[], finalNodes: ExtendedNode[], @@ -208,8 +237,8 @@ const GraphViewModal: React.FunctionComponent = ({ finalRels ?? [], schemeVal ); - setNodes(filteredNodes); - setRelationships(filteredRelations); + setNode(filteredNodes); + setRelationship(filteredRelations); setNewScheme(filteredScheme); } }; @@ -219,45 +248,42 @@ const GraphViewModal: React.FunctionComponent = ({ return undefined; } if (selected.type === 'node') { - return nodes.find((node) => node.id === selected.id); + return node.find((nodeVal) => nodeVal.id === selected.id); } - return relationships.find((relationship) => relationship.id === selected.id); - }, [selected, relationships, nodes]); + return relationship.find((relationshipVal) => relationshipVal.id === selected.id); + }, [selected, relationship, node]); - // The search and update nodes const handleSearch = useCallback( (value: string) => { const query = value.toLowerCase(); - const updatedNodes = nodes.map((node) => { + const updatedNodes = node.map((nodeVal) => { if (query === '') { return { - ...node, + ...nodeVal, selected: false, size: graphLabels.nodeSize, }; } - const { id, properties, caption } = node; + const { id, properties, caption } = nodeVal; const propertiesMatch = properties?.id?.toLowerCase().includes(query); const match = id.toLowerCase().includes(query) || propertiesMatch || caption?.toLowerCase().includes(query); return { - ...node, + ...nodeVal, selected: match, }; }); - // deactivating any active relationships - const updatedRelationships = relationships.map((rel) => { + const updatedRelationships = relationship.map((rel) => { return { ...rel, selected: false, }; }); - setNodes(updatedNodes); - setRelationships(updatedRelationships); + setNode(updatedNodes); + setRelationship(updatedRelationships); }, - [nodes, relationships] + [node, relationship] ); - // Unmounting the component if (!open) { return <>; } @@ -266,12 +292,11 @@ const GraphViewModal: React.FunctionComponent = ({ viewPoint === graphLabels.showGraphView || viewPoint === graphLabels.chatInfoView ? graphLabels.generateGraph : viewPoint === graphLabels.showSchemaView - ? graphLabels.renderSchemaGraph - : `${graphLabels.inspectGeneratedGraphFrom} ${inspectedName}`; + ? graphLabels.renderSchemaGraph + : `${graphLabels.inspectGeneratedGraphFrom} ${inspectedName}`; const checkBoxView = viewPoint !== graphLabels.chatInfoView; - // the checkbox selection const handleCheckboxChange = (graph: GraphType) => { const currentIndex = graphType.indexOf(graph); const newGraphSelected = [...graphType]; @@ -323,29 +348,26 @@ const GraphViewModal: React.FunctionComponent = ({ setGraphViewOpen(false); setScheme({}); setGraphType([]); - setNodes([]); - setRelationships([]); + setNode([]); + setRelationship([]); setAllNodes([]); setAllRelationships([]); setSearchQuery(''); setSelected(undefined); }; - const mouseEventCallbacks = { - onNodeClick: (clickedNode: Node) => { - setSelected({ type: 'node', id: clickedNode.id }); - }, - onRelationshipClick: (clickedRelationship: Relationship) => { - setSelected({ type: 'relationship', id: clickedRelationship.id }); - }, - onCanvasClick: () => { - setSelected(undefined); - }, - onPan: true, - onZoom: true, - onDrag: true, + const handleSchemaView = async ( + rawNodes: any[], + rawRelationships: any[] + ) => { + const { nodes, relationships } = extractGraphSchemaFromRawData(rawNodes, rawRelationships); + setSchemaNodes(nodes as any); + setSchemaRels(relationships as any); + setViewcheck('viz'); + setOpenGraphView(true); }; + return ( <> = ({ {graphLabels.chunksInfo} )} - + {checkBoxView && ( = ({ {...getCheckboxConditions(allNodes)} /> )} + {/* */} @@ -392,7 +415,7 @@ const GraphViewModal: React.FunctionComponent = ({
- ) : nodes.length === 0 && relationships.length === 0 && graphType.length !== 0 ? ( + ) : node.length === 0 && relationship.length === 0 && graphType.length !== 0 ? (
@@ -405,8 +428,8 @@ const GraphViewModal: React.FunctionComponent = ({
= ({ }} nvlCallbacks={nvlCallbacks} /> + + handleSchemaView(node, relationship)} + placement='left' + > + + + {viewPoint !== 'chatInfoView' && ( = ({ /> ) : ( )} @@ -467,7 +500,18 @@ const GraphViewModal: React.FunctionComponent = ({
+ {openGraphView && ( + + )} + ); }; export default GraphViewModal; diff --git a/frontend/src/components/Graph/SchemaDropdown.tsx b/frontend/src/components/Graph/SchemaDropdown.tsx new file mode 100644 index 000000000..57c989543 --- /dev/null +++ b/frontend/src/components/Graph/SchemaDropdown.tsx @@ -0,0 +1,82 @@ +import { DropdownButton, Menu } from '@neo4j-ndl/react'; +import { useRef, useState } from 'react'; +import TooltipWrapper from '../UI/TipWrapper'; +import { useFileContext } from '../../context/UsersFiles'; +interface SchemaDropdownProps { + isDisabled: boolean; + onSchemaSelect?: (source: string, nodes: any[], rels: any[]) => void; // <-- NEW +} +const SchemaDropdown: React.FunctionComponent = ({ isDisabled, onSchemaSelect }) => { + const [isOpen, setIsOpen] = useState(false); + const btnRef = useRef(null); + const { + userDefinedNodes, + userDefinedRels, + dbNodes, + dbRels, + schemaValNodes, + schemaValRels, + preDefinedNodes, + preDefinedRels, + } = useFileContext(); + const handleSelect = (source: string, nodes: any[], rels: any[]) => { + setIsOpen(false); + if (onSchemaSelect) { + onSchemaSelect(source, nodes, rels); + } + }; + return ( +
+ { + e.preventDefault(); + setIsOpen((prev) => !prev); + }, + }} + > + Show Schema from ... + + setIsOpen(false)}> + + + Predefined Schema + + } + onClick={() => handleSelect('predefined', preDefinedNodes, preDefinedRels)} + /> + + DB Schema + + } + onClick={() => handleSelect('db', dbNodes, dbRels)} + /> + + Get Schema From Text + + } + onClick={() => handleSelect('text', schemaValNodes, schemaValRels)} + /> + + User Configured + + } + onClick={() => handleSelect('user', userDefinedNodes, userDefinedRels)} + /> + + +
+ ); +}; +export default SchemaDropdown; \ No newline at end of file diff --git a/frontend/src/components/Graph/SchemaViz.tsx b/frontend/src/components/Graph/SchemaViz.tsx new file mode 100644 index 000000000..4991b82bf --- /dev/null +++ b/frontend/src/components/Graph/SchemaViz.tsx @@ -0,0 +1,287 @@ +import { Banner, Dialog, IconButtonArray, LoadingSpinner, useDebounceValue } from '@neo4j-ndl/react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { + BasicNode, + BasicRelationship, + EntityType, + ExtendedNode, + ExtendedRelationship, + SchemaViewModalProps, + Scheme, + OptionType, +} from '../../types'; +import { InteractiveNvlWrapper } from '@neo4j-nvl/react'; +import NVL from '@neo4j-nvl/base'; +import type { Node, Relationship } from '@neo4j-nvl/base'; +import { + FitToScreenIcon, + MagnifyingGlassMinusIconOutline, + MagnifyingGlassPlusIconOutline, +} from '@neo4j-ndl/react/icons'; +import { IconButtonWithToolTip } from '../UI/IconButtonToolTip'; +import { userDefinedGraphSchema, generateGraphFromNodeAndRelVals } from '../../utils/Utils'; +import { graphLabels, nvlOptions } from '../../utils/Constants'; +import ResultOverview from './ResultOverview'; +import { ResizePanelDetails } from './ResizePanel'; +import GraphPropertiesPanel from './GraphPropertiesPanel'; + +const SchemaViz: React.FunctionComponent = ({ + open, + setGraphViewOpen, + nodeValues, + relationshipValues, + schemaLoading, + view +}) => { + const nvlRef = useRef(null); + const [nodes, setNodes] = useState([]); + const [relationships, setRelationships] = useState([]); + const [loading, setLoading] = useState(false); + const [status, setStatus] = useState<'unknown' | 'success' | 'danger'>('unknown'); + const [statusMessage, setStatusMessage] = useState(''); + const [newScheme, setNewScheme] = useState({}); + const [searchQuery, setSearchQuery] = useState(''); + const [debouncedQuery] = useDebounceValue(searchQuery, 300); + const [selected, setSelected] = useState<{ type: EntityType; id: string } | undefined>(undefined); + const graphQueryAbortControllerRef = useRef(); + + const handleZoomToFit = () => { + nvlRef.current?.fit( + nodes.map((node) => node.id), + {} + ); + }; + + useEffect(() => { + const timeoutId = setTimeout(() => { + handleZoomToFit(); + }, 10); + return () => { + if (nvlRef.current) { + nvlRef.current?.destroy(); + } + clearTimeout(timeoutId); + setNodes([]); + setRelationships([]); + setSearchQuery(''); + setSelected(undefined); + }; + }, []); + + useEffect(() => { + if (open) { + if (view !== 'viz') { + setLoading(true); + const { nodes, relationships, scheme } = userDefinedGraphSchema( + (nodeValues as OptionType[]) ?? [], + (relationshipValues as OptionType[]) ?? [] + ); + + setNodes(nodes); + setRelationships(relationships); + setNewScheme(scheme); + setLoading(false); + } + else { + const { nodes, relationships, scheme } = + generateGraphFromNodeAndRelVals(nodeValues as any, relationshipValues as any); + + setNodes(nodes); + setRelationships(relationships); + setNewScheme(scheme); + } + } + }, [open]); + + useEffect(() => { + if (debouncedQuery) { + handleSearch(debouncedQuery); + } + }, [debouncedQuery]); + + const selectedItem = useMemo(() => { + if (selected === undefined) { + return undefined; + } + if (selected.type === 'node') { + return nodes.find((node) => node.id === selected.id); + } + return relationships.find((relationship) => relationship.id === selected.id); + }, [selected, relationships, nodes]); + + // The search and update nodes + const handleSearch = useCallback( + (value: string) => { + const query = value.toLowerCase(); + const updatedNodes = nodes.map((node) => { + if (query === '') { + return { + ...node, + selected: false, + size: graphLabels.nodeSize, + }; + } + const { id, properties, caption } = node; + const propertiesMatch = properties?.id?.toLowerCase().includes(query); + const match = id.toLowerCase().includes(query) || propertiesMatch || caption?.toLowerCase().includes(query); + return { + ...node, + selected: match, + }; + }); + // deactivating any active relationships + const updatedRelationships = relationships.map((rel) => { + return { + ...rel, + selected: false, + }; + }); + setNodes(updatedNodes); + setRelationships(updatedRelationships); + }, + [nodes, relationships] + ); + + // Unmounting the component + if (!open) { + return <>; + } + + const headerTitle = graphLabels.generateGraph; + + // Callback + const nvlCallbacks = { + onLayoutComputing(isComputing: boolean) { + if (!isComputing) { + handleZoomToFit(); + } + }, + }; + + // To handle the current zoom in function of graph + const handleZoomIn = () => { + nvlRef.current?.setZoom(nvlRef.current.getScale() * 1.3); + }; + + // To handle the current zoom out function of graph + const handleZoomOut = () => { + nvlRef.current?.setZoom(nvlRef.current.getScale() * 0.7); + }; + + // when modal closes reset all states to default + const onClose = () => { + graphQueryAbortControllerRef?.current?.abort(); + setStatus('unknown'); + setStatusMessage(''); + setGraphViewOpen(false); + setNodes([]); + setRelationships([]); + setSearchQuery(''); + setSelected(undefined); + }; + + const mouseEventCallbacks = { + onNodeClick: (clickedNode: Node) => { + setSelected({ type: 'node', id: clickedNode.id }); + }, + onRelationshipClick: (clickedRelationship: Relationship) => { + setSelected({ type: 'relationship', id: clickedRelationship.id }); + }, + onCanvasClick: () => { + setSelected(undefined); + }, + onPan: true, + onZoom: true, + onDrag: true, + }; + + return ( + <> + + {headerTitle} + +
+ {loading || schemaLoading ? ( +
+ +
+ ) : status !== 'unknown' ? ( +
+ +
+ ) : nodes.length === 0 && relationships.length === 0 ? ( +
+ +
+ ) : ( + <> +
+
+ + + + + + + + + + + + +
+ + {selectedItem !== undefined ? ( + + ) : ( + + )} + +
+ + )} +
+
+
+ + ); +}; +export default SchemaViz; diff --git a/frontend/src/components/Layout/PageLayout.tsx b/frontend/src/components/Layout/PageLayout.tsx index 734041b7c..d4b1e8568 100644 --- a/frontend/src/components/Layout/PageLayout.tsx +++ b/frontend/src/components/Layout/PageLayout.tsx @@ -5,11 +5,11 @@ import DrawerChatbot from './DrawerChatbot'; import Content from '../Content'; import { clearChatAPI } from '../../services/QnaAPI'; import { useCredentials } from '../../context/UserCredentials'; -import { connectionState } from '../../types'; +import { connectionState, OptionType } from '../../types'; import { useMessageContext } from '../../context/UserMessages'; import { useMediaQuery, Spotlight, SpotlightTour, useSpotlightContext } from '@neo4j-ndl/react'; import { useFileContext } from '../../context/UsersFiles'; -import SchemaFromTextDialog from '../Popups/Settings/SchemaFromText'; +import SchemaFromTextDialog from '../../components/Popups/GraphEnhancementDialog/EnitityExtraction/SchemaFromTextDialog'; import useSpeechSynthesis from '../../hooks/useSpeech'; import FallBackDialog from '../UI/FallBackDialog'; import { envConnectionAPI } from '../../services/ConnectAPI'; @@ -19,6 +19,9 @@ import { useAuth0 } from '@auth0/auth0-react'; import { showErrorToast } from '../../utils/Toasts'; import { APP_SOURCES } from '../../utils/Constants'; import { createDefaultFormData } from '../../API/Index'; +import LoadExistingSchemaDialog from '../Popups/GraphEnhancementDialog/EnitityExtraction/LoadExistingSchema'; +import PredefinedSchemaDialog from '../Popups/GraphEnhancementDialog/EnitityExtraction/PredefinedSchemaDialog'; + const GCSModal = lazy(() => import('../DataSources/GCS/GCSModal')); const S3Modal = lazy(() => import('../DataSources/AWS/S3Modal')); const GenericModal = lazy(() => import('../WebSources/GenericSourceModal')); @@ -202,7 +205,24 @@ const PageLayout: React.FC = () => { [] ); const { messages, setClearHistoryData, clearHistoryData, setMessages, setIsDeleteChatLoading } = useMessageContext(); - const { setShowTextFromSchemaDialog, showTextFromSchemaDialog } = useFileContext(); + const { + setShowTextFromSchemaDialog, + showTextFromSchemaDialog, + setSchemaTextPattern, + schemaLoadDialog, + setSchemaLoadDialog, + setPredefinedSchemaDialog, + setDbPattern, + setSchemaValNodes, + predefinedSchemaDialog, + setSchemaValRels, + setDbNodes, + setDbRels, + setSchemaView, + setPreDefinedNodes, + setPreDefinedRels, + setPreDefinedPattern, + } = useFileContext(); const { cancel } = useSpeechSynthesis(); const { setActiveSpotlight } = useSpotlightContext(); const isFirstTimeUser = useMemo(() => { @@ -317,14 +337,50 @@ const PageLayout: React.FC = () => { } }; + const handleApplyPatternsFromText = (newPatterns: string[], nodes: OptionType[], rels: OptionType[]) => { + setSchemaTextPattern((prevPatterns: string[]) => { + const uniquePatterns = Array.from(new Set([...newPatterns, ...prevPatterns])); + return uniquePatterns; + }); + setShowTextFromSchemaDialog({ + triggeredFrom: 'schematextApply', + show: true, + }); + setSchemaValNodes(nodes); + setSchemaValRels(rels); + setSchemaView('text'); + }; + + const handleDbApply = (newPatterns: string[], nodes: OptionType[], rels: OptionType[]) => { + setDbPattern((prevPatterns: string[]) => { + const uniquePatterns = Array.from(new Set([...newPatterns, ...prevPatterns])); + return uniquePatterns; + }); + setSchemaLoadDialog({ + triggeredFrom: 'loadExistingSchemaApply', + show: true, + }); + setSchemaView('db'); + setDbNodes(nodes); + setDbRels(rels); + }; + + const handlePredinedApply = (newPatterns: string[], nodes: OptionType[], rels: OptionType[]) => { + setPreDefinedPattern((prevPatterns: string[]) => { + const uniquePatterns = Array.from(new Set([...newPatterns, ...prevPatterns])); + return uniquePatterns; + }); + setPredefinedSchemaDialog({ + triggeredFrom: 'predefinedSchemaApply', + show: true, + }); + setSchemaView('preDefined'); + setPreDefinedNodes(nodes); + setPreDefinedRels(rels); + }; + return ( <> - {/* {!isAuthenticated && isFirstTimeUser && ( - - )} - {isAuthenticated && isFirstTimeUser && ( - - )} */} {!isAuthenticated && !SKIP_AUTH && isFirstTimeUser ? ( { console.log(`Action ${action} was performed in spotlight ${target}`); }} /> - ) : isAuthenticated || (SKIP_AUTH && isFirstTimeUser) ? ( + ) : (isAuthenticated || SKIP_AUTH) && isFirstTimeUser ? ( { @@ -379,7 +435,36 @@ const PageLayout: React.FC = () => { break; } }} + onApply={handleApplyPatternsFromText} > + { + setSchemaLoadDialog({ triggeredFrom: '', show: false }); + switch (schemaLoadDialog.triggeredFrom) { + case 'enhancementtab': + toggleEnhancementDialog(); + break; + default: + break; + } + }} + onApply={handleDbApply} + /> + { + setPredefinedSchemaDialog({ triggeredFrom: '', show: false }); + switch (predefinedSchemaDialog.triggeredFrom) { + case 'enhancementtab': + toggleEnhancementDialog(); + break; + default: + break; + } + }} + onApply={handlePredinedApply} + > {isLargeDesktop ? (
{ openTextSchema={() => { setShowTextFromSchemaDialog({ triggeredFrom: 'schemadialog', show: true }); }} + openLoadSchema={() => { + setSchemaLoadDialog({ triggeredFrom: 'loadDialog', show: true }); + }} + openPredefinedSchema={() => { + setPredefinedSchemaDialog({ triggeredFrom: 'predefinedDialog', show: true }); + }} showEnhancementDialog={showEnhancementDialog} toggleEnhancementDialog={toggleEnhancementDialog} setOpenConnection={setOpenConnection} @@ -472,7 +563,13 @@ const PageLayout: React.FC = () => { openChatBot={() => setShowChatBot(true)} showChatBot={showChatBot} openTextSchema={() => { - setShowTextFromSchemaDialog({ triggeredFrom: 'schemadialog', show: true }); + setShowTextFromSchemaDialog({ triggeredFrom: 'schemaDialog', show: true }); + }} + openLoadSchema={() => { + setShowTextFromSchemaDialog({ triggeredFrom: 'loadDialog', show: true }); + }} + openPredefinedSchema={() => { + setPredefinedSchemaDialog({ triggeredFrom: 'prdefinedDialog', show: true }); }} showEnhancementDialog={showEnhancementDialog} toggleEnhancementDialog={toggleEnhancementDialog} diff --git a/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx b/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx index fc4876a49..f4d5c6322 100644 --- a/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx +++ b/frontend/src/components/Popups/ConnectionModal/ConnectionModal.tsx @@ -9,6 +9,8 @@ import { ConnectionModalProps, Message, UserCredentials } from '../../../types'; import VectorIndexMisMatchAlert from './VectorIndexMisMatchAlert'; import { useAuth0 } from '@auth0/auth0-react'; import { createDefaultFormData } from '../../../API/Index'; +import { getNodeLabelsAndRelTypesFromText } from '../../../services/SchemaFromTextAPI'; +import { useFileContext } from '../../../context/UsersFiles'; export default function ConnectionModal({ open, @@ -58,7 +60,7 @@ export default function ConnectionModal({ const [searchParams, setSearchParams] = useSearchParams(); const [userDbVectorIndex, setUserDbVectorIndex] = useState(initialuserdbvectorindex ?? undefined); const [vectorIndexLoading, setVectorIndexLoading] = useState(false); - + const { model } = useFileContext(); const connectRef = useRef(null); const uriRef = useRef(null); const databaseRef = useRef(null); @@ -237,10 +239,24 @@ export default function ConnectionModal({ const isReadOnlyUser = !response.data.data.write_access; const isGCSActive = response.data.data.gcs_file_cache === 'True'; const chunksTobeProcess = Number(response.data.data.chunk_to_be_created); + const existingRels = JSON.parse(localStorage.getItem('selectedRelationshipLabels') ?? 'null'); + const existingNodes = JSON.parse(localStorage.getItem('selectedNodeLabels') ?? 'null'); + const pattern = /^[^,]+,[^,]+,[^,]+$/; + if (existingRels && existingRels.selectedOptions.length) { + if (!pattern.test(existingRels.selectedOptions[0].value)) { + const response = await getNodeLabelsAndRelTypesFromText( + model, + JSON.stringify({ nodes: existingNodes.selectedOptions, rels: existingRels.selectedOptions }), + false, + true + ); + console.log(response); + } + } setIsGCSActive(isGCSActive); setGdsActive(isgdsActive); setIsReadOnlyUser(isReadOnlyUser); - // setChunksToBeProces(chunksTobeProcess); + localStorage.setItem( 'neo4j.connection', JSON.stringify({ diff --git a/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/EntityExtractionSetting.tsx b/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/EntityExtractionSetting.tsx index a2fcf7452..d780cad85 100644 --- a/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/EntityExtractionSetting.tsx +++ b/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/EntityExtractionSetting.tsx @@ -1,382 +1,380 @@ -import { MouseEventHandler, useCallback, useEffect, useMemo, useState } from 'react'; -import ButtonWithToolTip from '../../../UI/ButtonWithToolTip'; -import { appLabels, buttonCaptions, getDefaultSchemaExamples, tooltips } from '../../../../utils/Constants'; -import { Select, Flex, Typography, useMediaQuery } from '@neo4j-ndl/react'; -import { useCredentials } from '../../../../context/UserCredentials'; -import { useFileContext } from '../../../../context/UsersFiles'; -import { OnChangeValue, ActionMeta } from 'react-select'; -import { OptionType, schema } from '../../../../types'; -import { getNodeLabelsAndRelTypes } from '../../../../services/GetNodeLabelsRelTypes'; -import { tokens } from '@neo4j-ndl/base'; -import { showNormalToast } from '../../../../utils/Toasts'; -import { useHasSelections } from '../../../../hooks/useHasSelections'; -import { Hierarchy1Icon } from '@neo4j-ndl/react/icons'; -import GraphViewModal from '../../../Graph/GraphViewModal'; +// import { MouseEventHandler, useCallback, useEffect, useMemo, useState } from 'react'; +// import ButtonWithToolTip from '../../../UI/ButtonWithToolTip'; +// import { appLabels, buttonCaptions, getDefaultSchemaExamples, tooltips } from '../../../../utils/Constants'; +// import { Select, Flex, Typography, useMediaQuery } from '@neo4j-ndl/react'; +// import { useCredentials } from '../../../../context/UserCredentials'; +// import { useFileContext } from '../../../../context/UsersFiles'; +// import { OnChangeValue, ActionMeta } from 'react-select'; +// import { OptionType, schema } from '../../../../types'; +// import { getNodeLabelsAndRelTypes } from '../../../../services/GetNodeLabelsRelTypes'; +// import { tokens } from '@neo4j-ndl/base'; +// import { showNormalToast } from '../../../../utils/Toasts'; +// import { useHasSelections } from '../../../../hooks/useHasSelections'; +// import { Hierarchy1Icon } from '@neo4j-ndl/react/icons'; +// import GraphViewModal from '../../../Graph/GraphViewModal'; -export default function EntityExtractionSetting({ - view, - open, - onClose, - openTextSchema, - settingView, - onContinue, - closeEnhanceGraphSchemaDialog, -}: { - view: 'Dialog' | 'Tabs'; - open?: boolean; - onClose?: () => void; - openTextSchema: () => void; - settingView: 'contentView' | 'headerView'; - onContinue?: () => void; - closeEnhanceGraphSchemaDialog?: () => void; -}) { - const { breakpoints } = tokens; - const { setSelectedRels, setSelectedNodes, selectedNodes, selectedRels, selectedSchemas, setSelectedSchemas } = - useFileContext(); - const { userCredentials } = useCredentials(); - const [loading, setLoading] = useState(false); - const isTablet = useMediaQuery(`(min-width:${breakpoints.xs}) and (max-width: ${breakpoints.lg})`); - const hasSelections = useHasSelections(selectedNodes, selectedRels); - const [openGraphView, setOpenGraphView] = useState(false); - const [viewPoint, setViewPoint] = useState('tableView'); - const removeNodesAndRels = (nodelabels: string[], relationshipTypes: string[]) => { - const labelsToRemoveSet = new Set(nodelabels); - const relationshipLabelsToremoveSet = new Set(relationshipTypes); - setSelectedNodes((prevState) => { - const filterednodes = prevState.filter((item) => !labelsToRemoveSet.has(item.label)); - localStorage.setItem( - 'selectedNodeLabels', - JSON.stringify({ db: userCredentials?.uri, selectedOptions: filterednodes }) - ); - return filterednodes; - }); - setSelectedRels((prevState) => { - const filteredrels = prevState.filter((item) => !relationshipLabelsToremoveSet.has(item.label)); - localStorage.setItem( - 'selectedRelationshipLabels', - JSON.stringify({ db: userCredentials?.uri, selectedOptions: filteredrels }) - ); - return filteredrels; - }); - }; - const onChangeSchema = (selectedOptions: OnChangeValue, actionMeta: ActionMeta) => { - if (actionMeta.action === 'remove-value') { - const removedSchema: schema = JSON.parse(actionMeta.removedValue.value); - const { nodelabels, relationshipTypes } = removedSchema; - removeNodesAndRels(nodelabels, relationshipTypes); - } else if (actionMeta.action === 'clear') { - const removedSchemas = actionMeta.removedValues.map((s) => JSON.parse(s.value)); - const removedNodelabels = removedSchemas.map((s) => s.nodelabels).flatMap((k) => k); - const removedRelations = removedSchemas.map((s) => s.relationshipTypes).flatMap((k) => k); - removeNodesAndRels(removedNodelabels, removedRelations); - } - setSelectedSchemas(selectedOptions); - localStorage.setItem( - 'selectedSchemas', - JSON.stringify({ db: userCredentials?.uri, selectedOptions: selectedOptions }) - ); - const nodesFromSchema = selectedOptions.map((s) => JSON.parse(s.value).nodelabels).flat(); - const relationsFromSchema = selectedOptions.map((s) => JSON.parse(s.value).relationshipTypes).flat(); - let nodeOptionsFromSchema: OptionType[] = []; - for (let index = 0; index < nodesFromSchema.length; index++) { - const n = nodesFromSchema[index]; - nodeOptionsFromSchema.push({ label: n, value: n }); - } - let relationshipOptionsFromSchema: OptionType[] = []; - for (let index = 0; index < relationsFromSchema.length; index++) { - const r = relationsFromSchema[index]; - relationshipOptionsFromSchema.push({ label: r, value: r }); - } - setSelectedNodes((prev) => { - const combinedData = [...prev, ...nodeOptionsFromSchema]; - const uniqueLabels = new Set(); - const updatedOptions = combinedData.filter((item) => { - if (!uniqueLabels.has(item.label)) { - uniqueLabels.add(item.label); - return true; - } - return false; - }); - localStorage.setItem( - 'selectedNodeLabels', - JSON.stringify({ db: userCredentials?.uri, selectedOptions: updatedOptions }) - ); - return updatedOptions; - }); - setSelectedRels((prev) => { - const combinedData = [...prev, ...relationshipOptionsFromSchema]; - const uniqueLabels = new Set(); - const updatedOptions = combinedData.filter((item) => { - if (!uniqueLabels.has(item.label)) { - uniqueLabels.add(item.label); - return true; - } - return false; - }); - localStorage.setItem( - 'selectedRelationshipLabels', - JSON.stringify({ db: userCredentials?.uri, selectedOptions: updatedOptions }) - ); - return updatedOptions; - }); - }; - const onChangenodes = (selectedOptions: OnChangeValue, actionMeta: ActionMeta) => { - if (actionMeta.action === 'clear') { - localStorage.setItem('selectedNodeLabels', JSON.stringify({ db: userCredentials?.uri, selectedOptions: [] })); - } - setSelectedNodes(selectedOptions); - localStorage.setItem('selectedNodeLabels', JSON.stringify({ db: userCredentials?.uri, selectedOptions })); - }; - const onChangerels = (selectedOptions: OnChangeValue, actionMeta: ActionMeta) => { - if (actionMeta.action === 'clear') { - localStorage.setItem( - 'selectedRelationshipLabels', - JSON.stringify({ db: userCredentials?.uri, selectedOptions: [] }) - ); - } - setSelectedRels(selectedOptions); - localStorage.setItem('selectedRelationshipLabels', JSON.stringify({ db: userCredentials?.uri, selectedOptions })); - }; - const [nodeLabelOptions, setnodeLabelOptions] = useState([]); - const [relationshipTypeOptions, setrelationshipTypeOptions] = useState([]); - const defaultExamples = useMemo(() => getDefaultSchemaExamples(), []); +// export default function EntityExtractionSetting({ +// view, +// open, +// onClose, +// openTextSchema, +// settingView, +// onContinue, +// closeEnhanceGraphSchemaDialog, +// }: { +// view: 'Dialog' | 'Tabs'; +// open?: boolean; +// onClose?: () => void; +// openTextSchema: () => void; +// settingView: 'contentView' | 'headerView'; +// onContinue?: () => void; +// closeEnhanceGraphSchemaDialog?: () => void; +// }) { +// const { breakpoints } = tokens; +// const { setSelectedRels, setSelectedNodes, selectedNodes, selectedRels, selectedSchemas, setSelectedSchemas } = +// useFileContext(); +// const { userCredentials } = useCredentials(); +// const [loading, setLoading] = useState(false); +// const isTablet = useMediaQuery(`(min-width:${breakpoints.xs}) and (max-width: ${breakpoints.lg})`); +// const hasSelections = useHasSelections(selectedNodes, selectedRels); +// const [openGraphView, setOpenGraphView] = useState(false); +// const [viewPoint, setViewPoint] = useState('tableView'); +// const removeNodesAndRels = (nodelabels: string[], relationshipTypes: string[]) => { +// const labelsToRemoveSet = new Set(nodelabels); +// const relationshipLabelsToremoveSet = new Set(relationshipTypes); +// setSelectedNodes((prevState) => { +// const filterednodes = prevState.filter((item) => !labelsToRemoveSet.has(item.label)); +// localStorage.setItem( +// 'selectedNodeLabels', +// JSON.stringify({ db: userCredentials?.uri, selectedOptions: filterednodes }) +// ); +// return filterednodes; +// }); +// setSelectedRels((prevState) => { +// const filteredrels = prevState.filter((item) => !relationshipLabelsToremoveSet.has(item.label)); +// localStorage.setItem( +// 'selectedRelationshipLabels', +// JSON.stringify({ db: userCredentials?.uri, selectedOptions: filteredrels }) +// ); +// return filteredrels; +// }); +// }; +// const onChangeSchema = (selectedOptions: OnChangeValue, actionMeta: ActionMeta) => { +// if (actionMeta.action === 'remove-value') { +// const removedSchema: schema = JSON.parse(actionMeta.removedValue.value); +// const { nodelabels, relationshipTypes } = removedSchema; +// removeNodesAndRels(nodelabels, relationshipTypes); +// } else if (actionMeta.action === 'clear') { +// const removedSchemas = actionMeta.removedValues.map((s) => JSON.parse(s.value)); +// const removedNodelabels = removedSchemas.map((s) => s.nodelabels).flatMap((k) => k); +// const removedRelations = removedSchemas.map((s) => s.relationshipTypes).flatMap((k) => k); +// removeNodesAndRels(removedNodelabels, removedRelations); +// } +// setSelectedSchemas(selectedOptions); +// localStorage.setItem( +// 'selectedSchemas', +// JSON.stringify({ db: userCredentials?.uri, selectedOptions: selectedOptions }) +// ); +// const nodesFromSchema = selectedOptions.map((s) => JSON.parse(s.value).nodelabels).flat(); +// const relationsFromSchema = selectedOptions.map((s) => JSON.parse(s.value).relationshipTypes).flat(); +// let nodeOptionsFromSchema: OptionType[] = []; +// for (let index = 0; index < nodesFromSchema.length; index++) { +// const n = nodesFromSchema[index]; +// nodeOptionsFromSchema.push({ label: n, value: n }); +// } +// let relationshipOptionsFromSchema: OptionType[] = []; +// for (let index = 0; index < relationsFromSchema.length; index++) { +// const r = relationsFromSchema[index]; +// relationshipOptionsFromSchema.push({ label: r, value: r }); +// } +// setSelectedNodes((prev) => { +// const combinedData = [...prev, ...nodeOptionsFromSchema]; +// const uniqueLabels = new Set(); +// const updatedOptions = combinedData.filter((item) => { +// if (!uniqueLabels.has(item.label)) { +// uniqueLabels.add(item.label); +// return true; +// } +// return false; +// }); +// localStorage.setItem( +// 'selectedNodeLabels', +// JSON.stringify({ db: userCredentials?.uri, selectedOptions: updatedOptions }) +// ); +// return updatedOptions; +// }); +// setSelectedRels((prev) => { +// const combinedData = [...prev, ...relationshipOptionsFromSchema]; +// const uniqueLabels = new Set(); +// const updatedOptions = combinedData.filter((item) => { +// if (!uniqueLabels.has(item.label)) { +// uniqueLabels.add(item.label); +// return true; +// } +// return false; +// }); +// localStorage.setItem( +// 'selectedRelationshipLabels', +// JSON.stringify({ db: userCredentials?.uri, selectedOptions: updatedOptions }) +// ); +// return updatedOptions; +// }); +// }; +// const onChangenodes = (selectedOptions: OnChangeValue, actionMeta: ActionMeta) => { +// if (actionMeta.action === 'clear') { +// localStorage.setItem('selectedNodeLabels', JSON.stringify({ db: userCredentials?.uri, selectedOptions: [] })); +// } +// setSelectedNodes(selectedOptions); +// localStorage.setItem('selectedNodeLabels', JSON.stringify({ db: userCredentials?.uri, selectedOptions })); +// }; +// const onChangerels = (selectedOptions: OnChangeValue, actionMeta: ActionMeta) => { +// if (actionMeta.action === 'clear') { +// localStorage.setItem( +// 'selectedRelationshipLabels', +// JSON.stringify({ db: userCredentials?.uri, selectedOptions: [] }) +// ); +// } +// setSelectedRels(selectedOptions); +// localStorage.setItem('selectedRelationshipLabels', JSON.stringify({ db: userCredentials?.uri, selectedOptions })); +// }; +// const [nodeLabelOptions, setnodeLabelOptions] = useState([]); +// const [relationshipTypeOptions, setrelationshipTypeOptions] = useState([]); +// const defaultExamples = useMemo(() => getDefaultSchemaExamples(), []); - useEffect(() => { - if (userCredentials) { - if (open && view === 'Dialog') { - const getOptions = async () => { - setLoading(true); - try { - const response = await getNodeLabelsAndRelTypes(); - setLoading(false); - if (response.data.data.length) { - const nodelabels = response.data?.data[0]?.labels.map((l) => ({ value: l, label: l })); - const reltypes = response.data?.data[0]?.relationshipTypes.map((t) => ({ value: t, label: t })); - setnodeLabelOptions(nodelabels); - setrelationshipTypeOptions(reltypes); - } - } catch (error) { - setLoading(false); - console.log(error); - } - }; - getOptions(); - return; - } - if (view == 'Tabs') { - const getOptions = async () => { - setLoading(true); - try { - const response = await getNodeLabelsAndRelTypes(); - setLoading(false); - if (response.data.data.length) { - const nodelabels = response.data?.data[0]?.labels.map((l) => ({ value: l, label: l })); - const reltypes = response.data?.data[0]?.relationshipTypes.map((t) => ({ value: t, label: t })); - setnodeLabelOptions(nodelabels); - setrelationshipTypeOptions(reltypes); - } - } catch (error) { - setLoading(false); - console.log(error); - } - }; - getOptions(); - } - } - }, [userCredentials, open]); +// useEffect(() => { +// if (userCredentials) { +// if (open && view === 'Dialog') { +// const getOptions = async () => { +// setLoading(true); +// try { +// const response = await getNodeLabelsAndRelTypes(); +// setLoading(false); +// if (response.data.data.length) { +// const nodelabels = response.data?.data[0]?.labels.map((l) => ({ value: l, label: l })); +// const reltypes = response.data?.data[0]?.relationshipTypes.map((t) => ({ value: t, label: t })); +// setnodeLabelOptions(nodelabels); +// setrelationshipTypeOptions(reltypes); +// } +// } catch (error) { +// setLoading(false); +// console.log(error); +// } +// }; +// getOptions(); +// return; +// } +// if (view == 'Tabs') { +// const getOptions = async () => { +// setLoading(true); +// try { +// const response = await getNodeLabelsAndRelTypes(); +// setLoading(false); +// if (response.data.data.length) { +// const nodelabels = response.data?.data[0]?.labels.map((l) => ({ value: l, label: l })); +// const reltypes = response.data?.data[0]?.relationshipTypes.map((t) => ({ value: t, label: t })); +// setnodeLabelOptions(nodelabels); +// setrelationshipTypeOptions(reltypes); +// } +// } catch (error) { +// setLoading(false); +// console.log(error); +// } +// }; +// getOptions(); +// } +// } +// }, [userCredentials, open]); - const clickHandler: MouseEventHandler = useCallback(() => { - setSelectedSchemas([]); - setSelectedNodes(nodeLabelOptions); - setSelectedRels(relationshipTypeOptions); - localStorage.setItem( - 'selectedNodeLabels', - JSON.stringify({ db: userCredentials?.uri, selectedOptions: nodeLabelOptions }) - ); - localStorage.setItem( - 'selectedRelationshipLabels', - JSON.stringify({ db: userCredentials?.uri, selectedOptions: relationshipTypeOptions }) - ); - }, [nodeLabelOptions, relationshipTypeOptions]); +// const clickHandler: MouseEventHandler = useCallback(() => { +// setSelectedSchemas([]); +// setSelectedNodes(nodeLabelOptions); +// setSelectedRels(relationshipTypeOptions); +// localStorage.setItem( +// 'selectedNodeLabels', +// JSON.stringify({ db: userCredentials?.uri, selectedOptions: nodeLabelOptions }) +// ); +// localStorage.setItem( +// 'selectedRelationshipLabels', +// JSON.stringify({ db: userCredentials?.uri, selectedOptions: relationshipTypeOptions }) +// ); +// }, [nodeLabelOptions, relationshipTypeOptions]); - const handleClear = () => { - setSelectedNodes([]); - setSelectedRels([]); - setSelectedSchemas([]); - localStorage.setItem('selectedNodeLabels', JSON.stringify({ db: userCredentials?.uri, selectedOptions: [] })); - localStorage.setItem( - 'selectedRelationshipLabels', - JSON.stringify({ db: userCredentials?.uri, selectedOptions: [] }) - ); - localStorage.setItem('selectedSchemas', JSON.stringify({ db: userCredentials?.uri, selectedOptions: [] })); - showNormalToast(`Successfully Removed the Schema settings`); - if (view === 'Tabs' && closeEnhanceGraphSchemaDialog != undefined) { - closeEnhanceGraphSchemaDialog(); - } - }; - const handleApply = () => { - showNormalToast(`Successfully Applied the Schema settings`); - if (view === 'Tabs' && closeEnhanceGraphSchemaDialog != undefined) { - closeEnhanceGraphSchemaDialog(); - } - localStorage.setItem( - 'selectedNodeLabels', - JSON.stringify({ db: userCredentials?.uri, selectedOptions: selectedNodes }) - ); - localStorage.setItem( - 'selectedRelationshipLabels', - JSON.stringify({ db: userCredentials?.uri, selectedOptions: selectedRels }) - ); - localStorage.setItem( - 'selectedSchemas', - JSON.stringify({ db: userCredentials?.uri, selectedOptions: selectedSchemas }) - ); - }; +// const handleClear = () => { +// setSelectedNodes([]); +// setSelectedRels([]); +// setSelectedSchemas([]); +// localStorage.setItem('selectedNodeLabels', JSON.stringify({ db: userCredentials?.uri, selectedOptions: [] })); +// localStorage.setItem( +// 'selectedRelationshipLabels', +// JSON.stringify({ db: userCredentials?.uri, selectedOptions: [] }) +// ); +// localStorage.setItem('selectedSchemas', JSON.stringify({ db: userCredentials?.uri, selectedOptions: [] })); +// showNormalToast(`Successfully Removed the Schema settings`); +// if (view === 'Tabs' && closeEnhanceGraphSchemaDialog != undefined) { +// closeEnhanceGraphSchemaDialog(); +// } +// }; +// const handleApply = () => { +// showNormalToast(`Successfully Applied the Schema settings`); +// if (view === 'Tabs' && closeEnhanceGraphSchemaDialog != undefined) { +// closeEnhanceGraphSchemaDialog(); +// } +// localStorage.setItem( +// 'selectedNodeLabels', +// JSON.stringify({ db: userCredentials?.uri, selectedOptions: selectedNodes }) +// ); +// localStorage.setItem( +// 'selectedRelationshipLabels', +// JSON.stringify({ db: userCredentials?.uri, selectedOptions: selectedRels }) +// ); +// localStorage.setItem( +// 'selectedSchemas', +// JSON.stringify({ db: userCredentials?.uri, selectedOptions: selectedSchemas }) +// ); +// }; - const handleSchemaView = () => { - setOpenGraphView(true); - setViewPoint('showSchemaView'); - }; +// const handleSchemaView = () => { +// setOpenGraphView(true); +// setViewPoint('showSchemaView'); +// }; - return ( -
- - - 1.Predefine the structure of your knowledge graph by selecting specific node and relationship labels. - -

- - 2.Focus your analysis by extracting only the relationships and entities that matter most to your use case. - Achieve a cleaner and more insightful graph representation tailored to your domain. - -
-
-
-
{appLabels.predefinedSchema}
-
- - +//
+//
{appLabels.ownSchema}
+//
+// +// +// +// +// Load Existing Schema +// +// +// +// +// { +// if (view === 'Dialog' && onClose != undefined) { +// onClose(); +// } +// if (view === 'Tabs' && closeEnhanceGraphSchemaDialog != undefined) { +// closeEnhanceGraphSchemaDialog(); +// } +// openTextSchema(); +// }} +// label='Get Existing Schema From Text' +// > +// Get Schema From Text +// +// {settingView === 'contentView' ? ( +// +// {buttonCaptions.continueSettings} +// +// ) : ( +// +// {buttonCaptions.clearSettings} +// +// )} +// +// {buttonCaptions.applyGraphSchema} +// +// +// +//
+// +//
+// ); +// } diff --git a/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/GraphPattern.tsx b/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/GraphPattern.tsx new file mode 100644 index 000000000..86584a34d --- /dev/null +++ b/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/GraphPattern.tsx @@ -0,0 +1,174 @@ +import React, { useState, useRef, useEffect } from 'react'; +import { Select } from '@neo4j-ndl/react'; +import ButtonWithToolTip from '../../../UI/ButtonWithToolTip'; +import { OptionType, TupleCreationProps } from '../../../../types'; +import { + appLabels, + LOCAL_KEYS, + sourceOptions as initialSourceOptions, + targetOptions as initialTargetOptions, + typeOptions as initialTypeOptions, +} from '../../../../utils/Constants'; + +const GraphPattern: React.FC = ({ + selectedSource, + selectedType, + selectedTarget, + onPatternChange, + onAddPattern, +}) => { + const [sourceOptions, setSourceOptions] = useState(initialSourceOptions); + const [typeOptions, setTypeOptions] = useState(initialTypeOptions); + const [targetOptions, setTargetOptions] = useState(initialTargetOptions); + const [inputValues, setInputValues] = useState<{ source: string; type: string; target: string }>({ + source: '', + type: '', + target: '', + }); + const sourceRef = useRef(null); + + useEffect(() => { + const savedSources = JSON.parse(localStorage.getItem('customSourceOptions') ?? 'null'); + const savedTypes = JSON.parse(localStorage.getItem('customTypeOptions') ?? 'null'); + const savedTargets = JSON.parse(localStorage.getItem('customTargetOptions') ?? 'null'); + if (savedSources) { + setSourceOptions(savedSources); + } + if (savedTypes) { + setTypeOptions(savedTypes); + } + if (savedTargets) { + setTargetOptions(savedTargets); + } + }, []); + + const handleNewValue = (newValue: string, type: 'source' | 'type' | 'target') => { + if (!newValue.trim()) { + return; + } + const newOption: OptionType = { value: newValue.trim(), label: newValue.trim() }; + const checkUniqueValue = (list: OptionType[], value: OptionType) => + (list.some((opt) => opt.value === value.value) ? list : [...list, value]); + switch (type) { + case 'source': + setSourceOptions((prev) => checkUniqueValue(prev, newOption)); + onPatternChange(newOption, selectedType as OptionType, selectedTarget as OptionType); + break; + case 'type': + setTypeOptions((prev) => checkUniqueValue(prev, newOption)); + onPatternChange(selectedSource as OptionType, newOption, selectedTarget as OptionType); + break; + case 'target': + setTargetOptions((prev) => checkUniqueValue(prev, newOption)); + onPatternChange(selectedSource as OptionType, selectedType as OptionType, newOption); + break; + default: + console.log('wrong type added'); + break; + } + setInputValues((prev) => ({ ...prev, [type]: '' })); + }; + + const handleInputChange = (newValue: string, type: 'source' | 'type' | 'target') => { + setInputValues((prev) => ({ ...prev, [type]: newValue })); + }; + + const handleAddPattern = () => { + onAddPattern(); + localStorage.setItem(LOCAL_KEYS.source, JSON.stringify(sourceOptions)); + localStorage.setItem(LOCAL_KEYS.type, JSON.stringify(typeOptions)); + localStorage.setItem(LOCAL_KEYS.target, JSON.stringify(targetOptions)); + setTimeout(() => { + const selectInput = sourceRef.current?.querySelector('input'); + selectInput?.focus(); + }, 100); + }; + + const isDisabled = !selectedSource?.value.length || !selectedTarget?.value.length || !selectedType?.value.length; + return ( +
+
+
{appLabels.graphPatternTuple}
+
+
+
+ { + if (!selected) { + onPatternChange(selectedSource, null, selectedTarget); + } else { + handleNewValue(selected.value, 'type'); + } + }, + value: selectedType, + inputValue: inputValues.type, + onInputChange: (newValue) => handleInputChange(newValue, 'type'), + onCreateOption: (newOption) => handleNewValue(newOption, 'type'), + }} + type='creatable' + className='w-1/4' + /> + + + + + + + + + {openGraphView && ( + + )} + + ); +}; +export default PredefinedSchemaDialog; diff --git a/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/SchemaFromTextDialog.tsx b/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/SchemaFromTextDialog.tsx new file mode 100644 index 000000000..bffeac4b6 --- /dev/null +++ b/frontend/src/components/Popups/GraphEnhancementDialog/EnitityExtraction/SchemaFromTextDialog.tsx @@ -0,0 +1,217 @@ +import { Checkbox, Dialog, TextArea, Button } from '@neo4j-ndl/react'; +import { useCallback, useState } from 'react'; +import { getNodeLabelsAndRelTypesFromText } from '../../../../services/SchemaFromTextAPI'; +import { useFileContext } from '../../../../context/UsersFiles'; +import { buttonCaptions } from '../../../../utils/Constants'; +import ButtonWithToolTip from '../../../UI/ButtonWithToolTip'; +import { showNormalToast } from '../../../../utils/Toasts'; +import PatternContainer from './PatternContainer'; +import { OptionType, TupleType } from '../../../../types'; +import SchemaViz from '../../../Graph/SchemaViz'; +import { extractOptions } from '../../../../utils/Utils'; + +interface SchemaFromTextProps { + open: boolean; + onClose: () => void; + onApply: (patterns: string[], nodes: OptionType[], rels: OptionType[], view: string) => void; +} + +const SchemaFromTextDialog = ({ open, onClose, onApply }: SchemaFromTextProps) => { + const [userText, setUserText] = useState(''); + const [loading, setLoading] = useState(false); + const [isSchemaText, setIsSchemaText] = useState(false); + const { model } = useFileContext(); + const { + schemaValNodes, + setSchemaValNodes, + schemaValRels, + setSchemaValRels, + schemaTextPattern, + setSchemaTextPattern, + } = useFileContext(); + const [openGraphView, setOpenGraphView] = useState(false); + const [viewPoint, setViewPoint] = useState(''); + + const clickHandler = useCallback(async () => { + setLoading(true); + try { + const response = await getNodeLabelsAndRelTypesFromText(model, userText, isSchemaText, false); + setLoading(false); + const { status, message, data } = response.data; + if (status === 'Success' && data?.triplets?.length) { + const schemaData: string[] = data.triplets; + const schemaTuples: TupleType[] = schemaData + .map((item: string) => { + const matchResult = item.match(/^(.+?)-([A-Z_]+)->(.+)$/); + if (matchResult) { + const [source, rel, target] = matchResult.slice(1).map((s) => s.trim()); + return { + value: `${source},${rel},${target}`, + label: `${source} -[:${rel}]-> ${target}`, + source, + target, + type: rel, + }; + } + return null; + }) + .filter(Boolean) as TupleType[]; + const { nodeLabelOptions, relationshipTypeOptions } = extractOptions(schemaTuples); + setSchemaValNodes(nodeLabelOptions); + setSchemaValRels(relationshipTypeOptions); + setSchemaTextPattern(schemaTuples.map((t) => t.label)); + } else if (status === 'Failed') { + showNormalToast(message as string); + } + else{ + showNormalToast('Please provide meaningful text.'); + } + } catch (error: any) { + setLoading(false); + console.error('Error processing schema:', error); + showNormalToast(error?.message || 'Unexpected error occurred.'); + } + }, [model, userText, isSchemaText]); + + const handleRemovePattern = (pattern: string) => { + const updatedPatterns = schemaTextPattern.filter((p) => p !== pattern); + if (updatedPatterns.length === 0) { + setSchemaTextPattern([]); + setSchemaValNodes([]); + setSchemaValRels([]); + setUserText(''); + return; + } + // Otherwise, recalculate nodes and rels from updated patterns + const updatedTuples: TupleType[] = updatedPatterns + .map((item: string) => { + const matchResult = item.match(/^(.+?)-\[:([A-Z_]+)\]->(.+)$/); + if (matchResult) { + const [source, rel, target] = matchResult.slice(1).map((s) => s.trim()); + return { + value: `${source},${rel},${target}`, + label: `${source} -[:${rel}]-> ${target}`, + source, + target, + type: rel, + }; + } + return null; + }) + .filter(Boolean) as TupleType[]; + const { nodeLabelOptions, relationshipTypeOptions } = extractOptions(updatedTuples); + setSchemaTextPattern(updatedPatterns); + setSchemaValNodes(nodeLabelOptions); + setSchemaValRels(relationshipTypeOptions); + }; + + const handleSchemaView = () => { + setOpenGraphView(true); + setViewPoint('showSchemaView'); + }; + + const handleSchemaTextApply = () => { + if (onApply) { + onApply(schemaTextPattern, schemaValNodes, schemaValRels, 'text'); + } + onClose(); + }; + + const handleCancel = () => { + setSchemaValNodes([]); + setSchemaValRels([]); + setSchemaTextPattern([]); + setUserText(''); + setIsSchemaText(false); + onClose(); + }; + + return ( + <> + { + setLoading(false); + handleCancel(); + }} + htmlAttributes={{ + 'aria-labelledby': 'form-dialog-title', + }} + > + Entity Graph Extraction Settings + +