Merge branch 'STAGING' of https://github.com/neo4j-labs/llm-graph-builder

Author: prakriti-solankey

README.md

@@ -127,8 +127,6 @@ Allow unauthenticated request : Yes
 ## ENV
 | Env Variable Name       | Mandatory/Optional | Default Value | Description                                                                                      |
 |-------------------------|--------------------|---------------|--------------------------------------------------------------------------------------------------|
-| OPENAI_API_KEY          | Mandatory          |               | API key for OpenAI                                                                               |
-| DIFFBOT_API_KEY         | Mandatory          |               | API key for Diffbot                                                                              |
 | EMBEDDING_MODEL         | Optional           | all-MiniLM-L6-v2 | Model for generating the text embedding (all-MiniLM-L6-v2 , openai , vertexai)                |
 | IS_EMBEDDING            | Optional           | true          | Flag to enable text embedding                                                                    |
 | KNN_MIN_SCORE           | Optional           | 0.94          | Minimum score for KNN algorithm                                                                  |
@@ -155,9 +153,35 @@ Allow unauthenticated request : Yes
 | GCS_FILE_CACHE          | Optional           | False         | If set to True, will save the files to process into GCS. If set to False, will save the files locally   |
 | ENTITY_EMBEDDING        | Optional           | False         | If set to True, It will add embeddings for each entity in database |
 | LLM_MODEL_CONFIG_ollama_<model_name>         | Optional      |               | Set ollama config as - model_name,model_local_url for local deployments |
+| RAGAS_EMBEDDING_MODEL         | Optional      | openai              | embedding model used by ragas evaluation framework                               |
 
 
-
+## For local llms (Ollama)
+1. Pull the docker imgage of ollama
+```bash
+docker pull ollama/ollama
+```
+2. Run the ollama docker image
+```bash
+docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
+```
+3. Execute any llm model ex🦙3
+```bash
+docker exec -it ollama ollama run llama3
+```
+4. Configure  env variable in docker compose or backend enviournment.
+```env
+LLM_MODEL_CONFIG_ollama_<model_name>
+#example
+LLM_MODEL_CONFIG_ollama_llama3=${LLM_MODEL_CONFIG_ollama_llama3-llama3,
+http://host.docker.internal:11434}
+```
+5. Configure the backend API url
+```env
+VITE_BACKEND_API_URL=${VITE_BACKEND_API_URL-backendurl}
+```
+6. Open the application in browser and select the ollama model for the extraction.
+7. Enjoy Graph Building.
 
 
 ## Usage

backend/example.env

@@ -2,6 +2,7 @@ OPENAI_API_KEY = ""
 DIFFBOT_API_KEY = ""
 GROQ_API_KEY = ""
 EMBEDDING_MODEL = "all-MiniLM-L6-v2"
+RAGAS_EMBEDDING_MODEL = "openai"
 IS_EMBEDDING = "true"
 KNN_MIN_SCORE = "0.94"
 # Enable Gemini (default is False) | Can be False or True
@@ -28,11 +29,17 @@ ENTITY_EMBEDDING="" True or False
 DUPLICATE_SCORE_VALUE = ""
 DUPLICATE_TEXT_DISTANCE = ""
 #examples
+LLM_MODEL_CONFIG_openai_gpt_3.5="gpt-3.5-turbo-0125,openai_api_key"
+LLM_MODEL_CONFIG_openai_gpt_4o_mini="gpt-4o-mini-2024-07-18,openai_api_key"
+LLM_MODEL_CONFIG_gemini_1.5_pro="gemini-1.5-pro-002"
+LLM_MODEL_CONFIG_gemini_1.5_flash="gemini-1.5-flash-002"
+LLM_MODEL_CONFIG_diffbot="diffbot,diffbot_api_key"
 LLM_MODEL_CONFIG_azure_ai_gpt_35="azure_deployment_name,azure_endpoint or base_url,azure_api_key,api_version"
 LLM_MODEL_CONFIG_azure_ai_gpt_4o="gpt-4o,https://YOUR-ENDPOINT.openai.azure.com/,azure_api_key,api_version"
 LLM_MODEL_CONFIG_groq_llama3_70b="model_name,base_url,groq_api_key"
 LLM_MODEL_CONFIG_anthropic_claude_3_5_sonnet="model_name,anthropic_api_key"
 LLM_MODEL_CONFIG_fireworks_llama_v3_70b="model_name,fireworks_api_key"
 LLM_MODEL_CONFIG_bedrock_claude_3_5_sonnet="model_name,aws_access_key_id,aws_secret__access_key,region_name"
 LLM_MODEL_CONFIG_ollama_llama3="model_name,model_local_url"
+YOUTUBE_TRANSCRIPT_PROXY="https://user:pass@domain:port"
 

backend/requirements.txt

@@ -69,22 +69,22 @@ jsonpath-python==1.0.6
 jsonpointer==2.4
 json-repair==0.25.2
 kiwisolver==1.4.5
-langchain
-langchain-aws
-langchain-anthropic
-langchain-fireworks
-langchain-google-genai
-langchain-community
-langchain-core
-langchain-experimental
-langchain-google-vertexai
-langchain-groq
-langchain-openai
-langchain-text-splitters
+langchain==0.3.0
+langchain-aws==0.2.1
+langchain-anthropic==0.2.1
+langchain-fireworks==0.2.0
+langchain-google-genai==2.0.0
+langchain-community==0.3.0
+langchain-core==0.3.5
+langchain-experimental==0.3.1
+langchain-google-vertexai==2.0.1
+langchain-groq==0.2.0
+langchain-openai==0.2.0
+langchain-text-splitters==0.3.0
 langdetect==1.0.9
-langsmith==0.1.83
+langsmith==0.1.128
 layoutparser==0.3.4
-langserve==0.2.2
+langserve==0.3.0
 #langchain-cli==0.0.25
 lxml==5.1.0
 MarkupSafe==2.1.5
@@ -100,7 +100,7 @@ numpy==1.26.4
 omegaconf==2.3.0
 onnx==1.16.1
 onnxruntime==1.18.1
-openai==1.35.10
+openai==1.47.1
 opencv-python==4.8.0.76
 orjson==3.9.15
 packaging==23.2
@@ -140,12 +140,10 @@ requests==2.32.3
 rsa==4.9
 s3transfer==0.10.1
 safetensors==0.4.1
-scipy==1.10.1
 shapely==2.0.3
 six==1.16.0
 sniffio==1.3.1
 soupsieve==2.5
-SQLAlchemy==2.0.28
 starlette==0.37.2
 sse-starlette==2.1.2
 starlette-session==0.4.3
@@ -160,7 +158,7 @@ transformers==4.42.3
 types-protobuf
 types-requests
 typing-inspect==0.9.0
-typing_extensions==4.9.0
+typing_extensions==4.12.2
 tzdata==2024.1
 unstructured==0.14.9
 unstructured-client==0.23.8
@@ -179,3 +177,7 @@ sentence-transformers==3.0.1
 google-cloud-logging==3.10.0
 PyMuPDF==1.24.5
 pypandoc==1.13
+graphdatascience==1.10
+Secweb==1.11.0
+ragas==0.1.14
+