Merge pull request #535 from vedovati-matteo/pre/beta

Refactoring of prompts

Author: VinciGit00

examples/local_models/smart_scraper_ollama.py

@@ -22,13 +22,8 @@
 # Create the SmartScraperGraph instance and run it
 # ************************************************
 smart_scraper_graph = SmartScraperGraph(
-<<<<<<< Updated upstream
     prompt="Find some information about what does the company do, the name and a contact email.",
     source="https://scrapegraphai.com/",
-=======
-    prompt="List all the projects with their descriptions",
-    source="https://perinim.github.io/projects/",
->>>>>>> Stashed changes
     config=graph_config
 )
 

pyproject.toml

@@ -3,7 +3,6 @@ name = "scrapegraphai"
 
 
 version = "1.13.3"
-version = "1.13.0b9"
 
 
 

requirements-dev.lock

@@ -6,6 +6,8 @@
 #   features: []
 #   all-features: false
 #   with-sources: false
+#   generate-hashes: false
+#   universal: false
 
 -e file:.
 aiofiles==24.1.0
@@ -110,6 +112,7 @@ filelock==3.15.4
     # via huggingface-hub
     # via torch
     # via transformers
+    # via triton
 fireworks-ai==0.14.0
     # via langchain-fireworks
 fonttools==4.53.1
@@ -185,6 +188,7 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
+    # via sqlalchemy
 groq==0.9.0
     # via langchain-groq
 grpc-google-iam-v1==0.13.1
@@ -358,6 +362,34 @@ numpy==1.26.4
     # via shapely
     # via streamlit
     # via transformers
+nvidia-cublas-cu12==12.1.3.1
+    # via nvidia-cudnn-cu12
+    # via nvidia-cusolver-cu12
+    # via torch
+nvidia-cuda-cupti-cu12==12.1.105
+    # via torch
+nvidia-cuda-nvrtc-cu12==12.1.105
+    # via torch
+nvidia-cuda-runtime-cu12==12.1.105
+    # via torch
+nvidia-cudnn-cu12==8.9.2.26
+    # via torch
+nvidia-cufft-cu12==11.0.2.54
+    # via torch
+nvidia-curand-cu12==10.3.2.106
+    # via torch
+nvidia-cusolver-cu12==11.4.5.107
+    # via torch
+nvidia-cusparse-cu12==12.1.0.106
+    # via nvidia-cusolver-cu12
+    # via torch
+nvidia-nccl-cu12==2.19.3
+    # via torch
+nvidia-nvjitlink-cu12==12.6.20
+    # via nvidia-cusolver-cu12
+    # via nvidia-cusparse-cu12
+nvidia-nvtx-cu12==12.1.105
+    # via torch
 openai==1.37.0
     # via burr
     # via langchain-fireworks
@@ -599,6 +631,8 @@ tqdm==4.66.4
 transformers==4.43.3
     # via langchain-huggingface
     # via sentence-transformers
+triton==2.2.0
+    # via torch
 typer==0.12.3
     # via fastapi-cli
 typing-extensions==4.12.2
@@ -642,6 +676,8 @@ uvicorn==0.30.3
     # via fastapi
 uvloop==0.19.0
     # via uvicorn
+watchdog==4.0.2
+    # via streamlit
 watchfiles==0.22.0
     # via uvicorn
 websockets==12.0

requirements.lock

@@ -6,6 +6,8 @@
 #   features: []
 #   all-features: false
 #   with-sources: false
+#   generate-hashes: false
+#   universal: false
 
 -e file:.
 aiohttp==3.9.5
@@ -67,6 +69,7 @@ filelock==3.15.4
     # via huggingface-hub
     # via torch
     # via transformers
+    # via triton
 fireworks-ai==0.14.0
     # via langchain-fireworks
 free-proxy==1.1.1
@@ -133,6 +136,7 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
+    # via sqlalchemy
 groq==0.9.0
     # via langchain-groq
 grpc-google-iam-v1==0.13.1
@@ -263,6 +267,34 @@ numpy==1.26.4
     # via sentence-transformers
     # via shapely
     # via transformers
+nvidia-cublas-cu12==12.1.3.1
+    # via nvidia-cudnn-cu12
+    # via nvidia-cusolver-cu12
+    # via torch
+nvidia-cuda-cupti-cu12==12.1.105
+    # via torch
+nvidia-cuda-nvrtc-cu12==12.1.105
+    # via torch
+nvidia-cuda-runtime-cu12==12.1.105
+    # via torch
+nvidia-cudnn-cu12==8.9.2.26
+    # via torch
+nvidia-cufft-cu12==11.0.2.54
+    # via torch
+nvidia-curand-cu12==10.3.2.106
+    # via torch
+nvidia-cusolver-cu12==11.4.5.107
+    # via torch
+nvidia-cusparse-cu12==12.1.0.106
+    # via nvidia-cusolver-cu12
+    # via torch
+nvidia-nccl-cu12==2.19.3
+    # via torch
+nvidia-nvjitlink-cu12==12.6.20
+    # via nvidia-cusolver-cu12
+    # via nvidia-cusparse-cu12
+nvidia-nvtx-cu12==12.1.105
+    # via torch
 openai==1.37.0
     # via langchain-fireworks
     # via langchain-openai
@@ -414,6 +446,8 @@ tqdm==4.66.4
 transformers==4.43.3
     # via langchain-huggingface
     # via sentence-transformers
+triton==2.2.0
+    # via torch
 typing-extensions==4.12.2
     # via anthropic
     # via anyio

scrapegraphai/helpers/__init__.py

@@ -1,13 +1,8 @@
 """ 
-__init__.py for th e helpers folder
+__init__.py for the helpers folder
 """
 
 from .nodes_metadata import nodes_metadata
 from .schemas import graph_schema
 from .models_tokens import models_tokens
 from .robots import robots_dictionary
-from .generate_answer_node_prompts import template_chunks, template_no_chunks, template_merge, template_chunks_md, template_no_chunks_md, template_merge_md
-from .generate_answer_node_csv_prompts import template_chunks_csv, template_no_chunks_csv, template_merge_csv  
-from .generate_answer_node_pdf_prompts import template_chunks_pdf, template_no_chunks_pdf, template_merge_pdf
-from .generate_answer_node_omni_prompts import template_chunks_omni, template_no_chunk_omni, template_merge_omni
-from .merge_answer_node_prompts import template_combined

scrapegraphai/helpers/merge_answer_node_prompts.py

@@ -32,9 +32,9 @@ def __init__(self):
         """
         Initializes an empty ConditionalNode.
         """
-
-       #super().__init__(node_name, "node", input, output, 2, node_config)
-       pass
+        
+        #super().__init__(node_name, "node", input, output, 2, node_config)
+        pass
 
 
     def execute(self, state: dict) -> dict:

scrapegraphai/nodes/conditional_node.py

@@ -10,7 +10,7 @@
 from tqdm import tqdm
 from ..utils.logging import get_logger
 from .base_node import BaseNode
-from ..helpers.generate_answer_node_csv_prompts import template_chunks_csv, template_no_chunks_csv, template_merge_csv
+from ..prompts.generate_answer_node_csv_prompts import template_chunks_csv, template_no_chunks_csv, template_merge_csv
 
 
 class GenerateAnswerCSVNode(BaseNode):

scrapegraphai/nodes/generate_answer_csv_node.py

@@ -10,7 +10,7 @@
 from tqdm import tqdm
 from ..utils.logging import get_logger
 from .base_node import BaseNode
-from ..helpers import template_chunks, template_no_chunks, template_merge, template_chunks_md, template_no_chunks_md, template_merge_md
+from ..prompts import template_chunks, template_no_chunks, template_merge, template_chunks_md, template_no_chunks_md, template_merge_md
 
 class GenerateAnswerNode(BaseNode):
     """

scrapegraphai/nodes/generate_answer_node.py

@@ -8,7 +8,7 @@
 from tqdm import tqdm
 from langchain_community.chat_models import ChatOllama
 from .base_node import BaseNode
-from ..helpers.generate_answer_node_omni_prompts import template_no_chunk_omni, template_chunks_omni, template_merge_omni
+from ..prompts.generate_answer_node_omni_prompts import template_no_chunk_omni, template_chunks_omni, template_merge_omni
 
 
 class GenerateAnswerOmniNode(BaseNode):