removed duplicates

Author: VinciGit00

.python-version

@@ -1 +1 @@
-3.9.19
+3.10.14

examples/openai/deep_scraper_openai.py

@@ -19,7 +19,7 @@
 graph_config = {
     "llm": {
         "api_key": openai_key,
-        "model": "gpt-4",
+        "model": "gpt-3.5-turbo",
     },
     "verbose": True,
 }
@@ -29,9 +29,9 @@
 # ************************************************
 
 deep_scraper_graph = DeepScraperGraph(
-    prompt="List me all the job titles and detailed job description.",
+    prompt="List me all the contacts",
     # also accepts a string with the already downloaded HTML code
-    source="https://www.google.com/about/careers/applications/jobs/results/?location=Bangalore%20India",
+    source="https://davittoriogift.com/en/home/a",
     config=graph_config
 )
 

requirements-dev.lock

@@ -93,7 +93,6 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
-    # via sqlalchemy
 groq==0.5.0
     # via langchain-groq
 grpcio==1.63.0

requirements.lock

@@ -92,7 +92,6 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
-    # via sqlalchemy
 groq==0.5.0
     # via langchain-groq
 grpcio==1.63.0

scrapegraphai/graphs/__init__.py

@@ -15,4 +15,4 @@
 from .pdf_scraper_graph import PDFScraperGraph
 from .omni_scraper_graph import OmniScraperGraph
 from .omni_search_graph import OmniSearchGraph
-from .turbo_scraper import TurboScraperGraph
+#from .turbo_scraper import TurboScraperGraph

scrapegraphai/nodes/search_link_node.py

@@ -103,5 +103,5 @@ def execute(self, state: dict) -> dict:
             answer = merge_chain.invoke(
                 {"content": chunk.page_content, "user_prompt": user_prompt})
             relevant_links += answer
-        state.update({self.output[0]: relevant_links})
+        state.update({self.output[0]: list(set(relevant_links))})
         return state