add examples

VinciGit00 · VinciGit00 · commit 004d03a0ff43 · 2024-05-27T12:50:51.000+02:00
diff --git a/examples/bedrock/csv_scraper_bedrock.py b/examples/bedrock/csv_scraper_bedrock.py
@@ -30,14 +30,14 @@
 
 graph_config = {
     "llm": {
+        "client": "client_name",
         "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
         "temperature": 0.0
     },
     "embeddings": {
         "model": "bedrock/cohere.embed-multilingual-v3"
     }
 }
-
 # ************************************************
 # Create the CSVScraperGraph instance and run it
 # ************************************************
diff --git a/examples/bedrock/custom_graph_bedrock.py b/examples/bedrock/custom_graph_bedrock.py
@@ -25,6 +25,7 @@
 
 graph_config = {
     "llm": {
+        "client": "client_name",
         "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
         "temperature": 0.0
     },
diff --git a/examples/bedrock/json_scraper_bedrock.py b/examples/bedrock/json_scraper_bedrock.py
@@ -29,6 +29,7 @@
 
 graph_config = {
     "llm": {
+        "client": "client_name",
         "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
         "temperature": 0.0
     },
diff --git a/examples/bedrock/pdf_scraper_graph_bedrock.py b/examples/bedrock/pdf_scraper_graph_bedrock.py
@@ -0,0 +1,63 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+
+import os, json
+from dotenv import load_dotenv
+from scrapegraphai.utils import prettify_exec_info
+from scrapegraphai.graphs import PDFScraperGraph
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "client": "client_name",
+        "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+        "temperature": 0.0
+    },
+    "embeddings": {
+        "model": "bedrock/cohere.embed-multilingual-v3"
+    }
+}
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+schema = """
+    {
+        "type": "object",
+        "properties": {
+            "summary": {
+                "type": "string"
+            },
+            "topics": {
+                "type": "array",
+                "items": {
+                    "type": "string"
+                }
+            }
+        }
+    }
+"""
+
+pdf_scraper_graph = PDFScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+    schema=schema,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))
diff --git a/examples/bedrock/scrape_plain_text_bedrock.py b/examples/bedrock/scrape_plain_text_bedrock.py
@@ -30,6 +30,7 @@
 
 graph_config = {
     "llm": {
+        "client": "client_name",
         "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
         "temperature": 0.0
     },
diff --git a/examples/bedrock/script_generator_bedrock.py b/examples/bedrock/script_generator_bedrock.py
@@ -15,13 +15,14 @@
 
 graph_config = {
     "llm": {
+        "client": "client_name",
         "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
         "temperature": 0.0
     },
     "embeddings": {
         "model": "bedrock/cohere.embed-multilingual-v3"
     },
-    "library": "beautifulsoup"
+        "library": "beautifulsoup"
 }
 
 # ************************************************
diff --git a/examples/bedrock/search_graph_bedrock.py b/examples/bedrock/search_graph_bedrock.py
@@ -14,14 +14,14 @@
 
 graph_config = {
     "llm": {
+        "client": "client_name",
         "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
         "temperature": 0.0
     },
     "embeddings": {
-        "model": "bedrock/amazon.titan-embed-text-v2:0"
+        "model": "bedrock/cohere.embed-multilingual-v3"
     }
 }
-
 # ************************************************
 # Create the SearchGraph instance and run it
 # ************************************************
diff --git a/examples/bedrock/smart_scraper_bedrock.py b/examples/bedrock/smart_scraper_bedrock.py
@@ -14,15 +14,15 @@
 # Define the configuration for the graph
 # ************************************************
 
-openai_key = os.getenv("OPENAI_APIKEY")
-
 graph_config = {
     "llm": {
-        "api_key": openai_key,
-        "model": "gpt-4o",
+        "client": "client_name",
+        "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+        "temperature": 0.0
     },
-    "verbose": True,
-    "headless": False,
+    "embeddings": {
+        "model": "bedrock/cohere.embed-multilingual-v3"
+    }
 }
 
 # ************************************************
diff --git a/examples/bedrock/smart_scraper_multi_bedrock.py b/examples/bedrock/smart_scraper_multi_bedrock.py
@@ -0,0 +1,41 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+
+import os, json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperMultiGraph
+
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "client": "client_name",
+        "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+        "temperature": 0.0
+    },
+    "embeddings": {
+        "model": "bedrock/cohere.embed-multilingual-v3"
+    }
+}
+
+# *******************************************************
+# Create the SmartScraperMultiGraph instance and run it
+# *******************************************************
+
+multiple_search_graph = SmartScraperMultiGraph(
+    prompt="Who is Marco Perini?",
+    source= [
+        "https://perinim.github.io/",
+        "https://perinim.github.io/cv/"
+        ],
+    schema=None,
+    config=graph_config
+)
+
+result = multiple_search_graph.run()
+print(json.dumps(result, indent=4))
diff --git a/examples/bedrock/smart_scraper_schema_bedrock.py b/examples/bedrock/smart_scraper_schema_bedrock.py
@@ -33,15 +33,15 @@
 # Define the configuration for the graph
 # ************************************************
 
-openai_key = os.getenv("OPENAI_APIKEY")
-
 graph_config = {
     "llm": {
-        "api_key": openai_key,
-        "model": "gpt-4o",
+        "client": "client_name",
+        "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+        "temperature": 0.0
     },
-    "verbose": True,
-    "headless": False,
+    "embeddings": {
+        "model": "bedrock/cohere.embed-multilingual-v3"
+    }
 }
 
 # ************************************************
diff --git a/examples/bedrock/xml_scraper_bedrock.py b/examples/bedrock/xml_scraper_bedrock.py
@@ -28,6 +28,7 @@
 
 graph_config = {
     "llm": {
+        "client": "client_name",
         "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
         "temperature": 0.0
     },
@@ -59,4 +60,3 @@
 # Save to json or csv
 convert_to_csv(result, "result")
 convert_to_json(result, "result")
-
diff --git a/examples/deepseek/custom_graph_deepseek.py b/examples/deepseek/custom_graph_deepseek.py
@@ -0,0 +1,84 @@
+"""
+Example of custom graph using Gemini Google model
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.models import Gemini
+from scrapegraphai.graphs import BaseGraph
+from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+deepseek_key = os.getenv("DEEPSEEK_APIKEY")
+
+graph_config = {
+    "llm": {
+        "model": "deepseek-chat",
+        "openai_api_key": deepseek_key,
+        "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+    "verbose": True,
+}
+
+# ************************************************
+# Define the graph nodes
+# ************************************************
+
+llm_model = Gemini(graph_config["llm"])
+
+# define the nodes for the graph
+fetch_node = FetchNode(
+    input="url | local_dir",
+    output=["doc"],
+)
+parse_node = ParseNode(
+    input="doc",
+    output=["parsed_doc"],
+    node_config={"chunk_size": 4096}
+)
+rag_node = RAGNode(
+    input="user_prompt & (parsed_doc | doc)",
+    output=["relevant_chunks"],
+    node_config={"llm": llm_model},
+)
+generate_answer_node = GenerateAnswerNode(
+    input="user_prompt & (relevant_chunks | parsed_doc | doc)",
+    output=["answer"],
+    node_config={"llm": llm_model},
+)
+
+# ************************************************
+# Create the graph by defining the connections
+# ************************************************
+
+graph = BaseGraph(
+    nodes={
+        fetch_node,
+        parse_node,
+        rag_node,
+        generate_answer_node,
+    },
+    edges={
+        (fetch_node, parse_node),
+        (parse_node, rag_node),
+        (rag_node, generate_answer_node)
+    },
+    entry_point=fetch_node
+)
+
+# ************************************************
+# Execute the graph
+# ************************************************
+
+result, execution_info = graph.execute({
+    "user_prompt": "List me the projects with their description",
+    "url": "https://perinim.github.io/projects/"
+})
+
+# get the answer from the result
+result = result.get("answer", "No answer found.")
+print(result)
diff --git a/examples/deepseek/pdf_scraper_graph_deepseek.py b/examples/deepseek/pdf_scraper_graph_deepseek.py
@@ -0,0 +1,63 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+
+import os, json
+from dotenv import load_dotenv
+from scrapegraphai.utils import prettify_exec_info
+from scrapegraphai.graphs import PDFScraperGraph
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+deepseek_key = os.getenv("DEEPSEEK_APIKEY")
+
+graph_config = {
+    "llm": {
+        "model": "deepseek-chat",
+        "openai_api_key": deepseek_key,
+        "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+    "verbose": True,
+}
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+schema = """
+    {
+        "type": "object",
+        "properties": {
+            "summary": {
+                "type": "string"
+            },
+            "topics": {
+                "type": "array",
+                "items": {
+                    "type": "string"
+                }
+            }
+        }
+    }
+"""
+
+pdf_scraper_graph = PDFScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+    schema=schema,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))
diff --git a/examples/deepseek/scrape_plain_text_deepseek.py b/examples/deepseek/scrape_plain_text_deepseek.py
diff --git a/examples/deepseek/smart_scraper_deepseek.py b/examples/deepseek/smart_scraper_deepseek.py
diff --git a/examples/gemini/pdf_scraper_graph_gemini.py b/examples/gemini/pdf_scraper_graph_gemini.py
diff --git a/examples/gemini/smart_scraper_multi_gemini.py b/examples/gemini/smart_scraper_multi_gemini.py
diff --git a/examples/huggingfacehub/smart_scraper_schema_huggingfacehub.py b/examples/huggingfacehub/smart_scraper_schema_huggingfacehub.py
