Merge pull request #639 from ScrapeGraphAI/scrape_do_integration

Scrape do integration

Author: f-aguzzi

examples/extras/browser_base_integration.py

@@ -18,7 +18,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("OPENAI_API_KEY"),
-        "model": "gpt-4o",
+        "model": "openai/gpt-4o",
     },
     "browser_base": {
         "api_key": os.getenv("BROWSER_BASE_API_KEY"),

examples/extras/scrape_do.py

@@ -0,0 +1,40 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperGraph
+
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("OPENAI_API_KEY"),
+        "model": "openai/gpt-4o",
+    },
+    "scrape_do": {
+        "api_key": os.getenv("SCRAPE_DO_API_KEY"),
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="List me all the projects",
+    source="https://perinim.github.io/projects/",
+    config=graph_config
+)
+
+result = smart_scraper_graph.run()
+print(json.dumps(result, indent=4))

scrapegraphai/docloaders/__init__.py

@@ -2,3 +2,4 @@
 
 from .chromium import ChromiumLoader
 from .browser_base import browser_base_fetch
+from .scrape_do import scrape_do_fetch

scrapegraphai/docloaders/scrape_do.py

@@ -0,0 +1,41 @@
+"""
+Scrape_do module
+"""
+import urllib.parse
+import requests
+import urllib3
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+def scrape_do_fetch(token, target_url, use_proxy=False, geoCode=None, super_proxy=False):
+    """
+    Fetches the IP address of the machine associated with the given URL using Scrape.do.
+
+    Args:
+        token (str): The API token for Scrape.do service.
+        target_url (str): A valid web page URL to fetch its associated IP address.
+        use_proxy (bool): Whether to use Scrape.do proxy mode. Default is False.
+        geoCode (str, optional): Specify the country code for 
+        geolocation-based proxies. Default is None.
+        super_proxy (bool): If True, use Residential & Mobile Proxy Networks. Default is False.
+
+    Returns:
+        str: The raw response from the target URL.
+    """
+    encoded_url = urllib.parse.quote(target_url)
+    if use_proxy:
+        # Create proxy mode URL
+        proxyModeUrl = f"http://{token}:@proxy.scrape.do:8080"
+        proxies = {
+            "http": proxyModeUrl,
+            "https": proxyModeUrl,
+        }
+        # Add optional geoCode and super proxy parameters if provided
+        params = {"geoCode": geoCode, "super": str(super_proxy).lower()} if geoCode else {}
+        response = requests.get(target_url, proxies=proxies, verify=False, params=params)
+    else:
+        # API Mode URL
+        url = f"http://api.scrape.do?token={token}&url={encoded_url}"
+        response = requests.get(url)
+
+    return response.text

scrapegraphai/graphs/abstract_graph.py

@@ -63,6 +63,7 @@ def __init__(self, prompt: str, config: dict,
         self.loader_kwargs = self.config.get("loader_kwargs", {})
         self.cache_path = self.config.get("cache_path", False)
         self.browser_base = self.config.get("browser_base")
+        self.scrape_do = self.config.get("scrape_do")
 
         self.graph = self._create_graph()
         self.final_state = None

scrapegraphai/graphs/smart_scraper_graph.py

@@ -67,7 +67,8 @@ def _create_graph(self) -> BaseGraph:
                 "force": self.config.get("force", False),
                 "cut": self.config.get("cut", True),
                 "loader_kwargs": self.config.get("loader_kwargs", {}),
-                "browser_base": self.config.get("browser_base")
+                "browser_base": self.config.get("browser_base"),
+                "scrape_do": self.config.get("scrape_do")
             }
         )
         parse_node = ParseNode(

scrapegraphai/nodes/fetch_node.py

@@ -76,6 +76,10 @@ def __init__(
             None if node_config is None else node_config.get("browser_base", None)
         )
 
+        self.scrape_do = (
+            None if node_config is None else node_config.get("scrape_do", None)
+        )
+
     def execute(self, state):
         """
         Executes the node's logic to fetch HTML content from a specified URL and
@@ -102,7 +106,7 @@ def execute(self, state):
 
         source = input_data[0]
         input_type = input_keys[0]
-        
+
         handlers = {
             "json_dir": self.handle_directory,
             "xml_dir": self.handle_directory,
@@ -271,19 +275,34 @@ def handle_web_source(self, state, source):
                 try:
                     from ..docloaders.browser_base import browser_base_fetch
                 except ImportError:
-                    raise ImportError("The browserbase module is not installed. Please install it using `pip install browserbase`.")
+                    raise ImportError("""The browserbase module is not installed. 
+                                      Please install it using `pip install browserbase`.""")
 
                 data =  browser_base_fetch(self.browser_base.get("api_key"),
                                             self.browser_base.get("project_id"), [source])
 
                 document = [Document(page_content=content,
                                     metadata={"source": source}) for content in data]
+            elif self.scrape_do is not None:
+                from ..docloaders.scrape_do import scrape_do_fetch
+                if self.scrape_do.get("use_proxy") is None or self.scrape_do.get("geoCode") is None or self.scrape_do.get("super_proxy") is None:
+                    data =  scrape_do_fetch(self.scrape_do.get("api_key"),
+                                                source)
+                else:
+                    data =  scrape_do_fetch(self.scrape_do.get("api_key"),
+                                                source, self.scrape_do.get("use_proxy"),
+                                                self.scrape_do.get("geoCode"),
+                                                self.scrape_do.get("super_proxy"))
+
+                document = [Document(page_content=data,
+                                    metadata={"source": source})]
             else:
                 loader = ChromiumLoader([source], headless=self.headless, **loader_kwargs)
                 document = loader.load()
 
             if not document or not document[0].page_content.strip():
-                raise ValueError("No HTML body content found in the document fetched by ChromiumLoader.")
+                raise ValueError("""No HTML body content found in
+                                 the document fetched by ChromiumLoader.""")
             parsed_content = document[0].page_content
 
             if (isinstance(self.llm_model, ChatOpenAI) or isinstance(self.llm_model, AzureChatOpenAI))  and not self.script_creator or self.force and not self.script_creator and not self.openai_md_enabled:
@@ -292,7 +311,7 @@ def handle_web_source(self, state, source):
             compressed_document = [
                 Document(page_content=parsed_content, metadata={"source": "html file"})
             ]
-        
+
         return self.update_state(state, compressed_document)
 
     def update_state(self, state, compressed_document):