Merge pull request #838 from ScrapeGraphAI/pre/beta

Pre/beta

Author: VinciGit00

CHANGELOG.md

@@ -1,9 +1,8 @@
+## [1.32.0-beta.5](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.32.0-beta.4...v1.32.0-beta.5) (2024-12-02)
 ## [1.32.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.31.1...v1.32.0) (2024-12-02)
 
 
-### Features
 
-* add API integration ([46373af](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/46373afe6d8c05ad26039e68190f13d82b20a349))
 
 ## [1.32.0-beta.4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.32.0-beta.3...v1.32.0-beta.4) (2024-12-02)
 

examples/extras/authenticated_playwright.py

@@ -0,0 +1,93 @@
+"""
+Example leveraging a state file containing session cookies which
+might be leveraged to authenticate to a website and scrape protected
+content.
+"""
+
+import os
+import random
+from dotenv import load_dotenv
+
+# import playwright so we can use it to create the state file
+from playwright.async_api import async_playwright
+
+from scrapegraphai.graphs import OmniScraperGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+# ************************************************
+# Leveraging Playwright external to the invocation of the graph to
+# login and create the state file
+# ************************************************
+
+
+# note this is just an example and probably won't actually work on
+# LinkedIn, the implementation of the login is highly dependent on the website
+async def do_login():
+    async with async_playwright() as playwright:
+        browser = await playwright.chromium.launch(
+            timeout=30000,
+            headless=False,
+            slow_mo=random.uniform(500, 1500),
+        )
+    page = await browser.new_page()
+
+    # very basic implementation of a login, in reality it may be trickier
+    await page.goto("https://www.linkedin.com/login")
+    await page.get_by_label("Email or phone").fill("some_bloke@some_domain.com")
+    await page.get_by_label("Password").fill("test1234")
+    await page.get_by_role("button", name="Sign in").click()
+    await page.wait_for_timeout(3000)
+
+    # assuming a successful login, we save the cookies to a file
+    await page.context.storage_state(path="./state.json")
+
+
+async def main():
+    await do_login()
+
+    # ************************************************
+    # Define the configuration for the graph
+    # ************************************************
+
+    openai_api_key = os.getenv("OPENAI_APIKEY")
+
+    graph_config = {
+        "llm": {
+            "api_key": openai_api_key,
+            "model": "openai/gpt-4o",
+        },
+        "max_images": 10,
+        "headless": False,
+        # provide the path to the state file
+        "storage_state": "./state.json",
+    }
+
+    # ************************************************
+    # Create the OmniScraperGraph instance and run it
+    # ************************************************
+
+    omni_scraper_graph = OmniScraperGraph(
+        prompt="List me all the projects with their description.",
+        source="https://www.linkedin.com/feed/",
+        config=graph_config,
+    )
+
+    # the storage_state is used to load the cookies from the state file
+    # so we are authenticated and able to scrape protected content
+    result = omni_scraper_graph.run()
+    print(result)
+
+    # ************************************************
+    # Get graph execution info
+    # ************************************************
+
+    graph_exec_info = omni_scraper_graph.get_execution_info()
+    print(prettify_exec_info(graph_exec_info))
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

examples/extras/undected_playwrigth.py renamed to examples/extras/undected_playwright.py

@@ -3,7 +3,8 @@ name = "scrapegraphai"
 
 
 
-version = "1.32.0"
+version = "1.32.0b5"
+
 
 
 

pyproject.toml

@@ -8,6 +8,7 @@
 
 logger = get_logger("web-loader")
 
+
 class ChromiumLoader(BaseLoader):
     """Scrapes HTML pages from URLs using a (headless) instance of the
     Chromium web driver with proxy protection.
@@ -33,6 +34,7 @@ def __init__(
         proxy: Optional[Proxy] = None,
         load_state: str = "domcontentloaded",
         requires_js_support: bool = False,
+        storage_state: Optional[str] = None,
         **kwargs: Any,
     ):
         """Initialize the loader with a list of URL paths.
@@ -62,6 +64,7 @@ def __init__(
         self.urls = urls
         self.load_state = load_state
         self.requires_js_support = requires_js_support
+        self.storage_state = storage_state
 
     async def ascrape_undetected_chromedriver(self, url: str) -> str:
         """
@@ -91,7 +94,9 @@ async def ascrape_undetected_chromedriver(self, url: str) -> str:
                 attempt += 1
                 logger.error(f"Attempt {attempt} failed: {e}")
                 if attempt == self.RETRY_LIMIT:
-                    results = f"Error: Network error after {self.RETRY_LIMIT} attempts - {e}"
+                    results = (
+                        f"Error: Network error after {self.RETRY_LIMIT} attempts - {e}"
+                    )
             finally:
                 driver.quit()
 
@@ -113,7 +118,9 @@ async def ascrape_playwright(self, url: str) -> str:
                     browser = await p.chromium.launch(
                         headless=self.headless, proxy=self.proxy, **self.browser_config
                     )
-                    context = await browser.new_context()
+                    context = await browser.new_context(
+                        storage_state=self.storage_state
+                    )
                     await Malenia.apply_stealth(context)
                     page = await context.new_page()
                     await page.goto(url, wait_until="domcontentloaded")
@@ -125,10 +132,12 @@ async def ascrape_playwright(self, url: str) -> str:
                 attempt += 1
                 logger.error(f"Attempt {attempt} failed: {e}")
                 if attempt == self.RETRY_LIMIT:
-                    raise RuntimeError(f"Failed to fetch {url} after {self.RETRY_LIMIT} attempts: {e}")
+                    raise RuntimeError(
+                        f"Failed to fetch {url} after {self.RETRY_LIMIT} attempts: {e}"
+                    )
             finally:
-                if 'browser' in locals():
-                    await browser.close()
+                if "browser" in locals():
+
 
     async def ascrape_with_js_support(self, url: str) -> str:
         """
@@ -138,7 +147,7 @@ async def ascrape_with_js_support(self, url: str) -> str:
             url (str): The URL to scrape.
 
         Returns:
-            str: The fully rendered HTML content after JavaScript execution, 
+            str: The fully rendered HTML content after JavaScript execution,
             or an error message if an exception occurs.
         """
         from playwright.async_api import async_playwright
@@ -153,7 +162,9 @@ async def ascrape_with_js_support(self, url: str) -> str:
                     browser = await p.chromium.launch(
                         headless=self.headless, proxy=self.proxy, **self.browser_config
                     )
-                    context = await browser.new_context()
+                    context = await browser.new_context(
+                        storage_state=self.storage_state
+                    )
                     page = await context.new_page()
                     await page.goto(url, wait_until="networkidle")
                     results = await page.content()
@@ -163,7 +174,9 @@ async def ascrape_with_js_support(self, url: str) -> str:
                 attempt += 1
                 logger.error(f"Attempt {attempt} failed: {e}")
                 if attempt == self.RETRY_LIMIT:
-                    results = f"Error: Network error after {self.RETRY_LIMIT} attempts - {e}"
+                    results = (
+                        f"Error: Network error after {self.RETRY_LIMIT} attempts - {e}"
+                    )
             finally:
                 await browser.close()
 
@@ -180,7 +193,9 @@ def lazy_load(self) -> Iterator[Document]:
             Document: The scraped content encapsulated within a Document object.
         """
         scraping_fn = (
-            self.ascrape_with_js_support if self.requires_js_support else getattr(self, f"ascrape_{self.backend}")
+            self.ascrape_with_js_support
+            if self.requires_js_support
+            else getattr(self, f"ascrape_{self.backend}")
         )
 
         for url in self.urls:
@@ -202,7 +217,9 @@ async def alazy_load(self) -> AsyncIterator[Document]:
             source URL as metadata.
         """
         scraping_fn = (
-            self.ascrape_with_js_support if self.requires_js_support else getattr(self, f"ascrape_{self.backend}")
+            self.ascrape_with_js_support
+            if self.requires_js_support
+            else getattr(self, f"ascrape_{self.backend}")
         )
 
         tasks = [scraping_fn(url) for url in self.urls]