Merge branch 'main' into main

Author: VinciGit00

CHANGELOG.md

@@ -1,3 +1,15 @@
+## [1.17.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.16.0...v1.17.0) (2024-09-08)
+
+
+### Features
+
+* **docloaders:** Enhance browser_base_fetch function flexibility ([57fd01f](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/57fd01f9a76ea8ea69ec04b7238ab58ca72ac8f4))
+
+
+### Docs
+
+* **sponsor:** 🅱️ Browserbase sponsor 🅱️ ([a540139](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/a5401394cc939d9a5fc58b8a9145141c2f047bab))
+
 ## [1.16.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.15.2...v1.16.0) (2024-09-01)
 
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapegraphai"
-version = "1.16.0"
+version = "1.17.0"
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
 authors = [
     { name = "Marco Vinciguerra", email = "[email protected]" },

scrapegraphai/docloaders/browser_base.py

@@ -3,7 +3,7 @@
 """
 from typing import List
 
-def browser_base_fetch(api_key: str, project_id: str, link: List[str]) -> List[str]:
+def browser_base_fetch(api_key: str, project_id: str, link: List[str], text_content: bool = True) -> List[str]:
     """
     BrowserBase Fetch
 
@@ -53,22 +53,19 @@ def browser_base_fetch(api_key: str, project_id: str, link: List[str]) -> List[s
     browserbase = Browserbase(api_key=api_key, project_id=project_id)
 
     result = []
-    # Define the async fetch logic for individual links
     async def _async_fetch_link(l):
         return await asyncio.to_thread(browserbase.load, l, text_content=text_content)
 
     if async_mode:
-        # Asynchronously process each link
         async def _async_browser_base_fetch():
             for l in link:
                 result.append(await _async_fetch_link(l))
             return result
 
-        # Run the async fetch function
         result = asyncio.run(_async_browser_base_fetch())
     else:
-        # Synchronous logic
         for l in link:
             result.append(browserbase.load(l, text_content=text_content))
 
+
     return result