feat: ad infinite scrolling

Author: VinciGit00

scrapegraph-py/examples/sync/smartscraper_infinite_scroll.py

@@ -14,41 +14,20 @@ def main():
     # Initialize the client with your API key
     sgai_client = Client(api_key="your-api-key-here")
 
-    try:
-        # Example 1: Basic infinite scrolling with default settings
-        response1 = sgai_client.smartscraper(
-            website_url="https://example.com/infinite-scroll",
-            user_prompt="Extract all product names and prices from the page",
-            infinite_scrolling=True  # Uses default max_pages=10
-        )
-        print("\nExample 1 - Basic infinite scrolling:")
-        print(f"Request ID: {response1['request_id']}")
-        print(f"Result: {response1['result']}")
-
-        # Example 2: Custom infinite scrolling with specific max pages
-        response2 = sgai_client.smartscraper(
-            website_url="https://example.com/long-list",
-            user_prompt="Extract all article titles and their publication dates",
-            infinite_scrolling=True,
-            max_pages=50  # Custom maximum number of pages to scroll
-        )
-        print("\nExample 2 - Custom max pages:")
-        print(f"Request ID: {response2['request_id']}")
-        print(f"Result: {response2['result']}")
-
-        # Example 3: Without infinite scrolling (for comparison)
-        response3 = sgai_client.smartscraper(
-            website_url="https://example.com/static-page",
-            user_prompt="Extract the main heading and first paragraph",
-            infinite_scrolling=False
-        )
-        print("\nExample 3 - Without infinite scrolling:")
-        print(f"Request ID: {response3['request_id']}")
-        print(f"Result: {response3['result']}")
-
-    finally:
-        # Always close the client when done
-        sgai_client.close()
+    response1 = sgai_client.smartscraper(
+        website_url="https://www.ycombinator.com/companies",
+        user_prompt="Extract all the companies and their info",
+        infinite_scrolling=True,
+        max_pages=10,
+    )
+
+    print("\nExample 1 - Basic infinite scrolling:")
+    print(f"Request ID: {response1['request_id']}")
+    print(f"Result: {response1['result']}")
+
+
+    # Always close the client when done
+    sgai_client.close()
 
 if __name__ == "__main__":
-    main() 
+    main()

scrapegraph-py/scrapegraph_py/async_client.py

@@ -174,6 +174,8 @@ async def smartscraper(
         website_html: Optional[str] = None,
         headers: Optional[dict[str, str]] = None,
         output_schema: Optional[BaseModel] = None,
+        infinite_scrolling: bool = False,
+        max_pages: int = 10,
     ):
         """Send a smartscraper request"""
         logger.info("🔍 Starting smartscraper request")
@@ -184,13 +186,17 @@ async def smartscraper(
         if headers:
             logger.debug("🔧 Using custom headers")
         logger.debug(f"📝 Prompt: {user_prompt}")
+        if infinite_scrolling:
+            logger.debug(f"🔄 Infinite scrolling enabled with max_pages={max_pages}")
 
         request = SmartScraperRequest(
             website_url=website_url,
             website_html=website_html,
             headers=headers,
             user_prompt=user_prompt,
             output_schema=output_schema,
+            infinite_scrolling=infinite_scrolling,
+            max_pages=max_pages,
         )
         logger.debug("✅ Request validation passed")
 

scrapegraph-py/scrapegraph_py/client.py

@@ -182,6 +182,8 @@ def smartscraper(
         website_html: Optional[str] = None,
         headers: Optional[dict[str, str]] = None,
         output_schema: Optional[BaseModel] = None,
+        infinite_scrolling: bool = False,
+        max_pages: int = 10,
     ):
         """Send a smartscraper request"""
         logger.info("🔍 Starting smartscraper request")
@@ -192,13 +194,17 @@ def smartscraper(
         if headers:
             logger.debug("🔧 Using custom headers")
         logger.debug(f"📝 Prompt: {user_prompt}")
+        if infinite_scrolling:
+            logger.debug(f"🔄 Infinite scrolling enabled with max_pages={max_pages}")
 
         request = SmartScraperRequest(
             website_url=website_url,
             website_html=website_html,
             headers=headers,
             user_prompt=user_prompt,
             output_schema=output_schema,
+            infinite_scrolling=infinite_scrolling,
+            max_pages=max_pages,
         )
         logger.debug("✅ Request validation passed")