add fiels

Author: VinciGit00

scrapegraph-py/scrapegraph_py/async_client.py

@@ -63,7 +63,8 @@ def __init__(
         """Initialize AsyncClient with configurable parameters.
 
         Args:
-            api_key: API key for authentication. If None, will try to load from environment
+            api_key: API key for authentication. If None, will try to
+                     load from environment
             verify_ssl: Whether to verify SSL certificates
             timeout: Request timeout in seconds. None means no timeout (infinite)
             max_retries: Maximum number of retry attempts
@@ -83,7 +84,8 @@ def __init__(
 
         validate_api_key(api_key)
         logger.debug(
-            f"🛠️ Configuration: verify_ssl={verify_ssl}, timeout={timeout}, max_retries={max_retries}"
+            f"🛠️ Configuration: verify_ssl={verify_ssl}, "
+            f"timeout={timeout}, max_retries={max_retries}"
         )
         self.api_key = api_key
         self.headers = {**DEFAULT_HEADERS, "SGAI-APIKEY": api_key}
@@ -104,7 +106,8 @@ async def _make_request(self, method: str, url: str, **kwargs) -> Any:
         for attempt in range(self.max_retries):
             try:
                 logger.info(
-                    f"🚀 Making {method} request to {url} (Attempt {attempt + 1}/{self.max_retries})"
+                    f"🚀 Making {method} request to {url} "
+                    f"(Attempt {attempt + 1}/{self.max_retries})"
                 )
                 logger.debug(f"🔍 Request parameters: {kwargs}")
 
@@ -255,7 +258,8 @@ async def get_credits(self):
             f"{API_BASE_URL}/credits",
         )
         logger.info(
-            f"✨ Credits info retrieved: {result.get('remaining_credits')} credits remaining"
+            f"✨ Credits info retrieved: "
+            f"{result.get('remaining_credits')} credits remaining"
         )
         return result
 
@@ -271,8 +275,9 @@ async def searchscraper(
         Args:
             user_prompt: The search prompt string
             num_results: Number of websites to scrape (3-20). Default is 3.
-                        More websites provide better research depth but cost more credits.
-                        Credit calculation: 30 base + 10 per additional website beyond 3.
+                        More websites provide better research depth but cost more
+                        credits. Credit calculation: 30 base + 10 per additional
+                        website beyond 3.
             headers: Optional headers to send with the request
             output_schema: Optional schema to structure the output
         """
@@ -323,7 +328,8 @@ async def crawl(
         batch_size: Optional[int] = None,
         sitemap: bool = False,
     ):
-        """Send a crawl request with support for both AI extraction and markdown conversion modes"""
+        """Send a crawl request with support for both AI extraction and
+        markdown conversion modes"""
         logger.info("🔍 Starting crawl request")
         logger.debug(f"🌐 URL: {url}")
         logger.debug(

scrapegraph-py/scrapegraph_py/client.py

@@ -64,7 +64,8 @@ def __init__(
         """Initialize Client with configurable parameters.
 
         Args:
-            api_key: API key for authentication. If None, will try to load from environment
+            api_key: API key for authentication. If None, will try to load
+                     from environment
             verify_ssl: Whether to verify SSL certificates
             timeout: Request timeout in seconds. None means no timeout (infinite)
             max_retries: Maximum number of retry attempts
@@ -84,7 +85,8 @@ def __init__(
 
         validate_api_key(api_key)
         logger.debug(
-            f"🛠️ Configuration: verify_ssl={verify_ssl}, timeout={timeout}, max_retries={max_retries}"
+            f"🛠️ Configuration: verify_ssl={verify_ssl}, timeout={timeout}, "
+            f"max_retries={max_retries}"
         )
 
         self.api_key = api_key
@@ -260,7 +262,8 @@ def get_credits(self):
             f"{API_BASE_URL}/credits",
         )
         logger.info(
-            f"✨ Credits info retrieved: {result.get('remaining_credits')} credits remaining"
+            f"✨ Credits info retrieved: {result.get('remaining_credits')} "
+            f"credits remaining"
         )
         return result
 
@@ -276,8 +279,9 @@ def searchscraper(
         Args:
             user_prompt: The search prompt string
             num_results: Number of websites to scrape (3-20). Default is 3.
-                        More websites provide better research depth but cost more credits.
-                        Credit calculation: 30 base + 10 per additional website beyond 3.
+                        More websites provide better research depth but cost more
+                        credits. Credit calculation: 30 base + 10 per additional
+                        website beyond 3.
             headers: Optional headers to send with the request
             output_schema: Optional schema to structure the output
         """
@@ -326,7 +330,8 @@ def crawl(
         batch_size: Optional[int] = None,
         sitemap: bool = False,
     ):
-        """Send a crawl request with support for both AI extraction and markdown conversion modes"""
+        """Send a crawl request with support for both AI extraction and
+        markdown conversion modes"""
         logger.info("🔍 Starting crawl request")
         logger.debug(f"🌐 URL: {url}")
         logger.debug(

scrapegraph-py/scrapegraph_py/models/crawl.py

@@ -14,16 +14,20 @@ class CrawlRequest(BaseModel):
     )
     extraction_mode: bool = Field(
         default=True,
-        description="True for AI extraction mode, False for markdown conversion mode (no AI/LLM processing)",
+        description="True for AI extraction mode, False for markdown conversion "
+        "mode (no AI/LLM processing)",
     )
     prompt: Optional[str] = Field(
         default=None,
-        example="What does the company do? and I need text content from there privacy and terms",
-        description="The prompt to guide the crawl and extraction (required when extraction_mode=True)",
+        example="What does the company do? and I need text content from there "
+        "privacy and terms",
+        description="The prompt to guide the crawl and extraction (required when "
+        "extraction_mode=True)",
     )
     data_schema: Optional[Dict[str, Any]] = Field(
         default=None,
-        description="JSON schema defining the structure of the extracted data (required when extraction_mode=True)",
+        description="JSON schema defining the structure of the extracted data "
+        "(required when extraction_mode=True)",
     )
     cache_website: bool = Field(
         default=True, description="Whether to cache the website content"
@@ -74,11 +78,13 @@ def validate_extraction_mode_requirements(self) -> "CrawlRequest":
             # Markdown conversion mode - prompt and data_schema should be None
             if self.prompt is not None:
                 raise ValueError(
-                    "Prompt should not be provided when extraction_mode=False (markdown mode)"
+                    "Prompt should not be provided when extraction_mode=False "
+                    "(markdown mode)"
                 )
             if self.data_schema is not None:
                 raise ValueError(
-                    "Data schema should not be provided when extraction_mode=False (markdown mode)"
+                    "Data schema should not be provided when extraction_mode=False "
+                    "(markdown mode)"
                 )
 
         return self

scrapegraph-py/scrapegraph_py/models/markdownify.py

@@ -11,10 +11,12 @@ class MarkdownifyRequest(BaseModel):
     headers: Optional[dict[str, str]] = Field(
         None,
         example={
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36",
             "Cookie": "cookie1=value1; cookie2=value2",
         },
-        description="Optional headers to send with the request, including cookies and user agent",
+        description="Optional headers to send with the request, including cookies "
+        "and user agent",
     )
 
     @model_validator(mode="after")

scrapegraph-py/scrapegraph_py/models/searchscraper.py

@@ -13,15 +13,18 @@ class SearchScraperRequest(BaseModel):
         ge=3,
         le=20,
         example=5,
-        description="Number of websites to scrape (3-20). Default is 3. More websites provide better research depth but cost more credits.",
+        description="Number of websites to scrape (3-20). Default is 3. More "
+        "websites provide better research depth but cost more credits.",
     )
     headers: Optional[dict[str, str]] = Field(
         None,
         example={
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36",
             "Cookie": "cookie1=value1; cookie2=value2",
         },
-        description="Optional headers to send with the request, including cookies and user agent",
+        description="Optional headers to send with the request, including cookies "
+        "and user agent",
     )
     output_schema: Optional[Type[BaseModel]] = None
 

scrapegraph-py/scrapegraph_py/models/smartscraper.py

@@ -23,25 +23,30 @@ class SmartScraperRequest(BaseModel):
     headers: Optional[dict[str, str]] = Field(
         None,
         example={
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36",
             "Cookie": "cookie1=value1; cookie2=value2",
         },
-        description="Optional headers to send with the request, including cookies and user agent",
+        description="Optional headers to send with the request, including cookies "
+        "and user agent",
     )
     cookies: Optional[Dict[str, str]] = Field(
         None,
         example={"session_id": "abc123", "user_token": "xyz789"},
-        description="Dictionary of cookies to send with the request for authentication or session management",
+        description="Dictionary of cookies to send with the request for "
+        "authentication or session management",
     )
     output_schema: Optional[Type[BaseModel]] = None
     number_of_scrolls: Optional[conint(ge=0, le=100)] = Field(
         default=None,
-        description="Number of times to scroll the page (0-100). If None, no scrolling will be performed.",
+        description="Number of times to scroll the page (0-100). If None, no "
+        "scrolling will be performed.",
         example=10,
     )
     total_pages: Optional[conint(ge=1, le=10)] = Field(
         default=None,
-        description="Number of pages to scrape (1-10). If None, only the first page will be scraped.",
+        description="Number of pages to scrape (1-10). If None, only the first "
+        "page will be scraped.",
         example=5,
     )
 

scrapegraph-py/scrapegraph_py/utils/helpers.py

@@ -17,7 +17,8 @@ def validate_api_key(api_key: str) -> bool:
         UUID(uuid_part)
     except ValueError:
         raise ValueError(
-            "Invalid API key format. API key must be 'sgai-' followed by a valid UUID. You can get one at https://dashboard.scrapegraphai.com/"
+            "Invalid API key format. API key must be 'sgai-' followed by a valid UUID. "
+            "You can get one at https://dashboard.scrapegraphai.com/"
         )
     return True