Merge pull request #7 from vzucher/Code-Improvements

refactor: improve code quality with constants and best practices

Author: vzucher

src/brightdata/api/base.py

@@ -1,5 +1,6 @@
 """Base API class for all API implementations."""
 
+import asyncio
 from abc import ABC, abstractmethod
 from typing import Any
 from ..core.engine import AsyncEngine
@@ -38,8 +39,6 @@ def _execute_sync(self, *args: Any, **kwargs: Any) -> Any:
         
         Wraps async method using asyncio.run() for sync compatibility.
         """
-        import asyncio
-        
         try:
             loop = asyncio.get_running_loop()
             raise RuntimeError(

src/brightdata/api/serp/base.py

@@ -11,6 +11,7 @@
 from ...core.engine import AsyncEngine
 from ...models import SearchResult
 from ...types import NormalizedSERPData
+from ...constants import HTTP_OK
 from ...exceptions import ValidationError, APIError
 from ...utils.validation import validate_zone_name
 from ...utils.retry import retry_with_backoff
@@ -132,10 +133,14 @@ async def _search_single_async(
             **kwargs
         )
 
+        # Use "json" format when brd_json=1 is in URL (enables Bright Data parsing)
+        # Otherwise use "raw" to get HTML response
+        response_format = "json" if "brd_json=1" in search_url else "raw"
+        
         payload = {
             "zone": zone,
             "url": search_url,
-            "format": "raw",
+            "format": response_format,
             "method": "GET",
         }
 
@@ -151,14 +156,33 @@ async def _make_request():
             ) as response:
                 data_fetched_at = datetime.now(timezone.utc)
 
-                if response.status == 200:
-                    # With brd_json=1, response is JSON text (not wrapped in status_code/body)
+                if response.status == HTTP_OK:
+                    # Try to parse response - could be direct JSON or wrapped in status_code/body
                     text = await response.text()
                     try:
                         data = json.loads(text)
                     except json.JSONDecodeError:
                         # Fallback to regular JSON response
-                        data = await response.json()
+                        try:
+                            data = await response.json()
+                        except Exception:
+                            # If all else fails, treat as raw text/HTML
+                            data = {"raw_html": text}
+                    
+                    # Handle wrapped response format (status_code/headers/body)
+                    if isinstance(data, dict) and "body" in data and "status_code" in data:
+                        # This is a wrapped HTTP response - extract body
+                        body = data.get("body", "")
+                        if isinstance(body, str) and body.strip().startswith("<"):
+                            # Body is HTML - pass to normalizer which will handle it
+                            data = {"body": body, "status_code": data.get("status_code")}
+                        else:
+                            # Body might be JSON string - try to parse it
+                            try:
+                                data = json.loads(body) if isinstance(body, str) else body
+                            except (json.JSONDecodeError, TypeError):
+                                data = {"body": body, "status_code": data.get("status_code")}
+                    
                     normalized_data = self.data_normalizer.normalize(data)
 
                     return SearchResult(

src/brightdata/api/serp/data_normalizer.py

@@ -1,5 +1,6 @@
 """Data normalization for SERP responses."""
 
+import warnings
 from abc import ABC, abstractmethod
 from typing import Any, Dict, List
 from ...types import NormalizedSERPData
@@ -16,6 +17,9 @@ def normalize(self, data: Any) -> NormalizedSERPData:
 
 class GoogleDataNormalizer(BaseDataNormalizer):
     """Data normalizer for Google SERP responses."""
+    
+    # Length of prefix to check for HTML detection
+    HTML_DETECTION_PREFIX_LENGTH = 200
 
     def normalize(self, data: Any) -> NormalizedSERPData:
         """Normalize Google SERP data."""
@@ -30,11 +34,31 @@ def normalize(self, data: Any) -> NormalizedSERPData:
 
         # Handle raw HTML response (body field)
         if "body" in data and isinstance(data.get("body"), str):
-            return {
-                "results": [],
-                "raw_html": data["body"],
-                "status_code": data.get("status_code"),
-            }
+            body = data["body"]
+            # Check if body is HTML with improved detection
+            body_lower = body.strip().lower()
+            is_html = (
+                body_lower.startswith(("<html", "<!doctype", "<!DOCTYPE")) or
+                "<html" in body_lower[:self.HTML_DETECTION_PREFIX_LENGTH]
+            )
+            
+            if is_html:
+                warnings.warn(
+                    "SERP API returned raw HTML instead of parsed JSON. "
+                    "This usually means:\n"
+                    "1. The zone doesn't support automatic parsing\n"
+                    "2. The brd_json=1 parameter didn't work as expected\n"
+                    "3. You may need to use a different zone type or endpoint\n\n"
+                    "The raw HTML is available in the 'raw_html' field of the response. "
+                    "Consider using an HTML parser (e.g., BeautifulSoup) to extract results.",
+                    UserWarning,
+                    stacklevel=3
+                )
+                return {
+                    "results": [],
+                    "raw_html": body,
+                    "status_code": data.get("status_code"),
+                }
 
         results = []
         organic = data.get("organic", [])

src/brightdata/api/web_unlocker.py

@@ -17,6 +17,7 @@
 )
 from ..utils.url import extract_root_domain
 from ..utils.function_detection import get_caller_function_name
+from ..constants import HTTP_OK
 from ..exceptions import ValidationError, APIError
 
 
@@ -130,11 +131,11 @@ async def _scrape_single_async(
             ) as response:
                 data_fetched_at = datetime.now(timezone.utc)
 
-                if response.status == 200:
+                if response.status == HTTP_OK:
                     if response_format == "json":
                         try:
                             data = await response.json()
-                        except Exception as e:
+                        except (ValueError, TypeError) as e:
                             raise APIError(f"Failed to parse JSON response: {str(e)}")
                     else:
                         data = await response.text()

src/brightdata/client.py

@@ -10,6 +10,7 @@
 
 import os
 import asyncio
+import warnings
 from typing import Optional, Dict, Any, Union, List
 from datetime import datetime, timezone
 
@@ -27,6 +28,11 @@
 from .api.crawler_service import CrawlerService
 from .models import ScrapeResult, SearchResult
 from .types import AccountInfo, URLParam, OptionalURLParam
+from .constants import (
+    HTTP_OK,
+    HTTP_UNAUTHORIZED,
+    HTTP_FORBIDDEN,
+)
 from .exceptions import (
     ValidationError,
     AuthenticationError,
@@ -62,9 +68,9 @@ class BrightDataClient:
 
     # Default configuration
     DEFAULT_TIMEOUT = 30
-    DEFAULT_WEB_UNLOCKER_ZONE = "sdk_unlocker"
-    DEFAULT_SERP_ZONE = "sdk_serp"
-    DEFAULT_BROWSER_ZONE = "sdk_browser"
+    DEFAULT_WEB_UNLOCKER_ZONE = "web_unlocker1"
+    DEFAULT_SERP_ZONE = "serp_api1"
+    DEFAULT_BROWSER_ZONE = "browser_api1"
 
     # Environment variable name for API token
     TOKEN_ENV_VAR = "BRIGHTDATA_API_TOKEN"
@@ -93,9 +99,9 @@ def __init__(
                   (supports .env files via python-dotenv)
             customer_id: Customer ID (optional, can also be set via BRIGHTDATA_CUSTOMER_ID)
             timeout: Default timeout in seconds for all requests (default: 30)
-            web_unlocker_zone: Zone name for web unlocker (default: "sdk_unlocker")
-            serp_zone: Zone name for SERP API (default: "sdk_serp")
-            browser_zone: Zone name for browser API (default: "sdk_browser")
+            web_unlocker_zone: Zone name for web unlocker (default: "web_unlocker1")
+            serp_zone: Zone name for SERP API (default: "serp_api1")
+            browser_zone: Zone name for browser API (default: "browser_api1")
             auto_create_zones: Automatically create zones if they don't exist (default: False)
             validate_token: Validate token by testing connection on init (default: False)
             rate_limit: Maximum requests per rate_period (default: 10). Set to None to disable.
@@ -324,14 +330,14 @@ async def test_connection(self) -> bool:
                 async with self.engine.get_from_url(
                     f"{self.engine.BASE_URL}/zone/get_active_zones"
                 ) as response:
-                    if response.status == 200:
+                    if response.status == HTTP_OK:
                         self._is_connected = True
                         return True
                     else:
                         self._is_connected = False
                         return False
 
-        except Exception:
+        except (asyncio.TimeoutError, OSError, Exception):
             self._is_connected = False
             return False
 
@@ -366,21 +372,34 @@ async def get_account_info(self) -> AccountInfo:
                 async with self.engine.get_from_url(
                     f"{self.engine.BASE_URL}/zone/get_active_zones"
                 ) as zones_response:
-                    if zones_response.status == 200:
+                    if zones_response.status == HTTP_OK:
                         zones = await zones_response.json()
+                        zones = zones or []
+                        
+                        # Warn user if no active zones found (they might be inactive)
+                        if not zones:
+                            warnings.warn(
+                                "No active zones found. This could mean:\n"
+                                "1. Your zones might be inactive - activate them in the Bright Data dashboard\n"
+                                "2. You might need to create zones first\n"
+                                "3. Check your dashboard at https://brightdata.com for zone status\n\n"
+                                "Note: The API only returns active zones. Inactive zones won't appear here.",
+                                UserWarning,
+                                stacklevel=2
+                            )
 
                         account_info = {
                             "customer_id": self.customer_id,
-                            "zones": zones or [],
-                            "zone_count": len(zones or []),
+                            "zones": zones,
+                            "zone_count": len(zones),
                             "token_valid": True,
                             "retrieved_at": datetime.now(timezone.utc).isoformat(),
                         }
 
                         self._account_info = account_info
                         return account_info
 
-                    elif zones_response.status in (401, 403):
+                    elif zones_response.status in (HTTP_UNAUTHORIZED, HTTP_FORBIDDEN):
                         error_text = await zones_response.text()
                         raise AuthenticationError(
                             f"Invalid token (HTTP {zones_response.status}): {error_text}"

src/brightdata/constants.py

@@ -23,3 +23,38 @@
 
 DEFAULT_COST_PER_RECORD: float = 0.001
 """Default cost per record for base scrapers."""
+
+# Platform-specific costs (when different from default)
+COST_PER_RECORD_LINKEDIN: float = 0.002
+"""Cost per record for LinkedIn scrapers."""
+
+COST_PER_RECORD_FACEBOOK: float = 0.002
+"""Cost per record for Facebook scrapers."""
+
+COST_PER_RECORD_INSTAGRAM: float = 0.002
+"""Cost per record for Instagram scrapers."""
+
+COST_PER_RECORD_CHATGPT: float = 0.005
+"""Cost per record for ChatGPT scrapers (higher due to AI processing)."""
+
+# HTTP Status Codes
+HTTP_OK: int = 200
+"""HTTP 200 OK - Request succeeded."""
+
+HTTP_CREATED: int = 201
+"""HTTP 201 Created - Resource created successfully."""
+
+HTTP_BAD_REQUEST: int = 400
+"""HTTP 400 Bad Request - Invalid request parameters."""
+
+HTTP_UNAUTHORIZED: int = 401
+"""HTTP 401 Unauthorized - Authentication required or failed."""
+
+HTTP_FORBIDDEN: int = 403
+"""HTTP 403 Forbidden - Access denied."""
+
+HTTP_CONFLICT: int = 409
+"""HTTP 409 Conflict - Resource conflict (e.g., duplicate)."""
+
+HTTP_INTERNAL_SERVER_ERROR: int = 500
+"""HTTP 500 Internal Server Error - Server error."""

src/brightdata/core/engine.py

@@ -6,6 +6,7 @@
 from typing import Optional, Dict, Any
 from datetime import datetime, timezone
 from ..exceptions import APIError, AuthenticationError, NetworkError, TimeoutError, SSLError
+from ..constants import HTTP_UNAUTHORIZED, HTTP_FORBIDDEN
 from ..utils.ssl_helpers import is_ssl_certificate_error, get_ssl_error_message
 
 # Rate limiting support
@@ -304,14 +305,14 @@ async def __aenter__(self):
                         timeout=self._timeout,
                     )
                     # Check status codes that should raise exceptions
-                    if self._response.status == 401:
+                    if self._response.status == HTTP_UNAUTHORIZED:
                         text = await self._response.text()
                         await self._response.release()
-                        raise AuthenticationError(f"Unauthorized (401): {text}")
-                    elif self._response.status == 403:
+                        raise AuthenticationError(f"Unauthorized ({HTTP_UNAUTHORIZED}): {text}")
+                    elif self._response.status == HTTP_FORBIDDEN:
                         text = await self._response.text()
                         await self._response.release()
-                        raise AuthenticationError(f"Forbidden (403): {text}")
+                        raise AuthenticationError(f"Forbidden ({HTTP_FORBIDDEN}): {text}")
 
                     return self._response
                 except (aiohttp.ClientError, ssl.SSLError, OSError) as e: