Merge pull request #287 from autoscrape-labs/refactor/add-logging

Add logging through all codebase

Author: thalissonvs

pydoll/browser/chromium/base.py

@@ -1,3 +1,4 @@
+import logging
 import platform
 from typing import Optional
 
@@ -7,6 +8,8 @@
 from pydoll.exceptions import UnsupportedOS
 from pydoll.utils import validate_browser_paths
 
+logger = logging.getLogger(__name__)
+
 
 class Chrome(Browser):
     """Chrome browser implementation for CDP automation."""
@@ -39,6 +42,7 @@ def _get_default_binary_location():
             ValueError: If executable not found at default location.
         """
         os_name = platform.system()
+        logger.debug(f'Resolving default Chrome binary for OS: {os_name}')
 
         browser_paths = {
             'Windows': [
@@ -57,6 +61,9 @@ def _get_default_binary_location():
         browser_path = browser_paths.get(os_name)
 
         if not browser_path:
+            logger.error(f'Unsupported OS: {os_name}')
             raise UnsupportedOS(f'Unsupported OS: {os_name}')
 
-        return validate_browser_paths(browser_path)
+        path = validate_browser_paths(browser_path)
+        logger.debug(f'Using Chrome binary: {path}')
+        return path

pydoll/browser/chromium/chrome.py

@@ -1,3 +1,4 @@
+import logging
 import platform
 from typing import Optional
 
@@ -7,6 +8,8 @@
 from pydoll.exceptions import UnsupportedOS
 from pydoll.utils import validate_browser_paths
 
+logger = logging.getLogger(__name__)
+
 
 class Edge(Browser):
     """Edge browser implementation for CDP automation."""
@@ -39,6 +42,7 @@ def _get_default_binary_location():
             ValueError: If executable not found at default location.
         """
         os_name = platform.system()
+        logger.debug(f'Resolving default Edge binary for OS: {os_name}')
 
         browser_paths = {
             'Windows': [
@@ -62,6 +66,9 @@ def _get_default_binary_location():
         browser_path = browser_paths.get(os_name)
 
         if not browser_path:
+            logger.error(f'Unsupported OS: {os_name}')
             raise UnsupportedOS()
 
-        return validate_browser_paths(browser_path)
+        path = validate_browser_paths(browser_path)
+        logger.debug(f'Using Edge binary: {path}')
+        return path

pydoll/browser/chromium/edge.py

@@ -1,9 +1,12 @@
+import logging
 from typing import Optional
 
 from pydoll.browser.interfaces import BrowserOptionsManager, Options
 from pydoll.browser.options import ChromiumOptions
 from pydoll.exceptions import InvalidOptionsObject
 
+logger = logging.getLogger(__name__)
+
 
 class ChromiumOptionsManager(BrowserOptionsManager):
     """
@@ -15,6 +18,10 @@ class ChromiumOptionsManager(BrowserOptionsManager):
 
     def __init__(self, options: Optional[Options] = None):
         self.options = options
+        logger.debug(
+            f'ChromiumOptionsManager initialized with options='
+            f'{type(options).__name__ if options is not None else "None"}'
+        )
 
     def initialize_options(
         self,
@@ -33,14 +40,18 @@ def initialize_options(
         """
         if self.options is None:
             self.options = ChromiumOptions()
+            logger.debug('No options provided; created default ChromiumOptions')
 
         if not isinstance(self.options, ChromiumOptions):
+            logger.error(f'Invalid options type: {type(self.options)}; expected ChromiumOptions')
             raise InvalidOptionsObject(f'Expected ChromiumOptions, got {type(self.options)}')
 
         self.add_default_arguments()
+        logger.debug('Options initialized and default arguments applied')
         return self.options
 
     def add_default_arguments(self):
         """Add default arguments required for CDP integration."""
+        logger.debug('Adding default arguments for Chromium-based browsers')
         self.options.add_argument('--no-first-run')
         self.options.add_argument('--no-default-browser-check')

pydoll/browser/managers/browser_options_manager.py

@@ -1,6 +1,9 @@
+import logging
 import subprocess
 from typing import Callable, Optional
 
+logger = logging.getLogger(__name__)
+
 
 class BrowserProcessManager:
     """
@@ -24,6 +27,9 @@ def __init__(
         """
         self._process_creator = process_creator or self._default_process_creator
         self._process: Optional[subprocess.Popen] = None
+        logger.debug(
+            f'BrowserProcessManager initialized; custom process_creator={bool(process_creator)}'
+        )
 
     def start_browser_process(
         self,
@@ -45,16 +51,23 @@ def start_browser_process(
         Note:
             Automatically adds --remote-debugging-port argument.
         """
-        self._process = self._process_creator([
+        logger.info(f'Starting browser process: {binary_location} on port {port}')
+        command = [
             binary_location,
             f'--remote-debugging-port={port}',
             *arguments,
-        ])
+        ]
+        logger.debug(f'Command: {command}')
+        self._process = self._process_creator(command)
+        logger.debug(
+            f'Browser process started: pid={self._process.pid if self._process else "unknown"}'
+        )
         return self._process
 
     @staticmethod
     def _default_process_creator(command: list[str]) -> subprocess.Popen:
         """Create browser process with output capture to prevent console clutter."""
+        logger.debug(f'Creating process: {command}')
         return subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     def stop_process(self):
@@ -65,8 +78,12 @@ def stop_process(self):
         Safe to call even if no process is running.
         """
         if self._process:
+            logger.info(f'Stopping browser process pid={self._process.pid}')
             self._process.terminate()
             try:
                 self._process.wait(timeout=15)
+                logger.debug('Process terminated gracefully')
             except subprocess.TimeoutExpired:
+                logger.warning('Process did not terminate in 15s; sending SIGKILL')
                 self._process.kill()
+                logger.debug('Process killed')

pydoll/browser/managers/browser_process_manager.py

@@ -1,7 +1,10 @@
+import logging
 from typing import Optional
 
 from pydoll.browser.options import Options
 
+logger = logging.getLogger(__name__)
+
 
 class ProxyManager:
     """
@@ -20,6 +23,7 @@ def __init__(self, options: Options):
                 Will be modified if credentials are found.
         """
         self.options = options
+        logger.debug('ProxyManager initialized with options')
 
     def get_proxy_credentials(self) -> tuple[bool, tuple[Optional[str], Optional[str]]]:
         """
@@ -44,6 +48,11 @@ def get_proxy_credentials(self) -> tuple[bool, tuple[Optional[str], Optional[str
                 self._update_proxy_argument(index, clean_proxy)
                 private_proxy = True
                 credentials = (username, password)
+                logger.debug(
+                    f'Proxy credentials extracted (user_set={bool(username)}); argument sanitized'
+                )
+            else:
+                logger.debug('Proxy configured without embedded credentials')
 
         return private_proxy, credentials
 
@@ -56,7 +65,9 @@ def _find_proxy_argument(self) -> Optional[tuple[int, str]]:
         """
         for index, arg in enumerate(self.options.arguments):
             if arg.startswith('--proxy-server='):
-                return index, arg.split('=', 1)[1]
+                value = arg.split('=', 1)[1]
+                logger.debug(f'Found proxy argument at index {index}: {value}')
+                return index, value
         return None
 
     @staticmethod
@@ -83,11 +94,12 @@ def _parse_proxy(proxy_value: str) -> tuple[bool, Optional[str], Optional[str],
             creds_part, server_part = proxy_value.split('@', 1)
             username, password = creds_part.split(':', 1)
 
-            clean_proxy = f"{scheme}://{server_part}" if has_scheme else server_part
+            clean_proxy = f'{scheme}://{server_part}' if has_scheme else server_part
             return True, username, password, clean_proxy
         except ValueError:
             return False, None, None, proxy_value
 
     def _update_proxy_argument(self, index: int, clean_proxy: str) -> None:
         """Replace proxy argument with credential-free version."""
         self.options.arguments[index] = f'--proxy-server={clean_proxy}'
+        logger.debug(f'Proxy argument updated at index {index}: {clean_proxy}')

pydoll/browser/managers/proxy_manager.py

@@ -1,9 +1,12 @@
+import logging
 import shutil
 import time
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Callable
 
+logger = logging.getLogger(__name__)
+
 
 class TempDirectoryManager:
     """
@@ -23,6 +26,7 @@ def __init__(self, temp_dir_factory: Callable[[], TemporaryDirectory] = Temporar
         """
         self._temp_dir_factory = temp_dir_factory
         self._temp_dirs: list[TemporaryDirectory] = []
+        logger.debug('TempDirectoryManager initialized')
 
     def create_temp_dir(self) -> TemporaryDirectory:
         """
@@ -33,6 +37,7 @@ def create_temp_dir(self) -> TemporaryDirectory:
         """
         temp_dir = self._temp_dir_factory()
         self._temp_dirs.append(temp_dir)
+        logger.debug(f'Created temp directory: {temp_dir.name}')
         return temp_dir
 
     @staticmethod
@@ -56,6 +61,9 @@ def retry_process_file(func: Callable[[str], None], path: str, retry_times: int
                 break
             except PermissionError:
                 time.sleep(0.1)
+                logger.debug(
+                    f'Retrying file operation due to PermissionError (attempt {retry_time})'
+                )
         else:
             raise PermissionError()
 
@@ -80,6 +88,7 @@ def handle_cleanup_error(self, func: Callable[[str], None], path: str, exc_info:
                     self.retry_process_file(func, path)
                     return
                 except PermissionError:
+                    logger.warning(f'Failed retrying cleanup for locked file: {path}')
                     raise exc_value
         elif exc_type is OSError:
             return
@@ -93,4 +102,5 @@ def cleanup(self):
         Continues cleanup even if some files resist deletion.
         """
         for temp_dir in self._temp_dirs:
+            logger.info(f'Cleaning up temp directory: {temp_dir.name}')
             shutil.rmtree(temp_dir.name, onerror=self.handle_cleanup_error)