fix cookie auth

Author: amankumarsingh77

src/linkedin_spider/cli/main.py

@@ -31,6 +31,9 @@ def search(
         Parameter(name=["-o", "--output"], help="Output file path (.json or .csv format)"),
     ] = None,
     headless: bool | None = None,
+    user_agent: Annotated[
+        str | None, Parameter(help="Custom user agent string for requests")
+    ] = None,
     email: Annotated[str | None, Parameter(help="LinkedIn email for authentication")] = None,
     password: Annotated[str | None, Parameter(help="LinkedIn password for authentication")] = None,
     cookie: Annotated[
@@ -41,12 +44,14 @@ def search(
     try:
         config = _create_config(headless)
         credentials = _get_credentials(email, password, cookie)
+        custom_user_agent = _get_user_agent(user_agent)
 
         scraper = LinkedinSpider(
             email=credentials.get("email"),
             password=credentials.get("password"),
             li_at_cookie=credentials.get("cookie"),
             config=config,
+            user_agent=custom_user_agent,
         )
 
         results = scraper.search_profiles(query, max_results)
@@ -73,6 +78,9 @@ def profile(
         Parameter(name=["-o", "--output"], help="Output file path (.json or .csv format)"),
     ] = None,
     headless: bool | None = None,
+    user_agent: Annotated[
+        str | None, Parameter(help="Custom user agent string for requests")
+    ] = None,
     email: Annotated[str | None, Parameter(help="LinkedIn email for authentication")] = None,
     password: Annotated[str | None, Parameter(help="LinkedIn password for authentication")] = None,
     cookie: Annotated[
@@ -83,12 +91,14 @@ def profile(
     try:
         config = _create_config(headless)
         credentials = _get_credentials(email, password, cookie)
+        custom_user_agent = _get_user_agent(user_agent)
 
         scraper = LinkedinSpider(
             email=credentials.get("email"),
             password=credentials.get("password"),
             li_at_cookie=credentials.get("cookie"),
             config=config,
+            user_agent=custom_user_agent,
         )
 
         result = scraper.scrape_profile(url)
@@ -119,6 +129,9 @@ def company(
         Parameter(name=["-o", "--output"], help="Output file path (.json or .csv format)"),
     ] = None,
     headless: bool | None = None,
+    user_agent: Annotated[
+        str | None, Parameter(help="Custom user agent string for requests")
+    ] = None,
     email: Annotated[str | None, Parameter(help="LinkedIn email for authentication")] = None,
     password: Annotated[str | None, Parameter(help="LinkedIn password for authentication")] = None,
     cookie: Annotated[
@@ -129,12 +142,14 @@ def company(
     try:
         config = _create_config(headless)
         credentials = _get_credentials(email, password, cookie)
+        custom_user_agent = _get_user_agent(user_agent)
 
         scraper = LinkedinSpider(
             email=credentials.get("email"),
             password=credentials.get("password"),
             li_at_cookie=credentials.get("cookie"),
             config=config,
+            user_agent=custom_user_agent,
         )
 
         result = scraper.scrape_company(url)
@@ -167,6 +182,9 @@ def connections(
         Parameter(name=["-o", "--output"], help="Output file path (.json or .csv format)"),
     ] = None,
     headless: bool | None = None,
+    user_agent: Annotated[
+        str | None, Parameter(help="Custom user agent string for requests")
+    ] = None,
     email: Annotated[str | None, Parameter(help="LinkedIn email for authentication")] = None,
     password: Annotated[str | None, Parameter(help="LinkedIn password for authentication")] = None,
     cookie: Annotated[
@@ -177,12 +195,14 @@ def connections(
     try:
         config = _create_config(headless)
         credentials = _get_credentials(email, password, cookie)
+        custom_user_agent = _get_user_agent(user_agent)
 
         scraper = LinkedinSpider(
             email=credentials.get("email"),
             password=credentials.get("password"),
             li_at_cookie=credentials.get("cookie"),
             config=config,
+            user_agent=custom_user_agent,
         )
 
         results = scraper.scrape_incoming_connections(max_results)
@@ -231,6 +251,9 @@ def search_posts(
         Parameter(name=["-o", "--output"], help="Output file path (.json or .csv format)"),
     ] = None,
     headless: bool | None = None,
+    user_agent: Annotated[
+        str | None, Parameter(help="Custom user agent string for requests")
+    ] = None,
     email: Annotated[str | None, Parameter(help="LinkedIn email for authentication")] = None,
     password: Annotated[str | None, Parameter(help="LinkedIn password for authentication")] = None,
     cookie: Annotated[
@@ -241,12 +264,14 @@ def search_posts(
     try:
         config = _create_config(headless)
         credentials = _get_credentials(email, password, cookie)
+        custom_user_agent = _get_user_agent(user_agent)
 
         scraper = LinkedinSpider(
             email=credentials.get("email"),
             password=credentials.get("password"),
             li_at_cookie=credentials.get("cookie"),
             config=config,
+            user_agent=custom_user_agent,
         )
 
         print(f"Searching for posts with keywords: '{keywords}'")
@@ -285,20 +310,25 @@ def _create_config(headless: bool | None) -> ScraperConfig:
     return ScraperConfig(headless=headless)
 
 
+def _get_user_agent(user_agent: str | None) -> str | None:
+    """Get user agent from argument or environment variable."""
+    return user_agent or os.getenv("USER_AGENT")
+
+
 def _get_credentials(email: str | None, password: str | None, cookie: str | None) -> dict:
     """Get authentication credentials from arguments or environment."""
     credentials = {
         "email": email or os.getenv("LINKEDIN_EMAIL"),
         "password": password or os.getenv("LINKEDIN_PASSWORD"),
-        "cookie": cookie or os.getenv("LINKEDIN_COOKIE"),
+        "cookie": cookie or os.getenv("cookie"),
     }
 
     if not any(credentials.values()):
         raise ValueError(
             "Authentication required. Provide either:\n"
             "1. Email and password (--email, --password)\n"
             "2. LinkedIn cookie (--cookie)\n"
-            "3. Set environment variables: LINKEDIN_EMAIL, LINKEDIN_PASSWORD, or LINKEDIN_COOKIE"
+            "3. Set environment variables: LINKEDIN_EMAIL, LINKEDIN_PASSWORD, or cookie"
         )
 
     return credentials

src/linkedin_spider/core/auth.py

@@ -1,3 +1,4 @@
+import logging
 import os
 
 from selenium.common.exceptions import NoSuchElementException, TimeoutException
@@ -7,6 +8,8 @@
 from linkedin_spider.core.driver import DriverManager
 from linkedin_spider.utils.human_behavior import HumanBehavior
 
+logger = logging.getLogger(__name__)
+
 
 class AuthManager:
     """Handles LinkedIn authentication via cookies or credentials."""
@@ -28,36 +31,59 @@ def __init__(
         self.li_at_cookie = li_at_cookie
 
     def authenticate(self) -> bool:
-        """Authenticate using available methods in priority order."""
-        if self._is_authenticated():
-            return True
+        """
+        Authenticate using available methods with smart fallback priority.
 
-        login_with_cred = os.getenv("LOGIN_WITH_CRED", "true").lower() == "true"
+        Priority order:
+        1. Try li_at_cookie from parameter (if provided)
+        2. Check if already authenticated
+        3. Try credentials from parameters (if provided)
+        4. Raise error if no authentication method available
 
-        if login_with_cred:
-            if self.email and self.password and self._login_with_credentials():
-                self.driver_manager.save_cookies()
-                return True
+        Returns:
+            bool: True if authentication succeeded
 
-            if self.li_at_cookie and self._authenticate_with_cookie():
-                self.driver_manager.save_cookies()
-                return True
+        Raises:
+            Exception: If all authentication methods fail or no credentials provided
+        """
 
-            if self._try_saved_cookies():
-                return True
-        else:
-            if self._try_saved_cookies():
-                return True
+        # Priority 1: If cookie parameter is explicitly provided, use it
 
-            if self.li_at_cookie and self._authenticate_with_cookie():
+        if self.li_at_cookie:
+            logger.info("Using LinkedIn cookie from parameter")
+            if self._authenticate_with_cookie(self.li_at_cookie):
                 self.driver_manager.save_cookies()
                 return True
+            else:
+                logger.error("Cookie from parameter failed")
+                raise Exception("Provided li_at cookie is invalid or expired")
+
+        # Priority 2: Check if already authenticated with saved cookies
+        logger.info("Checking for saved cookies in profile directory...")
+        if self._try_saved_cookies():
+            logger.info("Successfully authenticated using saved cookies")
+            return True
+        else:
+            logger.info("No valid saved cookies found or authentication failed")
 
-            if self.email and self.password and self._login_with_credentials():
+        # Priority 3: Try credentials from parameters if provided
+        if self.email and self.password:
+            logger.info("Using LinkedIn credentials from parameter")
+            if self._login_with_credentials(self.email, self.password):
                 self.driver_manager.save_cookies()
                 return True
-
-        raise Exception("All authentication methods failed")
+            else:
+                logger.error("Login with provided credentials failed")
+                raise Exception("Login failed with provided credentials")
+
+        # No authentication method available
+        logger.error("No valid authentication method found")
+        raise Exception(
+            "Authentication required. Saved cookies not found or invalid. Please provide either:\n"
+            "  - li_at cookie (--cookie parameter)\n"
+            "  - Email and password (--email and --password parameters)\n"
+            "  - Set environment variables: LINKEDIN_EMAIL, LINKEDIN_PASSWORD, or cookie"
+        )
 
     def _is_authenticated(self) -> bool:
         """Check if already authenticated by examining current page."""
@@ -113,44 +139,53 @@ def _try_saved_cookies(self) -> bool:
 
         return self._is_authenticated()
 
-    def _authenticate_with_cookie(self) -> bool:
-        """Authenticate using li_at cookie."""
-        try:
-            self.driver.get("https://www.linkedin.com")
-            self.human_behavior.delay()
+    def _authenticate_with_cookie(self, cookie: str) -> bool:
+        """
+        Authenticate using li_at cookie with improved validation.
 
-            self.driver.add_cookie(
-                {
-                    "name": "li_at",
-                    "value": self.li_at_cookie,
-                    "domain": ".linkedin.com",
-                    "path": "/",
-                    "secure": True,
-                }
-            )
+        Args:
+            cookie: LinkedIn session cookie value
 
-            self.driver.refresh()
-            self.human_behavior.delay(2, 3)
+        Returns:
+            bool: True if authentication succeeded
+        """
+        try:
+            # Use the improved login_with_cookie method from driver_manager
+            success = self.driver_manager.login_with_cookie(cookie)
 
-            self._handle_welcome_page()
-            return self._is_authenticated()
+            if success:
+                self.human_behavior.delay(1, 2)
+                self._handle_welcome_page()
+                return self._is_authenticated()
 
-        except Exception:
             return False
 
-    def _login_with_credentials(self) -> bool:
-        """Login using email and password."""
+        except Exception as e:
+            logger.error(f"Cookie authentication exception: {e}")
+            return False
+
+    def _login_with_credentials(self, email: str, password: str) -> bool:
+        """
+        Login using email and password.
+
+        Args:
+            email: LinkedIn email
+            password: LinkedIn password
+
+        Returns:
+            bool: True if login succeeded
+        """
         try:
             self.driver.get("https://www.linkedin.com/login")
             self.human_behavior.delay(2, 3)
 
             email_field = self.wait.until(EC.presence_of_element_located((By.ID, "username")))
             self.human_behavior.click(email_field)
-            self.human_behavior.type_text(email_field, self.email)
+            self.human_behavior.type_text(email_field, email)
 
             password_field = self.driver.find_element(By.ID, "password")
             self.human_behavior.click(password_field)
-            self.human_behavior.type_text(password_field, self.password)
+            self.human_behavior.type_text(password_field, password)
 
             self.human_behavior.delay()
 
@@ -160,14 +195,16 @@ def _login_with_credentials(self) -> bool:
             self.human_behavior.delay(3, 5)
 
             if self._check_login_errors():
+                logger.error("Login errors detected")
                 return False
 
             if self._is_challenge_present():
                 return self._handle_challenge()
 
             return self._is_authenticated()
 
-        except Exception:
+        except Exception as e:
+            logger.error(f"Credential login exception: {e}")
             return False
 
     def _is_challenge_present(self) -> bool: