⚡ Add option to control no of comments to be parsed while scraping a linkedin post url

Author: kgritesh

src/linkedin_spider/core/scraper.py

@@ -168,7 +168,7 @@ def search_posts(
             keywords, max_results, scroll_pause, max_comments, date_posted
         )
 
-    def open_link(self, url: str) -> dict[str, Any] | None:
+    def open_link(self, url: str, max_comments: int = 10) -> dict[str, Any] | None:
         """
         Open a LinkedIn post URL and extract its content.
 
@@ -178,7 +178,7 @@ def open_link(self, url: str) -> dict[str, Any] | None:
         Returns:
             Dictionary containing post data (same structure as search_posts), or None if failed
         """
-        return self.search_scraper.open_link(url)
+        return self.search_scraper.open_link(url, max_comments=max_comments)
 
     def scrape_company(self, company_url: str) -> dict[str, Any] | None:
         """Scrape a LinkedIn company page."""

src/linkedin_spider/scrapers/search.py

@@ -1227,7 +1227,7 @@ def _load_more_comments(self, container: WebElement, max_comments: int) -> None:
         except Exception as e:
             self.log_action("DEBUG", f"Error loading more comments: {e!s}")
 
-    def open_link(self, url: str) -> dict[str, Any] | None:
+    def open_link(self, url: str, max_comments: int = 10) -> dict[str, Any] | None:
         """
         Open a LinkedIn post URL and extract its content.
 
@@ -1255,7 +1255,7 @@ def open_link(self, url: str) -> dict[str, Any] | None:
                 return None
 
             # Extract data from the first post
-            post_data = self._extract_post_data(post_containers[0])
+            post_data = self._extract_post_data(post_containers[0], max_comments=max_comments)
 
             # Set post_url if not already set
             if post_data.get("post_url") == "N/A":