feat: add LinkedIn scraping and search capabilities, restructure ChatGPT API, and improve download functionality

Major Features:
- Add comprehensive LinkedIn data scraping (profiles, companies, jobs, posts)
- Add LinkedIn search capabilities with keyword and URL-based discovery
- Restructure ChatGPT API with sync/async support and rename to search_chatGPT
- Refactor download functions into separate module for better organization
- Add new example files for LinkedIn scraping and searching

API Enhancements:
- New scrape_linkedin class with specialized methods for different data types
- New search_linkedin class for discovering LinkedIn content
- Enhanced ChatGPT API with synchronous and asynchronous processing options
- Improved NDJSON response parsing for better data handling
- Better parameter validation and error handling across all APIs

Code Organization:
- Move download functionality to dedicated api/download.py module
- Create specialized API modules for LinkedIn and ChatGPT operations
- Maintain backward compatibility for existing download methods
- Clean up code for production readiness with optimized performance

Version bump to 1.0.7

Author: עידן וילנסקי

brightdata/__init__.py

@@ -3,13 +3,32 @@
 
 A comprehensive SDK for Bright Data's Web Scraping and SERP APIs, providing
 easy-to-use methods for web scraping, search engine result parsing, and data management.
-### Functions:
+## Functions:
+First import the package and create a client:
+```python
+from brightdata import bdclient
+client = bdclient(your-apy-key)
+```
+Then use the client to call the desired functions:  
 #### scrape()
 - Scrapes a website using Bright Data Web Unblocker API with proxy support (or multiple websites sequentially)
+- syntax: `results = client.scrape(url, country, max_workers, ...)`
+#### .scrape_linkedin. class
+- Scrapes LinkedIn data including posts, jobs, companies, and profiles, recieve structured data as a result
+- syntax: `results = client.scrape_linkedin.posts()/jobs()/companies()/profiles() # insert parameters per function`
 #### search()
 - Performs web searches using Bright Data SERP API with customizable search engines (or multiple search queries sequentially)
-#### download_content()
+- syntax: `results = client.search(query, search_engine, country, ...)`
+#### .search_linkedin. class
+- Search LinkedIn data including for specific posts, jobs, profiles. recieve the relevent data as a result
+- syntax: `results = client.search_linkedin.posts()/jobs()/profiles() # insert parameters per function`
+#### search_chatGPT()
+- Interact with ChatGPT using Bright Data's ChatGPT API, sending prompts and receiving responses
+- syntax: `results = client.search_chatGPT(prompt, additional_prompt, max_workers, ...)`
+#### download_content() / download_snapshot()
 - Saves the scraped content to local files in various formats (JSON, CSV, etc.)
+- syntax: `client.download_content(results)`
+- syntax: `client.download_snapshot(results)`
 
 ### Features:
 - Web Scraping: Scrape websites using Bright Data Web Unlocker API with proxy support
@@ -32,7 +51,7 @@
     APIError
 )
 
-__version__ = "1.0.6"
+__version__ = "1.0.7"
 __author__ = "Bright Data"
 __email__ = "[email protected]"
 

brightdata/api/__init__.py

@@ -1,9 +1,11 @@
 from .scraper import WebScraper
 from .search import SearchAPI
 from .chatgpt import ChatGPTAPI
+from .linkedin import LinkedInAPI
 
 __all__ = [
     'WebScraper',
     'SearchAPI',
-    'ChatGPTAPI'
+    'ChatGPTAPI',
+    'LinkedInAPI'
 ]

brightdata/api/chatgpt.py

@@ -24,6 +24,7 @@ def scrape_chatgpt(
         countries: List[str],
         additional_prompts: List[str],
         web_searches: List[bool],
+        sync: bool = True,
         timeout: int = None
     ) -> Dict[str, Any]:
         """
@@ -34,12 +35,13 @@ def scrape_chatgpt(
         - countries: List of country codes matching prompts
         - additional_prompts: List of follow-up prompts matching prompts
         - web_searches: List of web_search flags matching prompts
+        - sync: If True, uses synchronous API for immediate results
         - timeout: Request timeout in seconds
         
         Returns:
-        - Dict containing response with snapshot_id
+        - Dict containing response with snapshot_id or direct data (if sync=True)
         """
-        url = "https://api.brightdata.com/datasets/v3/trigger"
+        url = "https://api.brightdata.com/datasets/v3/scrape" if sync else "https://api.brightdata.com/datasets/v3/trigger"
         headers = {
             "Authorization": f"Bearer {self.api_token}",
             "Content-Type": "application/json"
@@ -49,38 +51,59 @@ def scrape_chatgpt(
             "include_errors": "true"
         }
 
-        data = []
-        for i in range(len(prompts)):
-            data.append({
+        data = [
+            {
                 "url": "https://chatgpt.com/",
                 "prompt": prompts[i],
                 "country": countries[i],
                 "additional_prompt": additional_prompts[i],
                 "web_search": web_searches[i]
-            })
+            }
+            for i in range(len(prompts))
+        ]
 
         try:
             response = self.session.post(
                 url, 
                 headers=headers, 
                 params=params, 
                 json=data, 
-                timeout=timeout or self.default_timeout
+                timeout=timeout or (65 if sync else self.default_timeout)
             )
 
             if response.status_code == 401:
                 raise AuthenticationError("Invalid API token or insufficient permissions")
             elif response.status_code != 200:
                 raise APIError(f"ChatGPT scraping request failed with status {response.status_code}: {response.text}")
 
-            result = response.json()
-            snapshot_id = result.get('snapshot_id')
-            if snapshot_id:
-                logger.info(f"ChatGPT scraping job initiated successfully for {len(prompts)} prompt(s)")
-                print("")
-                print("Snapshot ID:")
-                print(snapshot_id)
-                print("")
+            if sync:
+                response_text = response.text
+                if '\n{' in response_text and response_text.strip().startswith('{'):
+                    json_objects = []
+                    for line in response_text.strip().split('\n'):
+                        if line.strip():
+                            try:
+                                json_objects.append(json.loads(line))
+                            except json.JSONDecodeError:
+                                continue
+                    result = json_objects
+                else:
+                    try:
+                        result = response.json()
+                    except json.JSONDecodeError:
+                        result = response_text
+                
+                logger.info(f"ChatGPT data retrieved synchronously for {len(prompts)} prompt(s)")
+                print(f"Retrieved {len(result) if isinstance(result, list) else 1} ChatGPT response(s)")
+            else:
+                result = response.json()
+                snapshot_id = result.get('snapshot_id')
+                if snapshot_id:
+                    logger.info(f"ChatGPT scraping job initiated successfully for {len(prompts)} prompt(s)")
+                    print("")
+                    print("Snapshot ID:")
+                    print(snapshot_id)
+                    print("")
 
             return result