Skip to content

Commit b048c2e

Browse files
committed
Fixed sync calls
1 parent cfaba9b commit b048c2e

File tree

10 files changed

+1222
-50
lines changed

10 files changed

+1222
-50
lines changed

README.md

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -607,8 +607,7 @@ brightdata --help
607607

608608
# Scrape Amazon product (URL is positional argument)
609609
brightdata scrape amazon products \
610-
"https://amazon.com/dp/B0CRMZHDG8" \
611-
--output-format json
610+
"https://amazon.com/dp/B0CRMZHDG8"
612611

613612
# Search LinkedIn jobs
614613
brightdata search linkedin jobs \
@@ -625,7 +624,7 @@ brightdata search google \
625624
# Generic web scraping (URL is positional argument)
626625
brightdata scrape generic \
627626
"https://example.com" \
628-
--output-format pretty
627+
--response-format pretty
629628
```
630629

631630
### Available Commands

pyproject.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,4 +61,7 @@ python_files = ["test_*.py"]
6161
python_classes = ["Test*"]
6262
python_functions = ["test_*"]
6363
asyncio_mode = "auto"
64+
markers = [
65+
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
66+
]
6467

src/brightdata/scrapers/amazon/scraper.py

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,10 @@ def products(
107107
... timeout=240
108108
... )
109109
"""
110-
return asyncio.run(self.products_async(url, timeout=timeout))
110+
async def _run():
111+
async with self.engine:
112+
return await self.products_async(url, timeout=timeout)
113+
return asyncio.run(_run())
111114

112115
# ============================================================================
113116
# PRODUCTS TRIGGER/STATUS/FETCH (Manual Control)
@@ -280,7 +283,10 @@ def reviews(
280283
... timeout=240
281284
... )
282285
"""
283-
return asyncio.run(self.reviews_async(url, pastDays, keyWord, numOfReviews, timeout))
286+
async def _run():
287+
async with self.engine:
288+
return await self.reviews_async(url, pastDays, keyWord, numOfReviews, timeout)
289+
return asyncio.run(_run())
284290

285291
# ============================================================================
286292
# REVIEWS TRIGGER/STATUS/FETCH (Manual Control)
@@ -394,7 +400,10 @@ def sellers(
394400
395401
See sellers_async() for documentation.
396402
"""
397-
return asyncio.run(self.sellers_async(url, timeout))
403+
async def _run():
404+
async with self.engine:
405+
return await self.sellers_async(url, timeout)
406+
return asyncio.run(_run())
398407

399408
# ============================================================================
400409
# SELLERS TRIGGER/STATUS/FETCH (Manual Control)

src/brightdata/scrapers/chatgpt/scraper.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,10 @@ def prompt(
121121
Example:
122122
>>> result = scraper.prompt("Explain Python asyncio")
123123
"""
124-
return asyncio.run(self.prompt_async(prompt, **kwargs))
124+
async def _run():
125+
async with self.engine:
126+
return await self.prompt_async(prompt, **kwargs)
127+
return asyncio.run(_run())
125128

126129
# ============================================================================
127130
# PROMPT TRIGGER/STATUS/FETCH (Manual Control)
@@ -267,7 +270,10 @@ def prompts(
267270
268271
See prompts_async() for full documentation.
269272
"""
270-
return asyncio.run(self.prompts_async(prompts, **kwargs))
273+
async def _run():
274+
async with self.engine:
275+
return await self.prompts_async(prompts, **kwargs)
276+
return asyncio.run(_run())
271277

272278
# ============================================================================
273279
# PROMPTS TRIGGER/STATUS/FETCH (Manual Control for batch)

src/brightdata/scrapers/facebook/scraper.py

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -129,9 +129,12 @@ def posts_by_profile(
129129
timeout: int = DEFAULT_TIMEOUT_MEDIUM,
130130
) -> Union[ScrapeResult, List[ScrapeResult]]:
131131
"""Collect posts from Facebook profile URL (sync wrapper)."""
132-
return asyncio.run(self.posts_by_profile_async(
133-
url, num_of_posts, posts_to_not_include, start_date, end_date, timeout
134-
))
132+
async def _run():
133+
async with self.engine:
134+
return await self.posts_by_profile_async(
135+
url, num_of_posts, posts_to_not_include, start_date, end_date, timeout
136+
)
137+
return asyncio.run(_run())
135138

136139
# --- Trigger Interface (Manual Control) ---
137140

@@ -253,9 +256,12 @@ def posts_by_group(
253256
timeout: int = DEFAULT_TIMEOUT_MEDIUM,
254257
) -> Union[ScrapeResult, List[ScrapeResult]]:
255258
"""Collect posts from Facebook group URL (sync wrapper)."""
256-
return asyncio.run(self.posts_by_group_async(
257-
url, num_of_posts, posts_to_not_include, start_date, end_date, timeout
258-
))
259+
async def _run():
260+
async with self.engine:
261+
return await self.posts_by_group_async(
262+
url, num_of_posts, posts_to_not_include, start_date, end_date, timeout
263+
)
264+
return asyncio.run(_run())
259265

260266
# --- Trigger Interface (Manual Control) ---
261267

@@ -339,7 +345,10 @@ def posts_by_url(
339345
timeout: int = DEFAULT_TIMEOUT_MEDIUM,
340346
) -> Union[ScrapeResult, List[ScrapeResult]]:
341347
"""Collect detailed data from specific Facebook post URLs (sync wrapper)."""
342-
return asyncio.run(self.posts_by_url_async(url, timeout))
348+
async def _run():
349+
async with self.engine:
350+
return await self.posts_by_url_async(url, timeout)
351+
return asyncio.run(_run())
343352

344353
# --- Trigger Interface (Manual Control) ---
345354

@@ -434,9 +443,12 @@ def comments(
434443
timeout: int = DEFAULT_TIMEOUT_MEDIUM,
435444
) -> Union[ScrapeResult, List[ScrapeResult]]:
436445
"""Collect comments from Facebook post URL (sync wrapper)."""
437-
return asyncio.run(self.comments_async(
438-
url, num_of_comments, comments_to_not_include, start_date, end_date, timeout
439-
))
446+
async def _run():
447+
async with self.engine:
448+
return await self.comments_async(
449+
url, num_of_comments, comments_to_not_include, start_date, end_date, timeout
450+
)
451+
return asyncio.run(_run())
440452

441453
# --- Trigger Interface (Manual Control) ---
442454

@@ -537,9 +549,12 @@ def reels(
537549
timeout: int = DEFAULT_TIMEOUT_MEDIUM,
538550
) -> Union[ScrapeResult, List[ScrapeResult]]:
539551
"""Collect reels from Facebook profile URL (sync wrapper)."""
540-
return asyncio.run(self.reels_async(
541-
url, num_of_posts, posts_to_not_include, start_date, end_date, timeout
542-
))
552+
async def _run():
553+
async with self.engine:
554+
return await self.reels_async(
555+
url, num_of_posts, posts_to_not_include, start_date, end_date, timeout
556+
)
557+
return asyncio.run(_run())
543558

544559
# --- Trigger Interface (Manual Control) ---
545560

src/brightdata/scrapers/instagram/scraper.py

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,10 @@ def profiles(
109109
timeout: int = DEFAULT_TIMEOUT_MEDIUM,
110110
) -> Union[ScrapeResult, List[ScrapeResult]]:
111111
"""Collect profile details from Instagram profile URL (sync wrapper)."""
112-
return asyncio.run(self.profiles_async(url, timeout))
112+
async def _run():
113+
async with self.engine:
114+
return await self.profiles_async(url, timeout)
115+
return asyncio.run(_run())
113116

114117
# --- Trigger Interface (Manual Control) ---
115118

@@ -185,7 +188,10 @@ def posts(
185188
timeout: int = DEFAULT_TIMEOUT_MEDIUM,
186189
) -> Union[ScrapeResult, List[ScrapeResult]]:
187190
"""Collect detailed data from Instagram post URLs (sync wrapper)."""
188-
return asyncio.run(self.posts_async(url, timeout))
191+
async def _run():
192+
async with self.engine:
193+
return await self.posts_async(url, timeout)
194+
return asyncio.run(_run())
189195

190196
# --- Trigger Interface (Manual Control) ---
191197

@@ -261,7 +267,10 @@ def comments(
261267
timeout: int = DEFAULT_TIMEOUT_MEDIUM,
262268
) -> Union[ScrapeResult, List[ScrapeResult]]:
263269
"""Collect comments from Instagram post URL (sync wrapper)."""
264-
return asyncio.run(self.comments_async(url, timeout))
270+
async def _run():
271+
async with self.engine:
272+
return await self.comments_async(url, timeout)
273+
return asyncio.run(_run())
265274

266275
# --- Trigger Interface (Manual Control) ---
267276

@@ -337,7 +346,10 @@ def reels(
337346
timeout: int = DEFAULT_TIMEOUT_MEDIUM,
338347
) -> Union[ScrapeResult, List[ScrapeResult]]:
339348
"""Collect detailed data from Instagram reel URLs (sync wrapper)."""
340-
return asyncio.run(self.reels_async(url, timeout))
349+
async def _run():
350+
async with self.engine:
351+
return await self.reels_async(url, timeout)
352+
return asyncio.run(_run())
341353

342354
# --- Trigger Interface (Manual Control) ---
343355

src/brightdata/scrapers/instagram/search.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -128,9 +128,12 @@ def posts(
128128
timeout: int = DEFAULT_TIMEOUT_MEDIUM,
129129
) -> Union[ScrapeResult, List[ScrapeResult]]:
130130
"""Discover recent Instagram posts from a public profile (sync wrapper)."""
131-
return asyncio.run(self.posts_async(
132-
url, num_of_posts, posts_to_not_include, start_date, end_date, post_type, timeout
133-
))
131+
async def _run():
132+
async with self.engine:
133+
return await self.posts_async(
134+
url, num_of_posts, posts_to_not_include, start_date, end_date, post_type, timeout
135+
)
136+
return asyncio.run(_run())
134137

135138
# ============================================================================
136139
# REELS DISCOVERY (by profile or search URL with filters)
@@ -197,9 +200,12 @@ def reels(
197200
timeout: int = DEFAULT_TIMEOUT_MEDIUM,
198201
) -> Union[ScrapeResult, List[ScrapeResult]]:
199202
"""Discover Instagram Reels from profile or search URL (sync wrapper)."""
200-
return asyncio.run(self.reels_async(
201-
url, num_of_posts, posts_to_not_include, start_date, end_date, timeout
202-
))
203+
async def _run():
204+
async with self.engine:
205+
return await self.reels_async(
206+
url, num_of_posts, posts_to_not_include, start_date, end_date, timeout
207+
)
208+
return asyncio.run(_run())
203209

204210
# ============================================================================
205211
# CORE DISCOVERY LOGIC

src/brightdata/scrapers/linkedin/scraper.py

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,10 @@ def posts(
112112
113113
See posts_async() for documentation.
114114
"""
115-
return asyncio.run(self.posts_async(url, timeout))
115+
async def _run():
116+
async with self.engine:
117+
return await self.posts_async(url, timeout)
118+
return asyncio.run(_run())
116119

117120
# ============================================================================
118121
# POSTS TRIGGER/STATUS/FETCH (Manual Control)
@@ -191,7 +194,10 @@ def jobs(
191194
timeout: int = DEFAULT_TIMEOUT_SHORT,
192195
) -> Union[ScrapeResult, List[ScrapeResult]]:
193196
"""Scrape LinkedIn jobs (sync wrapper)."""
194-
return asyncio.run(self.jobs_async(url, timeout))
197+
async def _run():
198+
async with self.engine:
199+
return await self.jobs_async(url, timeout)
200+
return asyncio.run(_run())
195201

196202
# ============================================================================
197203
# JOBS TRIGGER/STATUS/FETCH (Manual Control)
@@ -270,7 +276,10 @@ def profiles(
270276
timeout: int = DEFAULT_TIMEOUT_SHORT,
271277
) -> Union[ScrapeResult, List[ScrapeResult]]:
272278
"""Scrape LinkedIn profiles (sync wrapper)."""
273-
return asyncio.run(self.profiles_async(url, timeout))
279+
async def _run():
280+
async with self.engine:
281+
return await self.profiles_async(url, timeout)
282+
return asyncio.run(_run())
274283

275284
# --- Trigger Interface (Manual Control) ---
276285

@@ -346,7 +355,10 @@ def companies(
346355
timeout: int = DEFAULT_TIMEOUT_SHORT,
347356
) -> Union[ScrapeResult, List[ScrapeResult]]:
348357
"""Scrape LinkedIn companies (sync wrapper)."""
349-
return asyncio.run(self.companies_async(url, timeout))
358+
async def _run():
359+
async with self.engine:
360+
return await self.companies_async(url, timeout)
361+
return asyncio.run(_run())
350362

351363
# ============================================================================
352364
# COMPANIES TRIGGER/STATUS/FETCH (Manual Control)

src/brightdata/scrapers/linkedin/search.py

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,10 @@ def posts(
128128
129129
See posts_async() for documentation.
130130
"""
131-
return asyncio.run(self.posts_async(profile_url, start_date, end_date, timeout))
131+
async def _run():
132+
async with self.engine:
133+
return await self.posts_async(profile_url, start_date, end_date, timeout)
134+
return asyncio.run(_run())
132135

133136
# ============================================================================
134137
# PROFILES DISCOVERY (by name)
@@ -188,7 +191,10 @@ def profiles(
188191
189192
See profiles_async() for documentation.
190193
"""
191-
return asyncio.run(self.profiles_async(firstName, lastName, timeout))
194+
async def _run():
195+
async with self.engine:
196+
return await self.profiles_async(firstName, lastName, timeout)
197+
return asyncio.run(_run())
192198

193199
# ============================================================================
194200
# JOBS DISCOVERY (by keyword + extensive filters)
@@ -325,19 +331,22 @@ def jobs(
325331
... remote=True
326332
... )
327333
"""
328-
return asyncio.run(self.jobs_async(
329-
url=url,
330-
location=location,
331-
keyword=keyword,
332-
country=country,
333-
timeRange=timeRange,
334-
jobType=jobType,
335-
experienceLevel=experienceLevel,
336-
remote=remote,
337-
company=company,
338-
locationRadius=locationRadius,
339-
timeout=timeout
340-
))
334+
async def _run():
335+
async with self.engine:
336+
return await self.jobs_async(
337+
url=url,
338+
location=location,
339+
keyword=keyword,
340+
country=country,
341+
timeRange=timeRange,
342+
jobType=jobType,
343+
experienceLevel=experienceLevel,
344+
remote=remote,
345+
company=company,
346+
locationRadius=locationRadius,
347+
timeout=timeout
348+
)
349+
return asyncio.run(_run())
341350

342351
# ============================================================================
343352
# HELPER METHODS

0 commit comments

Comments
 (0)