Skip to content

Commit 22a81ba

Browse files
committed
add test
1 parent 6b014a7 commit 22a81ba

File tree

1 file changed

+32
-0
lines changed

1 file changed

+32
-0
lines changed

tests/unit/crawlers/_basic/test_basic_crawler.py

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1768,3 +1768,35 @@ async def handler(_: BasicCrawlingContext) -> None:
17681768

17691769
# Wait for crawler to finish
17701770
await crawler_task
1771+
1772+
1773+
async def test_protect_request_and_session_in_run_handlers() -> None:
1774+
"""Test that session and request in crawling context are protected in run handlers."""
1775+
request_queue = await RequestQueue.open(name='state-test')
1776+
1777+
async with SessionPool(max_pool_size=1) as session_pool:
1778+
session = await session_pool.get_session()
1779+
session.user_data['session_state'] = ['initial']
1780+
1781+
request = Request.from_url('https://test.url/', user_data={'request_state': ['initial']}, session_id=session.id)
1782+
1783+
crawler = BasicCrawler(session_pool=session_pool, request_manager=request_queue, max_request_retries=0)
1784+
1785+
@crawler.router.default_handler
1786+
async def handler(context: BasicCrawlingContext) -> None:
1787+
if context.session:
1788+
context.session.user_data['session_state'].append('modified')
1789+
if isinstance(context.request.user_data['request_state'], list):
1790+
context.request.user_data['request_state'].append('modified')
1791+
raise ValueError('Simulated error after modifying request and session')
1792+
1793+
await crawler.run([request])
1794+
1795+
check_request = await request_queue.get_request(request.unique_key)
1796+
assert check_request is not None
1797+
assert check_request.user_data['request_state'] == ['initial']
1798+
1799+
check_session = await session_pool.get_session()
1800+
assert check_session.user_data['session_state'] == ['initial']
1801+
1802+
await request_queue.drop()

0 commit comments

Comments
 (0)