3636
3737
3838async def fetch_html (
39- url : str ,
40- session : ClientSession ,
41- timeout : ClientTimeout ,
42- ** kwargs ,
39+ url : str , session : ClientSession , timeout : ClientTimeout , ** kwargs
4340) -> str :
4441 """GET request wrapper to fetch page HTML.
4542
@@ -63,27 +60,31 @@ async def parse(
6360 url : str ,
6461 session : ClientSession ,
6562 timeout : ClientTimeout = DEFAULT_GET_TIMEOUT ,
66- ** kwargs ,
63+ ** kwargs
6764) -> set :
6865 """Find HREFs in the HTML of `url`."""
6966 found = set ()
7067 try :
71- html = await fetch_html (url = url , session = session , timeout = timeout ,
72- ** kwargs )
68+ html = await fetch_html (
69+ url = url , session = session , timeout = timeout , ** kwargs
70+ )
7371 except (
7472 aiohttp .ClientError ,
75- aiohttp .http_exceptions .HttpProcessingError
73+ aiohttp .http_exceptions .HttpProcessingError ,
7674 ) as e :
7775 logger .error (
78- 'aiohttp exception for %s [%s]: %s' ,
79- url , getattr (e , 'status' , None ), getattr (e , 'message' , None )
76+ "aiohttp exception for %s [%s]: %s" ,
77+ url ,
78+ getattr (e , "status" , None ),
79+ getattr (e , "message" , None ),
8080 )
8181 return found
8282 except Exception as e :
8383 # May be raised from other libraries, such as chardet or yarl.
8484 # logger.exception will show the full traceback.
85- logger .exception ('Non-aiohttp exception occured: %s' ,
86- getattr (e , '__dict__' , {}))
85+ logger .exception (
86+ "Non-aiohttp exception occured: %s" , getattr (e , "__dict__" , {})
87+ )
8788 return found
8889 else :
8990 # This portion is not really async, but it is the request/response
@@ -116,7 +117,7 @@ async def bulk_crawl_and_write(
116117 file : BinaryIO ,
117118 urls : set ,
118119 timeout : Union [object , ClientTimeout ] = sentinel ,
119- ** kwargs
120+ ** kwargs ,
120121) -> None :
121122 """Crawl & write concurrently to `file` for multiple `urls`."""
122123 async with ClientSession () as session :
0 commit comments