@@ -1460,7 +1460,7 @@ async def is_crawl_stopping(
14601460
14611461 async def get_redis_crawl_stats (
14621462 self , redis : Redis , crawl_id : str
1463- ) -> tuple [CrawlStats , dict [str , Any ]]:
1463+ ) -> tuple [CrawlStats , dict [str , str ]]:
14641464 """get page stats"""
14651465 try :
14661466 # crawler >0.9.0, done key is a value
@@ -1513,9 +1513,10 @@ async def update_crawl_state(
15131513 crawl .db_crawl_id , crawl .is_qa , stats
15141514 )
15151515
1516- for key , value in sizes .items ():
1516+ for key , str_value in sizes .items ():
15171517 increase_storage = False
1518- value = int (value )
1518+ pod_info = None
1519+ value = int (str_value )
15191520 if value > 0 and status .podStatus :
15201521 pod_info = status .podStatus [key ]
15211522 pod_info .used .storage = value
@@ -1530,11 +1531,11 @@ async def update_crawl_state(
15301531 increase_storage = True
15311532
15321533 # out of storage
1533- if pod_info .isNewExit and pod_info .exitCode == 3 :
1534+ if pod_info and pod_info .isNewExit and pod_info .exitCode == 3 :
15341535 pod_info .used .storage = pod_info .allocated .storage
15351536 increase_storage = True
15361537
1537- if increase_storage :
1538+ if pod_info and increase_storage :
15381539 new_storage = math .ceil (
15391540 pod_info .used .storage * self .min_avail_storage_ratio / 1_000_000_000
15401541 )
0 commit comments