22import os
33import os .path
44import time
5+ import typing
56
67import psutil
78import base64
@@ -121,12 +122,12 @@ def download_blocks(local, bag: dict):
121122
122123def download_master_blocks (local , bag : dict ):
123124 local .add_log (f"Downloading master blocks from { bag ['from' ]} to { bag ['to' ]} " , "info" )
124- if not download_bag (local , bag ['bag' ], download_all = False ):
125+ if not download_bag (local , bag ['bag' ], download_all = False , download_file = lambda f : ':' not in f [ 'name' ] ):
125126 local .add_log ("Error downloading master bag" , "error" )
126127 return
127128
128129
129- def download_bag (local , bag_id : str , download_all : bool = True ):
130+ def download_bag (local , bag_id : str , download_all : bool = True , download_file : typing . Callable = None ):
130131 indexes = []
131132 local_ts_url = f"http://127.0.0.1:{ local .buffer .ton_storage .api_port } "
132133 downloads_path = '/tmp/ts-downloads/'
@@ -141,7 +142,7 @@ def download_bag(local, bag_id: str, download_all: bool = True):
141142 time .sleep (1 )
142143 resp = requests .get (local_ts_url + f'/api/v1/details?bag_id={ bag_id } ' ).json ()
143144 for f in resp ['files' ]:
144- if ':' not in f [ 'name' ]: # do not download shardblock packs
145+ if download_file ( f ):
145146 indexes .append (f ['index' ])
146147 resp = requests .post (local_ts_url + '/api/v1/add' , json = {'bag_id' : bag_id , 'download_all' : download_all , 'path' : downloads_path , 'files' : indexes })
147148 if not resp .json ()['ok' ]:
@@ -241,8 +242,9 @@ def download_archive_from_ts(local):
241242 return
242243
243244 update_init_block (local , state_bag ['at_block' ])
245+ estimated_size = len (block_bags ) * 4 * 2 ** 30 + len (master_block_bags ) * 4 * 2 ** 30 * 0.2 # 4 GB per bag, 20% for master blocks
244246
245- local .add_log ("Downloading archive blocks" , "info" )
247+ local .add_log (f "Downloading archive blocks. Rough estimate total blocks size is { estimated_size / 2 ** 30 } GB " , "info" )
246248 with ThreadPoolExecutor (max_workers = 4 ) as executor :
247249 futures = [executor .submit (download_blocks , local , bag ) for bag in block_bags ]
248250 futures += [executor .submit (download_master_blocks , local , bag ) for bag in master_block_bags ]
0 commit comments