1313
1414import rasterio
1515from loguru import logger as log
16- from wurlitzer import pipes
1716
1817fmt = "{time} | TILEBENCH | {message}"
1918log .remove ()
2019log .add (sys .stderr , format = fmt )
2120
2221
23- def parse_logs (rio_lines : List [ str ], curl_lines : List [str ]) -> Dict [str , Any ]:
22+ def parse_logs (logs : List [str ]) -> Dict [str , Any ]:
2423 """Parse Rasterio and CURL logs."""
25- # LIST
26- list_requests = [line for line in rio_lines if " VSICURL: GetFileList" in line ]
27- list_summary = {
28- "count" : len (list_requests ),
29- }
30-
3124 # HEAD
32- curl_head_requests = [line for line in curl_lines if line . startswith ( "> HEAD")]
25+ head_requests = len ( [line for line in logs if "CURL_INFO_HEADER_OUT: HEAD" in line ])
3326 head_summary = {
34- "count" : len ( curl_head_requests ) ,
27+ "count" : head_requests ,
3528 }
3629
37- # CURL GET
38- # CURL logs failed requests
39- curl_get_requests = [line for line in curl_lines if line .startswith ("> GET" )]
30+ # GET
31+ all_get_requests = len (
32+ [line for line in logs if "CURL_INFO_HEADER_OUT: GET" in line ]
33+ )
4034
41- # Rasterio GET
42- # Rasterio only log successfull requests
43- get_requests = [line for line in rio_lines if ": Downloading" in line ]
35+ get_requests = [line for line in logs if ": Downloading" in line ]
4436 get_values = [
4537 map (int , get .split (" Downloading " )[1 ].split (" " )[0 ].split ("-" ))
4638 for get in get_requests
@@ -49,17 +41,14 @@ def parse_logs(rio_lines: List[str], curl_lines: List[str]) -> Dict[str, Any]:
4941 data_transfer = sum ([j - i + 1 for i , j in get_values ])
5042
5143 get_summary = {
52- "count" : len ( curl_get_requests ) ,
44+ "count" : all_get_requests ,
5345 "bytes" : data_transfer ,
5446 "ranges" : get_values_str ,
5547 }
5648
57- warp_kernel = [
58- line .split (" " )[- 2 :] for line in rio_lines if "GDALWarpKernel" in line
59- ]
49+ warp_kernel = [line .split (" " )[- 2 :] for line in logs if "GDALWarpKernel" in line ]
6050
6151 return {
62- "LIST" : list_summary ,
6352 "HEAD" : head_summary ,
6453 "GET" : get_summary ,
6554 "WarpKernels" : warp_kernel ,
@@ -88,25 +77,23 @@ def wrapped_f(*args, **kwargs):
8877 logger .addHandler (handler )
8978
9079 gdal_config = config or {}
91- gdal_config .update ({"CPL_DEBUG" : "ON" , "CPL_CURL_VERBOSE" : "TRUE " })
80+ gdal_config .update ({"CPL_DEBUG" : "ON" , "CPL_CURL_VERBOSE" : "YES " })
9281
93- with pipes () as (_ , curl_stream ):
94- with rasterio .Env (** gdal_config ):
95- with Timer () as t :
96- prof = cProfile .Profile ()
97- retval = prof .runcall (func , * args , ** kwargs )
98- profile_stream = StringIO ()
99- ps = pstats .Stats (prof , stream = profile_stream )
100- ps .strip_dirs ().sort_stats ("time" , "ncalls" ).print_stats ()
82+ with rasterio .Env (** gdal_config ):
83+ with Timer () as t :
84+ prof = cProfile .Profile ()
85+ retval = prof .runcall (func , * args , ** kwargs )
86+ profile_stream = StringIO ()
87+ ps = pstats .Stats (prof , stream = profile_stream )
88+ ps .strip_dirs ().sort_stats ("time" , "ncalls" ).print_stats ()
10189
10290 logger .removeHandler (handler )
10391 handler .close ()
10492
105- rio_lines = rio_stream .getvalue ().splitlines ()
106- curl_lines = curl_stream .read ().splitlines ()
93+ logs = rio_stream .getvalue ().splitlines ()
10794 profile_lines = [p for p in profile_stream .getvalue ().splitlines () if p ]
10895
109- results = parse_logs (rio_lines , curl_lines )
96+ results = parse_logs (logs )
11097 results ["Timing" ] = t .elapsed
11198
11299 if cprofile :
@@ -119,8 +106,7 @@ def wrapped_f(*args, **kwargs):
119106 results .pop ("WarpKernels" )
120107
121108 if raw :
122- results ["curl" ] = curl_lines
123- results ["rasterio" ] = rio_lines
109+ results ["logs" ] = logs
124110
125111 if not quiet :
126112 log .info (json .dumps (results ))
0 commit comments