@@ -29,48 +29,47 @@ class ExecutionMetrics:
29
29
execution_count : int = 0
30
30
slot_millis : int = 0
31
31
bytes_processed : int = 0
32
+ execution_secs : float = 0
32
33
33
34
def count_job_stats (self , query_job : bq_job .QueryJob ):
34
35
stats = get_performance_stats (query_job )
35
36
if stats is not None :
36
- bytes_processed , slot_millis , exec_seconds = stats
37
+ bytes_processed , slot_millis , execution_secs = stats
37
38
self .execution_count += 1
38
39
self .bytes_processed += bytes_processed
39
40
self .slot_millis += slot_millis
41
+ self .execution_secs += execution_secs
40
42
if LOGGING_NAME_ENV_VAR in os .environ :
41
43
# when running notebooks via pytest nbmake
42
- write_stats_to_disk (bytes_processed , slot_millis , exec_seconds )
44
+ write_stats_to_disk (bytes_processed , slot_millis , execution_secs )
43
45
44
46
45
47
def get_performance_stats (
46
48
query_job : bigquery .QueryJob ,
47
- ) -> Optional [Tuple [int , int , Optional [ float ] ]]:
49
+ ) -> Optional [Tuple [int , int , float ]]:
48
50
"""Parse the query job for performance stats.
49
51
50
52
Return None if the stats do not reflect real work done in bigquery.
51
53
"""
54
+
55
+ if (
56
+ query_job .configuration .dry_run
57
+ or query_job .created is None
58
+ or query_job .ended is None
59
+ ):
60
+ return None
61
+
52
62
bytes_processed = query_job .total_bytes_processed
53
63
if not isinstance (bytes_processed , int ):
54
64
return None # filter out mocks
55
- if query_job .configuration .dry_run :
56
- # dry run stats are just predictions of the real run
57
- bytes_processed = 0
58
65
59
66
slot_millis = query_job .slot_millis
60
67
if not isinstance (slot_millis , int ):
61
68
return None # filter out mocks
62
69
63
- if query_job .configuration .dry_run :
64
- # dry run stats are just predictions of the real run
65
- slot_millis = 0
66
-
67
- exec_seconds = (
68
- (query_job .ended - query_job .created ).total_seconds ()
69
- if query_job .created is not None and query_job .ended is not None
70
- else None
71
- )
70
+ execution_secs = (query_job .ended - query_job .created ).total_seconds ()
72
71
73
- return bytes_processed , slot_millis , exec_seconds
72
+ return bytes_processed , slot_millis , execution_secs
74
73
75
74
76
75
def write_stats_to_disk (
0 commit comments