@@ -185,7 +185,7 @@ def export_experiment_data(
185185
186186 _partition_by_unit = dataset .has_unit and (partition_by_unit or dataset .always_partition_by_unit )
187187 _partition_by_experiment = dataset .has_experiment and partition_by_experiment
188- filenames : list [str ] = []
188+ path_to_files : list [Path ] = []
189189 placeholders : dict [str , str ] = {}
190190
191191 order_by_col = dataset .default_order_by
@@ -243,15 +243,17 @@ def export_experiment_data(
243243 )
244244
245245 if rows_partition not in parition_to_writer_map :
246+ # create a new csv writer for this partition since it doesn't exist yet
246247 filename = f"{ dataset_name } -" + "-" .join (rows_partition ) + f"-{ time } .csv"
247248 filename = filename .replace (" " , "_" )
248- filenames .append (filename )
249- path_to_file = Path (Path (output ).parent / filename )
249+ path_to_file = Path (output ).parent / filename
250250 parition_to_writer_map [rows_partition ] = csv .writer (
251251 stack .enter_context (open (path_to_file , "w" )), delimiter = ","
252252 )
253253 parition_to_writer_map [rows_partition ].writerow (headers )
254254
255+ path_to_files .append (path_to_file )
256+
255257 parition_to_writer_map [rows_partition ].writerow (row )
256258
257259 if count % 10_000 == 0 :
@@ -262,10 +264,9 @@ def export_experiment_data(
262264 logger .warning (f"No data present in { dataset_name } . Check database?" )
263265
264266 zf .mkdir (dataset_name )
265- for filename in filenames :
266- path_to_file = Path (output , filename )
267+ for path_to_file in path_to_files :
267268 zf .write (path_to_file , arcname = f"{ dataset_name } /{ filename } " )
268- Path ( path_to_file ) .unlink ()
269+ path_to_file .unlink ()
269270
270271 logger .info ("Finished export." )
271272 return
0 commit comments