disabled the logic to remove the cached files

ChaoPang · ChaoPang · commit 914ac2c932ab · 2025-09-21T17:37:01.000-04:00
diff --git a/src/cehrbert/runners/data_utils.py b/src/cehrbert/runners/data_utils.py
@@ -97,6 +97,4 @@ def extract_cohort_sequences(
         num_proc=data_args.preprocessing_num_workers,
         remove_columns=filtered_tokenized_dataset["train"].column_names,
     )
-    if cache_file_collector:
-        cache_file_collector.add_cache_files(filtered_tokenized_dataset)
     return processed_dataset

Original file line number	Diff line number	Diff line change
`@@ -97,6 +97,4 @@ def extract_cohort_sequences(`
`97`	`97`	`num_proc=data_args.preprocessing_num_workers,`
`98`	`98`	`remove_columns=filtered_tokenized_dataset["train"].column_names,`
`99`	`99`	`)`
`100`		`- if cache_file_collector:`
`101`		`- cache_file_collector.add_cache_files(filtered_tokenized_dataset)`
`102`	`100`	`return processed_dataset`