Removing duplicate data

Author: MaithriRao

augmented_data/dataset.py

@@ -63,8 +63,11 @@ def read_dataset_mms():
     return dataset
 
 
-def write_dataset_text(dataset, main_folder):
+def write_dataset_text(original_dataset, dataset, main_folder):
     for folder_name, file_contents in dataset.items():
+        original_file_contents = original_dataset[folder_name]
+        if original_file_contents == file_contents:
+            continue
         file_folder = os.path.join(main_folder, folder_name)
         os.makedirs(file_folder, exist_ok = True)
         file_path = os.path.join(file_folder, text_file_name)
@@ -74,11 +77,14 @@ def write_dataset_text(dataset, main_folder):
                 f.write(text_line + '\n')
 
 
-def write_dataset_mms(dataset, main_folder):
+def write_dataset_mms(original_dataset, dataset, main_folder):
     fieldnames = ['maingloss', 'framestart', 'frameend', 'duration', 'transition', 'domgloss', 'ndomgloss', 'domreloc', 'ndomreloc', 'headpos', 'headmov', 'cheecks', 'nose', 'mouthgest', 'mouthing', 'eyegaze', 'eyeaperture', 'eyebrows', 'neck', 'shoulders', 'torso', 'domhandrelocx', 'domhandrelocy', 'domhandrelocz', 'domhandrelocax', 'domhandrelocay', 'domhandrelocaz', 'domhandrelocsx', 'domhandrelocsy', 'domhandrelocsz', 'domhandrotx', 'domhandroty', 'domhandrotz', 'ndomhandrelocx', 'ndomhandrelocy', 'ndomhandrelocz', 'ndomhandrelocax', 'ndomhandrelocay', 'ndomhandrelocaz', 'ndomhandrelocsx', 'ndomhandrelocsy', 'ndomhandrelocsz', 'ndomhandrotx', 'ndomhandroty', 'ndomhandrotz']
     os.makedirs(main_folder, exist_ok = True)
 
     for file_name, file_contents in dataset.items():
+        original_file_contents = original_dataset[file_name]
+        if original_file_contents == file_contents:
+            continue
         file_path = os.path.join(main_folder, file_name + '.mms')
         with open(file_path, 'w', encoding='utf-8', newline='') as f:
             writer = csv.DictWriter(f, fieldnames=fieldnames)

augmented_data/location.py

@@ -104,5 +104,5 @@ def replace_location_entities(dataset_text, dataset_mms):
     dataset_mms = read_dataset_mms()
 
     result_text, result_mms = replace_location_entities(dataset_text, dataset_mms)
-    write_dataset_text(result_text, main_folder = 'modified/location/text')
-    write_dataset_mms(result_mms, main_folder = 'modified/location/mms')
+    write_dataset_text(dataset_text, result_text, main_folder = 'modified/location/text')
+    write_dataset_mms(dataset_mms, result_mms, main_folder = 'modified/location/mms')

augmented_data/platform.py

@@ -97,5 +97,5 @@ class State(Enum):
     dataset_mms = read_dataset_mms()
 
     result_text, result_mms = replace_platform_entities(dataset_text, dataset_mms)
-    write_dataset_text(result_text, main_folder = 'modified/platform/text')
-    write_dataset_mms(result_mms, main_folder = 'modified/platform/mms')
+    write_dataset_text(dataset_text, result_text, main_folder = 'modified/platform/text')
+    write_dataset_mms(dataset_mms, result_mms, main_folder = 'modified/platform/mms')

augmented_data/time.py

@@ -338,5 +338,5 @@ def process_time(state, time_data, time_positions):
     dataset_mms = read_dataset_mms()
 
     result_text, result_mms = replace_time_entities(dataset_text, dataset_mms)
-    write_dataset_text(result_text, main_folder = 'modified/time/text')
-    write_dataset_mms(result_mms, main_folder = 'modified/time/mms')
+    write_dataset_text(dataset_text, result_text, main_folder = 'modified/time/text')
+    write_dataset_mms(dataset_mms, result_mms, main_folder = 'modified/time/mms')

augmented_data/train_name.py

@@ -356,5 +356,5 @@ def process_train(state, train_data, train_positions):
     dataset_mms = read_dataset_mms()
 
     result_text, result_mms = replace_train_entities(dataset_text, dataset_mms)
-    write_dataset_text(result_text, main_folder = 'modified/train_name/text')
-    write_dataset_mms(result_mms, main_folder = 'modified/train_name/mms')
+    write_dataset_text(dataset_text, result_text, main_folder = 'modified/train_name/text')
+    write_dataset_mms(dataset_mms, result_mms, main_folder = 'modified/train_name/mms')