Remove previously tokenized files if concat occurs (#21)

* fix concat on regex

* temp git ignore change

* remove dir

* remove check internally

* fix tokenize

* fix filesystem tweaks

* rm filesystem stuff

* remove

Author: andrewaikens87

bin/concatenate_all.py

@@ -8,6 +8,7 @@
 import os
 import json
 import sys
+import shutil
 import fnmatch
 
 CONFIG_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', 'config')
@@ -71,13 +72,8 @@ def main():
             if not os.path.isdir(my_concatenated_dir):
                 os.makedirs(my_concatenated_dir)
             my_concatenated_file=os.path.join(my_concatenated_dir,"submission.concatenated")
+            total_concat = 0
             with open(my_concatenated_file,'w') as my_cf:
-                # print a brief header of information
-                my_cf.write("SEMESTER: "+semester+"\n")
-                my_cf.write("COURSE: "+course+"\n")
-                my_cf.write("GRADEABLE: "+gradeable+"\n")
-                my_cf.write("USER: "+user+"\n")
-                my_cf.write("VERSION: "+version+"\n")
                 # loop over all files in all subdirectories
                 base_path = os.path.join(submission_dir,user,version)
                 for my_dir,dirs,my_files in os.walk(base_path):
@@ -88,19 +84,24 @@ def main():
                         for e in expressions:
                             files_filtered.extend(fnmatch.filter(files, e.strip()))
                         files = files_filtered
+                    total_concat += len(files)
                     for my_file in files:
                         # skip the timestep
                         if my_file == ".submit.timestamp":
                             continue
                         absolute_path=os.path.join(my_dir,my_file)
                         relative_path=absolute_path[len(base_path):]
                         # print a separator & filename
-                        my_cf.write("----------------------------------------------------\n")
-                        my_cf.write("FILE: "+relative_path+"\n\n")
                         with open(absolute_path, encoding='ISO-8859-1') as tmp:
                             # append the contents of the file
                             my_cf.write(tmp.read())
-                        my_cf.write("\n")
+            # Remove concat file if there no content...
+            if total_concat == 0:
+                os.remove(my_concatenated_file)
+                p2 = os.path.join(course_dir, "lichen", "tokenized", gradeable, user, version)
+                if os.path.isdir(p2):
+                    shutil.rmtree(p2)
+                os.rmdir(my_concatenated_dir)
 
     print ("done")
 

bin/hash_all.py

@@ -13,7 +13,6 @@
 import json
 import hashlib
 
-
 CONFIG_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', 'config')
 with open(os.path.join(CONFIG_PATH, 'submitty.json')) as open_file:
     OPEN_JSON = json.load(open_file)

bin/tokenize_all.py

@@ -83,9 +83,7 @@ def main():
             if not os.path.isdir(my_tokenized_dir):
                 os.makedirs(my_tokenized_dir)
             my_tokenized_file=os.path.join(my_tokenized_dir,"tokens.json")
-            # Check to see if we have already tokenized...
-            if(not os.path.isfile(my_tokenized_file)):
-                tokenize(args,my_concatenated_file,my_tokenized_file)
+            tokenize(args,my_concatenated_file,my_tokenized_file)
 
     print ("done")