[Feature:Plagiarism] Support arbitrary other gradeable paths (#68)

* renaming sequence length to hash size and prior term to other gradeables

* suggested edits

* linting

* 99% implemented (syntax error????)

* Fix compilation issues

* fix ranking files

* Make requested changes

* Fix tests

* progress

* Implement arbitrary other gradeables

* lint

* lint (v2.0)

Co-authored-by: sbelsk <[email protected]>

Author: williamjallen

.gitignore

@@ -3,6 +3,7 @@ tools/assignments/*
 __pycache__
 tests/__pycache__
 vendor/
-cmake-build-debug/*
+cmake-build-debug
 .idea
 .DS_Store
+.vscode

bin/concatenate_all.py

@@ -11,6 +11,7 @@
 import time
 import humanize
 import fnmatch
+import hashlib
 from pathlib import Path
 
 IGNORED_FILES = [
@@ -57,6 +58,65 @@ def getConcatFilesInDir(input_dir, regex_patterns):
     return result
 
 
+# This function is passed a path to a gradeable and an output path to place files in and
+# concatenates all of the files for each submission into a single file in the output directory
+# returns the total size of the files concatenated
+def processGradeable(basepath, config, input_dir, output_dir, total_concat):
+    # basic error checking
+    if not Path(input_dir).exists():
+        raise SystemExit(f"ERROR: Unable to find directory {input_dir}")
+
+    if Path(input_dir).group() != Path(basepath).group():
+        raise SystemExit(f"ERROR: Group for directory {input_dir} does not"
+                         f"match group for {basepath} directory")
+
+    # loop over each user
+    for user in sorted(os.listdir(input_dir)):
+        user_path = os.path.join(input_dir, user)
+        if not os.path.isdir(user_path):
+            continue
+        elif user in config["ignore_submissions"]:
+            continue
+
+        if config["version"] == "active_version":
+            # get the user's active version from their settings file if it exists, else get
+            # most recent version for compatibility with early versions of Submitty
+            submissions_details_path = os.path.join(user_path, 'user_assignment_settings.json')
+            if os.path.exists(submissions_details_path):
+                with open(submissions_details_path) as details_file:
+                    details_json = json.load(details_file)
+                    my_active_version = int(details_json["active_version"])
+            else:
+                # get the most recent version
+                my_active_version = sorted(os.listdir(user_path))[-1]
+
+        # loop over each version
+        for version in sorted(os.listdir(user_path)):
+            version_path = os.path.join(user_path, version)
+            if dir == "results":
+                # only the "details" folder within "results" contains files relevant to Lichen
+                version_path = os.path.join(version_path, "details")
+            if not os.path.isdir(version_path):
+                continue
+            if config["version"] == "active_version" and int(version) != my_active_version:
+                continue
+
+            output_file_path = os.path.join(output_dir, user, version, "submission.concatenated")
+
+            if not os.path.exists(os.path.dirname(output_file_path)):
+                os.makedirs(os.path.dirname(output_file_path))
+
+            # append to concatenated file
+            with open(output_file_path, "a") as output_file:
+                concatenated_contents = getConcatFilesInDir(version_path, config["regex"])
+                output_file.write(concatenated_contents)
+                total_concat += sys.getsizeof(concatenated_contents)
+
+            # If we've exceeded the concatenation limit, kill program
+            checkTotalSize(total_concat)
+    return total_concat
+
+
 def checkTotalSize(total_concat):
     if total_concat > LICHEN_CONFIG['concat_max_total_bytes']:
         raise SystemExit("ERROR! exceeded"
@@ -85,37 +145,44 @@ def validate(config, args):
     with open(langs_data_json_path, 'r') as langs_data_file:
         langs_data = json.load(langs_data_file)
         if language not in langs_data:
-            raise SystemExit(f"ERROR! tokenizing not supported for language {language}")
+            raise SystemExit(f"ERROR: tokenizing not supported for language {language}")
 
     # Check values of common code threshold and hash size
     if (threshold < 2):
-        raise SystemExit("ERROR! threshold must be >= 2")
+        raise SystemExit("ERROR: threshold must be >= 2")
 
     if (hash_size < 1):
-        raise SystemExit("ERROR! hash_size must be >= 1")
+        raise SystemExit("ERROR: hash_size must be >= 1")
 
     # Check for backwards crawling
     for e in regex_patterns:
         if ".." in e:
-            raise SystemExit('ERROR! Invalid path component ".." in regex')
+            raise SystemExit('ERROR: Invalid path component ".." in regex')
 
     for gradeable in other_gradeables:
         for field in gradeable:
             if ".." in field:
-                raise SystemExit('ERROR! Invalid component ".." in other_gradeable path')
+                raise SystemExit('ERROR: Invalid component ".." in other_gradeable path')
 
     # check permissions to make sure we have access to the other gradeables
     my_course_group_perms = Path(args.basepath).group()
     for gradeable in other_gradeables:
         if Path(args.datapath, gradeable["other_semester"], gradeable["other_course"]).group()\
            != my_course_group_perms:
-            raise SystemExit("ERROR! Invalid permissions to access course "
+            raise SystemExit("ERROR: Invalid permissions to access course "
                              f"{gradeable['other_semester']}/{gradeable['other_course']}")
 
+    # check permissions for each path we are given (if any are provided)
+    if config.get("other_gradeable_paths") is not None:
+        for path in config["other_gradeable_paths"]:
+            if Path(path).group() != my_course_group_perms:
+                raise SystemExit(f"ERROR: Group for directory {path} does not"
+                                 f"match group for {args.basepath} directory")
+
     # make sure the regex directory is one of the acceptable directories
     for dir in regex_dirs:
         if dir not in ["submissions", "results", "checkout"]:
-            raise SystemExit("ERROR! ", dir, " is not a valid input directory for Lichen")
+            raise SystemExit(f"ERROR: {dir} is not a valid input directory for Lichen")
 
 
 def main():
@@ -138,145 +205,80 @@ def main():
     semester = config["semester"]
     course = config["course"]
     gradeable = config["gradeable"]
-    version_mode = config["version"]
     regex_patterns = config["regex"]
     regex_dirs = config["regex_dirs"]
     other_gradeables = config["other_gradeables"]
-    users_to_ignore = config["ignore_submissions"]
+    # optional field -> other_gradeable_paths=None if key doesn't exist
+    other_gradeable_paths = config.get("other_gradeable_paths")
 
     # ==========================================================================
     # loop through and concatenate the selected files for each user in this gradeable
     total_concat = 0
 
     for dir in regex_dirs:
-        gradeable_path = os.path.join(args.datapath, semester, course, dir, gradeable)
-        # loop over each user
-        for user in sorted(os.listdir(gradeable_path)):
-            user_path = os.path.join(gradeable_path, user)
-            if not os.path.isdir(user_path):
-                continue
-            elif user in users_to_ignore:
-                continue
-
-            if version_mode == "active_version":
-                # get the user's active version from their settings file if it exists, else get
-                # most recent version for compatibility with early versions of Submitty
-                submissions_details_path = os.path.join(user_path, 'user_assignment_settings.json')
-                if os.path.exists(submissions_details_path):
-                    with open(submissions_details_path) as details_file:
-                        details_json = json.load(details_file)
-                        my_active_version = int(details_json["active_version"])
-                else:
-                    # get the most recent version
-                    my_active_version = sorted(os.listdir(user_path))[-1]
-
-            # loop over each version
-            for version in sorted(os.listdir(user_path)):
-                version_path = os.path.join(user_path, version)
-                if dir == "results":
-                    # only the "details" folder within "results" contains files relevant to Lichen
-                    version_path = os.path.join(version_path, "details")
-                if not os.path.isdir(version_path):
-                    continue
-                if version_mode == "active_version" and int(version) != my_active_version:
-                    continue
-
-                output_file_path = os.path.join(args.basepath, "users", user,
-                                                version, "submission.concatenated")
-
-                if not os.path.exists(os.path.dirname(output_file_path)):
-                    os.makedirs(os.path.dirname(output_file_path))
-
-                # append to concatenated file
-                with open(output_file_path, "a") as output_file:
-                    concatenated_contents = getConcatFilesInDir(version_path, regex_patterns)
-                    output_file.write(concatenated_contents)
-                    total_concat += sys.getsizeof(concatenated_contents)
-
-                checkTotalSize(total_concat)
+        input_path = os.path.join(args.datapath, semester, course, dir, gradeable)
+        output_path = os.path.join(args.basepath, "users")
+        total_concat = processGradeable(args.basepath, config,
+                                        input_path, output_path, total_concat)
 
     # ==========================================================================
     # loop over all of the other gradeables and concatenate their submissions
     for other_gradeable in other_gradeables:
         for dir in regex_dirs:
-            other_gradeable_path = os.path.join(args.datapath,
-                                                other_gradeable["other_semester"],
-                                                other_gradeable["other_course"],
-                                                dir,
-                                                other_gradeable["other_gradeable"])
-            # loop over each user
-            for other_user in sorted(os.listdir(other_gradeable_path)):
-                other_user_path = os.path.join(other_gradeable_path, other_user)
-                if not os.path.isdir(other_user_path):
-                    continue
-
-                if version_mode == "active_version":
-                    # get the user's active version from their settings file if it exists, else get
-                    # most recent version for compatibility with early versions of Submitty
-                    other_submissions_details_path = os.path.join(other_user_path,
-                                                                  'user_assignment_settings.json')
-                    if os.path.exists(other_submissions_details_path):
-                        with open(other_submissions_details_path) as other_details_file:
-                            other_details_json = json.load(other_details_file)
-                            my_active_version = int(other_details_json["active_version"])
-                    else:
-                        # get the most recent version
-                        my_active_version = sorted(os.listdir(other_user_path))[-1]
-
-                # loop over each version
-                for other_version in sorted(os.listdir(other_user_path)):
-                    other_version_path = os.path.join(other_user_path, other_version)
-                    if dir == "results":
-                        # only the "details" dir within "results" contains files relevant to Lichen
-                        other_version_path = os.path.join(other_version_path, "details")
-                    if not os.path.isdir(other_version_path):
-                        continue
-
-                    other_output_file_path = os.path.join(args.basepath, "other_gradeables",
-                                                          f"{other_gradeable['other_semester']}__{other_gradeable['other_course']}__{other_gradeable['other_gradeable']}",  # noqa: E501
-                                                          other_user, other_version,
-                                                          "submission.concatenated")
-
-                    if not os.path.exists(os.path.dirname(other_output_file_path)):
-                        os.makedirs(os.path.dirname(other_output_file_path))
-
-                    # append to concatenated file
-                    with open(other_output_file_path, "a") as other_output_file:
-                        other_concatenated_contents = getConcatFilesInDir(other_version_path,
-                                                                          regex_patterns)
-                        other_output_file.write(other_concatenated_contents)
-                        total_concat += sys.getsizeof(other_concatenated_contents)
-
-                    checkTotalSize(total_concat)
+            input_path = os.path.join(args.datapath,
+                                      other_gradeable["other_semester"],
+                                      other_gradeable["other_course"],
+                                      dir,
+                                      other_gradeable["other_gradeable"])
+
+            output_path = os.path.join(args.basepath, "other_gradeables",
+                                       f"{other_gradeable['other_semester']}__{other_gradeable['other_course']}__{other_gradeable['other_gradeable']}")  # noqa: E501
+            total_concat = processGradeable(args.basepath, config,
+                                            input_path, output_path, total_concat)
+
+    # take care of any manually-specified paths if they exist
+    if other_gradeable_paths is not None:
+        for path in other_gradeable_paths:
+            # We hash the path as the name of the gradeable
+            dir_name = hashlib.md5(path.encode('utf-8')).hexdigest()
+            output_path = os.path.join(args.basepath, "other_gradeables", dir_name)
+            total_concat = processGradeable(args.basepath, config, path,
+                                            output_path, total_concat)
 
     # ==========================================================================
     # iterate over all of the created submissions, checking to see if they are empty
     # and printing a message if so
 
+    empty_directories = []  # holds a list of users who had no files concatenated
+
     for user in os.listdir(os.path.join(args.basepath, "users")):
         user_path = os.path.join(args.basepath, "users", user)
         for version in os.listdir(user_path):
             version_path = os.path.join(user_path, version)
             my_concatenated_file = os.path.join(version_path, "submission.concatenated")
             with open(my_concatenated_file, "r") as my_cf:
                 if my_cf.read() == "":
-                    print("Warning: No files matched provided regex in selected directories "
-                          f"for user {user} version {version}")
+                    empty_directories.append(f"{user}:{version}")
+    if len(empty_directories) > 0:
+        print("Warning: No files matched provided regex in selected directories for user(s):",
+              ", ".join(empty_directories))
 
     # do the same for the other gradeables
-    for other_gradeable in other_gradeables:
-        other_gradeable_dir_name = f"{other_gradeable['other_semester']}__{other_gradeable['other_course']}__{other_gradeable['other_gradeable']}"  # noqa: E501
-        for other_user in os.listdir(os.path.join(args.basepath, "other_gradeables",
-                                                  other_gradeable_dir_name)):
+    for other_gradeable in os.listdir(os.path.join(args.basepath, "other_gradeables")):
+        empty_directories = []
+        for other_user in os.listdir(os.path.join(args.basepath,
+                                                  "other_gradeables", other_gradeable)):
             other_user_path = os.path.join(args.basepath, "other_gradeables",
-                                           other_gradeable_dir_name, other_user)
+                                           other_gradeable, other_user)
             for other_version in os.listdir(other_user_path):
                 other_version_path = os.path.join(other_user_path, other_version)
                 my_concatenated_file = os.path.join(other_version_path, "submission.concatenated")
                 with open(my_concatenated_file, "r") as my_cf:
                     if my_cf.read() == "":
-                        print("Warning: No files matched provided regex in selected directories "
-                              f"for user {other_user} version {other_version}")
+                        empty_directories.append(f"{other_user}:{other_version}")
+        if len(empty_directories) > 0:
+            print("Warning: No files matched provided regex in selected directories for user(s):",
+                  ", ".join(empty_directories), "in gradeable", other_gradeable)
 
     # ==========================================================================
     # concatenate provided code