[Refactor:Plagiarism] Discard overlap with common code (#63)

* initial draft

* Fix errors

Co-authored-by: sbelsk <[email protected]>

Author: williamjallen

compare_hashes/compare_hashes.cpp

@@ -14,6 +14,7 @@
 #include "boost/filesystem/path.hpp"
 #include "nlohmann/json.hpp"
 
+#include "lichen_config.h"
 #include "submission.h"
 #include "hash_location.h"
 
@@ -101,11 +102,13 @@ int main(int argc, char* argv[]) {
   time_t overall_start, overall_end;
   time(&overall_start);
 
+
   // ===========================================================================
   // load Lichen config data
   std::ifstream lichen_config_istr("./lichen_config.json");
   assert(lichen_config_istr.good());
   nlohmann::json lichen_config = nlohmann::json::parse(lichen_config_istr);
+  LichenConfig config;
 
   // ===========================================================================
   // load config info
@@ -119,11 +122,11 @@ int main(int argc, char* argv[]) {
   assert(istr.good());
   nlohmann::json config_file_json = nlohmann::json::parse(istr);
 
-  std::string semester = config_file_json.value("semester", "ERROR");
-  std::string course = config_file_json.value("course", "ERROR");
-  std::string gradeable = config_file_json.value("gradeable", "ERROR");
-  int sequence_length = config_file_json.value("sequence_length", 1);
-  int threshold = config_file_json.value("threshold", 5);
+  config.semester = config_file_json.value("semester", "ERROR");
+  config.course = config_file_json.value("course", "ERROR");
+  config.gradeable = config_file_json.value("gradeable", "ERROR");
+  config.sequence_length = config_file_json.value("sequence_length", 1);
+  config.threshold = config_file_json.value("threshold", 5);
 
   // error checking, confirm there are hashes to work with
   boost::filesystem::path users_root_directory = lichen_gradeable_path / "users";
@@ -136,7 +139,7 @@ int main(int argc, char* argv[]) {
   // the file path where we expect to find the hashed instructor provided code file
   boost::filesystem::path provided_code_file = lichen_gradeable_path / "provided_code" / "hashes.txt";
   // if file exists in that location, the provided code mode is enabled.
-  bool provided_code_enabled = boost::filesystem::exists(provided_code_file);
+  config.provided_code_enabled = boost::filesystem::exists(provided_code_file);
   // path to prior gradeables' data
   boost::filesystem::path prior_terms_dir = lichen_gradeable_path / "other_gradeables";
 
@@ -158,7 +161,7 @@ int main(int argc, char* argv[]) {
   time_t start, end;
   time(&start);
 
-  if (provided_code_enabled) {
+  if (config.provided_code_enabled) {
     // load the instructor provided code's hashes
     std::ifstream istr(provided_code_file.string());
     assert(istr.good());
@@ -221,7 +224,7 @@ int main(int argc, char* argv[]) {
       assert (version > 0);
 
       // create a submission object and load to the main submissions structure
-      Submission* curr_submission = new Submission(username, version);
+      Submission* curr_submission = new Submission(username, version, config);
 
       // load the hashes from this submission
       boost::filesystem::path hash_file = version_path;
@@ -233,7 +236,7 @@ int main(int argc, char* argv[]) {
       while (istr >> input_hash_str) {
         hash input_hash = (unsigned int)(stoul(input_hash_str, 0, 16));
         location++;
-        all_hashes[input_hash][username].push_back(HashLocation(username, version, location, semester+"__"+course+"__"+gradeable));
+        all_hashes[input_hash][username].push_back(HashLocation(username, version, location, config.semester + "__" + config.course + "__" + config.gradeable));
         curr_submission->addHash(input_hash, location);
       }
 
@@ -267,7 +270,7 @@ int main(int argc, char* argv[]) {
 
       // if provided code was enabled, look for the submission hash in the provided code's hashes
       bool provided_match_found = false;
-      if (provided_code_enabled) {
+      if (config.provided_code_enabled) {
         std::unordered_set<hash>::iterator provided_match_itr = provided_code.find(hash_itr->first);
         if (provided_match_itr != provided_code.end()) {
           provided_match_found = true;
@@ -292,7 +295,7 @@ int main(int argc, char* argv[]) {
           std::vector<HashLocation>::iterator itr = occurences_itr->second.begin();
           for (; itr != occurences_itr->second.end(); ++itr) {
 
-            if (occurences.size() > (unsigned int)threshold) {
+            if (occurences.size() > (unsigned int) config.threshold) {
               // if the number of students with matching code is more
               // than the threshold, it is considered common code
               (*submission_itr)->addCommonMatch(hash_itr->second);
@@ -368,7 +371,7 @@ int main(int argc, char* argv[]) {
         std::vector<nlohmann::json> matchingpositions;
         nlohmann::json position;
         position["start"] = matching_positions_itr->location;
-        position["end"] = matching_positions_itr->location + sequence_length - 1;
+        position["end"] = matching_positions_itr->location + config.sequence_length - 1;
         matchingpositions.push_back(position);
 
         // search for all matching positions of the suspicious match in other submissions
@@ -400,7 +403,7 @@ int main(int argc, char* argv[]) {
               other["source_gradeable"] = matching_positions_itr->source_gradeable;
             }
             position["start"] = matching_positions_itr->location;
-            position["end"] = matching_positions_itr->location + sequence_length - 1;
+            position["end"] = matching_positions_itr->location + config.sequence_length - 1;
             matchingpositions.push_back(position);
           }
         }
@@ -411,7 +414,7 @@ int main(int argc, char* argv[]) {
 
       nlohmann::json info;
       info["start"] = location_itr->first;
-      info["end"] = location_itr->first + sequence_length - 1;
+      info["end"] = location_itr->first + config.sequence_length - 1;
       info["type"] = "match";
       info["others"] = others;
 
@@ -428,7 +431,7 @@ int main(int argc, char* argv[]) {
 
       nlohmann::json info;
       info["start"] = *location_itr;
-      info["end"] = *location_itr + sequence_length - 1;
+      info["end"] = *location_itr + config.sequence_length - 1;
       info["type"] = "common";
 
       result.push_back(info);
@@ -444,7 +447,7 @@ int main(int argc, char* argv[]) {
 
       nlohmann::json info;
       info["start"] = *location_itr;
-      info["end"] = *location_itr + sequence_length - 1;
+      info["end"] = *location_itr + config.sequence_length - 1;
       info["type"] = "provided";
 
       result.push_back(info);
@@ -456,7 +459,7 @@ int main(int argc, char* argv[]) {
     // Done creating the JSON file/objects, now we merge them to shrink them in size
 
     // Merge matching regions:
-    if (result.size() > 0) { // check to make sure that there are more than 1 positions (if it's 1, we can't merge anyway)
+    if (!result.empty()) { // check to make sure that there are more than 1 positions (if it's 1, we can't merge anyway)
       // loop through all positions
       for (unsigned int position = 1; position < result.size(); position++) {
         nlohmann::json* prevPosition = &result[position - 1];

compare_hashes/hash_location.h

@@ -19,7 +19,8 @@ struct HashLocation {
   std::string source_gradeable;
 };
 
-bool operator < (const HashLocation &hl1, const HashLocation &hl2) {
+// inline keyword is necessary to prevent linker errors when multiple .cpp files include this header and are then linked
+inline bool operator < (const HashLocation &hl1, const HashLocation &hl2) {
   return hl1.student > hl2.student ||
          (hl1.student == hl2.student && hl1.version < hl2.version) ||
          (hl1.student == hl2.student && hl1.version == hl2.version && hl1.location < hl2.location);

compare_hashes/lichen_config.h

@@ -0,0 +1,13 @@
+#ifndef LICHEN_CONFIG_H
+#define LICHEN_CONFIG_H
+
+struct LichenConfig {
+    std::string semester;
+    std::string course;
+    std::string gradeable;
+    int sequence_length;
+    int threshold;
+    bool provided_code_enabled;
+};
+
+#endif

compare_hashes/submission.cpp

@@ -0,0 +1,62 @@
+#include <map>
+#include <set>
+#include <vector>
+
+#include "hash_location.h"
+#include "submission.h"
+
+typedef int location_in_submission;
+typedef unsigned int hash;
+typedef std::string user_id;
+typedef unsigned int version_number;
+
+float Submission::getPercentage() const {
+  return (100.0 * (suspicious_matches.size())) / hashes.size();
+}
+
+void Submission::addSuspiciousMatch(location_in_submission location, const HashLocation &matching_location, const hash &matched_hash) {
+  // figure out if there is an overlap between this hash and a common/provided match
+  int sequence_length = config_.sequence_length;
+  for (int i = location - 1; i > location - sequence_length && i >= 0; i--) {
+    if (common_matches.find(i) != common_matches.end() || provided_matches.find(i) != provided_matches.end()) {
+      return;
+    }
+  }
+
+  // save the found match
+  suspicious_matches[location].insert(matching_location);
+  // update the students_matched container
+  students_matched[matching_location.source_gradeable][matching_location.student][matching_location.version].insert(matched_hash);
+}
+
+void Submission::addCommonMatch(location_in_submission location) {
+  // figure out if there is an overlap between this hash and a match
+  int sequence_length = config_.sequence_length;
+  for (int i = location - 1; i > location - sequence_length && i >= 0; i--) {
+    std::map<location_in_submission, std::set<HashLocation> >::const_iterator find_i = suspicious_matches.find(i);
+    // if there is an overlap, remove the suspicious match that overlaps
+    // hopefully this doesn't cause problems with other submissions thinking
+    // this hash still matches...
+    if (find_i != suspicious_matches.end()) {
+      suspicious_matches.erase(find_i);
+    }
+  }
+
+  common_matches.insert(location);
+}
+
+void Submission::addProvidedMatch(location_in_submission location) {
+  // figure out if there is an overlap between this hash and a match
+  int sequence_length = config_.sequence_length;
+  for (int i = location - 1; i > location - sequence_length && i >= 0; i--) {
+    std::map<location_in_submission, std::set<HashLocation> >::const_iterator find_i = suspicious_matches.find(i);
+    // if there is an overlap, remove the suspicious match that overlaps
+    // hopefully this doesn't cause problems with other submissions thinking
+    // this hash still matches...
+    if (find_i != suspicious_matches.end()) {
+      suspicious_matches.erase(find_i);
+    }
+  }
+
+  provided_matches.insert(location);
+}

compare_hashes/submission.h

@@ -9,6 +9,7 @@
 #include <vector>
 
 #include "hash_location.h"
+#include "lichen_config.h"
 
 typedef int location_in_submission;
 typedef unsigned int hash;
@@ -20,7 +21,7 @@ typedef unsigned int version_number;
 class Submission {
 public:
   // CONSTRUCTOR
-  Submission(const user_id &s, version_number v) : student_(s), version_(v) {}
+  Submission(const user_id &s, version_number v, const LichenConfig &c) : student_(s), version_(v), config_(c) {}
 
   // GETTERS
   const user_id& student() const { return student_; }
@@ -31,26 +32,18 @@ class Submission {
   const std::set<location_in_submission>& getProvidedMatches() const { return provided_matches; }
   const std::unordered_map<std::string, std::unordered_map<user_id, std::unordered_map<version_number, std::unordered_set<hash>>>>& getStudentsMatched() const { return students_matched; }
   const std::vector<std::pair<hash, location_in_submission>> & getHashes() const { return hashes; }
-  float getPercentage() const {
-    return (100.0 * (suspicious_matches.size())) / hashes.size();
-  }
+  float getPercentage() const;
 
   // MODIFIERS
   void addHash(const hash &h, location_in_submission l) { hashes.push_back(std::make_pair(h, l)); }
-
-  void addSuspiciousMatch(location_in_submission location, const HashLocation &matching_location, const hash &matched_hash) {
-    // save the found match
-    suspicious_matches[location].insert(matching_location);
-    // update the students_matched container
-    students_matched[matching_location.source_gradeable][matching_location.student][matching_location.version].insert(matched_hash);
-  }
-
-  void addCommonMatch(location_in_submission location) { common_matches.insert(location); }
-  void addProvidedMatch(location_in_submission location) { provided_matches.insert(location); }
+  void addSuspiciousMatch(location_in_submission location, const HashLocation &matching_location, const hash &matched_hash);
+  void addCommonMatch(location_in_submission location);
+  void addProvidedMatch(location_in_submission location);
 
 private:
   user_id student_;
   version_number version_;
+  LichenConfig config_;
   std::vector<std::pair<hash, location_in_submission> > hashes;
   std::map<location_in_submission, std::set<HashLocation> > suspicious_matches;
   std::set<location_in_submission> common_matches;

install_lichen.sh

@@ -62,7 +62,7 @@ mkdir -p ${lichen_installation_dir}/tools/assignments
 #-------------------------------------------
 # compile & install the hash comparison tool
 pushd ${lichen_repository_dir}  > /dev/null
-clang++ -I ${lichen_vendor_dir} -lboost_system -lboost_filesystem -Wall -Wextra -Werror -g -O3 -flto -funroll-loops -std=c++11 compare_hashes/compare_hashes.cpp -o ${lichen_installation_dir}/bin/compare_hashes.out
+clang++ -I ${lichen_vendor_dir} -lboost_system -lboost_filesystem -Wall -Wextra -Werror -g -Ofast -flto -funroll-loops -std=c++11 compare_hashes/compare_hashes.cpp compare_hashes/submission.cpp -o ${lichen_installation_dir}/bin/compare_hashes.out
 if [ $? -ne 0 ]; then
     echo -e "ERROR: FAILED TO BUILD HASH COMPARISON TOOL\n"
     exit 1