Ensures miBf is deterministic (#150)

* Ensures miBf is deterministic

* remove extra space

* add dev only function description

* ensure all insert functions are equivalent

* remove unused variable

Author: jwcodee

goldrush_path/MIBFConstructSupport.hpp

@@ -202,13 +202,13 @@ class MIBFConstructSupport
     values.set_empty_key(miBF.size());
     while (itr != itr.end()) {
       for (unsigned i = 0; i < m_h; ++i) {
-        values.insert((*itr)[i]);
+        values.insert(miBF.getRankPos((*itr)[i]));
       }
       ++itr;
     }
     for (hashSet::iterator itr = values.begin(); itr != values.end(); itr++) {
       uint64_t randomSeed = *itr ^ id;
-      uint64_t rank = miBF.getRankPos(*itr);
+      uint64_t rank = *itr;
       T count = __sync_add_and_fetch(&m_counts[rank], 1);
       T randomNum = std::hash<T>{}(randomSeed) % count;
       if (randomNum == count - 1) {
@@ -227,15 +227,19 @@ class MIBFConstructSupport
 #pragma omp parallel for
 #endif
     for (size_t i = 0; i < hash_vec.size(); ++i) {
-      const auto& hash = hash_vec[i];
-      uint64_t randomSeed = hash ^ id;
-      uint64_t rank = miBF.getRankPos(hash);
-      T count = __sync_add_and_fetch(&m_counts[rank], 1);
-      T randomNum = std::hash<T>{}(randomSeed) % count;
-      // std::cerr << "id: " << id << " randomNum: " << randomNum << "
-      // randomSeed: " << randomSeed <<std::endl;
-      if (randomNum == count - 1) {
-        miBF.setData(rank, id);
+      hashSet values;
+      values.set_empty_key(miBF.size());
+      for (size_t i = 0; i < hash_vec.size(); ++i) {
+        values.insert(miBF.getRankPos(hash_vec[i]));
+      }
+      for (hashSet::iterator itr = values.begin(); itr != values.end(); itr++) {
+        uint64_t randomSeed = *itr ^ id;
+        uint64_t rank = *itr;
+        T count = __sync_add_and_fetch(&m_counts[rank], 1);
+        T randomNum = std::hash<T>{}(randomSeed) % count;
+        if (randomNum == count - 1) {
+          miBF.setData(rank, id);
+        }
       }
     }
   }
@@ -248,21 +252,28 @@ class MIBFConstructSupport
   {
     // assert(m_isBVMade & !m_isMIBFMade);
     // get positions
+    hashSet values;
+    values.set_empty_key(miBF.size());
     size_t vec_size = hash_vec[0].size();
     size_t num_elements = 0;
     for (size_t i = start; i < end; ++i) {
       num_elements += hash_vec[i].size();
     }
 
-#if _OPENMP
-#pragma omp parallel for
-#endif
     for (size_t i = 0; i < num_elements; ++i) {
       size_t vec_num = i / vec_size;
       size_t hash_loc = i % vec_size;
       const auto& hash = hash_vec[start + vec_num][hash_loc];
-      uint64_t randomSeed = hash ^ id;
-      uint64_t rank = miBF.getRankPos(hash);
+      values.insert(miBF.getRankPos(hash));
+    }
+
+    std::vector<uint64_t> unique_values(values.begin(), values.end());
+#if _OPENMP
+#pragma omp parallel for
+#endif
+    for (size_t i = 0; i < unique_values.size(); ++i) {
+      const auto& rank = unique_values[i];
+      uint64_t randomSeed = rank ^ id;
       T count = __sync_add_and_fetch(&m_counts[rank], 1);
       T randomNum = std::hash<T>{}(randomSeed) % count;
       if (randomNum == count - 1) {
@@ -283,14 +294,14 @@ class MIBFConstructSupport
       for (unsigned i = 0; i < m_h; ++i) {
         const std::vector<uint64_t> hash{ (*itr)[i] };
         if (solid_vec[i]->contains(hash)) {
-          values.insert(hash[0]);
+          values.insert(miBF.getRankPos(hash[0]));
         }
       }
       ++itr;
     }
     for (hashSet::iterator itr = values.begin(); itr != values.end(); itr++) {
       uint64_t randomSeed = *itr ^ id;
-      uint64_t rank = miBF.getRankPos(*itr);
+      uint64_t rank = *itr;
       T count = __sync_add_and_fetch(&m_counts[rank], 1);
       T randomNum = std::hash<T>{}(randomSeed) % count;
       if (randomNum == count - 1) {

goldrush_path/MIBloomFilter.hpp

@@ -120,6 +120,45 @@ class MIBloomFilter
     return colliCount;
   }
 
+	/*
+	 * Stores the filter as a binary file to the path specified
+	 * Stores uncompressed because the random data tends to
+	 * compress poorly anyway
+   * FOR DEVELOPMENT PURPOSES ONLY
+	 */
+	void store(string const& filterFilePath) const
+	{
+
+#pragma omp parallel for
+		for (unsigned i = 0; i < 2; ++i) {
+			if (i == 0) {
+				ofstream myFile(filterFilePath.c_str(), ios::out | ios::binary);
+
+				assert(myFile);
+        myFile.write(reinterpret_cast<const char*>(m_data.data()), m_dSize * sizeof(T));
+
+				myFile.close();
+				assert(myFile);
+
+				FILE* file = fopen(filterFilePath.c_str(), "rb");
+				if (file == NULL) {
+					cerr << "file \"" << filterFilePath << "\" could not be read." << endl;
+					exit(1);
+				}
+			} else {
+				string bvFilename = filterFilePath + ".sdsl";
+				//				cerr << "Storing sdsl interleaved bit vector to: " << bvFilename
+				//						<< endl;
+				store_to_file(m_bv, bvFilename);
+				//				cerr << "Number of bit vector buckets is " << m_bv.size()
+				//						<< endl;
+				//				cerr << "Uncompressed bit vector size is "
+				//						<< (m_bv.size() + m_bv.size() * 64 / BLOCKSIZE) / 8
+				//						<< " bytes" << endl;
+			}
+		}
+	}
+
   /*
    * Constructor using a prebuilt bitvector
    */