Decent.

Author: lemire

.github/workflows/ubuntu.yml

@@ -0,0 +1,28 @@
+name: Ubuntu 22.04 Sanitized CI (GCC 11)
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - {shared: ON}
+          - {shared: OFF}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use make
+        run: |
+          make &&
+          ./build_filter -o filter.bin sample.txt  &&
+          ./query_filter filter.bin AF8978B1797B72AC

README.md

@@ -57,7 +57,7 @@ Though the filter can use little memory (less than a GB), it seems unwarranted t
 
 There are two executables:
 
-- `build_filter` is the expensive program that parses the large text files containing password hashes.
+- `build_filter` is the expensive program that parses the large text files containing password hashes. If you want to check the filter at construction time, you can use the `-V` flag. You may also specify the filter type with the `-f` flag: `-f binaryfuse8`, `-f binaryfuse16`, `-f xor8`, `-f bloom12`.
 - `query_filter` is a simple program that takes a 64-bit hash in hexadecimal for (the first 16 hexadecimal characters from the SHA1 hash) and checks whether the hash is contained in our set.
 
 Do 
@@ -89,6 +89,7 @@ Expected number of queries per second: 17241.379
 ```
 
 
+
 ## Performance comparisons
 
 For a comparable false positive probability (about 0.3%), the Bloom filter uses more space

src/build_filter.cpp

@@ -236,6 +236,78 @@ int main(int argc, char **argv) {
       return EXIT_FAILURE;
     }
     binary_fuse8_free(&filter);
+  } else if (strcmp("binaryfuse16", filtername) == 0) {
+    start = clock();
+    binary_fuse16_t filter;
+    if(!binary_fuse16_allocate(array_size, &filter)) {
+      printf("failed to allocate memory.\n");
+      return EXIT_FAILURE;
+    }
+    if(!binary_fuse16_populate(array, array_size, &filter)){
+      printf("failed to build the filter, do you have sufficient memory?\n");
+      return EXIT_FAILURE;
+    }
+    end = clock();
+    printf("Done in %.3f seconds.\n", (float)(end - start) / CLOCKS_PER_SEC);
+    if (verify) {
+      printf("Checking for false negatives\n");
+      for (size_t i = 0; i < array_size; i++) {
+        if (!binary_fuse16_contain(array[i], &filter)) {
+          printf("Detected a false negative. You probably have a bug. "
+                 "Aborting.\n");
+          return EXIT_FAILURE;
+        }
+      }
+      printf("Verified with success: no false negatives\n");
+      size_t matches = 0;
+      size_t volume = 100000;
+      for (size_t t = 0; t < volume; t++) {
+        if (binary_fuse16_contain(t * 10001 + 13 + array_size, &filter)) {
+          matches++;
+        }
+      }
+      printf("estimated false positive rate: %.3f percent\n",
+             matches * 100.0 / volume);
+    }
+    free(array);
+
+    FILE *write_ptr;
+    write_ptr = fopen(outputfilename, "wb");
+    if (write_ptr == NULL) {
+      printf("Cannot write to the output file %s.", outputfilename);
+      return EXIT_FAILURE;
+    }
+    uint64_t cookie = 1234570;
+    bool isok = true;
+    size_t total_bytes =
+        sizeof(cookie) + sizeof(filter.Seed) + sizeof(filter.SegmentLength) +
+        sizeof(filter.SegmentLengthMask) + sizeof(filter.SegmentCount) +
+        sizeof(filter.SegmentCountLength) + sizeof(filter.ArrayLength) +
+        sizeof(uint16_t) * filter.ArrayLength;
+
+    isok &= fwrite(&cookie, sizeof(cookie), 1, write_ptr);
+    isok &= fwrite(&filter.Seed, sizeof(filter.Seed), 1, write_ptr);
+    isok &= fwrite(&filter.SegmentLength, sizeof(filter.SegmentLength), 1,
+                   write_ptr);
+    isok &= fwrite(&filter.SegmentLengthMask, sizeof(filter.SegmentLengthMask),
+                   1, write_ptr);
+    isok &=
+        fwrite(&filter.SegmentCount, sizeof(filter.SegmentCount), 1, write_ptr);
+    isok &= fwrite(&filter.SegmentCountLength,
+                   sizeof(filter.SegmentCountLength), 1, write_ptr);
+    isok &=
+        fwrite(&filter.ArrayLength, sizeof(filter.ArrayLength), 1, write_ptr);
+    isok &= fwrite(filter.Fingerprints, sizeof(uint16_t) * filter.ArrayLength, 1,
+                   write_ptr);
+    isok &= (fclose(write_ptr) == 0);
+    if (isok) {
+      printf("filter data saved to %s. Total bytes = %zu. \n", outputfilename,
+             total_bytes);
+    } else {
+      printf("failed to write filter data to %s.\n", outputfilename);
+      return EXIT_FAILURE;
+    }
+    binary_fuse16_free(&filter);
   } else if (strcmp("xor8", filtername) == 0) {
     start = clock();
     xor8_t filter;

src/query_filter.cpp

@@ -88,14 +88,17 @@ int main(int argc, char **argv) {
     return EXIT_FAILURE;
   }
   bool xor8 = false;
+  bool bin16 = false;
   bool bloom12 = false;
   if (fread(&cookie, sizeof(cookie), 1, fp) != 1)
     printf("failed read.\n");
   if (fread(&seed, sizeof(seed), 1, fp) != 1)
     printf("failed read.\n");
 
   if (cookie != 1234569) {
-    if (cookie == 1234567) {
+    if(cookie == 1234570) {
+      bin16 = true;
+    } else  if (cookie == 1234567) {
       xor8 = true;
       if (fread(&BlockLength, sizeof(BlockLength), 1, fp) != 1)
         printf("failed read.\n");
@@ -106,17 +109,38 @@ int main(int argc, char **argv) {
         printf("failed read.\n");
 
     } else {
-      printf("Not a filter file.\n");
+      printf("Not a filter file. Cookie found: %llu.\n", cookie);
       return EXIT_FAILURE;
     }
   }
   size_t length = 0;
   binary_fuse8_t binfilter;
+  binary_fuse16_t binfilter16;
 
   if (bloom12) {
     length = BlockLength * sizeof(uint64_t) + 3 * sizeof(uint64_t);
   } else if (xor8) {
     length = 3 * BlockLength * sizeof(uint8_t) + 3 * sizeof(uint64_t);
+  } else if(bin16) {
+    bool isok = true;
+    binfilter16.Seed = seed;
+    isok &=
+        fread(&binfilter16.SegmentLength, sizeof(binfilter16.SegmentLength), 1, fp);
+    isok &=
+        fread(&binfilter16.SegmentLengthMask, sizeof(binfilter16.SegmentLengthMask), 1, fp);
+    isok &=
+        fread(&binfilter16.SegmentCount, sizeof(binfilter16.SegmentCount), 1, fp);
+    isok &= fread(&binfilter16.SegmentCountLength,
+                  sizeof(binfilter16.SegmentCountLength), 1, fp);
+    isok &= fread(&binfilter16.ArrayLength, sizeof(binfilter16.ArrayLength), 1, fp);
+    if (!isok)
+      printf("failed read.\n");
+    length =
+        sizeof(cookie) + sizeof(binfilter16.Seed) +
+        sizeof(binfilter16.SegmentLength) + sizeof(binfilter16.SegmentLengthMask) +
+        sizeof(binfilter16.SegmentCount) + sizeof(binfilter16.SegmentCountLength) +
+        sizeof(binfilter16.ArrayLength) + sizeof(uint16_t) * binfilter16.ArrayLength;
+
   } else {
     bool isok = true;
     binfilter.Seed = seed;
@@ -142,8 +166,10 @@ int main(int argc, char **argv) {
     printf("Bloom filter detected.\n");
   else if (xor8)
     printf("Xor filter detected.\n");
+  else if(bin16)
+    printf("16-bit binary fuse filter detected.\n");
   else
-    printf("Binary fuse filter detected.\n");
+    printf("8-bit binary fuse filter detected.\n");
   fclose(fp);
   int fd = open(filename, O_RDONLY);
   bool shared = false;
@@ -183,6 +209,17 @@ int main(int argc, char **argv) {
     } else {
       printf("Surely not in the set.\n");
     }
+  } else if (bin16) {
+    binfilter16.Fingerprints = reinterpret_cast<uint16_t*>(
+        addr + sizeof(cookie) + sizeof(binfilter16.Seed) +
+        sizeof(binfilter16.SegmentLength) + sizeof(binfilter16.SegmentLengthMask) +
+        sizeof(binfilter16.SegmentCount) + sizeof(binfilter16.SegmentCountLength) +
+        sizeof(binfilter16.ArrayLength));
+    if (binary_fuse16_contain(hexval, &binfilter16)) {
+      printf("Probably in the set.\n");
+    } else {
+      printf("Surely not in the set.\n");
+    }
   } else {
     binfilter.Fingerprints =
         addr + sizeof(cookie) + sizeof(binfilter.Seed) +