Adjust unit tests to new _{un,}pack() interface

To exercise the new code without too much of a change to the existing
unit test, we change the signature of _{un,}serialize_gen() to take an
additional (const) size_t argument, which we ignore for
_{un,}serialize().

We add to the reported metrics absolute and relative size information
for the "in-core" and "wire" format, the latter jointly referencing to
_{un,}serialize() and _{un,}pack().

Author: nigoroll

tests/unit.c

@@ -8,16 +8,24 @@
 #define F1(t, a, rt, t1, p1) rt GFNAM(t, a)(t1 p1) { return FNAM(t, a)(p1); }
 #define F2(t, a, rt, t1, p1, t2, p2) rt GFNAM(t, a)(t1 p1, t2 p2) { return FNAM(t, a)(p1, p2); }
 #define F3(t, a, rt, t1, p1, t2, p2, t3, p3) rt GFNAM(t, a)(t1 p1, t2 p2, t3 p3) { return FNAM(t, a)(p1, p2, p3); }
+// map 3-argument _gen to 2-argument, discarding last
+#define F32(t, a, rt, t1, p1, t2, p2, t3, p3) rt GFNAM(t, a)(t1 p1, t2 p2, t3 p3) { (void)p3; return FNAM(t, a)(p1, p2); }
+// void return, ignore return value
+#define F3V(t, a, rt, t1, p1, t2, p2, t3, p3) rt GFNAM(t, a)(t1 p1, t2 p2, t3 p3) { (void)FNAM(t, a)(p1, p2, p3); }
 
 #define GEN_THUNKS(ftype)                                                                          \
   F2(ftype, allocate, bool, uint32_t, size, void*, filter)                                         \
   F1(ftype, free, void, void*, filter)                                                             \
   F1(ftype, size_in_bytes, size_t, const void*, filter)                                            \
   F1(ftype, serialization_bytes, size_t, void*, filter)                                            \
-  F2(ftype, serialize, void, void*, filter, char*, buffer)                                         \
-  F2(ftype, deserialize, bool, void*, filter, const char*, buffer)                                 \
+  F32(ftype, serialize, void, void*, filter, char*, buffer, size_t, len)                           \
+  F32(ftype, deserialize, bool, void*, filter, const char*, buffer, size_t, len)                    \
   F3(ftype, populate, bool, uint64_t*, keys, uint32_t, size, void*, filter)                        \
-  F2(ftype, contain, bool, uint64_t, key, const void*, filter)
+  F2(ftype, contain, bool, uint64_t, key, const void*, filter)                                     \
+  F1(ftype, pack_bytes, size_t, void*, filter)                                                     \
+  F3V(ftype, pack, void, void*, filter, char*, buffer, size_t, len)                                  \
+  F3(ftype, unpack, bool, void*, filter, const char*, buffer, size_t, len)
+
 
 GEN_THUNKS(xor8)
 GEN_THUNKS(xor16)
@@ -32,8 +40,8 @@ bool test(size_t size, size_t repeated_size, void *filter,
           void (*free_filter)(void *filter),
           size_t (*size_in_bytes)(const void *filter),
           size_t (*serialization_bytes)(void *filter),
-          void (*serialize)(void *filter, char *buffer),
-          bool (*deserialize)(void *filter, const char *buffer),
+          void (*serialize)(void *filter, char *buffer, size_t len),
+          bool (*deserialize)(void *filter, const char *buffer, size_t len),
           bool (*populate)(uint64_t *keys, uint32_t size, void *filter),
           bool (*contain)(uint64_t key, const void *filter)) {
   allocate((uint32_t)size, filter);
@@ -56,9 +64,9 @@ bool test(size_t size, size_t repeated_size, void *filter,
 
   size_t buffer_size = serialization_bytes(filter);
   char *buffer = (char*)malloc(buffer_size);
-  serialize(filter, buffer);
+  serialize(filter, buffer, buffer_size);
   free_filter(filter);
-  deserialize(filter, buffer);
+  deserialize(filter, buffer, buffer_size);
   free(buffer);
   for (size_t i = 0; i < size; i++) {
     if (!(contain)(big_set[i], filter)) {
@@ -79,10 +87,14 @@ bool test(size_t size, size_t repeated_size, void *filter,
   }
   double fpp = (double)random_matches * 1.0 / (double)trials;
   printf(" fpp %3.5f (estimated) \n", fpp);
-  double bpe = (double)size_in_bytes(filter) * 8.0 / (double)size;
-  printf(" bits per entry %3.2f\n", bpe);
-  printf(" bits per entry %3.2f (theoretical lower bound)\n", - log(fpp)/log(2));
-  printf(" efficiency ratio %3.3f \n", bpe /(- log(fpp)/log(2)));
+  size_t core_size = size_in_bytes(filter);
+  printf(" size in-core %zu wire %zu\n", core_size, buffer_size);
+  double cbpe = (double)core_size * 8.0 / (double)size;
+  double wbpe = (double)buffer_size * 8.0 / (double)size;
+  printf(" bits per entry in-core %3.2f wire %3.2f\n", cbpe, wbpe);
+  double bound = - log(fpp)/log(2);
+  printf(" bits per entry %3.2f (theoretical lower bound)\n", bound);
+  printf(" efficiency ratio in-core %3.3f wire %3.3f\n", cbpe/bound, wbpe/bound);
   free_filter(filter);
   free(big_set);
   return true;
@@ -132,6 +144,35 @@ bool testxor16(size_t size) {
 }
 
 
+bool testxor8pack(size_t size) {
+  printf("testing xor8 pack/unpack\n");
+  xor8_t filter;
+  return test(size, 0, &filter,
+              xor8_allocate_gen,
+              xor8_free_gen,
+              xor8_size_in_bytes_gen,
+              xor8_pack_bytes_gen,
+              xor8_pack_gen,
+              xor8_unpack_gen,
+              xor8_populate_gen,
+              xor8_contain_gen);
+}
+
+bool testxor16pack(size_t size) {
+  printf("testing xor16 pack/unpack\n");
+  xor8_t filter;
+  return test(size, 0, &filter,
+              xor16_allocate_gen,
+              xor16_free_gen,
+              xor16_size_in_bytes_gen,
+              xor16_pack_bytes_gen,
+              xor16_pack_gen,
+              xor16_unpack_gen,
+              xor16_populate_gen,
+              xor16_contain_gen);
+}
+
+
 
 bool testbufferedxor16(size_t size) {
   printf("testing buffered xor16\n");
@@ -161,8 +202,6 @@ bool testbinaryfuse8(size_t size, size_t repeated_size) {
               binary_fuse8_contain_gen);
 }
 
-
-
 bool testbinaryfuse16(size_t size, size_t repeated_size) {
   printf("testing binary fuse16 with size %zu and %zu duplicates\n", size, repeated_size);
   binary_fuse16_t filter;
@@ -177,6 +216,35 @@ bool testbinaryfuse16(size_t size, size_t repeated_size) {
               binary_fuse16_contain_gen);
 }
 
+
+bool testbinaryfuse8pack(size_t size, size_t repeated_size) {
+  printf("testing binary fuse8 pack/unpack with size %zu and %zu duplicates\n", size, repeated_size);
+  binary_fuse8_t filter;
+  return test(size, repeated_size, &filter,
+              binary_fuse8_allocate_gen,
+              binary_fuse8_free_gen,
+              binary_fuse8_size_in_bytes_gen,
+              binary_fuse8_pack_bytes_gen,
+              binary_fuse8_pack_gen,
+              binary_fuse8_unpack_gen,
+              binary_fuse8_populate_gen,
+              binary_fuse8_contain_gen);
+}
+
+bool testbinaryfuse16pack(size_t size, size_t repeated_size) {
+  printf("testing binary fuse16 pack/unpack with size %zu and %zu duplicates\n", size, repeated_size);
+  binary_fuse16_t filter;
+  return test(size, repeated_size, &filter,
+              binary_fuse16_allocate_gen,
+              binary_fuse16_free_gen,
+              binary_fuse16_size_in_bytes_gen,
+              binary_fuse16_pack_bytes_gen,
+              binary_fuse16_pack_gen,
+              binary_fuse16_unpack_gen,
+              binary_fuse16_populate_gen,
+              binary_fuse16_contain_gen);
+}
+
 void failure_rate_binary_fuse16() {
   printf("testing binary fuse16 for failure rate\n");
   // we construct many 5000-long input cases and check the probability of failure.
@@ -208,6 +276,10 @@ int main() {
     printf("\n");
     if(!testbinaryfuse16(size, 0)) { abort(); }
     printf("\n");
+    if(!testbinaryfuse8pack(size, 0)) { abort(); }
+    printf("\n");
+    if(!testbinaryfuse16pack(size, 0)) { abort(); }
+    printf("\n");
     if(!testbinaryfuse8(size, 10)) { abort(); }
     printf("\n");
     if(!testbinaryfuse16(size, 10)) { abort(); }
@@ -220,6 +292,10 @@ int main() {
     printf("\n");
     if(!testxor16(size)) { abort(); }
     printf("\n");
+    if(!testxor8pack(size)) { abort(); }
+    printf("\n");
+    if(!testxor16pack(size)) { abort(); }
+    printf("\n");
     printf("======\n");
   }