Fixing issue 35

Author: lemire

binaryfusefilter.go

@@ -1,9 +1,9 @@
 package xorfilter
 
 import (
+	"errors"
 	"math"
 	"math/bits"
-	"sort"
 )
 
 type BinaryFuse8 struct {
@@ -114,13 +114,8 @@ func PopulateBinaryFuse8(keys []uint64) (*BinaryFuse8, error) {
 		iterations += 1
 		if iterations > MaxIterations {
 			// The probability of this happening is lower than the
-			// the cosmic-ray probability (i.e., a cosmic ray corrupts your system),
-			// but if it happens, we just fill the fingerprint with ones which
-			// will flag all possible keys as 'possible', ensuring a correct result.
-			for i := 0; i < len(filter.Fingerprints); i++ {
-				filter.Fingerprints[i] = ^uint8(0)
-			}
-			return filter, nil
+			// the cosmic-ray probability (i.e., a cosmic ray corrupts your system).
+			return nil, errors.New("too many iterations")
 		}
 
 		blockBits := 1
@@ -252,7 +247,7 @@ func PopulateBinaryFuse8(keys []uint64) (*BinaryFuse8, error) {
 			// manage to remove them all. We may simply sort the key to
 			// solve the issue. This will run in time O(n log n) and it
 			// mutates the input.
-			sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
+			keys = pruneDuplicates(keys)
 		}
 		for i := uint32(0); i < size; i++ {
 			reverseOrder[i] = 0

binaryfusefilter_test.go

@@ -5,6 +5,7 @@ import (
 	"math/rand"
 	"testing"
 
+	"github.com/cespare/xxhash"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -305,3 +306,24 @@ func BenchmarkBinaryFuse8Contains50000000(b *testing.B) {
 		binaryfusedbig.Contains(rand.Uint64())
 	}
 }
+
+func Test_Issue35(t *testing.T) {
+	for test := 0; test < 100; test++ {
+		hashes := make([]uint64, 0)
+		for i := 0; i < 40000; i++ {
+			v := encode(int32(rand.Intn(10)), int32(rand.Intn(100000)))
+			hashes = append(hashes, xxhash.Sum64(v))
+		}
+		inner, err := PopulateBinaryFuse8(hashes)
+		if err != nil {
+			panic(err)
+		}
+		for i, d := range hashes {
+			e := inner.Contains(d)
+			if !e {
+				panic(i)
+			}
+
+		}
+	}
+}