add filter.Merge

Author: dsa0x

badgerdb.go

@@ -92,4 +92,8 @@ func (store *BadgerStore) isReady() bool {
 	return store.db != nil
 }
 
+func (store *BadgerStore) DB() interface{} {
+	return store.db
+}
+
 var _ Store = (*BadgerStore)(nil)

bloom.go

@@ -38,6 +38,9 @@ type BloomFilter struct {
 	// m is the number bits per slice(hashFn)
 	m int
 
+	// k is the number of hash functions
+	k int
+
 	// one seed per hash function
 	seeds []int64
 
@@ -132,6 +135,7 @@ func NewBloom(opts *BloomOptions) *BloomFilter {
 		lock:      sync.Mutex{},
 		byteSize:  byteSize,
 		path:      opts.Path,
+		k:         numHashFn,
 	}
 }
 
@@ -209,6 +213,30 @@ func (bf *BloomFilter) Get(key []byte) []byte {
 
 }
 
+// Merge merges the filter with another bloom filter.
+// Both filters must have the same capacity and error rate.
+// merging increases the false positive rate of the resulting filter
+func (bf *BloomFilter) Merge(bf2 *BloomFilter) error {
+	if bf.k != bf2.k {
+		return fmt.Errorf("BloomFilter k values do not match")
+	}
+	if bf.bit_width != bf2.bit_width {
+		return fmt.Errorf("BloomFilter bit_width values do not match")
+	}
+
+	bf.lock.Lock()
+	defer bf.lock.Unlock()
+
+	bf2.lock.Lock()
+	defer bf2.lock.Unlock()
+
+	for i := 0; i < bf.bit_width; i++ {
+		bf.mem[i] |= bf2.mem[i]
+	}
+
+	return nil
+}
+
 func (bf *BloomFilter) hasStore() bool {
 	return bf.db != nil && bf.db.isReady()
 }
@@ -297,6 +325,11 @@ func (bf *BloomFilter) FilterSize() int {
 	return bf.bit_width
 }
 
+// DB returns the underlying persistent store
+func (bf *BloomFilter) DB() interface{} {
+	return bf.db.DB()
+}
+
 // divmod returns the quotient and remainder of a/b
 func divmod(num, denom int64) (quot, rem int64) {
 	quot = num / denom

bloom_test.go

@@ -126,6 +126,63 @@ func TestBloomFilter_Add(t *testing.T) {
 		t.Errorf("Expected function to panic when there is no persistent store, got %s", val)
 	})
 }
+
+func TestBloomFilter_Merge(t *testing.T) {
+	opts := &BloomOptions{
+		Err_rate: 0.01,
+		Capacity: 1000,
+		Path:     "./test.db",
+	}
+	bf := NewBloom(opts)
+	bf2 := NewBloom(opts)
+
+	defer func() {
+		bf.Close()
+		bf2.Close()
+		os.Remove(opts.Path)
+	}()
+
+	t.Run("merge success", func(t *testing.T) {
+		err := bf.Merge(bf2)
+		if err != nil {
+			t.Errorf("Expected no error, got %v", err)
+		}
+	})
+
+	t.Run("merge should return an error when the filters dont match", func(t *testing.T) {
+		opts := &BloomOptions{
+			Err_rate: 0.01,
+			Capacity: 10000,
+			Path:     "./test.db",
+		}
+		bf2 := NewBloom(opts)
+		bf.Merge(bf2)
+		defer func() {
+			bf2.Close()
+			os.Remove(opts.Path)
+		}()
+
+		err := bf.Merge(bf2)
+		if err == nil {
+			t.Errorf("Expected error, got nil")
+		}
+	})
+
+	t.Run("object added to the single filters should be found in the resulting merge", func(t *testing.T) {
+		key, val := []byte("foo"), []byte("bar")
+		bf := NewBloom(opts)
+		bf2 := NewBloom(opts)
+		bf2.Add(key, val)
+		err := bf.Merge(bf2)
+		if err != nil {
+			t.Errorf("Expected no error, got %v", err)
+		}
+		if !bf.Contains(key) {
+			t.Errorf("Expected key %s to be found in the merged filter", string(key))
+		}
+	})
+
+}
 func TestBloomFilter_AddToDB(t *testing.T) {
 	store, cleanupFunc := DBSetupTest(t)
 	defer cleanupFunc()

boltdb.go

@@ -99,4 +99,8 @@ func (store *BoltStore) isReady() bool {
 	return store.db != nil
 }
 
+func (store *BoltStore) DB() interface{} {
+	return store.db
+}
+
 var _ Store = (*BoltStore)(nil)

readme.md

@@ -4,7 +4,7 @@ A bloom filter is a probabilistic data structure that is used to determine if an
 
 To fulfil the false positive rate, bloom filters are initialized with a capacity. The capacity is the number of elements that can be inserted into the bloom filter, and this cannot be changed.
 
-Sprout implements a bloom filter in Go, while using boltdb and badgerdb as optional in-memory persistent storage. Sprout writes the bloom filter to a memory-mapped file.
+Sprout implements a bloom filter in Go, while using boltdb and badgerdb as optional in-memory persistent storage for the values. The bloom filter is written to a memory-mapped file.
 
 Sprout also implement a scalable bloom filter described in a paper written by [P. Almeida, C.Baquero, N. Preguiça, D. Hutchison](https://haslab.uminho.pt/cbm/files/dbloom.pdf).
 

storage.go

@@ -1,23 +1,10 @@
 package sprout
 
-import (
-	"os"
-
-	badger "github.com/dgraph-io/badger/v3"
-	bolt "go.etcd.io/bbolt"
-)
-
 type Store interface {
 	open() error
 	Close() error
 	Get(key []byte) ([]byte, error)
 	Put(key, value []byte) error
 	isReady() bool
-}
-
-type StoreOptions struct {
-	BucketName string
-	Filemode   os.FileMode
-	boltOpts   *bolt.Options
-	badgerOpts *badger.Options
+	DB() interface{}
 }