update readme, minor fixes

Author: dsa0x

badgerdb.go

@@ -17,6 +17,7 @@ type BadgerStore struct {
 // default temp file path for badgerdb
 var badgerTmpFile = "/tmp/badger.db"
 
+// NewBadger instantiates a new BadgerStore.
 func NewBadger(opts ...badger.Options) *BadgerStore {
 	store := &BadgerStore{
 		dblock: sync.Mutex{},
@@ -86,8 +87,8 @@ func (store *BadgerStore) Put(key, value []byte) error {
 	return err
 }
 
-// IsReady returns true if the store is ready to use.
-func (store *BadgerStore) IsReady() bool {
+// isReady returns true if the store is ready to use.
+func (store *BadgerStore) isReady() bool {
 	return store.db != nil
 }
 

bloom.go

@@ -14,7 +14,7 @@ var ErrKeyNotFound = fmt.Errorf("Key not found")
 
 type BloomFilter struct {
 
-	// The desired false positive rate. e.g. 0.1 error rate implies 1 in 1000
+	// The desired false positive rate
 	err_rate float64
 
 	// the number of items intended to be added to the bloom filter (n)
@@ -53,11 +53,53 @@ func NewBloom(err_rate float64, capacity int, database Store) *BloomFilter {
 		panic("Capacity must be greater than 0")
 	}
 
+	// P = err_rate
+
+	// number of hash functions (k)
+	numHashFn := int(math.Ceil(math.Log2(1.0 / err_rate)))
+
+	//ln22 = ln2^2
+	ln22 := math.Pow(math.Ln2, 2)
+
+	// M
+	bit_width := int(math.Ceil((float64(capacity) * math.Abs(math.Log(err_rate))) /
+		ln22))
+	//m
+	bits_per_slice := bit_width / numHashFn
+
+	seeds := make([]int64, numHashFn)
+
+	for i := 0; i < len(seeds); i++ {
+		seeds[i] = int64((i + 1) << 16)
+	}
+
+	return &BloomFilter{
+		err_rate:  err_rate,
+		capacity:  capacity,
+		bit_width: bit_width,
+		bit_array: make([]bool, bit_width),
+		m:         bits_per_slice,
+		seeds:     seeds,
+		db:        database,
+	}
+}
+
+func NewBloomWithK(err_rate float64, capacity int, database Store, k int) *BloomFilter {
+	if err_rate <= 0 || err_rate >= 1 {
+		panic("Error rate must be between 0 and 1")
+	}
+	if capacity < 0 {
+		panic("Capacity must be greater than 0")
+	}
+
 	// P
 	err_rate /= 100.0
 
 	// number of hash functions (k)
 	numHashFn := int(math.Ceil(math.Log2(1.0 / err_rate)))
+	if k > 0 {
+		numHashFn = int(k)
+	}
 
 	//ln22 = ln2^2
 	ln22 := math.Pow(math.Ln2, 2)
@@ -135,7 +177,7 @@ func (bf *BloomFilter) Get(key []byte) []byte {
 }
 
 func (bf *BloomFilter) hasStore() bool {
-	return bf.db != nil && bf.db.IsReady()
+	return bf.db != nil && bf.db.isReady()
 }
 
 // every checks if each index in the indices array has a value of 1 in the bit array

boltdb.go

@@ -23,6 +23,7 @@ var (
 	bucketName  = "boltstore"
 )
 
+// NewBolt instantiates a new BoltStore.
 func NewBolt(filePath string, filemode os.FileMode, opts ...bolt.Options) *BoltStore {
 	store := &BoltStore{
 		filePath: filePath,
@@ -93,8 +94,8 @@ func (store *BoltStore) Put(key []byte, value []byte) error {
 	return err
 }
 
-// IsReady returns true if the store is ready to use.
-func (store *BoltStore) IsReady() bool {
+// isReady returns true if the store is ready to use.
+func (store *BoltStore) isReady() bool {
 	return store.db != nil
 }
 

cmd/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/binary"
-	"fmt"
 
 	"github.com/dsa0x/gobloomgo"
 )
@@ -18,30 +17,17 @@ func main() {
 	// opts := badger.DefaultOptions("/tmp/bloom.db")
 	// db := gobloomgo.NewBadger(opts)
 
-	bf := gobloomgo.NewBloom(0.1, 59000, nil)
-	// bf := gobloomgo.NewScalableBloom(0.1, 1000, nil)
+	// bf := gobloomgo.NewBloom(0.1, 50000, nil)
+	bf := gobloomgo.NewScalableBloom(0.000001, 10, nil)
 
-	// mp := map[bool]int{}
-
-	// start := time.Now()
-	// bf.Add([]byte("key"), []byte("bar"))
-	ii := 0
-	for i := 0; i < 50000; i++ {
+	for i := 0; i < 100000; i++ {
 		var by [4]byte
 		binary.LittleEndian.PutUint32(by[:], uint32(i))
 		bf.Add(by[:], []byte("bar"))
-		ii++
-		count := bf.Count()
-
-		fmt.Println(by, bf.Find(by[:]), count == ii, "bfCo=", count, "ii=", ii)
-
 	}
+	bf.Add([]byte("foo"), []byte("var"))
 
-	// fmt.Println((0 - bf.Count()), bf.Count(), ii)
-	// for i := 50000; i < 2000; i++ {
-	// 	mp[bf.Find([]byte(fmt.Sprintf("foo%d", i)))] += 1
-	// }
-	// fmt.Printf("%s %v\n", bf.Get([]byte("key")), mp)
-	// fmt.Println(bf.Capacity(), time.Since(start))
+	// fmt.Printf("Count: %d, Capacity: %d, ExpCap: %.f\n", bf.Count(), bf.Capacity(), bf.ExpCapacity())
+	// fmt.Println(bf.Capacity(), bf.ExpCapacity(), bf.Count(), bf.Prob())
 
 }

readme.md

@@ -6,7 +6,7 @@ Bloomdb implements a bloom filter in Go, while using boltdb and badgerdb as opti
 
 Bloomdb also implement a scalable bloom filter described in a paper written by [P. Almeida, C.Baquero, N. Preguiça, D. Hutchison](https://haslab.uminho.pt/cbm/files/dbloom.pdf).
 
-A scalable bloom filter removes the need for an apriori filter size as expected by the basic bloom filter, while preserving the desired false positive rate by scaling the filter as needed.
+A scalable bloom filter allows you to grow the filter as needed, and removes the need for an apriori filter size as expected by the basic bloom filter, while preserving the desired false positive rate by scaling the filter as needed.
 
 ### Installation
 
@@ -74,7 +74,7 @@ import (
 	"fmt"
 
 	"github.com/dgraph-io/badger/v3"
-	gobloomgo "github.com/dsa0x/gobloomgo"
+	"github.com/dsa0x/gobloomgo"
 )
 
 func main() {
@@ -86,3 +86,7 @@ func main() {
 	fmt.Printf("%s\n", bf.Get([]byte("key")))
 }
 ```
+
+#### References
+
+1. [P. Almeida, C.Baquero, N. Preguiça, D. Hutchison](https://haslab.uminho.pt/cbm/files/dbloom.pdf)

scalable_bloom.go

@@ -1,14 +1,11 @@
 package gobloomgo
 
 import (
-	"fmt"
 	"math"
 )
 
-// https://haslab.uminho.pt/cbm/files/dbloom.pdf
-
 type ScalableBloomFilter struct {
-	// The desired false positive rate. e.g. 0.1 error rate implies 1 in 1000
+	// The desired false positive rate
 	err_rate float64
 
 	// the number of items intended to be added to the bloom filter
@@ -27,9 +24,9 @@ type ScalableBloomFilter struct {
 type GrowthRate uint
 
 var (
-	// GrowthRateSmall represents a small expected growth rate
+	// GrowthRateSmall represents a small expected set growth
 	GrowthRateSmall GrowthRate = 2
-	// GrowthRateLarge represents a large expected growth rate
+	// GrowthRateLarge represents a large expected set growth
 	GrowthRateLarge GrowthRate = 4
 )
 
@@ -56,7 +53,7 @@ func NewScalableBloom(err_rate float64, initial_capacity int, database Store, gr
 		err_rate:    err_rate,
 		capacity:    initial_capacity,
 		growth_rate: _growth_rate,
-		ratio:       0.9,
+		ratio:       0.9, // Source: [1]
 		m0:          initialFilter.m,
 		filters:     []*BloomFilter{initialFilter},
 		db:          database,
@@ -100,10 +97,9 @@ func (sbf *ScalableBloomFilter) Top() *BloomFilter {
 // grow increases the capacity of the bloom filter by adding a new filter
 func (sbf *ScalableBloomFilter) grow() {
 	err_rate := sbf.err_rate * math.Pow(sbf.ratio, float64(len(sbf.filters)))
-
-	// newCapacity = m0 * growth_rate^i * ln2
-	newCapacity := sbf.m0 * int(math.Pow(float64(sbf.growth_rate), float64(len(sbf.filters))+1.0)*math.Ln2)
-	newFilter := NewBloom(err_rate, newCapacity, sbf.db)
+	i := float64(len(sbf.filters)) - 1.0
+	newCapacity := float64(sbf.m0) * float64(math.Pow(float64(sbf.growth_rate), i)) * math.Ln2
+	newFilter := NewBloom(err_rate, int(newCapacity), sbf.db)
 	sbf.filters = append(sbf.filters, newFilter)
 }
 
@@ -129,13 +125,35 @@ func (sbf *ScalableBloomFilter) filterSize() int {
 func (sbf *ScalableBloomFilter) getStore() Store {
 	return sbf.db
 }
+
+// Count returns the number of items added to the bloom filter
 func (sbf *ScalableBloomFilter) Count() int {
-	sum, smm := 0, 0
-	for i, filter := range sbf.filters {
+	sum := 0
+	for _, filter := range sbf.filters {
 		sum += filter.count
-		smm += filter.capacity
-		// fmt.Println(sum, smm, i)
-		fmt.Sprintln(filter.count, "i=", i)
 	}
 	return sum
 }
+func (sbf *ScalableBloomFilter) bitWidth() int {
+	sum := 0
+	for _, filter := range sbf.filters {
+		sum += filter.bit_width
+	}
+	return sum
+}
+
+func (sbf *ScalableBloomFilter) prob() float64 {
+	sum := 1.0
+	for i, _ := range sbf.filters {
+		sum *= 1.0 - (sbf.err_rate * math.Pow(sbf.ratio, float64(i)))
+	}
+	return 1.0 - sum
+}
+
+func (sbf *ScalableBloomFilter) expCapacity() float64 {
+	sum := 0
+	for i, _ := range sbf.filters {
+		sum += int(math.Pow(float64(sbf.growth_rate), float64(i)))
+	}
+	return float64(sum*sbf.m0) * math.Ln2
+}

storage.go

@@ -12,7 +12,7 @@ type Store interface {
 	Close() error
 	Get(key []byte) ([]byte, error)
 	Put(key, value []byte) error
-	IsReady() bool
+	isReady() bool
 }
 
 type StoreOptions struct {