diff --git a/dsqueue.go b/dsqueue.go index 2eb41b8..870f374 100644 --- a/dsqueue.go +++ b/dsqueue.go @@ -367,7 +367,9 @@ func (q *DSQueue) worker(ctx context.Context, bufferSize, dedupCacheSize int, id idle = false rmMemCount += inBuf.Len() inBuf.Clear() - dedupCache.Purge() + if dedupCache != nil { + dedupCache.Purge() + } rmDSCount, err := q.clearDatastore(ctx) if err != nil { log.Errorw("cannot clear datastore", "err", err, "qname", q.name) diff --git a/dsqueue_test.go b/dsqueue_test.go index ab0c5f7..a481e45 100644 --- a/dsqueue_test.go +++ b/dsqueue_test.go @@ -297,7 +297,7 @@ func TestPersistOneCid(t *testing.T) { func TestDeduplicateCids(t *testing.T) { ds := sync.MutexWrap(datastore.NewMapDatastore()) - queue := dsqueue.New(ds, dsqName) + queue := dsqueue.New(ds, dsqName, dsqueue.WithDedupCacheSize(2*1024)) defer queue.Close() cids := random.Cids(5) diff --git a/option.go b/option.go index 72518a2..13f7272 100644 --- a/option.go +++ b/option.go @@ -5,10 +5,9 @@ import ( ) const ( - DefaultBufferSize = 16 * 1024 - DefaultDedupCacheSize = 2 * 1024 - DefaultIdleWriteTime = time.Minute - DefaultCloseTimeout = 10 * time.Second + DefaultBufferSize = 16 * 1024 + DefaultIdleWriteTime = time.Minute + DefaultCloseTimeout = 10 * time.Second ) // config contains all options for DSQueue. @@ -25,10 +24,9 @@ type Option func(*config) // getOpts creates a config and applies Options to it. func getOpts(opts []Option) config { cfg := config{ - bufferSize: DefaultBufferSize, - dedupCacheSize: DefaultDedupCacheSize, - idleWriteTime: DefaultIdleWriteTime, - closeTimeout: DefaultCloseTimeout, + bufferSize: DefaultBufferSize, + idleWriteTime: DefaultIdleWriteTime, + closeTimeout: DefaultCloseTimeout, } for _, opt := range opts { @@ -52,7 +50,9 @@ func WithBufferSize(n int) Option { } // WithDedupCacheSize sets the size of the LRU cache used to deduplicate items -// in the queue. A value of 0 disables the dedup cache. +// in the queue. +// +// By default, the deduplication cache is disabled (size = 0). func WithDedupCacheSize(n int) Option { return func(c *config) { if n < 0 {