disable dedup cache by default

dsqueue.go

@@ -367,7 +367,9 @@ func (q *DSQueue) worker(ctx context.Context, bufferSize, dedupCacheSize int, id
 			idle = false
 			rmMemCount += inBuf.Len()
 			inBuf.Clear()
-			dedupCache.Purge()
+			if dedupCache != nil {
+				dedupCache.Purge()
+			}
 			rmDSCount, err := q.clearDatastore(ctx)
 			if err != nil {
 				log.Errorw("cannot clear datastore", "err", err, "qname", q.name)

dsqueue_test.go

@@ -297,7 +297,7 @@ func TestPersistOneCid(t *testing.T) {
 
 func TestDeduplicateCids(t *testing.T) {
 	ds := sync.MutexWrap(datastore.NewMapDatastore())
-	queue := dsqueue.New(ds, dsqName)
+	queue := dsqueue.New(ds, dsqName, dsqueue.WithDedupCacheSize(2*1024))
 	defer queue.Close()
 
 	cids := random.Cids(5)

option.go

@@ -5,10 +5,9 @@ import (
 )
 
 const (
-	DefaultBufferSize     = 16 * 1024
-	DefaultDedupCacheSize = 2 * 1024
-	DefaultIdleWriteTime  = time.Minute
-	DefaultCloseTimeout   = 10 * time.Second
+	DefaultBufferSize    = 16 * 1024
+	DefaultIdleWriteTime = time.Minute
+	DefaultCloseTimeout  = 10 * time.Second
 )
 
 // config contains all options for DSQueue.
@@ -25,10 +24,9 @@ type Option func(*config)
 // getOpts creates a config and applies Options to it.
 func getOpts(opts []Option) config {
 	cfg := config{
-		bufferSize:     DefaultBufferSize,
-		dedupCacheSize: DefaultDedupCacheSize,
-		idleWriteTime:  DefaultIdleWriteTime,
-		closeTimeout:   DefaultCloseTimeout,
+		bufferSize:    DefaultBufferSize,
+		idleWriteTime: DefaultIdleWriteTime,
+		closeTimeout:  DefaultCloseTimeout,
 	}
 
 	for _, opt := range opts {
@@ -52,7 +50,9 @@ func WithBufferSize(n int) Option {
 }
 
 // WithDedupCacheSize sets the size of the LRU cache used to deduplicate items
-// in the queue. A value of 0 disables the dedup cache.
+// in the queue.
+//
+// By default, the deduplication cache is disabled (size = 0).
 func WithDedupCacheSize(n int) Option {
 	return func(c *config) {
 		if n < 0 {