Badger for caching in s3 connector

Author: jakeloo

internal/storage/block_buffer.go

@@ -18,7 +18,22 @@ type BlockBuffer struct {
 	maxBlocks    int
 }
 
-// NewBlockBuffer creates a new block buffer
+// IBlockBuffer defines the interface for block buffer implementations
+type IBlockBuffer interface {
+	Add(blocks []common.BlockData, actualSizeBytes int64) bool
+	Flush() []common.BlockData
+	ShouldFlush() bool
+	Size() (int64, int)
+	IsEmpty() bool
+	GetData() []common.BlockData
+	GetBlocksInRange(chainId *big.Int, startBlock, endBlock *big.Int) []common.BlockData
+	GetBlockByNumber(chainId *big.Int, blockNumber *big.Int) *common.BlockData
+	GetMaxBlockNumber(chainId *big.Int) *big.Int
+	Clear()
+	Stats() BufferStats
+}
+
+// NewBlockBuffer creates a new in-memory block buffer
 func NewBlockBuffer(maxSizeMB int64, maxBlocks int) *BlockBuffer {
 	return &BlockBuffer{
 		data:         make([]common.BlockData, 0),
@@ -27,6 +42,12 @@ func NewBlockBuffer(maxSizeMB int64, maxBlocks int) *BlockBuffer {
 	}
 }
 
+// NewBlockBufferWithBadger creates a new Badger-backed block buffer for better memory management
+// This uses ephemeral storage with optimized settings for caching
+func NewBlockBufferWithBadger(maxSizeMB int64, maxBlocks int) (IBlockBuffer, error) {
+	return NewBadgerBlockBuffer(maxSizeMB, maxBlocks)
+}
+
 // Add adds blocks to the buffer and returns true if flush is needed
 func (b *BlockBuffer) Add(blocks []common.BlockData, actualSizeBytes int64) bool {
 	if len(blocks) == 0 {