Merge pull request #6285 from oasisprotocol/peternose/trivial/persistent-lb-store

go/consensus/cometbft/light: Use persistent store for stateless client

Author: peternose

.changelog/6285.trivial.md

@@ -45,7 +45,7 @@ func New(
 		}
 		return node, nil
 	case config.ModeStatelessClient:
-		node, err := createStatelessNode(ctx, identity, genesis, doc, genesisDoc, p2p)
+		node, err := createStatelessNode(ctx, dataDir, identity, genesis, doc, genesisDoc, p2p)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create stateless node: %w", err)
 		}
@@ -97,6 +97,7 @@ func createFullNode(
 
 func createStatelessNode(
 	ctx context.Context,
+	dataDir string,
 	identity *identity.Identity,
 	genesis genesisAPI.Provider,
 	doc *genesisAPI.Document,
@@ -108,7 +109,7 @@ func createStatelessNode(
 		return nil, fmt.Errorf("failed to create provider: %w", err)
 	}
 
-	lightClient, err := createLightClient(ctx, genesisDoc, doc, p2p)
+	lightClient, err := createLightClient(ctx, dataDir, genesisDoc, doc, p2p)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create light client: %w", err)
 	}
@@ -164,6 +165,7 @@ func createProvider(identity *identity.Identity) (consensusAPI.Backend, error) {
 
 func createLightClient(
 	ctx context.Context,
+	dataDir string,
 	genesisDoc *cmttypes.GenesisDoc,
 	doc *genesisAPI.Document,
 	p2p p2pAPI.Service,
@@ -180,6 +182,7 @@ func createLightClient(
 			Height: int64(config.GlobalConfig.Consensus.LightClient.Trust.Height),
 			Hash:   hash,
 		},
+		DataDir: dataDir,
 	}
 
 	return light.NewClient(ctx, doc.ChainContext(), p2p, cfg)

go/consensus/cometbft/cometbft.go

@@ -4,10 +4,11 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"path/filepath"
 	"sync"
 	"time"
 
-	dbm "github.com/cometbft/cometbft-db"
+	cmtdb "github.com/cometbft/cometbft-db"
 	cmtlight "github.com/cometbft/cometbft/light"
 	cmtlightprovider "github.com/cometbft/cometbft/light/provider"
 	cmtlightdb "github.com/cometbft/cometbft/light/store/db"
@@ -17,16 +18,36 @@ import (
 	"github.com/oasisprotocol/oasis-core/go/config"
 	consensus "github.com/oasisprotocol/oasis-core/go/consensus/api"
 	"github.com/oasisprotocol/oasis-core/go/consensus/cometbft/common"
+	cdb "github.com/oasisprotocol/oasis-core/go/consensus/cometbft/db"
 	"github.com/oasisprotocol/oasis-core/go/p2p/rpc"
 )
 
+const (
+	// dbName is the name of the database used to store trusted light blocks.
+	dbName = "consensus/light"
+
+	// storeHighWatermark is the maximum number of blocks the pruned store
+	// can hold before triggering pruning.
+	storeHighWatermark = 21_000
+
+	// storeLowWatermark is the number of blocks to retain in the pruned store
+	// after pruning is triggered.
+	storeLowWatermark = 20_000
+)
+
 // Config is the configuration for the light client.
 type Config struct {
 	// GenesisDocument is the CometBFT genesis document.
 	GenesisDocument *cmttypes.GenesisDoc
 
 	// TrustOptions are CometBFT light client trust options.
 	TrustOptions cmtlight.TrustOptions
+
+	// DataDir is the node's data directory.
+	//
+	// If set, a persistent store is used to store trusted light blocks.
+	// Otherwise, an in-memory store is used.
+	DataDir string
 }
 
 // Client is a CometBFT consensus light client that talks with remote oasis-nodes that are using
@@ -40,10 +61,7 @@ type Client struct {
 	lightClient *lazyClient
 }
 
-// NewClient creates an internal and non-persistent light client.
-//
-// This client is instantiated from the provided (obtained out of bound) trusted block
-// and is used internally for CometBFT's state sync protocol.
+// NewClient creates a new light client.
 func NewClient(ctx context.Context, chainContext string, p2p rpc.P2P, cfg Config) (*Client, error) {
 	pool := NewProviderPool(ctx, chainContext, p2p)
 	providers := make([]*Provider, 0, numWitnesses+1)
@@ -57,12 +75,27 @@ func NewClient(ctx context.Context, chainContext string, p2p rpc.P2P, cfg Config
 		witnesses = append(witnesses, provider)
 	}
 
+	var db cmtdb.DB
+	switch cfg.DataDir {
+	case "":
+		db = cmtdb.NewMemDB()
+	default:
+		fn := filepath.Join(cfg.DataDir, dbName)
+		cdb, err := cdb.New(fn, false)
+		if err != nil {
+			return nil, err
+		}
+		db = cmtdb.NewPrefixDB(cdb, []byte{})
+	}
+	store := cmtlightdb.New(db, "")
+	store = newPrunedStore(store, storeHighWatermark, storeLowWatermark)
+
 	lightClient, err := newLazyClient(
 		cfg.GenesisDocument.ChainID,
 		cfg.TrustOptions,
 		primary,
 		witnesses,
-		cmtlightdb.New(dbm.NewMemDB(), ""),
+		store,
 		cmtlight.MaxRetryAttempts(lcMaxRetryAttempts), // TODO: Make this configurable.
 		cmtlight.Logger(common.NewLogAdapter(!config.GlobalConfig.Consensus.LogDebug)),
 		cmtlight.DisableProviderRemoval(),

go/consensus/cometbft/light/client.go

@@ -0,0 +1,73 @@
+package light
+
+import (
+	"github.com/cometbft/cometbft/light/store"
+	cmttypes "github.com/cometbft/cometbft/types"
+)
+
+// prunedStore is a light block store that automatically prunes old blocks.
+//
+// When the number of stored blocks exceeds the high watermark, it prunes
+// blocks until the number of stored blocks reaches the low watermark.
+type prunedStore struct {
+	store store.Store
+	high  uint16
+	low   uint16
+}
+
+// newPrunedStore creates a new store backed by the given store,
+// configured with high and low watermarks.
+//
+// If high is set to zero, automatic pruning is disabled.
+func newPrunedStore(store store.Store, high uint16, low uint16) *prunedStore {
+	return &prunedStore{
+		store: store,
+		high:  high,
+		low:   low,
+	}
+}
+
+// DeleteLightBlock implements store.Store.
+func (p *prunedStore) DeleteLightBlock(height int64) error {
+	return p.store.DeleteLightBlock(height)
+}
+
+// FirstLightBlockHeight implements store.Store.
+func (p *prunedStore) FirstLightBlockHeight() (int64, error) {
+	return p.store.FirstLightBlockHeight()
+}
+
+// LastLightBlockHeight implements store.Store.
+func (p *prunedStore) LastLightBlockHeight() (int64, error) {
+	return p.store.LastLightBlockHeight()
+}
+
+// LightBlock implements store.Store.
+func (p *prunedStore) LightBlock(height int64) (*cmttypes.LightBlock, error) {
+	return p.store.LightBlock(height)
+}
+
+// LightBlockBefore implements store.Store.
+func (p *prunedStore) LightBlockBefore(height int64) (*cmttypes.LightBlock, error) {
+	return p.store.LightBlockBefore(height)
+}
+
+// Prune implements store.Store.
+func (p *prunedStore) Prune(size uint16) error {
+	return p.store.Prune(size)
+}
+
+// SaveLightBlock implements store.Store.
+func (p *prunedStore) SaveLightBlock(lb *cmttypes.LightBlock) error {
+	if p.high > 0 && p.Size() >= p.high {
+		if err := p.Prune(p.low); err != nil {
+			return err
+		}
+	}
+	return p.store.SaveLightBlock(lb)
+}
+
+// Size implements store.Store.
+func (p *prunedStore) Size() uint16 {
+	return p.store.Size()
+}

go/consensus/cometbft/light/store.go

@@ -0,0 +1,109 @@
+package light
+
+import (
+	"testing"
+
+	cmttypes "github.com/cometbft/cometbft/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPrunedStore(t *testing.T) {
+	t.Run("Pruning enabled", func(t *testing.T) {
+		high := uint16(5)
+		low := uint16(2)
+		store := newMockStore()
+		ps := newPrunedStore(store, high, low)
+
+		// Fill the store.
+		for i := range high {
+			lb := &cmttypes.LightBlock{
+				SignedHeader: &cmttypes.SignedHeader{
+					Header: &cmttypes.Header{
+						Height: int64(i),
+					},
+				},
+			}
+			err := ps.SaveLightBlock(lb)
+			require.NoError(t, err)
+			require.Equal(t, i+1, ps.Size())
+		}
+
+		// Trigger pruning.
+		lb := &cmttypes.LightBlock{
+			SignedHeader: &cmttypes.SignedHeader{
+				Header: &cmttypes.Header{
+					Height: int64(high),
+				},
+			},
+		}
+		err := ps.SaveLightBlock(lb)
+		require.NoError(t, err)
+		require.Equal(t, low+1, ps.Size())
+	})
+
+	t.Run("Pruning disabled", func(t *testing.T) {
+		high := uint16(0)
+		low := uint16(0)
+		store := newMockStore()
+		ps := newPrunedStore(store, high, low)
+
+		// Fill the store.
+		for i := range 2 * high {
+			lb := &cmttypes.LightBlock{
+				SignedHeader: &cmttypes.SignedHeader{
+					Header: &cmttypes.Header{
+						Height: int64(i),
+					},
+				},
+			}
+			err := ps.SaveLightBlock(lb)
+			require.NoError(t, err)
+			require.Equal(t, i+1, ps.Size())
+		}
+	})
+}
+
+type mockStore struct {
+	blocks []*cmttypes.LightBlock
+}
+
+func newMockStore() *mockStore {
+	return &mockStore{
+		blocks: make([]*cmttypes.LightBlock, 0),
+	}
+}
+
+func (s *mockStore) SaveLightBlock(lb *cmttypes.LightBlock) error {
+	s.blocks = append(s.blocks, lb)
+	return nil
+}
+
+func (s *mockStore) DeleteLightBlock(int64) error {
+	panic("not implemented")
+}
+
+func (s *mockStore) FirstLightBlockHeight() (int64, error) {
+	panic("not implemented")
+}
+
+func (s *mockStore) LastLightBlockHeight() (int64, error) {
+	panic("not implemented")
+}
+
+func (s *mockStore) LightBlock(int64) (*cmttypes.LightBlock, error) {
+	panic("not implemented")
+}
+
+func (s *mockStore) LightBlockBefore(int64) (*cmttypes.LightBlock, error) {
+	panic("not implemented")
+}
+
+func (s *mockStore) Size() uint16 {
+	return uint16(len(s.blocks))
+}
+
+func (s *mockStore) Prune(size uint16) error {
+	n := max(0, len(s.blocks)-int(size))
+	s.blocks = s.blocks[n:]
+	return nil
+}