From a57a79adb2923be64a46333aafdb54b12bcd0876 Mon Sep 17 00:00:00 2001 From: Eoous <38656355+Eoous@users.noreply.github.com> Date: Thu, 17 Apr 2025 16:36:18 +0000 Subject: [PATCH 1/4] basic config --- pkg/config/config.go | 27 +++++++++++++++++++++++++++ pkg/config/defaults.go | 6 ++++++ 2 files changed, 33 insertions(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index ee1a34bf08..12e108f4be 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -43,6 +43,11 @@ const ( // FlagLazyBlockTime is a flag for specifying the maximum interval between blocks in lazy aggregation mode FlagLazyBlockTime = "rollkit.node.lazy_block_time" + // Pruning configuration flags + FlagPruningStrategy = "rollkit.node.pruning.strategy" + FlagPruningKeepRecent = "rollkit.node.pruning.keep_recent" + FlagPruningKeepEvery = "rollkit.node.pruning.keep_every" + FlagPruningInterval = "rollkit.node.pruning.interval" // Data Availability configuration flags // FlagDAAddress is a flag for specifying the data availability layer address @@ -169,6 +174,22 @@ type NodeConfig struct { // Header configuration TrustedHash string `mapstructure:"trusted_hash" yaml:"trusted_hash" comment:"Initial trusted hash used to bootstrap the header exchange service. Allows nodes to start synchronizing from a specific trusted point in the chain instead of genesis. When provided, the node will fetch the corresponding header/block from peers using this hash and use it as a starting point for synchronization. If not provided, the node will attempt to fetch the genesis block instead."` + + // Pruning management configuration + Pruning PruningConfig `mapstructure:"pruning" yaml:"pruning"` +} + +// PruningConfig allows node operators to manage storage +type PruningConfig struct { + // todo: support volume-based strategy + Strategy string `mapstructure:"strategy" yaml:"strategy" comment:"Strategy determines the pruning approach (none, default, custom)"` + KeepRecent uint64 `mapstructure:"keep_recent" yaml:"keep_recent" comment:"Number of recent blocks to keep, used in \"default\" and \"custom\" strategies"` + KeepEvery uint64 `mapstructure:"keep_every" yaml:"keep_every" comment:"Periodic blocks to keep for historical reference, used in \"custom\" strategy"` + Interval uint64 `mapstructure:"interval" yaml:"interval" comment:"how offen the pruning process should run"` + + // todo: support volume-based strategy + // VolumeConfig specifies configuration for volume-based storage + // VolumeConfig *VolumeStorageConfig `mapstructure:"volume_config" yaml:"volume_config"` } // LogConfig contains all logging configuration parameters @@ -244,6 +265,12 @@ func AddFlags(cmd *cobra.Command) { cmd.Flags().Uint64(FlagMaxPendingBlocks, def.Node.MaxPendingBlocks, "maximum blocks pending DA confirmation before pausing block production (0 for no limit)") cmd.Flags().Duration(FlagLazyBlockTime, def.Node.LazyBlockTime.Duration, "maximum interval between blocks in lazy aggregation mode") + // Pruning configuration flags + cmd.Flags().String(FlagPruningStrategy, def.Node.Pruning.Strategy, "") + cmd.Flags().Uint64(FlagPruningKeepRecent, def.Node.Pruning.KeepRecent, "") + cmd.Flags().Uint64(FlagPruningKeepEvery, def.Node.Pruning.KeepEvery, "") + cmd.Flags().Uint64(FlagPruningInterval, def.Node.Pruning.Interval, "") + // Data Availability configuration flags cmd.Flags().String(FlagDAAddress, def.DA.Address, "DA address (host:port)") cmd.Flags().String(FlagDAAuthToken, def.DA.AuthToken, "DA auth token") diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index b3d52cd521..300879013a 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -51,6 +51,12 @@ var DefaultConfig = Config{ LazyBlockTime: DurationWrapper{60 * time.Second}, Light: false, TrustedHash: "", + Pruning: PruningConfig{ + Strategy: "none", + KeepRecent: 0, + KeepEvery: 0, + Interval: 0, + }, }, DA: DAConfig{ Address: "http://localhost:7980", From 47ccefc50cc924aa838b53fba6ad645324cd86b5 Mon Sep 17 00:00:00 2001 From: Eoous <38656355+Eoous@users.noreply.github.com> Date: Tue, 22 Apr 2025 02:33:00 +0800 Subject: [PATCH 2/4] add: PruningStore interface and DefaultPruningStore struct --- pkg/store/pruning.go | 90 ++++++++++++++++++++++++++++++++++++++++++++ pkg/store/store.go | 25 +++++++++++- pkg/store/types.go | 6 +++ 3 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 pkg/store/pruning.go diff --git a/pkg/store/pruning.go b/pkg/store/pruning.go new file mode 100644 index 0000000000..7daa856a0b --- /dev/null +++ b/pkg/store/pruning.go @@ -0,0 +1,90 @@ +package store + +import ( + "context" + + "github.com/rollkit/rollkit/types" +) + +type DefaultPruningStore struct { + Store +} + +var _ PruningStore = &DefaultPruningStore{} + +// NewDefaultPruningStore returns default pruning store. +func NewDefaultPruningStore(store Store) PruningStore { + return &DefaultPruningStore{ + Store: store, + } +} + +// Close safely closes underlying data storage, to ensure that data is actually saved. +func (s *DefaultPruningStore) Close() error { + return s.Store.Close() +} + +// SetHeight sets the height saved in the Store if it is higher than the existing height +func (s *DefaultPruningStore) SetHeight(ctx context.Context, height uint64) error { + return s.Store.SetHeight(ctx, height) +} + +// Height returns height of the highest block saved in the Store. +func (s *DefaultPruningStore) Height(ctx context.Context) (uint64, error) { + return s.Store.Height(ctx) +} + +// SaveBlockData adds block header and data to the store along with corresponding signature. +// Stored height is updated if block height is greater than stored value. +func (s *DefaultPruningStore) SaveBlockData(ctx context.Context, header *types.SignedHeader, data *types.Data, signature *types.Signature) error { + // todo: prune blocks + return s.Store.SaveBlockData(ctx, header, data, signature) +} + +// DeleteBlockData deletes block at given height. +func (s *DefaultPruningStore) DeleteBlockData(ctx context.Context, height uint64) error { + return s.Store.DeleteBlockData(ctx, height) +} + +// GetBlockData returns block header and data at given height, or error if it's not found in Store. +func (s *DefaultPruningStore) GetBlockData(ctx context.Context, height uint64) (*types.SignedHeader, *types.Data, error) { + return s.Store.GetBlockData(ctx, height) +} + +// GetBlockByHash returns block with given block header hash, or error if it's not found in Store. +func (s *DefaultPruningStore) GetBlockByHash(ctx context.Context, hash []byte) (*types.SignedHeader, *types.Data, error) { + return s.Store.GetBlockByHash(ctx, hash) +} + +// GetSignatureByHash returns signature for a block at given height, or error if it's not found in Store. +func (s *DefaultPruningStore) GetSignatureByHash(ctx context.Context, hash []byte) (*types.Signature, error) { + return s.Store.GetSignatureByHash(ctx, hash) +} + +// GetSignature returns signature for a block with given block header hash, or error if it's not found in Store. +func (s *DefaultPruningStore) GetSignature(ctx context.Context, height uint64) (*types.Signature, error) { + return s.Store.GetSignature(ctx, height) +} + +// UpdateState updates state saved in Store. Only one State is stored. +// If there is no State in Store, state will be saved. +func (s *DefaultPruningStore) UpdateState(ctx context.Context, state types.State) error { + return s.Store.UpdateState(ctx, state) +} + +// GetState returns last state saved with UpdateState. +func (s *DefaultPruningStore) GetState(ctx context.Context) (types.State, error) { + return s.Store.GetState(ctx) +} + +// SetMetadata saves arbitrary value in the store. +// +// Metadata is separated from other data by using prefix in KV. +func (s *DefaultPruningStore) SetMetadata(ctx context.Context, key string, value []byte) error { + return s.Store.SetMetadata(ctx, key, value) +} + +// GetMetadata returns values stored for given key with SetMetadata. +func (s *DefaultPruningStore) GetMetadata(ctx context.Context, key string) ([]byte, error) { + return s.Store.GetMetadata(ctx, key) +} diff --git a/pkg/store/store.go b/pkg/store/store.go index ccb1268dec..583e63363c 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -92,7 +92,7 @@ func (s *DefaultStore) SaveBlockData(ctx context.Context, header *types.SignedHe batch, err := s.db.Batch(ctx) if err != nil { - return fmt.Errorf("failed to create a new batch: %w", err) + return fmt.Errorf("failed to create a new batch for saving block data: %w", err) } if err := batch.Put(ctx, ds.NewKey(getHeaderKey(height)), headerBlob); err != nil { @@ -114,6 +114,29 @@ func (s *DefaultStore) SaveBlockData(ctx context.Context, header *types.SignedHe return nil } +// DeleteBlockData deletes block at given height. +func (s *DefaultStore) DeleteBlockData(ctx context.Context, height uint64) error { + batch, err := s.db.Batch(ctx) + if err != nil { + return fmt.Errorf("failed to create a new batch for deleting block data: %w", err) + } + + if err := batch.Delete(ctx, ds.NewKey(getHeaderKey(height))); err != nil { + return fmt.Errorf("failed to delete header blob in batch: %w", err) + } + if err := batch.Delete(ctx, ds.NewKey(getDataKey(height))); err != nil { + return fmt.Errorf("failed to delete data blob in batch: %w", err) + } + if err := batch.Delete(ctx, ds.NewKey(getSignatureKey(height))); err != nil { + return fmt.Errorf("failed to delete signature of block blob in batch: %w", err) + } + if err := batch.Commit(ctx); err != nil { + return fmt.Errorf("failed to commit batch: %w", err) + } + + return nil +} + // TODO: We unmarshal the header and data here, but then we re-marshal them to proto to hash or send them to DA, we should not unmarshal them here and allow the caller to handle them as needed. // GetBlockData returns block header and data at given height, or error if it's not found in Store. diff --git a/pkg/store/types.go b/pkg/store/types.go index efd768ddb3..b997b9b62b 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -16,6 +16,8 @@ type Store interface { // SaveBlock saves block along with its seen signature (which will be included in the next block). SaveBlockData(ctx context.Context, header *types.SignedHeader, data *types.Data, signature *types.Signature) error + // DeleteBlockData deletes block at given height. + DeleteBlockData(ctx context.Context, height uint64) error // GetBlock returns block at given height, or error if it's not found in Store. GetBlockData(ctx context.Context, height uint64) (*types.SignedHeader, *types.Data, error) @@ -44,3 +46,7 @@ type Store interface { // Close safely closes underlying data storage, to ensure that data is actually saved. Close() error } + +type PruningStore interface { + Store +} From 5a8fb4a6c6527135897b32bf522e899480eb19e9 Mon Sep 17 00:00:00 2001 From: Eoous <38656355+Eoous@users.noreply.github.com> Date: Thu, 17 Apr 2025 16:36:18 +0000 Subject: [PATCH 3/4] basic config --- pkg/config/config.go | 8 ++++++ pkg/config/defaults.go | 1 + pkg/config/pruning_config.go | 54 ++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 pkg/config/pruning_config.go diff --git a/pkg/config/config.go b/pkg/config/config.go index ee1a34bf08..ab193fe686 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -169,6 +169,9 @@ type NodeConfig struct { // Header configuration TrustedHash string `mapstructure:"trusted_hash" yaml:"trusted_hash" comment:"Initial trusted hash used to bootstrap the header exchange service. Allows nodes to start synchronizing from a specific trusted point in the chain instead of genesis. When provided, the node will fetch the corresponding header/block from peers using this hash and use it as a starting point for synchronization. If not provided, the node will attempt to fetch the genesis block instead."` + + // Pruning management configuration + Pruning PruningConfig `mapstructure:"pruning" yaml:"pruning"` } // LogConfig contains all logging configuration parameters @@ -244,6 +247,11 @@ func AddFlags(cmd *cobra.Command) { cmd.Flags().Uint64(FlagMaxPendingBlocks, def.Node.MaxPendingBlocks, "maximum blocks pending DA confirmation before pausing block production (0 for no limit)") cmd.Flags().Duration(FlagLazyBlockTime, def.Node.LazyBlockTime.Duration, "maximum interval between blocks in lazy aggregation mode") + // Pruning configuration flags + cmd.Flags().String(FlagPruningStrategy, def.Node.Pruning.Strategy, "") + cmd.Flags().Uint64(FlagPruningKeepRecent, def.Node.Pruning.KeepRecent, "") + cmd.Flags().Uint64(FlagPruningInterval, def.Node.Pruning.Interval, "") + // Data Availability configuration flags cmd.Flags().String(FlagDAAddress, def.DA.Address, "DA address (host:port)") cmd.Flags().String(FlagDAAuthToken, def.DA.AuthToken, "DA auth token") diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index b3d52cd521..cca0fee87e 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -51,6 +51,7 @@ var DefaultConfig = Config{ LazyBlockTime: DurationWrapper{60 * time.Second}, Light: false, TrustedHash: "", + Pruning: PruningConfigDefault, }, DA: DAConfig{ Address: "http://localhost:7980", diff --git a/pkg/config/pruning_config.go b/pkg/config/pruning_config.go new file mode 100644 index 0000000000..a9475457ed --- /dev/null +++ b/pkg/config/pruning_config.go @@ -0,0 +1,54 @@ +package config + +const ( + // Pruning configuration flags + + // FlagPruningStrategy is a flag for specifying strategy for pruning block store + FlagPruningStrategy = "rollkit.node.pruning.strategy" + // FlagPruningKeepRecent is a flag for specifying how many blocks need to keep in store + FlagPruningKeepRecent = "rollkit.node.pruning.keep_recent" + // FlagPruningInterval is a flag for specifying how offen prune blocks store + FlagPruningInterval = "rollkit.node.pruning.interval" +) + +const ( + PruningConfigStrategyNone = "none" + PruningConfigStrategyDefault = "default" + PruningConfigStrategyEverything = "everything" + PruningConfigStrategyCustom = "custom" +) + +var ( + PruningConfigNone = PruningConfig{ + Strategy: PruningConfigStrategyNone, + KeepRecent: 0, + Interval: 0, + } + PruningConfigDefault = PruningConfig{ + Strategy: PruningConfigStrategyDefault, + KeepRecent: 362880, + Interval: 10, + } + PruningConfigEverything = PruningConfig{ + Strategy: PruningConfigStrategyEverything, + KeepRecent: 2, + Interval: 10, + } + PruningConfigCustom = PruningConfig{ + Strategy: PruningConfigStrategyCustom, + KeepRecent: 100, + Interval: 100, + } +) + +// PruningConfig allows node operators to manage storage +type PruningConfig struct { + // todo: support volume-based strategy + Strategy string `mapstructure:"strategy" yaml:"strategy" comment:"Strategy determines the pruning approach (none, default, custom)"` + KeepRecent uint64 `mapstructure:"keep_recent" yaml:"keep_recent" comment:"Number of recent blocks to keep, used in \"default\" and \"custom\" strategies"` + Interval uint64 `mapstructure:"interval" yaml:"interval" comment:"how offen the pruning process should run"` + + // todo: support volume-based strategy + // VolumeConfig specifies configuration for volume-based storage + // VolumeConfig *VolumeStorageConfig `mapstructure:"volume_config" yaml:"volume_config"` +} From 523a6ce4a1b43146fa28d7903fae2fafea93db80 Mon Sep 17 00:00:00 2001 From: Eoous <38656355+Eoous@users.noreply.github.com> Date: Tue, 22 Apr 2025 04:17:13 +0800 Subject: [PATCH 4/4] prune block data when save --- pkg/store/pruning.go | 48 +++++++++++++++++++++++++++++++++++++++++--- pkg/store/types.go | 2 ++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/pkg/store/pruning.go b/pkg/store/pruning.go index 7daa856a0b..cf88de06d0 100644 --- a/pkg/store/pruning.go +++ b/pkg/store/pruning.go @@ -2,20 +2,25 @@ package store import ( "context" + "fmt" + "github.com/rollkit/rollkit/pkg/config" "github.com/rollkit/rollkit/types" ) type DefaultPruningStore struct { Store + + Config config.PruningConfig } var _ PruningStore = &DefaultPruningStore{} // NewDefaultPruningStore returns default pruning store. -func NewDefaultPruningStore(store Store) PruningStore { +func NewDefaultPruningStore(store Store, config config.PruningConfig) PruningStore { return &DefaultPruningStore{ - Store: store, + Store: store, + Config: config, } } @@ -37,7 +42,10 @@ func (s *DefaultPruningStore) Height(ctx context.Context) (uint64, error) { // SaveBlockData adds block header and data to the store along with corresponding signature. // Stored height is updated if block height is greater than stored value. func (s *DefaultPruningStore) SaveBlockData(ctx context.Context, header *types.SignedHeader, data *types.Data, signature *types.Signature) error { - // todo: prune blocks + if err := s.PruneBlockData(ctx); err != nil { + return fmt.Errorf("failed to prune block data: %w", err) + } + return s.Store.SaveBlockData(ctx, header, data, signature) } @@ -88,3 +96,37 @@ func (s *DefaultPruningStore) SetMetadata(ctx context.Context, key string, value func (s *DefaultPruningStore) GetMetadata(ctx context.Context, key string) ([]byte, error) { return s.Store.GetMetadata(ctx, key) } + +func (s *DefaultPruningStore) PruneBlockData(ctx context.Context) error { + var ( + err error + ) + + // Skip if strategy is none. + if s.Config.Strategy == config.PruningConfigStrategyNone { + return nil + } + + height, err := s.Height(ctx) + if err != nil { + return err + } + + // Skip if it's a correct interval or latest height is less or equal than number of blocks need to keep. + if height%s.Config.Interval != 0 || height <= s.Config.KeepRecent { + return nil + } + + // Must keep latest 2 blocks. + endHeight := height - 1 - s.Config.KeepRecent + startHeight := min(0, endHeight-s.Config.KeepRecent) + + for i := startHeight; i <= endHeight; i++ { + err = s.DeleteBlockData(ctx, i) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/store/types.go b/pkg/store/types.go index b997b9b62b..51304af186 100644 --- a/pkg/store/types.go +++ b/pkg/store/types.go @@ -49,4 +49,6 @@ type Store interface { type PruningStore interface { Store + + PruneBlockData(ctx context.Context) error }