Merge pull request #139 from coinbase/patrick/block-pruning

Add support for storage pruning

Author: patrick-ogrady

cmd/check_data.go

@@ -133,6 +133,10 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) {
 		return dataTester.StartSyncing(ctx)
 	})
 
+	g.Go(func() error {
+		return dataTester.StartPruning(ctx)
+	})
+
 	g.Go(func() error {
 		return dataTester.WatchEndConditions(ctx)
 	})

cmd/root.go

@@ -132,6 +132,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print rosetta-cli version",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("v0.5.2")
+		fmt.Println("v0.5.3")
 	},
 }

configuration/configuration.go

@@ -274,6 +274,12 @@ type DataConfiguration struct {
 	// ResultsOutputFile is the absolute filepath of where to save
 	// the results of a check:data run.
 	ResultsOutputFile string `json:"results_output_file"`
+
+	// PruningDisabled is a bolean that indicates storage pruning should
+	// not be attempted. This should really only ever be set to true if you
+	// wish to use `start_index` at a later point to restart from some
+	// previously synced block.
+	PruningDisabled bool `json:"pruning_disabled"`
 }
 
 // Configuration contains all configuration settings for running

examples/configuration/default.json

@@ -29,6 +29,7 @@
   "inactive_discrepency_search_disabled": false,
   "balance_tracking_disabled": false,
   "coin_tracking_disabled": false,
-  "results_output_file": ""
+  "results_output_file": "",
+  "pruning_disabled": false
  }
 }

go.mod

@@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli
 go 1.13
 
 require (
-	github.com/coinbase/rosetta-sdk-go v0.4.4
+	github.com/coinbase/rosetta-sdk-go v0.4.5
 	github.com/fatih/color v1.9.0
 	github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a
 	github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c

go.sum

@@ -141,6 +141,18 @@ github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916005026-5f676c313cfc h1:huzfl
 github.com/coinbase/rosetta-sdk-go v0.4.4-0.20200916005026-5f676c313cfc/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck=
 github.com/coinbase/rosetta-sdk-go v0.4.4 h1:zTUS4bVlTfD4xq/o6JtsuU+g9sf3+S3Nnn2A24Ycow4=
 github.com/coinbase/rosetta-sdk-go v0.4.4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck=
+github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918214028-4d968344e1a2 h1:SqJ4KAfZeyGiJc5GG9DD2vsOY/gsegJK9q9F8WaQRTc=
+github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918214028-4d968344e1a2/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck=
+github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918214833-43eb6f18490b h1:BXy7lSdgwJeBRRo03QLzW4Bj5cRYGQJcjJKxRlUmicU=
+github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918214833-43eb6f18490b/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck=
+github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215107-9c608cf4c6da h1:4rH5i5ePaF5Q5IpX40ZO+0wUjvGY1WJoHfkYcbGcpYE=
+github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215107-9c608cf4c6da/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck=
+github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215315-15f3222d97c4 h1:NNAwQM58a1eh/5vfrneTMeWG9hNrROlSj8Rwrc4TQW8=
+github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215315-15f3222d97c4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck=
+github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215605-d46fa023df73 h1:MDM59d52JqwUQFw0/Ow35IWxFQNkcTNIVO71MDRW9HM=
+github.com/coinbase/rosetta-sdk-go v0.4.5-0.20200918215605-d46fa023df73/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck=
+github.com/coinbase/rosetta-sdk-go v0.4.5 h1:5Z+25mm/J1SStzSdTp5e8dlEMtCZRBZdOaTFCBfnP4Y=
+github.com/coinbase/rosetta-sdk-go v0.4.5/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=

pkg/tester/data.go

@@ -317,6 +317,18 @@ func (t *DataTester) StartSyncing(
 	return t.syncer.Sync(ctx, startIndex, endIndex)
 }
 
+// StartPruning attempts to prune block storage
+// every 10 seconds.
+func (t *DataTester) StartPruning(
+	ctx context.Context,
+) error {
+	if t.config.Data.PruningDisabled {
+		return nil
+	}
+
+	return t.syncer.Prune(ctx, statefulsyncer.DefaultPruningDepth)
+}
+
 // StartReconciler starts the reconciler if
 // reconciliation is enabled.
 func (t *DataTester) StartReconciler(