Revert snapshot changes (#131)

Author: meyer9

configs/examples/snapshot.yml

@@ -1,40 +1,44 @@
-name: Two-Tier Snapshot Functionality Test
+name: Snapshot Functionality Test
 description: |
-  IMPORTANT: This feature is currently in development, and the results are not yet reliable.
+  Snapshot Functionality Test - Tests client snapshot creation and loading capabilities to validate state snapshot performance for fast sync operations.
 
-  Two-Tier Snapshot Functionality Test - Demonstrates the new optimized snapshot system with initial snapshots and per-test copying.
+  This benchmark suite tests snapshot functionality with both Sepolia Alpha and development network data, including snapshot creation, loading, and validation processes. Features skip-if-nonempty optimization for development efficiency and tests multiple gas limit configurations.
 
-  This benchmark suite uses a two-tier snapshot approach:
-  1. Initial snapshots are downloaded once at benchmark startup and stored persistently
-  2. Per-test snapshots are copied from initial snapshots for each test run using rsync
-  3. Test-specific copies are cleaned up after each test while preserving initial snapshots
-
-  Use Case: Optimized snapshot performance for fast test execution, reduced network overhead, and efficient storage management across multiple node types.
+  Use Case: Validate state snapshot performance for fast sync operations, test snapshot creation and loading capabilities across different environments, and ensure snapshot performance remains consistent in development workflows.
 
 payloads:
   - name: Transfer-only
     id: transfer-only
     type: transfer-only
 
 benchmarks:
-  - initial_snapshots:
-      - node_type: reth
-        # Download an initial reth snapshot that can be copied for each test
-        command: ./scripts/setup-base-snapshot.sh --network=sepolia --node-type=reth --destination=/data/snapshots/reth/initial
-        destination: /data/snapshots/reth/initial
-        superchain_chain_id: 84532
-      - node_type: geth
-        # Download an initial geth snapshot that can be copied for each test
-        command: ./scripts/setup-base-snapshot.sh --network=sepolia --node-type=geth --destination=/data/snapshots/geth/initial
-        destination: /data/snapshots/geth/initial
-        superchain_chain_id: 84532
+  - snapshot:
+      # skip non-empty for testing so we don't copy every time we run this
+      # just delete the snapshot directory to force a full copy
+      command: ./scripts/copy-local-snapshot.sh --skip-if-nonempty
+      genesis_file: ../../sepolia-alpha/sepolia-alpha-genesis.json
+    # force_clean is true by default to ensure consistency, but we can skip it for testing
+    force_clean: false
     variables:
       - type: payload
         value: transfer-only
       - type: node_type
         values:
           - reth
-          - geth
+      - type: num_blocks
+        value: 10
+      - type: gas_limit
+        values:
+          - 15000000
+          - 30000000
+          - 60000000
+          - 90000000
+  - variables:
+      - type: payload
+        value: transfer-only
+      - type: node_type
+        values:
+          - reth
       - type: num_blocks
         value: 10
       - type: gas_limit

runner/benchmark/definition.go

@@ -2,6 +2,11 @@ package benchmark
 
 import (
 	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
 
 	"github.com/base/base-bench/runner/payload"
 )
@@ -31,6 +36,54 @@ type ProofProgramOptions struct {
 	Type    string `yaml:"type"`
 }
 
+// SnapshotDefinition is the user-facing YAML configuration for specifying
+// a snapshot to be restored before running a benchmark.
+type SnapshotDefinition struct {
+	Command           string  `yaml:"command"`
+	GenesisFile       *string `yaml:"genesis_file"`
+	SuperchainChainID *uint64 `yaml:"superchain_chain_id"`
+	ForceClean        *bool   `yaml:"force_clean"`
+}
+
+// CreateSnapshot copies the snapshot to the output directory for the given
+// node type.
+func (s SnapshotDefinition) CreateSnapshot(nodeType string, outputDir string) error {
+	// default to true if not set
+	forceClean := s.ForceClean == nil || *s.ForceClean
+	if _, err := os.Stat(outputDir); err == nil && forceClean {
+		// TODO: we could reuse it here potentially
+		if err := os.RemoveAll(outputDir); err != nil {
+			return fmt.Errorf("failed to remove existing snapshot: %w", err)
+		}
+	}
+
+	// get absolute path of outputDir
+	currentDir, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("failed to get absolute path of outputDir: %w", err)
+	}
+
+	outputDir = path.Join(currentDir, outputDir)
+
+	var cmdBin string
+	var args []string
+	// split out default args from command
+	parts := strings.SplitN(s.Command, " ", 2)
+	if len(parts) < 2 {
+		cmdBin = parts[0]
+	} else {
+		cmdBin = parts[0]
+		args = strings.Split(parts[1], " ")
+	}
+
+	args = append(args, nodeType, outputDir)
+
+	cmd := exec.Command(cmdBin, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
 type BenchmarkConfig struct {
 	Name                string               `yaml:"name"`
 	Description         *string              `yaml:"description"`
@@ -41,11 +94,11 @@ type BenchmarkConfig struct {
 // TestDefinition is the user-facing YAML configuration for specifying a
 // matrix of benchmark runs.
 type TestDefinition struct {
-	InitialSnapshots []SnapshotDefinition `yaml:"initial_snapshots"`
-	Metrics          *ThresholdConfig     `yaml:"metrics"`
-	Tags             *map[string]string   `yaml:"tags"`
-	Variables        []Param              `yaml:"variables"`
-	ProofProgram     *ProofProgramOptions `yaml:"proof_program"`
+	Snapshot     *SnapshotDefinition  `yaml:"snapshot"`
+	Metrics      *ThresholdConfig     `yaml:"metrics"`
+	Tags         *map[string]string   `yaml:"tags"`
+	Variables    []Param              `yaml:"variables"`
+	ProofProgram *ProofProgramOptions `yaml:"proof_program"`
 }
 
 func (bc *TestDefinition) Check() error {

runner/benchmark/matrix.go

@@ -12,10 +12,10 @@ type ThresholdConfig struct {
 
 // TestPlan represents a list of test runs to be executed.
 type TestPlan struct {
-	Runs             []TestRun
-	InitialSnapshots []SnapshotDefinition
-	ProofProgram     *ProofProgramOptions
-	Thresholds       *ThresholdConfig
+	Runs         []TestRun
+	Snapshot     *SnapshotDefinition
+	ProofProgram *ProofProgramOptions
+	Thresholds   *ThresholdConfig
 }
 
 func NewTestPlanFromConfig(c TestDefinition, testFileName string, config *BenchmarkConfig) (*TestPlan, error) {
@@ -36,24 +36,13 @@ func NewTestPlanFromConfig(c TestDefinition, testFileName string, config *Benchm
 	}
 
 	return &TestPlan{
-		Runs:             testRuns,
-		InitialSnapshots: c.InitialSnapshots,
-		ProofProgram:     proofProgram,
-		Thresholds:       c.Metrics,
+		Runs:         testRuns,
+		Snapshot:     c.Snapshot,
+		ProofProgram: proofProgram,
+		Thresholds:   c.Metrics,
 	}, nil
 }
 
-// GetInitialSnapshotForNodeType returns the initial snapshot definition for the given node type.
-// Returns nil if no initial snapshot is found for the node type.
-func (tp *TestPlan) GetInitialSnapshotForNodeType(nodeType string) *SnapshotDefinition {
-	for _, snapshot := range tp.InitialSnapshots {
-		if snapshot.NodeType == nodeType {
-			return &snapshot
-		}
-	}
-	return nil
-}
-
 // ResolveTestRunsFromMatrix constructs a new ParamsMatrix from a config.
 func ResolveTestRunsFromMatrix(c TestDefinition, testFileName string, config *BenchmarkConfig) ([]TestRun, error) {
 	seenParams := make(map[string]bool)
@@ -128,7 +117,7 @@ func ResolveTestRunsFromMatrix(c TestDefinition, testFileName string, config *Be
 		testParams[i] = TestRun{
 			ID:          id,
 			Params:      *params,
-			OutputDir:   fmt.Sprintf("%s/%s-%d", id, id, i),
+			OutputDir:   fmt.Sprintf("%s-%d", id, i),
 			Name:        params.Name,
 			Description: params.Description,
 			TestFile:    testFileName,