Skip to content

Commit 2e03300

Browse files
bitwiseguybitwiseguy
authored
add-chain: add compress-genesis subcommand (#355)
* Add compress-genesis subcommand * Adjust params in compress-genesis test * Add genesis_zorasep.json to testdata/ * Use variables instead of string literals
1 parent 52c557a commit 2e03300

File tree

7 files changed

+14769
-10
lines changed

7 files changed

+14769
-10
lines changed

‎add-chain/cmd/compress-genesis.go‎

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
package cmd
2+
3+
import (
4+
"bytes"
5+
"compress/flate"
6+
"compress/gzip"
7+
"encoding/json"
8+
"errors"
9+
"fmt"
10+
"os"
11+
"path/filepath"
12+
"runtime"
13+
14+
"github.com/ethereum-optimism/optimism/op-service/jsonutil"
15+
"github.com/ethereum-optimism/superchain-registry/add-chain/flags"
16+
"github.com/ethereum/go-ethereum/common"
17+
"github.com/ethereum/go-ethereum/common/hexutil"
18+
"github.com/ethereum/go-ethereum/core"
19+
"github.com/ethereum/go-ethereum/core/types"
20+
"github.com/ethereum/go-ethereum/crypto"
21+
"github.com/urfave/cli/v2"
22+
)
23+
24+
var CompressGenesisCmd = cli.Command{
25+
Name: "compress-genesis",
26+
Flags: []cli.Flag{
27+
flags.L2GenesisFlag,
28+
flags.L2GenesisHeaderFlag,
29+
flags.ChainShortNameFlag,
30+
flags.SuperchainTargetFlag,
31+
},
32+
Usage: "Generate a single gzipped data file from a bytecode hex string",
33+
Action: func(ctx *cli.Context) error {
34+
// Get the current script filepath
35+
_, thisFile, _, ok := runtime.Caller(0)
36+
if !ok {
37+
panic("error getting current filepath")
38+
}
39+
superchainRepoRoot := filepath.Dir(filepath.Dir(filepath.Dir(thisFile)))
40+
superchainTarget := ctx.String(flags.SuperchainTargetFlag.Name)
41+
if superchainTarget == "" {
42+
return fmt.Errorf("missing required flag: %s", flags.SuperchainTargetFlag.Name)
43+
}
44+
chainShortName := ctx.String(flags.ChainShortNameFlag.Name)
45+
if chainShortName == "" {
46+
return fmt.Errorf("missing required flag: %s", flags.ChainShortNameFlag.Name)
47+
}
48+
49+
zipOutputDir := filepath.Join(superchainRepoRoot, "/superchain/extra/genesis", superchainTarget, chainShortName+".json.gz")
50+
genesisPath := ctx.Path(flags.L2GenesisFlag.Name)
51+
if genesisPath == "" {
52+
// When the genesis-state is too large, or not meant to be available, only the header data is made available.
53+
// This allows the user to verify the header-chain starting from genesis, and state-sync the latest state,
54+
// skipping the historical state.
55+
// Archive nodes that depend on this historical state should instantiate the chain from a full genesis dump
56+
// with allocation data, or datadir.
57+
genesisHeaderPath := ctx.Path(flags.L2GenesisHeaderFlag.Name)
58+
genesisHeader, err := loadJSON[types.Header](genesisHeaderPath)
59+
if err != nil {
60+
return fmt.Errorf("genesis-header %q failed to load: %w", genesisHeaderPath, err)
61+
}
62+
if genesisHeader.TxHash != types.EmptyTxsHash {
63+
return errors.New("genesis-header based genesis must have no transactions")
64+
}
65+
if genesisHeader.ReceiptHash != types.EmptyReceiptsHash {
66+
return errors.New("genesis-header based genesis must have no receipts")
67+
}
68+
if genesisHeader.UncleHash != types.EmptyUncleHash {
69+
return errors.New("genesis-header based genesis must have no uncle hashes")
70+
}
71+
if genesisHeader.WithdrawalsHash != nil && *genesisHeader.WithdrawalsHash != types.EmptyWithdrawalsHash {
72+
return errors.New("genesis-header based genesis must have no withdrawals")
73+
}
74+
out := Genesis{
75+
Nonce: genesisHeader.Nonce.Uint64(),
76+
Timestamp: genesisHeader.Time,
77+
ExtraData: genesisHeader.Extra,
78+
GasLimit: genesisHeader.GasLimit,
79+
Difficulty: (*hexutil.Big)(genesisHeader.Difficulty),
80+
Mixhash: genesisHeader.MixDigest,
81+
Coinbase: genesisHeader.Coinbase,
82+
Number: genesisHeader.Number.Uint64(),
83+
GasUsed: genesisHeader.GasUsed,
84+
ParentHash: genesisHeader.ParentHash,
85+
BaseFee: (*hexutil.Big)(genesisHeader.BaseFee),
86+
ExcessBlobGas: genesisHeader.ExcessBlobGas, // EIP-4844
87+
BlobGasUsed: genesisHeader.BlobGasUsed, // EIP-4844
88+
Alloc: make(jsonutil.LazySortedJsonMap[common.Address, GenesisAccount]),
89+
StateHash: &genesisHeader.Root,
90+
}
91+
if err := writeGzipJSON(zipOutputDir, out); err != nil {
92+
return fmt.Errorf("failed to write output: %w", err)
93+
}
94+
return nil
95+
}
96+
97+
genesis, err := loadJSON[core.Genesis](genesisPath)
98+
if err != nil {
99+
return fmt.Errorf("failed to load L2 genesis: %w", err)
100+
}
101+
102+
// export all contract bytecodes, write them to bytecodes collection
103+
bytecodesDir := filepath.Join(superchainRepoRoot, "/superchain/extra/bytecodes")
104+
fmt.Printf("using output bytecodes dir: %s\n", bytecodesDir)
105+
if err := os.MkdirAll(bytecodesDir, 0o755); err != nil {
106+
return fmt.Errorf("failed to make bytecodes dir: %w", err)
107+
}
108+
for addr, account := range genesis.Alloc {
109+
if len(account.Code) > 0 {
110+
err = writeBytecode(bytecodesDir, account.Code, addr)
111+
if err != nil {
112+
return err
113+
}
114+
}
115+
}
116+
117+
// convert into allocation data
118+
out := Genesis{
119+
Nonce: genesis.Nonce,
120+
Timestamp: genesis.Timestamp,
121+
ExtraData: genesis.ExtraData,
122+
GasLimit: genesis.GasLimit,
123+
Difficulty: (*hexutil.Big)(genesis.Difficulty),
124+
Mixhash: genesis.Mixhash,
125+
Coinbase: genesis.Coinbase,
126+
Number: genesis.Number,
127+
GasUsed: genesis.GasUsed,
128+
ParentHash: genesis.ParentHash,
129+
BaseFee: (*hexutil.Big)(genesis.BaseFee),
130+
ExcessBlobGas: genesis.ExcessBlobGas, // EIP-4844
131+
BlobGasUsed: genesis.BlobGasUsed, // EIP-4844
132+
Alloc: make(jsonutil.LazySortedJsonMap[common.Address, GenesisAccount]),
133+
}
134+
135+
// write genesis, but only reference code by code-hash, and don't encode the L2 predeploys to save space.
136+
for addr, account := range genesis.Alloc {
137+
var codeHash common.Hash
138+
if len(account.Code) > 0 {
139+
codeHash = crypto.Keccak256Hash(account.Code)
140+
}
141+
outAcc := GenesisAccount{
142+
CodeHash: codeHash,
143+
Nonce: account.Nonce,
144+
}
145+
if account.Balance != nil && account.Balance.Cmp(common.Big0) != 0 {
146+
outAcc.Balance = (*hexutil.Big)(account.Balance)
147+
}
148+
if len(account.Storage) > 0 {
149+
outAcc.Storage = make(jsonutil.LazySortedJsonMap[common.Hash, common.Hash])
150+
for k, v := range account.Storage {
151+
outAcc.Storage[k] = v
152+
}
153+
}
154+
out.Alloc[addr] = outAcc
155+
}
156+
157+
// write genesis alloc
158+
if err := writeGzipJSON(zipOutputDir, out); err != nil {
159+
return fmt.Errorf("failed to write output: %w", err)
160+
}
161+
return nil
162+
},
163+
}
164+
165+
func writeBytecode(bytecodesDir string, code []byte, addr common.Address) error {
166+
codeHash := crypto.Keccak256Hash(code)
167+
name := filepath.Join(bytecodesDir, fmt.Sprintf("%s.bin.gz", codeHash))
168+
_, err := os.Stat(name)
169+
if err == nil {
170+
// file already exists
171+
return nil
172+
}
173+
if !os.IsNotExist(err) {
174+
return fmt.Errorf("failed to check for pre-existing bytecode %s for address %s: %w", codeHash, addr, err)
175+
}
176+
var buf bytes.Buffer
177+
w, err := gzip.NewWriterLevel(&buf, 9)
178+
if err != nil {
179+
return fmt.Errorf("failed to construct gzip writer for bytecode %s: %w", codeHash, err)
180+
}
181+
if _, err := w.Write(code); err != nil {
182+
return fmt.Errorf("failed to write bytecode %s to gzip writer: %w", codeHash, err)
183+
}
184+
if err := w.Close(); err != nil {
185+
return fmt.Errorf("failed to close gzip writer: %w", err)
186+
}
187+
// new bytecode
188+
if err := os.WriteFile(name, buf.Bytes(), 0o755); err != nil {
189+
return fmt.Errorf("failed to write bytecode %s of account %s: %w", codeHash, addr, err)
190+
}
191+
fmt.Printf("created new bytecodes file: %s\n", filepath.Base(name))
192+
return nil
193+
}
194+
195+
type GenesisAccount struct {
196+
CodeHash common.Hash `json:"codeHash,omitempty"`
197+
Storage jsonutil.LazySortedJsonMap[common.Hash, common.Hash] `json:"storage,omitempty"`
198+
Balance *hexutil.Big `json:"balance,omitempty"`
199+
Nonce uint64 `json:"nonce,omitempty"`
200+
}
201+
202+
type Genesis struct {
203+
Nonce uint64 `json:"nonce"`
204+
Timestamp uint64 `json:"timestamp"`
205+
ExtraData []byte `json:"extraData"`
206+
GasLimit uint64 `json:"gasLimit"`
207+
Difficulty *hexutil.Big `json:"difficulty"`
208+
Mixhash common.Hash `json:"mixHash"`
209+
Coinbase common.Address `json:"coinbase"`
210+
Number uint64 `json:"number"`
211+
GasUsed uint64 `json:"gasUsed"`
212+
ParentHash common.Hash `json:"parentHash"`
213+
BaseFee *hexutil.Big `json:"baseFeePerGas"`
214+
ExcessBlobGas *uint64 `json:"excessBlobGas"` // EIP-4844
215+
BlobGasUsed *uint64 `json:"blobGasUsed"` // EIP-4844
216+
217+
Alloc jsonutil.LazySortedJsonMap[common.Address, GenesisAccount] `json:"alloc"`
218+
// For genesis definitions without full state (OP-Mainnet, OP-Goerli)
219+
StateHash *common.Hash `json:"stateHash,omitempty"`
220+
}
221+
222+
func loadJSON[X any](inputPath string) (*X, error) {
223+
if inputPath == "" {
224+
return nil, errors.New("no path specified")
225+
}
226+
f, err := os.OpenFile(inputPath, os.O_RDONLY, 0)
227+
if err != nil {
228+
return nil, fmt.Errorf("failed to open file %q: %w", inputPath, err)
229+
}
230+
defer f.Close()
231+
var obj X
232+
if err := json.NewDecoder(f).Decode(&obj); err != nil {
233+
return nil, fmt.Errorf("failed to decode file %q: %w", inputPath, err)
234+
}
235+
return &obj, nil
236+
}
237+
238+
func writeGzipJSON(outputPath string, value any) error {
239+
fmt.Printf("using output gzip filepath: %s\n", outputPath)
240+
f, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
241+
if err != nil {
242+
return fmt.Errorf("failed to open output file: %w", err)
243+
}
244+
defer f.Close()
245+
w, err := gzip.NewWriterLevel(f, flate.BestCompression)
246+
if err != nil {
247+
return fmt.Errorf("failed to create gzip writer: %w", err)
248+
}
249+
defer w.Close()
250+
enc := json.NewEncoder(w)
251+
if err := enc.Encode(value); err != nil {
252+
return fmt.Errorf("failed to encode to JSON: %w", err)
253+
}
254+
return nil
255+
}

‎add-chain/e2e_test.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,24 @@ func TestAddChain_Main(t *testing.T) {
9090
compareJsonFiles(t, "superchain/extra/genesis-system-configs/sepolia/", tt.name, tt.chainShortName)
9191
})
9292
}
93+
94+
t.Run("compress-genesis", func(t *testing.T) {
95+
// Must run this test to produce the .json.gz output artifact for the
96+
// subsequent CheckGenesisConfig test
97+
t.Parallel()
98+
err := os.Setenv("SCR_RUN_TESTS", "true")
99+
require.NoError(t, err, "failed to set SCR_RUN_TESTS env var")
100+
101+
args := []string{
102+
"add-chain",
103+
"compress-genesis",
104+
"--l2-genesis=" + "./testdata/monorepo/op-node/genesis_zorasep.json",
105+
"--superchain-target=" + "sepolia",
106+
"--chain-short-name=" + "testchain_zs",
107+
}
108+
err = runApp(args)
109+
require.NoError(t, err, "add-chain compress-genesis failed")
110+
})
93111
}
94112

95113
func TestAddChain_CheckRollupConfig(t *testing.T) {

‎add-chain/flags/flags.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,3 +97,18 @@ var (
9797
Required: true,
9898
}
9999
)
100+
101+
var (
102+
L2GenesisFlag = &cli.PathFlag{
103+
Name: "l2-genesis",
104+
Value: "genesis.json",
105+
Usage: "Path to genesis json (go-ethereum format)",
106+
EnvVars: prefixEnvVars("L2_GENESIS"),
107+
}
108+
L2GenesisHeaderFlag = &cli.PathFlag{
109+
Name: "l2-genesis-header",
110+
Value: "genesis-header.json",
111+
Usage: "Alternative to l2-genesis flag, if genesis-state is omitted. Path to block header at genesis",
112+
EnvVars: prefixEnvVars("L2_GENESIS_HEADER"),
113+
}
114+
)

‎add-chain/go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ require (
4444
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
4545
github.com/google/uuid v1.6.0 // indirect
4646
github.com/gorilla/websocket v1.5.0 // indirect
47+
github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
4748
github.com/holiman/uint256 v1.2.4 // indirect
4849
github.com/klauspost/compress v1.17.2 // indirect
4950
github.com/kr/pretty v0.3.1 // indirect

‎add-chain/main.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,12 @@ var app = &cli.App{
3333
flags.DeploymentsDirFlag,
3434
flags.StandardChainCandidateFlag,
3535
},
36-
Action: entrypoint,
37-
Commands: []*cli.Command{&cmd.PromoteToStandardCmd, &cmd.CheckRollupConfigCmd},
36+
Action: entrypoint,
37+
Commands: []*cli.Command{
38+
&cmd.PromoteToStandardCmd,
39+
&cmd.CheckRollupConfigCmd,
40+
&cmd.CompressGenesisCmd,
41+
},
3842
}
3943

4044
func main() {

0 commit comments

Comments
 (0)