Skip to content

Commit 5dc054c

Browse files
bitwiseguybitwiseguy
authored
ops: integrate op-fetcher (#951)
* ops: integrate op-fetcher * use L1ProxyAdminOwner instead of OpChainProxyAdminOwner * use cmd/codegen to wrap fetch_onchain and codegen_syncer * revert .gitignore change * refactor CodegenSyncer and fix all ops tests * fetch_onchain: remove unnecessary abstractions * add tests for new helper functions * circleci: add daily-codegen-sync job * UpdateChainList handle new chain addition * update op-fetcher import: use pointer for FaultProofsStatus * go mod tidy * codegen: chain-ids flag is optional comma separated list * remove unused helper function * circleci: update check-codegen job * circleci: fix codegen-sync-all reference * transform op-fetcher proofs struct to unique chainList struct * remove 'just codegen' (needs args now) * circleci: remove daily codegen-sync-all * rebase and revert unnecessary changes * simplify zero-address comparison * pass context from cli handler instead of creating one in helpers * use uint64 for chainId everywhere except cli handlers * remove unused FindChainConfig. Add FindChainConfigs tests * add FS helpers for chainList, CHAINS.md files * remove hardcoded SuperchainIds map * remove FindChainConfigs, add Superchain field to DiskChainConfig * use helper to encapsulate common ChecksummedAddress Unmarshal code * add paths.EnsureDir helper * codegen: add optional superchains flag * mise: add yq (for usage in ci) * cleanup log messages
1 parent 84838da commit 5dc054c

File tree

23 files changed

+1072
-195
lines changed

23 files changed

+1072
-195
lines changed

‎.circleci/config.yml‎

Lines changed: 70 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,74 @@ jobs:
124124
cd ops
125125
go run ./cmd/print_staging_report/main.go
126126
127+
check-codegen:
128+
circleci_ip_ranges: true
129+
docker:
130+
- image: <<pipeline.parameters.default_docker_image>>
131+
steps:
132+
- utils/checkout-with-mise
133+
- run:
134+
name: check-codegen-for-changed-chains
135+
command: |
136+
set -e
137+
138+
# Get list of changed files
139+
CHANGED_FILES=$(git diff --name-only origin/main... | grep "^superchain/configs/.*\.toml$" || true)
140+
if [ -z "$CHANGED_FILES" ]; then
141+
echo "No .toml files changed in superchain/configs, skipping codegen check"
142+
exit 0
143+
fi
144+
145+
# Extract chain_ids from changed .toml files
146+
CHAIN_IDS=""
147+
CHAIN_ID_COUNT=0
148+
CHANGED_FILE_COUNT=0
149+
150+
for file in $CHANGED_FILES; do
151+
CHANGED_FILE_COUNT=$((CHANGED_FILE_COUNT + 1))
152+
# Use yq to extract chain_id from TOML
153+
CHAIN_ID=$(yq -p=toml -o=json '.chain_id' "$file" | grep -v "null" | tr -d '"')
154+
155+
if [ -n "$CHAIN_ID" ]; then
156+
if [ -z "$CHAIN_IDS" ]; then
157+
CHAIN_IDS="$CHAIN_ID"
158+
else
159+
CHAIN_IDS="$CHAIN_IDS,$CHAIN_ID"
160+
fi
161+
CHAIN_ID_COUNT=$((CHAIN_ID_COUNT + 1))
162+
echo "Found chain_id $CHAIN_ID in $file"
163+
fi
164+
done
165+
echo "Found $CHAIN_ID_COUNT chain_ids in $CHANGED_FILE_COUNT files"
166+
167+
cd ops
168+
if [ "$CHAIN_ID_COUNT" -ne "$CHANGED_FILE_COUNT" ] && [ "$CHANGED_FILE_COUNT" -gt 0 ]; then
169+
# This accounts for changes to superchain.toml files
170+
echo "Running codegen for all chains\n"
171+
go run ./cmd/codegen \
172+
--l1-rpc-urls="<< pipeline.parameters.sepolia_rpc_url >>,<< pipeline.parameters.mainnet_rpc_url >>"
173+
else
174+
echo "Running codegen for the following chain_ids: $CHAIN_IDS\n"
175+
go run ./cmd/codegen \
176+
--l1-rpc-urls="<< pipeline.parameters.sepolia_rpc_url >>,<< pipeline.parameters.mainnet_rpc_url >>" \
177+
--chain-ids="$CHAIN_IDS"
178+
fi
179+
180+
if [ -n "$(git status --porcelain)" ] ; then
181+
echo "\n❌ Changes detected after running codegen. Run the following command locally and commit the changes:\n"
182+
echo "go run ./cmd/codegen \\"
183+
# Show the appropriate command flags based on which mode was run
184+
if [ "$CHAIN_ID_COUNT" -ne "$CHANGED_FILE_COUNT" ] && [ "$CHANGED_FILE_COUNT" -gt 0 ]; then
185+
echo " --l1-rpc-urls=\"<urls>\""
186+
else
187+
echo " --l1-rpc-urls=\"<urls>\" \\"
188+
echo " --chain-ids=\"$CHAIN_IDS\""
189+
fi
190+
exit 1
191+
else
192+
echo "\n✅ All codegen files are up to date"
193+
fi
194+
127195
workflows:
128196
main:
129197
jobs:
@@ -136,13 +204,12 @@ workflows:
136204
- run-tool:
137205
name: check-genesis-integrity
138206
tool: check_genesis_integrity
139-
- run-tool:
207+
- check-codegen:
140208
name: check-codegen
141-
tool: codegen
142-
check_diff: true
143209
- run-tool:
144210
name: check-staging-synced
145211
tool: sync_staging
212+
args: --l1-rpc-urls="<< pipeline.parameters.sepolia_rpc_url >>,<< pipeline.parameters.mainnet_rpc_url >>"
146213
check_diff: true
147214
- check-staging-empty:
148215
name: check-staging-empty
@@ -155,4 +222,3 @@ workflows:
155222
tool: check_chainlist
156223
- run-staging-report:
157224
name: run-staging-report
158-

‎justfile‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,6 @@ print-staging-report: (_run_ops_bin 'print_staging_report')
4747

4848
check-genesis-integrity: (_run_ops_bin 'check_genesis_integrity')
4949

50-
codegen: (_run_ops_bin 'codegen')
51-
5250
create-config SHORTNAME FILENAME:
5351
@just _run_ops_bin "create_config" "--shortname {{SHORTNAME}} --state-filename $(realpath {{FILENAME}})"
5452

‎mise.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# Core dependencies
44
go = "1.23.1"
55
just = "1.37.0"
6+
yq = "4.44.5"
67

78
# Go dependencies
89
"go:gotest.tools/gotestsum" = "1.12.0"

‎ops/cmd/codegen/main.go‎

Lines changed: 83 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,28 +3,105 @@ package main
33
import (
44
"fmt"
55
"os"
6+
"strconv"
7+
"strings"
68

9+
"github.com/ethereum-optimism/optimism/op-fetcher/pkg/fetcher/fetch/script"
10+
"github.com/ethereum-optimism/superchain-registry/ops/internal/config"
711
"github.com/ethereum-optimism/superchain-registry/ops/internal/manage"
812
"github.com/ethereum-optimism/superchain-registry/ops/internal/output"
913
"github.com/ethereum-optimism/superchain-registry/ops/internal/paths"
14+
"github.com/ethereum/go-ethereum/log"
15+
"github.com/urfave/cli/v2"
16+
)
17+
18+
var (
19+
L1RPCURLsFlag = &cli.StringFlag{
20+
Name: "l1-rpc-urls",
21+
Usage: "comma-separated list of L1 RPC URLs (only need multiple if fetching from multiple superchains)",
22+
EnvVars: []string{"L1_RPC_URLS"},
23+
Required: true,
24+
}
25+
ChainIDFlag = &cli.StringFlag{
26+
Name: "chain-ids",
27+
Usage: "comma-separated list of l2 chainIds to update (optional, fetches all chains if not provided)",
28+
}
29+
SuperchainsFlag = &cli.StringFlag{
30+
Name: "superchains",
31+
Usage: "comma-separated list of superchains to update (cannot provide both chain-ids and superchains flags, default to all superchains if not provided)",
32+
}
1033
)
1134

1235
func main() {
13-
if err := mainErr(); err != nil {
14-
output.WriteNotOK("application error: %v", err)
36+
app := &cli.App{
37+
Name: "codegen",
38+
Usage: "uses op-fetcher to fetch onchain config data for chain(s) in the superchain-registry, then propagates the data to codegen files",
39+
Flags: []cli.Flag{
40+
L1RPCURLsFlag,
41+
ChainIDFlag,
42+
SuperchainsFlag,
43+
},
44+
Action: CodegenCLI,
45+
}
46+
if err := app.Run(os.Args); err != nil {
47+
output.WriteStderr("%v", err)
1548
os.Exit(1)
1649
}
1750
}
1851

19-
func mainErr() error {
52+
func CodegenCLI(cliCtx *cli.Context) error {
53+
l1RpcUrls := strings.Split(cliCtx.String("l1-rpc-urls"), ",")
54+
chainIdStr := cliCtx.String("chain-ids")
55+
superchainsStr := cliCtx.String("superchains")
56+
if chainIdStr != "" && superchainsStr != "" {
57+
return fmt.Errorf("cannot provide both chain-ids and superchains flags")
58+
}
59+
60+
var superchains []config.Superchain
61+
if superchainsStr != "" {
62+
superchainStrs := strings.Split(superchainsStr, ",")
63+
for _, superchainStr := range superchainStrs {
64+
superchainStr = strings.TrimSpace(superchainStr)
65+
superchain, err := config.ParseSuperchain(superchainStr)
66+
if err != nil {
67+
return err
68+
}
69+
superchains = append(superchains, superchain)
70+
}
71+
}
72+
73+
var chainIds []uint64
74+
if chainIdStr != "" {
75+
chainIdStrs := strings.Split(chainIdStr, ",")
76+
// Convert each string to uint64
77+
for _, idStr := range chainIdStrs {
78+
idStr = strings.TrimSpace(idStr)
79+
id, err := strconv.ParseUint(idStr, 10, 64)
80+
if err != nil {
81+
return fmt.Errorf("invalid chain ID '%s': %w", idStr, err)
82+
}
83+
chainIds = append(chainIds, id)
84+
}
85+
}
86+
87+
lgr := log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, false))
2088
wd, err := paths.FindRepoRoot()
2189
if err != nil {
2290
return fmt.Errorf("failed to get working directory: %w", err)
2391
}
2492

25-
if err := manage.GenAllCode(wd); err != nil {
26-
return fmt.Errorf("error generating code: %w", err)
93+
var onchainCfgs map[uint64]script.ChainConfig
94+
ctx := cliCtx.Context
95+
onchainCfgs, err = manage.FetchChains(ctx, lgr, wd, l1RpcUrls, chainIds, superchains)
96+
if err != nil {
97+
return fmt.Errorf("error fetching onchain configs: %w", err)
98+
}
99+
syncer, err := manage.NewCodegenSyncer(lgr, wd, onchainCfgs)
100+
if err != nil {
101+
return fmt.Errorf("error creating codegen syncer: %w", err)
102+
}
103+
if err := syncer.SyncAll(); err != nil {
104+
return fmt.Errorf("error syncing codegen: %w", err)
27105
}
28-
29106
return nil
30107
}

‎ops/cmd/sync_staging/main.go‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,13 @@ import (
55
"fmt"
66
"os"
77
"path"
8+
"strings"
89

10+
"github.com/ethereum-optimism/superchain-registry/ops/internal/config"
911
"github.com/ethereum-optimism/superchain-registry/ops/internal/manage"
1012
"github.com/ethereum-optimism/superchain-registry/ops/internal/output"
1113
"github.com/ethereum-optimism/superchain-registry/ops/internal/paths"
14+
"github.com/ethereum/go-ethereum/log"
1215
"github.com/urfave/cli/v2"
1316
)
1417

@@ -21,6 +24,11 @@ var (
2124
Name: "preserve-input",
2225
Usage: "Skip cleanup of staging directory.",
2326
}
27+
FlagL1RPCURLs = &cli.StringFlag{
28+
Name: "l1-rpc-urls",
29+
Usage: "Comma-separated list of L1 RPC URLs",
30+
Required: true,
31+
}
2432
)
2533

2634
func main() {
@@ -30,6 +38,7 @@ func main() {
3038
Flags: []cli.Flag{
3139
FlagCheck,
3240
FlagPreserveInput,
41+
FlagL1RPCURLs,
3342
},
3443
Action: action,
3544
}
@@ -96,8 +105,20 @@ func action(cliCtx *cli.Context) error {
96105

97106
output.WriteOK("wrote genesis files")
98107

99-
if err := manage.GenAllCode(wd); err != nil {
100-
return fmt.Errorf("failed to generate code: %w", err)
108+
// Codegen
109+
lgr := log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, false))
110+
l1RpcUrls := strings.Split(cliCtx.String("l1-rpc-urls"), ",")
111+
ctx := cliCtx.Context
112+
onchainCfgs, err := manage.FetchChains(ctx, lgr, wd, l1RpcUrls, []uint64{chainCfg.ChainID}, []config.Superchain{})
113+
if err != nil {
114+
return fmt.Errorf("error fetching onchain configs: %w", err)
115+
}
116+
syncer, err := manage.NewCodegenSyncer(lgr, wd, onchainCfgs)
117+
if err != nil {
118+
return fmt.Errorf("error creating codegen syncer: %w", err)
119+
}
120+
if err := syncer.SyncAll(); err != nil {
121+
return fmt.Errorf("error syncing codegen: %w", err)
101122
}
102123

103124
if !noCleanup {

‎ops/go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.23.1
44

55
require (
66
github.com/BurntSushi/toml v1.4.0
7-
github.com/ethereum-optimism/optimism v1.13.1-0.20250402215013-5391bf29692d
7+
github.com/ethereum-optimism/optimism v1.12.3-0.20250402215013-5391bf29692d
88
github.com/ethereum/go-ethereum v1.15.3
99
github.com/lmittmann/w3 v0.17.8
1010
github.com/stretchr/testify v1.10.0
@@ -15,6 +15,7 @@ require (
1515
github.com/ethereum-optimism/superchain-registry/validation v0.0.0-20250314162817-2c60e5723c64
1616
github.com/google/go-github/v68 v68.0.0
1717
github.com/klauspost/compress v1.18.0
18+
golang.org/x/sync v0.10.0
1819
)
1920

2021
require (
@@ -74,7 +75,6 @@ require (
7475
github.com/yusufpapurcu/wmi v1.2.3 // indirect
7576
golang.org/x/crypto v0.32.0 // indirect
7677
golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect
77-
golang.org/x/sync v0.10.0 // indirect
7878
golang.org/x/sys v0.29.0 // indirect
7979
golang.org/x/term v0.28.0 // indirect
8080
golang.org/x/time v0.10.0 // indirect

‎ops/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,8 @@ github.com/ethereum-optimism/go-ethereum-hdwallet v0.1.3 h1:RWHKLhCrQThMfch+QJ1Z
8787
github.com/ethereum-optimism/go-ethereum-hdwallet v0.1.3/go.mod h1:QziizLAiF0KqyLdNJYD7O5cpDlaFMNZzlxYNcWsJUxs=
8888
github.com/ethereum-optimism/op-geth v1.101503.2-rc.5.0.20250401164435-02dfe8692a3c h1:2uU1Jx+wuqGtWR/qsZrmB+jfR+/hslowPS11bvBAda4=
8989
github.com/ethereum-optimism/op-geth v1.101503.2-rc.5.0.20250401164435-02dfe8692a3c/go.mod h1:QUo3fn+45vWqJWzJW+rIzRHUV7NmhhHLPdI87mAn1M8=
90-
github.com/ethereum-optimism/optimism v1.13.1-0.20250402215013-5391bf29692d h1:SWUqPFKjoh3WfAG2HaKNjwkwzxJ3Jm/ssvYQAF6AbY0=
91-
github.com/ethereum-optimism/optimism v1.13.1-0.20250402215013-5391bf29692d/go.mod h1:Pf1bUvfjbot5RyCmsfPKFR0kcBCT/HZvSlISRtB1+KE=
90+
github.com/ethereum-optimism/optimism v1.12.3-0.20250402215013-5391bf29692d h1:UGWjnYcPm7e0AOWZGyKhgBj+67oq8ZRrJYYu4Kfh7uw=
91+
github.com/ethereum-optimism/optimism v1.12.3-0.20250402215013-5391bf29692d/go.mod h1:Pf1bUvfjbot5RyCmsfPKFR0kcBCT/HZvSlISRtB1+KE=
9292
github.com/ethereum-optimism/superchain-registry/validation v0.0.0-20250314162817-2c60e5723c64 h1:teDhU4h4ryaE8rSBl+vJJiwKHjxdnnHPkKZ9iNr2R8k=
9393
github.com/ethereum-optimism/superchain-registry/validation v0.0.0-20250314162817-2c60e5723c64/go.mod h1:NZ816PzLU1TLv1RdAvYAb6KWOj4Zm5aInT0YpDVml2Y=
9494
github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA=

‎ops/internal/config/address.go‎

Lines changed: 41 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package config
22

33
import (
4+
"encoding/json"
45
"fmt"
56

67
"github.com/ethereum/go-ethereum/common"
@@ -19,20 +20,7 @@ func (a *ChecksummedAddress) UnmarshalTOML(data any) error {
1920
return fmt.Errorf("expected a string, got %T", data)
2021
}
2122

22-
if len(dataStr) != 42 {
23-
return fmt.Errorf("invalid address: %s", dataStr)
24-
}
25-
26-
if !common.IsHexAddress(dataStr) {
27-
return fmt.Errorf("invalid address: %s", dataStr)
28-
}
29-
30-
addr := common.HexToAddress(dataStr)
31-
if addr.Hex() != dataStr {
32-
return fmt.Errorf("invalid checksummed address: %s", dataStr)
33-
}
34-
*a = ChecksummedAddress(addr)
35-
return nil
23+
return a.parseAddress(dataStr)
3624
}
3725

3826
func (a ChecksummedAddress) MarshalTOML() ([]byte, error) {
@@ -43,6 +31,43 @@ func (a ChecksummedAddress) String() string {
4331
return common.Address(a).Hex()
4432
}
4533

46-
func (a ChecksummedAddress) MarshalJSON() ([]byte, error) {
47-
return []byte(fmt.Sprintf(`"%s"`, common.Address(a).Hex())), nil
34+
func (a *ChecksummedAddress) UnmarshalJSON(data []byte) error {
35+
var dataStr string
36+
if err := json.Unmarshal(data, &dataStr); err != nil {
37+
return fmt.Errorf("failed to unmarshal ChecksummedAddress: %w", err)
38+
}
39+
40+
return a.parseAddress(dataStr)
41+
}
42+
43+
func (a *ChecksummedAddress) MarshalJSON() ([]byte, error) {
44+
if common.Address(*a) == (common.Address{}) {
45+
// Return null for zero addresses so it doesn't pollute the json output
46+
return []byte("null"), nil
47+
}
48+
return []byte(fmt.Sprintf(`"%s"`, common.Address(*a).Hex())), nil
49+
}
50+
51+
// Helper function for validating and parsing Ethereum addresses
52+
func (a *ChecksummedAddress) parseAddress(addrStr string) error {
53+
// Validate length
54+
if len(addrStr) != 42 {
55+
return fmt.Errorf("invalid address length: %s", addrStr)
56+
}
57+
58+
// Validate hex format
59+
if !common.IsHexAddress(addrStr) {
60+
return fmt.Errorf("invalid hex address: %s", addrStr)
61+
}
62+
63+
// Convert to checksummed address
64+
addr := common.HexToAddress(addrStr)
65+
66+
// Validate that the address is properly checksummed
67+
if addr.Hex() != addrStr {
68+
return fmt.Errorf("invalid checksummed address: %s", addrStr)
69+
}
70+
71+
*a = ChecksummedAddress(addr)
72+
return nil
4873
}

0 commit comments

Comments
 (0)