Use toml instead of yaml for chain config files (#386)

* add-chain: create toml chain config file

* add-chain: import toml package

* add-chain: main writes toml chain config

* add-chain: add expected_*.toml test files

* add-chain: e2e_test uses toml chain config files

* add-chain: add convert-to-toml subcommand

* superchain: init reads from toml files instead of yaml

* circleci: set 'Install just' step name

* superchain: replace all chain config .yaml file with .toml

* superchain: test uses toml.Unmarshal

* add-chain: remove unused checkConfigYaml test function

* codegen: fix formatting for toml

* superchain: add toml tags for addresses, genesis.system_config

* add-chain: convert-to-toml writes addresses, genesis.system_config

* superchain: use superchain.toml instead of superchain.yaml

* superchain: fix unMarshalSuperchainConfig so it works with toml

* superchain: rename EnhanceTOML to GenerateTOMLComments

* add-chain: remove WriteChainConfig, last yaml dependency

* superchain: remove yaml config files

* superchain: fix types in genesis SystemConfig

* justfile: clean .toml instead of .yaml

* add-chain: write addresses to toml instead of json

* superchain: fix order of fields in ChainConfig

* superchain: move ChainConfig.Addresses field before plasma

* add-chain: remove all yaml and json testdata files

* validation: fix devnet config superchain_level and standard_candidate

* superchain: add addresses and genesis.system_config to all toml files

* add-chain: remove convert-to-toml subcommand

* justfile: remove unnecessary clean-add-chain commands

* superchain: use rawTOML instead of rawYAML in tests

* superchain: remove all yaml tags and references

* superchain: reorder ChainConfig.Plasma in toml file

* add-chain: tidy comments and function description

* superchain: standardize timestamp comment in superchain.toml files

* simplify AddressList.MarshalTOML (#392)

* simplify mapToAddressList (#393)

* superchain: convert superlumio mainnet, metal sepolia to toml

* justfile: update remove-chain to point at correct files

* superchain: use go codegen to create addresses.json

* run go linter

* remove javascript codegen

* superchain: remove all extra/addresses,genesis-system-config files

* superchain: remove embedded extra/addresses

* superchain: return error if ChainConfig fields are improperly ordered

* superchain: add explanatory comment for custom ChainConfig.MarshalTOML

* fix(bindings): Parse TOML Chain Configs (#394)

* fix(bindings): use toml configs

* fix(bindings): no need to read extra dir

* add-chain: cleanup unused code

* add-chain: remove 0x0 addresses via post-process instead of custom MarshalTOML

---------

Co-authored-by: George Knee <[email protected]>
Co-authored-by: refcell <[email protected]>

Author: 3 people

.circleci/config.yml

@@ -21,11 +21,13 @@ commands:
   install-just:
     description: "Install just"
     steps:
-      - run: |
-          wget -qO - 'https://proget.makedeb.org/debian-feeds/prebuilt-mpr.pub' | gpg --dearmor | sudo tee /usr/share/keyrings/prebuilt-mpr-archive-keyring.gpg 1> /dev/null
-          echo "deb [arch=all,$(dpkg --print-architecture) signed-by=/usr/share/keyrings/prebuilt-mpr-archive-keyring.gpg] https://proget.makedeb.org prebuilt-mpr $(lsb_release -cs)" | sudo tee /etc/apt/sources.list.d/prebuilt-mpr.list
-          sudo apt update
-          sudo apt install just
+      - run:
+          name: "Install just"
+          command: |
+            wget -qO - 'https://proget.makedeb.org/debian-feeds/prebuilt-mpr.pub' | gpg --dearmor | sudo tee /usr/share/keyrings/prebuilt-mpr-archive-keyring.gpg 1> /dev/null
+            echo "deb [arch=all,$(dpkg --print-architecture) signed-by=/usr/share/keyrings/prebuilt-mpr-archive-keyring.gpg] https://proget.makedeb.org prebuilt-mpr $(lsb_release -cs)" | sudo tee /etc/apt/sources.list.d/prebuilt-mpr.list
+            sudo apt update
+            sudo apt install just
 
 jobs:
   golang-lint:
@@ -106,9 +108,6 @@ jobs:
     steps:
       - checkout
       - install-just
-      - node/install:
-          install-yarn: false
-          node-version: '14.16.1'
       - run:
           name: Run codegen
           command: just codegen

add-chain/addresses.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/BurntSushi/toml"
 	"github.com/ethereum-optimism/superchain-registry/superchain"
 )
 
@@ -206,35 +207,11 @@ func readAddressesFromJSON(contractAddresses map[string]string, deploymentsDir s
 	return nil
 }
 
-func writeAddressesToJSON(contractsAddresses map[string]string, superchainRepoPath, target, chainName string) error {
-	checkSummedContractAddresses := make(map[string]string)
-	for k, v := range contractsAddresses {
-		checkSummedContractAddresses[k] = superchain.MustHexToAddress(v).String()
-	}
-
-	dirPath := filepath.Join(superchainRepoPath, "superchain", "extra", "addresses", target)
-	if err := os.MkdirAll(dirPath, 0o755); err != nil {
-		return fmt.Errorf("failed to create directory: %w", err)
-	}
-
-	filePath := filepath.Join(dirPath, chainName+".json")
-	file, err := os.Create(filePath)
-	if err != nil {
-		return fmt.Errorf("failed to create file: %w", err)
-	}
-	defer file.Close()
-
-	// Marshal the map to JSON
-	jsonData, err := json.MarshalIndent(checkSummedContractAddresses, "", "  ")
+func mapToAddressList(m map[string]string, result *superchain.AddressList) error {
+	out, err := toml.Marshal(m)
 	if err != nil {
-		return fmt.Errorf("failed to marshal json: %w", err)
+		return err
 	}
 
-	// Write the JSON data to the file
-	if _, err := file.Write(jsonData); err != nil {
-		return fmt.Errorf("failed to write json to file: %w", err)
-	}
-	fmt.Printf("Contract addresses written to: %s\n", filePath)
-
-	return nil
+	return toml.Unmarshal(out, result)
 }

add-chain/cmd/promote-to-standard.go

@@ -37,8 +37,8 @@ var PromoteToStandardCmd = cli.Command{
 
 		superchainRepoPath := filepath.Dir(filepath.Dir(filepath.Dir(thisFile)))
 		targetDir := filepath.Join(superchainRepoPath, "superchain", "configs", chain.Superchain)
-		targetFilePath := filepath.Join(targetDir, chain.Chain+".yaml")
-		err := config.WriteChainConfig(*chain, targetFilePath)
+		targetFilePath := filepath.Join(targetDir, chain.Chain+".toml")
+		err := config.WriteChainConfigTOML(*chain, targetFilePath)
 		if err != nil {
 			panic(err)
 		}

add-chain/config/config.go

@@ -1,15 +1,17 @@
 package config
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
+	"github.com/BurntSushi/toml"
 	"github.com/ethereum-optimism/superchain-registry/superchain"
-	"gopkg.in/yaml.v3"
 )
 
 type JSONChainConfig struct {
@@ -90,43 +92,50 @@ func ConstructChainConfig(
 	return chainConfig, nil
 }
 
-// WriteChainConfig accepts a rollupConfig, formats it, and writes some output files based on the given
-// target directories
-func WriteChainConfig(
-	rollupConfig superchain.ChainConfig,
-	targetDirectory string,
-) error {
-	yamlData, err := yaml.Marshal(rollupConfig)
+// WriteChainConfigTPOML accepts a rollupConfig, formats it, and writes a single output toml
+// file which includes the following:
+//   - general chain info/config
+//   - contract and role addresses
+//   - genesis system config
+//   - optional feature config info, if activated (e.g. plasma)
+func WriteChainConfigTOML(rollupConfig superchain.ChainConfig, targetDirectory string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	comments, err := rollupConfig.GenerateTOMLComments(ctx)
 	if err != nil {
-		return fmt.Errorf("failed to marshal yaml: %w", err)
+		return fmt.Errorf("failed to generate toml comments: %w", err)
 	}
 
-	// Unmarshal bytes into a yaml.Node for custom manipulation
-	var rootNode yaml.Node
-	if err = yaml.Unmarshal(yamlData, &rootNode); err != nil {
-		return err
+	// Marshal the struct to TOML
+	var buf bytes.Buffer
+	if err := toml.NewEncoder(&buf).Encode(rollupConfig); err != nil {
+		return fmt.Errorf("failed to marshal toml: %w", err)
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	if err = rollupConfig.EnhanceYAML(ctx, &rootNode); err != nil {
-		return err
-	}
+	// Create final content with comments
+	var finalContent strings.Builder
+	lines := strings.Split(buf.String(), "\n")
 
-	// Write the rollup config to a yaml file
-	filename := filepath.Join(targetDirectory)
-	file, err := os.Create(filename)
-	if err != nil {
-		return err
+	for i, line := range lines {
+		splits := strings.Split(line, "=")
+		lineKey := strings.TrimSpace(splits[0])
+		if len(splits) > 1 && strings.TrimSpace(splits[1]) == "\"0x0000000000000000000000000000000000000000\"" {
+			// Skip this line to exclude zero addresses from the output file. Makes the config .toml cleaner
+			continue
+		}
+		if comment, exists := comments[lineKey]; exists {
+			finalContent.WriteString(line + " " + comment + "\n")
+		} else if i != len(lines)-1 || line != "" {
+			// Prevent double empty line at the end of the file
+			finalContent.WriteString(line + "\n")
+		}
 	}
-	defer file.Close()
-
-	encoder := yaml.NewEncoder(file)
-	defer encoder.Close()
 
-	encoder.SetIndent(2)
-	if err := encoder.Encode(&rootNode); err != nil {
-		return fmt.Errorf("failed to write yaml file: %w", err)
+	// Write the enhanced TOML data to a file
+	filename := filepath.Join(targetDirectory)
+	if err := os.WriteFile(filename, []byte(finalContent.String()), 0o644); err != nil {
+		return fmt.Errorf("failed to write toml file: %w", err)
 	}
 
 	fmt.Printf("Rollup config written to: %s\n", filename)

add-chain/e2e_test.go

@@ -1,19 +1,17 @@
 package main
 
 import (
-	"encoding/json"
 	"os"
 	"strconv"
 	"testing"
 
-	"github.com/google/go-cmp/cmp"
+	"github.com/BurntSushi/toml"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v3"
 )
 
 const (
 	addressDir             = "../superchain/extra/addresses/sepolia/"
-	ymlConfigDir           = "../superchain/configs/sepolia/"
+	configDir              = "../superchain/configs/sepolia/"
 	genesisSystemConfigDir = "../superchain/extra/genesis-system-configs/sepolia/"
 )
 
@@ -90,9 +88,7 @@ func TestAddChain_Main(t *testing.T) {
 			err = runApp(args)
 			require.NoError(t, err, "add-chain app failed")
 
-			checkConfigYaml(t, tt.name, tt.chainShortName)
-			compareJsonFiles(t, "superchain/extra/addresses/sepolia/", tt.name, tt.chainShortName)
-			compareJsonFiles(t, "superchain/extra/genesis-system-configs/sepolia/", tt.name, tt.chainShortName)
+			checkConfigTOML(t, tt.name, tt.chainShortName)
 		})
 	}
 
@@ -151,44 +147,25 @@ func TestAddChain_CheckGenesis(t *testing.T) {
 	})
 }
 
-func compareJsonFiles(t *testing.T, dirPath, testName, chainShortName string) {
-	expectedBytes, err := os.ReadFile("./testdata/" + dirPath + "expected_" + testName + ".json")
-	require.NoError(t, err, "failed to read expected.json file from "+dirPath)
+func checkConfigTOML(t *testing.T, testName, chainShortName string) {
+	expectedBytes, err := os.ReadFile("./testdata/superchain/configs/sepolia/expected_" + testName + ".toml")
+	require.NoError(t, err, "failed to read expected.toml config file: %w", err)
 
-	var expectJSON map[string]interface{}
-	err = json.Unmarshal(expectedBytes, &expectJSON)
-	require.NoError(t, err, "failed to unmarshal expected.json file from "+dirPath)
+	var expectedTOML map[string]interface{}
+	err = toml.Unmarshal(expectedBytes, &expectedTOML)
+	require.NoError(t, err, "failed to unmarshal expected.toml config file: %w", err)
 
-	testBytes, err := os.ReadFile("../" + dirPath + chainShortName + ".json")
-	require.NoError(t, err, "failed to read test generated json file from "+dirPath)
+	testBytes, err := os.ReadFile(configDir + chainShortName + ".toml")
+	require.NoError(t, err, "failed to read testchain.toml config file: %w", err)
 
-	var testJSON map[string]interface{}
-	err = json.Unmarshal(testBytes, &testJSON)
-	require.NoError(t, err, "failed to read test generated json file from "+dirPath)
-
-	diff := cmp.Diff(expectJSON, testJSON)
-	require.Equal(t, diff, "", "expected json (-) does not match test json (+): %s", diff)
-}
-
-func checkConfigYaml(t *testing.T, testName, chainShortName string) {
-	expectedBytes, err := os.ReadFile("./testdata/superchain/configs/sepolia/expected_" + testName + ".yaml")
-	require.NoError(t, err, "failed to read expected.yaml config file: %w", err)
-
-	var expectedYaml map[string]interface{}
-	err = yaml.Unmarshal(expectedBytes, &expectedYaml)
-	require.NoError(t, err, "failed to unmarshal expected.yaml config file: %w", err)
-
-	testBytes, err := os.ReadFile(ymlConfigDir + chainShortName + ".yaml")
-	require.NoError(t, err, "failed to read testchain.yaml config file: %w", err)
-
-	require.Equal(t, string(expectedBytes), string(testBytes), "test .yaml contents do not meet expectation")
+	require.Equal(t, string(expectedBytes), string(testBytes), "test .toml contents do not meet expectation")
 }
 
 func cleanupTestFiles(t *testing.T, chainShortName string) {
 	paths := []string{
 		addressDir + chainShortName + ".json",
 		genesisSystemConfigDir + chainShortName + ".json",
-		ymlConfigDir + chainShortName + ".yaml",
+		configDir + chainShortName + ".toml",
 	}
 
 	for _, path := range paths {

add-chain/go.mod

@@ -5,14 +5,14 @@ go 1.21
 replace github.com/ethereum-optimism/superchain-registry/superchain => ../superchain
 
 require (
+	github.com/BurntSushi/toml v1.4.0
 	github.com/ethereum-optimism/optimism v1.7.7
 	github.com/ethereum-optimism/superchain-registry/superchain v0.0.0-20240614103325-d8902381f5d8
 	github.com/ethereum/go-ethereum v1.13.15
 	github.com/google/go-cmp v0.6.0
 	github.com/joho/godotenv v1.5.1
 	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.27.1
-	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -91,5 +91,6 @@ require (
 	golang.org/x/tools v0.21.0 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )

add-chain/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -135,34 +134,6 @@ func entrypoint(ctx *cli.Context) error {
 		return fmt.Errorf("failed to construct rollup config: %w", err)
 	}
 
-	targetFilePath := filepath.Join(targetDir, chainShortName+".yaml")
-	err = config.WriteChainConfig(rollupConfig, targetFilePath)
-	if err != nil {
-		return fmt.Errorf("error generating chain config .yaml file: %w", err)
-	}
-
-	fmt.Printf("Wrote config for new chain with identifier %s", rollupConfig.Identifier())
-
-	// Create genesis-system-config data
-	// (this is deprecated, users should load this from L1, when available via SystemConfig)
-	dirPath := filepath.Join(superchainRepoRoot, "superchain", "extra", "genesis-system-configs", superchainTarget)
-
-	if err := os.MkdirAll(dirPath, 0o755); err != nil {
-		return fmt.Errorf("failed to create directory: %w", err)
-	}
-
-	// Write the genesis system config JSON to a new file
-	systemConfigJSON, err := json.MarshalIndent(rollupConfig.Genesis.SystemConfig, "", "  ")
-	if err != nil {
-		return fmt.Errorf("failed to marshal genesis system config json: %w", err)
-	}
-
-	filePath := filepath.Join(dirPath, chainShortName+".json")
-	if err := os.WriteFile(filePath, systemConfigJSON, 0o644); err != nil {
-		return fmt.Errorf("failed to write genesis system config json: %w", err)
-	}
-	fmt.Printf("Genesis system config written to: %s\n", filePath)
-
 	err = readAddressesFromChain(addresses, l1RpcUrl, isFaultProofs)
 	if err != nil {
 		return fmt.Errorf("failed to read addresses from chain: %w", err)
@@ -172,11 +143,20 @@ func entrypoint(ctx *cli.Context) error {
 		addresses["DAChallengeAddress"] = rollupConfig.Plasma.DAChallengeAddress.String()
 	}
 
-	err = writeAddressesToJSON(addresses, superchainRepoRoot, superchainTarget, chainShortName)
+	addressList := superchain.AddressList{}
+	err = mapToAddressList(addresses, &addressList)
 	if err != nil {
-		return fmt.Errorf("failed to write contract addresses to JSON file: %w", err)
+		return fmt.Errorf("error converting map to AddressList: %w", err)
 	}
+	rollupConfig.Addresses = addressList
 
+	targetFilePath := filepath.Join(targetDir, chainShortName+".toml")
+	err = config.WriteChainConfigTOML(rollupConfig, targetFilePath)
+	if err != nil {
+		return fmt.Errorf("error generating chain config .yaml file: %w", err)
+	}
+
+	fmt.Printf("Wrote config for new chain with identifier %s", rollupConfig.Identifier())
 	return nil
 }