Write preimages and EVM state into erigon table

Makefile

@@ -19,7 +19,10 @@ opera-image:
 	docker build \
     	    --network=host \
     	    -f ./docker/Dockerfile.opera -t "opera:$(TAG)" .
-
+erigon:
+	./build/opera preimages write
+	./build/opera erigon write
+	./build/opera erigon read
 .PHONY: test
 test:
 	go test ./...

cmd/opera/launcher/chaincmd.go

@@ -3,6 +3,7 @@ package launcher
 import (
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"gopkg.in/urfave/cli.v1"
+	//"github.com/Fantom-foundation/go-opera/gossip/erigon"
 )
 
 var (
@@ -11,6 +12,7 @@ var (
 		Usage: `EVM export mode ("full" or "ext-mpt" or "mpt" or "none")`,
 		Value: "mpt",
 	}
+
 	importCommand = cli.Command{
 		Name:      "import",
 		Usage:     "Import a blockchain file",
@@ -116,4 +118,63 @@ Checks EVM storage roots and code hashes
 			},
 		},
 	}
+
+	erigonCommand = cli.Command{
+		Name:     "erigon",
+		Usage:    "Write/Read EVM data into/from Erigon kv.PlainState table",
+		Category: "MISCELLANEOUS COMMANDS",
+		//Action: utils.MigrateFlags(writeEVMToErigon),
+		Subcommands: []cli.Command{
+			{
+				Name:   "write",
+				Usage:  "Write EVM accounts to erigon database",
+				Action: utils.MigrateFlags(writeErigon),
+				Description: `
+   				opera erigon write
+				`,
+			},
+			{
+				Name:   "read",
+				Usage:  "Read EVM accounts from erigon database",
+				Action: utils.MigrateFlags(readErigon),
+				Description: `
+   				opera erigon read
+				`,
+			},
+		},
+
+		Description: `
+		opera erigon
+
+	The erigon command iterates over MPT data and flushes it into Erigon tables`,
+	}
+
+	preimagesCommand = cli.Command{
+		Name:     "preimages",
+		Usage:    "Write/Read preimages into/from Erigon kv.Senders table",
+		Category: "MISCELLANEOUS COMMANDS",
+		//Action: utils.MigrateFlags(writeEVMToErigon),
+		Subcommands: []cli.Command{
+			{
+				Name:   "write",
+				Usage:  "Write preimages to erigon table",
+				Action: utils.MigrateFlags(writePreimagesCmd),
+				Description: `
+   				opera preimages write
+				`,
+			},
+			{
+				Name:   "read",
+				Usage:  "Read EVM preimages from erigon table",
+				Action: utils.MigrateFlags(readPreimagesCmd),
+				Description: `
+   				opera preimages read
+				`,
+			},
+		},
+		Description: `
+		opera preimages
+
+	The erigon command iterates over MPT data and flushes it into Erigon tables`,
+	}
 )

cmd/opera/launcher/erigoncmd.go

@@ -0,0 +1,78 @@
+package launcher
+
+import (
+	"context"
+	//"fmt"
+	"fmt"
+	"path"
+	"time"
+
+	"gopkg.in/urfave/cli.v1"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+
+	"github.com/Fantom-foundation/go-opera/gossip/erigon"
+	"github.com/Fantom-foundation/go-opera/integration"
+	"github.com/Fantom-foundation/go-opera/logger"
+
+	"github.com/ledgerwatch/erigon-lib/kv"
+)
+
+func readErigon(_ *cli.Context) error {
+
+	db := erigon.MakeChainDatabase(logger.New("mdbx"))
+	defer db.Close()
+
+	tx, err := db.BeginRo(context.Background())
+	if err != nil {
+		return fmt.Errorf("unable to begin transaction, err: %q", err)
+	}
+	defer tx.Rollback()
+
+	if err := erigon.ReadErigonTableNoDups(kv.PlainState, tx); err != nil {
+		return fmt.Errorf("unable to read from Erigon table, err: %q", err)
+	}
+
+	// TODO handle flags
+	return nil
+}
+
+func writeErigon(ctx *cli.Context) error {
+
+	start := time.Now()
+	log.Info("Writing of EVM accounts into Erigon database started")
+
+	// initiate erigon database
+	db := erigon.MakeChainDatabase(logger.New("mdbx"))
+	defer db.Close()
+
+	cfg := makeAllConfigs(ctx)
+
+	rawProducer := integration.DBProducer(path.Join(erigon.DefaultDataDir(), "chaindata"), cacheScaler(ctx))
+
+	gdb, err := makeRawGossipStore(rawProducer, cfg)
+	if err != nil {
+		log.Crit("DB opening error", "datadir", cfg.Node.DataDir, "err", err)
+	}
+	if gdb.GetHighestLamport() != 0 {
+		log.Warn("Attempting genesis export not in a beginning of an epoch. Genesis file output may contain excessive data.")
+	}
+	defer gdb.Close()
+
+	log.Info("Getting EvmDb")
+	chaindb := gdb.EvmStore().EvmDb
+
+	log.Info("Getting FinalizedStateRoot")
+	root := common.Hash(gdb.GetBlockState().FinalizedStateRoot)
+
+	log.Info("Getting LastBlock")
+	lastBlockIdx := gdb.GetBlockState().LastBlock.Idx
+
+	log.Info("Generate Erigon Plain State...")
+	if err := erigon.GeneratePlainState(root, chaindb, db, lastBlockIdx); err != nil {
+		return err
+	}
+	log.Info("Generation of Erigon Plain State is complete", "elapsed", common.PrettyDuration(time.Since(start)))
+	return nil
+}

cmd/opera/launcher/genesiscmd.go

@@ -20,8 +20,6 @@ import (
 	"gopkg.in/urfave/cli.v1"
 
 	"github.com/Fantom-foundation/go-opera/gossip/evmstore"
-	"github.com/Fantom-foundation/go-opera/inter/ibr"
-	"github.com/Fantom-foundation/go-opera/inter/ier"
 	"github.com/Fantom-foundation/go-opera/opera/genesis"
 	"github.com/Fantom-foundation/go-opera/opera/genesisstore"
 	"github.com/Fantom-foundation/go-opera/opera/genesisstore/fileshash"
@@ -243,78 +241,84 @@ func exportGenesis(ctx *cli.Context) error {
 	if to > gdb.GetEpoch() {
 		to = gdb.GetEpoch()
 	}
+
 	toBlock := idx.Block(0)
 	fromBlock := idx.Block(0)
-	{
-		log.Info("Exporting epochs", "from", from, "to", to)
-		writer := newUnitWriter(plain)
-		err := writer.Start(header, genesisstore.EpochsSection, tmpPath)
-		if err != nil {
-			return err
-		}
-		for i := to; i >= from; i-- {
-			er := gdb.GetFullEpochRecord(i)
-			if er == nil {
-				log.Warn("No epoch record", "epoch", i)
-				break
-			}
-			b, _ := rlp.EncodeToBytes(ier.LlrIdxFullEpochRecord{
-				LlrFullEpochRecord: *er,
-				Idx:                i,
-			})
-			_, err := writer.Write(b)
+	/*
+		{
+			log.Info("Exporting epochs", "from", from, "to", to)
+			writer := newUnitWriter(plain)
+			err := writer.Start(header, genesisstore.EpochsSection, tmpPath)
 			if err != nil {
 				return err
 			}
-			if i == from {
-				fromBlock = er.BlockState.LastBlock.Idx
+			for i := to; i >= from; i-- {
+				er := gdb.GetFullEpochRecord(i)
+				if er == nil {
+					log.Warn("No epoch record", "epoch", i)
+					break
+				}
+				b, _ := rlp.EncodeToBytes(ier.LlrIdxFullEpochRecord{
+					LlrFullEpochRecord: *er,
+					Idx:                i,
+				})
+				_, err := writer.Write(b)
+				if err != nil {
+					return err
+				}
+				if i == from {
+					fromBlock = er.BlockState.LastBlock.Idx
+				}
+				if i == to {
+					toBlock = er.BlockState.LastBlock.Idx
+				}
 			}
-			if i == to {
-				toBlock = er.BlockState.LastBlock.Idx
+			epochsHash, err = writer.Flush()
+			if err != nil {
+				return err
 			}
+			log.Info("Exported epochs", "hash", epochsHash.String())
 		}
-		epochsHash, err = writer.Flush()
-		if err != nil {
-			return err
-		}
-		log.Info("Exported epochs", "hash", epochsHash.String())
-	}
+	*/
 
-	if fromBlock < 1 {
-		// avoid underflow
-		fromBlock = 1
-	}
-	{
-		log.Info("Exporting blocks", "from", fromBlock, "to", toBlock)
-		writer := newUnitWriter(plain)
-		err := writer.Start(header, genesisstore.BlocksSection, tmpPath)
-		if err != nil {
-			return err
+	/*
+		if fromBlock < 1 {
+			// avoid underflow
+			fromBlock = 1
 		}
-		for i := toBlock; i >= fromBlock; i-- {
-			br := gdb.GetFullBlockRecord(i)
-			if br == nil {
-				log.Warn("No block record", "block", i)
-				break
+		{
+			log.Info("Exporting blocks", "from", fromBlock, "to", toBlock)
+			writer := newUnitWriter(plain)
+			err := writer.Start(header, genesisstore.BlocksSection, tmpPath)
+			if err != nil {
+				return err
 			}
-			if i%200000 == 0 {
-				log.Info("Exporting blocks", "last", i)
+			for i := toBlock; i >= fromBlock; i-- {
+				br := gdb.GetFullBlockRecord(i)
+				if br == nil {
+					log.Warn("No block record", "block", i)
+					break
+				}
+				if i%200000 == 0 {
+					log.Info("Exporting blocks", "last", i)
+				}
+				b, _ := rlp.EncodeToBytes(ibr.LlrIdxFullBlockRecord{
+					LlrFullBlockRecord: *br,
+					Idx:                i,
+				})
+				_, err := writer.Write(b)
+				if err != nil {
+					return err
+				}
 			}
-			b, _ := rlp.EncodeToBytes(ibr.LlrIdxFullBlockRecord{
-				LlrFullBlockRecord: *br,
-				Idx:                i,
-			})
-			_, err := writer.Write(b)
+			blocksHash, err = writer.Flush()
 			if err != nil {
 				return err
 			}
+			log.Info("Exported blocks", "hash", blocksHash.String())
 		}
-		blocksHash, err = writer.Flush()
-		if err != nil {
-			return err
-		}
-		log.Info("Exported blocks", "hash", blocksHash.String())
-	}
+
+	*/
 
 	if mode != "none" {
 		log.Info("Exporting EVM data", "from", fromBlock, "to", toBlock)

cmd/opera/launcher/launcher.go

@@ -222,6 +222,10 @@ func init() {
 		snapshotCommand,
 		// See dbcmd.go
 		dbCommand,
+		// See fixdirty.go
+		//fixDirtyCommand,
+		erigonCommand,
+		preimagesCommand,
 	}
 	sort.Sort(cli.CommandsByName(app.Commands))
 

cmd/opera/launcher/preimagescmd.go

@@ -0,0 +1,51 @@
+package launcher
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"gopkg.in/urfave/cli.v1"
+
+	"github.com/Fantom-foundation/go-opera/gossip/erigon"
+	"github.com/Fantom-foundation/go-opera/logger"
+
+	"github.com/ethereum/go-ethereum/log"
+
+	"github.com/ledgerwatch/erigon/common"
+
+	"github.com/ledgerwatch/erigon-lib/kv"
+)
+
+func writePreimagesCmd(_ *cli.Context) error {
+	start := time.Now()
+
+	db := erigon.MakeChainDatabase(logger.New("mdbx"))
+	defer db.Close()
+
+	if err := erigon.WritePreimagesToSenders(db); err != nil {
+		return fmt.Errorf("Import error: %q", err)
+	}
+
+	log.Info("Writing preimages is complete", "elapsed", common.PrettyDuration(time.Since(start)))
+
+	return nil
+}
+
+func readPreimagesCmd(_ *cli.Context) error {
+
+	db := erigon.MakeChainDatabase(logger.New("mdbx"))
+	defer db.Close()
+
+	tx, err := db.BeginRo(context.Background())
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	if err := erigon.ReadErigonTableNoDups(kv.Senders, tx); err != nil {
+		return err
+	}
+
+	return nil
+}

debug/loudpanic.go

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
+//go:build go1.6
 // +build go1.6
 
 package debug

debug/loudpanic_fallback.go

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
+//go:build !go1.6
 // +build !go1.6
 
 package debug