Unified database frontend integration (#1670)

* Nimbus folder environment update

details:
* Integrated `CoreDbRef` for the sources in the `nimbus` sub-folder.
* The `nimbus` program does not compile yet as it needs the updates
  in the parallel `stateless` sub-folder.

* Stateless environment update

details:
* Integrated `CoreDbRef` for the sources in the `stateless` sub-folder.
* The `nimbus` program compiles now.

* Premix environment update

details:
* Integrated `CoreDbRef` for the sources in the `premix` sub-folder.

* Fluffy environment update

details:
* Integrated `CoreDbRef` for the sources in the `fluffy` sub-folder.

* Tools environment update

details:
* Integrated `CoreDbRef` for the sources in the `tools` sub-folder.

* Nodocker environment update

details:
* Integrated `CoreDbRef` for the sources in the
  `hive_integration/nodocker` sub-folder.

* Tests environment update

details:
* Integrated `CoreDbRef` for the sources in the `tests` sub-folder.
* The unit tests compile and run cleanly now.

* Generalise `CoreDbRef` to any `select_backend` supported database

why:
  Generalisation was just missed due to overcoming some compiler oddity
  which was tied to rocksdb for testing.

* Suppress compiler warning for `newChainDB()`

why:
  Warning was added to this function which must be wrapped so that
  any `CatchableError` is re-raised as `Defect`.

* Split off persistent `CoreDbRef` constructor into separate file

why:
  This allows to compile a memory only database version without linking
  the backend library.

* Use memory `CoreDbRef` database by default

detail:
 Persistent DB constructor needs to import `db/core_db/persistent

why:
 Most tests use memory DB anyway. This avoids linking `-lrocksdb` or
 any other backend by default.

* fix `toLegacyBackend()` availability check

why:
  got garbled after memory/persistent split.

* Clarify raw access to MPT for snap sync handler

why:
  Logically, `kvt` is not the raw access for the hexary trie (although
  this holds for the legacy database)

Author: mjfh

fluffy/conf.nim

@@ -10,10 +10,11 @@
 import
   std/os,
   uri, confutils, confutils/std/net, chronicles,
-  eth/keys, eth/net/nat, eth/p2p/discoveryv5/[enr, node, routing_table],
+  eth/keys, eth/p2p/discoveryv5/[enr, node, routing_table],
   json_rpc/rpcproxy,
   nimcrypto/hash,
   stew/byteutils,
+  eth/net/nat, # must be late (compilation annoyance)
   ./logging,
   ./network/wire/portal_protocol_config
 

fluffy/network/history/history_network.nim

@@ -9,12 +9,12 @@
 
 import
   stew/results, chronos, chronicles,
-  eth/[common/eth_types_rlp, rlp, trie, trie/db],
+  eth/[common/eth_types_rlp, rlp],
   eth/p2p/discoveryv5/[protocol, enr],
   ../../common/common_types,
   ../../content_db,
   ../../network_metadata,
-  ../../../nimbus/constants,
+  ../../../nimbus/[constants, db/core_db],
   ../wire/[portal_protocol, portal_stream, portal_protocol_config],
   "."/[history_content, accumulator]
 
@@ -184,7 +184,7 @@ func encode*(receipts: seq[Receipt]): seq[byte] =
 # for if/when peer scoring/banning is added.
 
 proc calcRootHash(items: Transactions | PortalReceipts | Withdrawals): Hash256 =
-  var tr = initHexaryTrie(newMemoryDB())
+  var tr = newCoreDbRef(LegacyDbMemory).mptPrune
   for i, item in items:
     try:
       tr.put(rlp.encode(i), item.asSeq())

fluffy/tests/test_state_network.nim

@@ -8,22 +8,22 @@
 import
   std/os,
   testutils/unittests, chronos,
-  eth/[common/eth_hash, keys, trie/db, trie/hexary],
+  eth/[common/eth_hash, keys],
   eth/p2p/discoveryv5/protocol as discv5_protocol, eth/p2p/discoveryv5/routing_table,
-  ../../nimbus/[config, db/db_chain, db/state_db],
+  ../../nimbus/[config, db/core_db, db/state_db],
   ../../nimbus/common/[chain_config, genesis],
   ../network/wire/[portal_protocol, portal_stream],
   ../network/state/[state_content, state_network],
   ../content_db,
   ./test_helpers
 
-proc genesisToTrie(filePath: string): HexaryTrie =
+proc genesisToTrie(filePath: string): CoreDbMptRef =
   # TODO: Doing our best here with API that exists, to be improved.
   var cn: NetworkParams
   if not loadNetworkParams(filePath, cn):
     quit(1)
 
-  let sdb  = newStateDB(newMemoryDB(), false)
+  let sdb  = newStateDB(newCoreDbRef LegacyDbMemory, false)
   let map  = toForkTransitionTable(cn.config)
   let fork = map.toHardFork(forkDeterminationInfo(0.toBlockNumber, cn.genesis.timestamp))
   discard toGenesisHeader(cn.genesis, sdb, fork)

fluffy/tools/beacon_lc_bridge/beacon_lc_bridge.nim

@@ -68,7 +68,7 @@
 import
   std/[os, strutils, options],
   chronicles, chronos, confutils,
-  eth/[keys, rlp], eth/[trie, trie/db],
+  eth/[keys, rlp],
   # Need to rename this because of web3 ethtypes and ambigious indentifier mess
   # for `BlockHeader`.
   eth/common/eth_types as etypes,
@@ -87,6 +87,7 @@ import
   ../../network/history/[history_content, history_network],
   ../../network/beacon_light_client/beacon_light_client_content,
   ../../common/common_types,
+  ../../nimbus/db/core_db,
   ./beacon_lc_bridge_conf
 
 from stew/objects import checkedEnumAssign
@@ -189,7 +190,7 @@ proc calculateTransactionData(
     items: openArray[TypedTransaction]):
     Hash256 {.raises: [].} =
 
-  var tr = initHexaryTrie(newMemoryDB())
+  var tr = newCoreDbRef(LegacyDbMemory).mptPrune
   for i, t in items:
     try:
       let tx = distinctBase(t)
@@ -207,7 +208,7 @@ proc calculateWithdrawalsRoot(
   items: openArray[WithdrawalV1]):
     Hash256 {.raises: [].} =
 
-  var tr = initHexaryTrie(newMemoryDB())
+  var tr = newCoreDbRef(LegacyDbMemory).mptPrune
   for i, w in items:
     try:
       let withdrawal = etypes.Withdrawal(

hive_integration/nodocker/consensus/consensus_sim.nim

@@ -19,7 +19,7 @@ import
 proc processChainData(cd: ChainData): TestStatus =
   let
     networkId = NetworkId(cd.params.config.chainId)
-    com = CommonRef.new(newMemoryDB(),
+    com = CommonRef.new(newCoreDbRef LegacyDbMemory,
       pruneTrie = false,
       networkId,
       cd.params

hive_integration/nodocker/engine/test_env.nim

@@ -78,7 +78,7 @@ proc setupELClient*(t: TestEnv, chainFile: string, enableAuth: bool) =
 
   t.ethNode = setupEthNode(t.conf, t.ctx, eth)
   t.com = CommonRef.new(
-      newMemoryDb(),
+      newCoreDbRef LegacyDbMemory,
       t.conf.pruneMode == PruneMode.Full,
       t.conf.networkId,
       t.conf.networkParams

hive_integration/nodocker/graphql/graphql_sim.nim

@@ -78,7 +78,7 @@ proc main() =
     conf    = makeConfig(@["--custom-network:" & genesisFile])
     ethCtx  = newEthContext()
     ethNode = setupEthNode(conf, ethCtx, eth)
-    com     = CommonRef.new(newMemoryDB(),
+    com     = CommonRef.new(newCoreDbRef LegacyDbMemory,
       pruneTrie = false,
       conf.networkId,
       conf.networkParams

hive_integration/nodocker/pyspec/test_env.nim

@@ -39,7 +39,7 @@ proc genesisHeader(node: JsonNode): BlockHeader =
   rlp.decode(genesisRLP, EthBlock).header
 
 proc setupELClient*(t: TestEnv, conf: ChainConfig, node: JsonNode) =
-  let memDB = newMemoryDb()
+  let memDB = newCoreDbRef LegacyDbMemory
   t.ctx  = newEthContext()
   t.ethNode = setupEthNode(t.conf, t.ctx, eth)
   t.com = CommonRef.new(

hive_integration/nodocker/rpc/test_env.nim

@@ -94,7 +94,7 @@ proc setupEnv*(): TestEnv =
   let
     ethCtx  = newEthContext()
     ethNode = setupEthNode(conf, ethCtx, eth)
-    com     = CommonRef.new(newMemoryDb(),
+    com     = CommonRef.new(newCoreDbRef LegacyDbMemory,
       conf.pruneMode == PruneMode.Full,
       conf.networkId,
       conf.networkParams

nimbus/common/common.nim

@@ -13,17 +13,17 @@ import
   std/[options, times],
   chronicles,
   eth/trie/trie_defs,
-  ./chain_config,
-  ./hardforks,
-  ./evmforks,
-  ./genesis,
+  ../core/[pow, clique, casper],
+  ../db/[core_db, storage_types],
   ../utils/[utils, ec_recover],
-  ../db/[db_chain, storage_types],
-  ../core/[pow, clique, casper]
+  ".."/[constants, errors],
+  "."/[chain_config, evmforks, genesis, hardforks]
 
 export
   chain_config,
-  db_chain,
+  core_db,
+  constants,
+  errors,
   options,
   evmforks,
   hardforks,
@@ -41,7 +41,7 @@ type
 
   CommonRef* = ref object
     # all purpose storage
-    db: ChainDBRef
+    db: CoreDbRef
 
     # prune underlying state db?
     pruneTrie: bool
@@ -130,15 +130,15 @@ proc daoCheck(conf: ChainConfig) =
     conf.daoForkBlock = conf.homesteadBlock
 
 proc init(com      : CommonRef,
-          db       : TrieDatabaseRef,
+          db       : CoreDbRef,
           pruneTrie: bool,
           networkId: NetworkId,
           config   : ChainConfig,
           genesis  : Genesis) {.gcsafe, raises: [CatchableError].} =
 
   config.daoCheck()
 
-  com.db          = ChainDBRef.new(db)
+  com.db          = db
   com.pruneTrie   = pruneTrie
   com.config      = config
   com.forkTransitionTable = config.toForkTransitionTable()
@@ -157,7 +157,7 @@ proc init(com      : CommonRef,
   # by setForkId
   if genesis.isNil.not:
     com.genesisHeader = toGenesisHeader(genesis,
-      com.currentFork, com.db.db)
+      com.currentFork, com.db)
     com.setForkId(com.genesisHeader)
 
   # Initalise the PoA state regardless of whether it is needed on the current
@@ -194,7 +194,7 @@ proc getTdIfNecessary(com: CommonRef, blockHash: Hash256): Option[DifficultyInt]
 # ------------------------------------------------------------------------------
 
 proc new*(_: type CommonRef,
-          db: TrieDatabaseRef,
+          db: CoreDbRef,
           pruneTrie: bool = true,
           networkId: NetworkId = MainNet,
           params = networkParams(MainNet)): CommonRef
@@ -211,7 +211,7 @@ proc new*(_: type CommonRef,
     params.genesis)
 
 proc new*(_: type CommonRef,
-          db: TrieDatabaseRef,
+          db: CoreDbRef,
           config: ChainConfig,
           pruneTrie: bool = true,
           networkId: NetworkId = MainNet): CommonRef
@@ -227,11 +227,11 @@ proc new*(_: type CommonRef,
     config,
     nil)
 
-proc clone*(com: CommonRef, db: TrieDatabaseRef): CommonRef =
+proc clone*(com: CommonRef, db: CoreDbRef): CommonRef =
   ## clone but replace the db
   ## used in EVM tracer whose db is CaptureDB
   CommonRef(
-    db           : ChainDBRef.new(db),
+    db           : db,
     pruneTrie    : com.pruneTrie,
     config       : com.config,
     forkTransitionTable: com.forkTransitionTable,
@@ -248,7 +248,7 @@ proc clone*(com: CommonRef, db: TrieDatabaseRef): CommonRef =
   )
 
 proc clone*(com: CommonRef): CommonRef =
-  com.clone(com.db.db)
+  com.clone(com.db)
 
 # ------------------------------------------------------------------------------
 # Public functions
@@ -361,14 +361,14 @@ proc consensus*(com: CommonRef, header: BlockHeader): ConsensusType
 
 proc initializeEmptyDb*(com: CommonRef)
     {.gcsafe, raises: [CatchableError].} =
-  let trieDB = com.db.db
-  if canonicalHeadHashKey().toOpenArray notin trieDB:
+  let kvt = com.db.kvt()
+  if canonicalHeadHashKey().toOpenArray notin kvt:
     trace "Writing genesis to DB"
     doAssert(com.genesisHeader.blockNumber.isZero,
       "can't commit genesis block with number > 0")
     discard com.db.persistHeaderToDb(com.genesisHeader,
       com.consensusType == ConsensusType.POS)
-    doAssert(canonicalHeadHashKey().toOpenArray in trieDB)
+    doAssert(canonicalHeadHashKey().toOpenArray in kvt)
 
 proc syncReqNewHead*(com: CommonRef; header: BlockHeader)
     {.gcsafe, raises: [].} =
@@ -396,7 +396,7 @@ func pos*(com: CommonRef): CasperRef =
   ## Getter
   com.pos
 
-func db*(com: CommonRef): ChainDBRef =
+func db*(com: CommonRef): CoreDbRef =
   com.db
 
 func consensus*(com: CommonRef): ConsensusType =