Store keys together with node data (#2849)

Currently, computed hash keys are stored in a separate column family
with respect to the MPT data they're generated from - this has several
disadvantages:

* A lot of space is wasted because the lookup key (`RootedVertexID`) is
repeated in both tables - this is 30% of the `AriKey` content!
* rocksdb must maintain in-memory bloom filters and LRU caches for said
keys, doubling its "minimal efficient cache size"
* An extra disk traversal must be made to check for existence of cached
hash key
* Doubles the amount of files on disk due to each column family being
its own set of files

Here, the two CFs are joined such that both key and data is stored in
`AriVtx`. This means:

* we save ~30% disk space on repeated lookup keys
* we save ~2gb of memory overhead that can be used to cache data instead
of indices
* we can skip storing hash keys for MPT leaf nodes - these are trivial
to compute and waste a lot of space - previously they had to present in
the `AriKey` CF to avoid having to look in two tables on the happy path.
* There is a small increase in write amplification because when a hash
value is updated for a branch node, we must write both key and branch
data - previously we would write only the key
* There's a small shift in CPU usage - instead of performing lookups in
the database, hashes for leaf nodes are (re)-computed on the fly
* We can return to slightly smaller on-disk SST files since there's
fewer of them, which should reduce disk traffic a bit

Internally, there are also other advantages:

* when clearing keys, we no longer have to store a zero hash in memory -
instead, we deduce staleness of the cached key from the presence of an
updated VertexRef - this saves ~1gb of mem overhead during import
* hash key cache becomes dedicated to branch keys since leaf keys are no
longer stored in memory, reducing churn
* key computation is a lot faster thanks to the skipped second disk
traversal - a key computation for mainnet can be completed in 11 hours
instead of ~2 days (!) thanks to better cache usage and less read
amplification - with additional improvements to the on-disk format, we
can probably get rid of the initial full traversal method of seeding the
key cache on first start after import

All in all, this PR reduces the size of a mainnet database from 160gb to
110gb and the peak memory footprint during import by ~1-2gb.

Author: arnetheduck

nimbus/db/aristo/aristo_api.nim

@@ -510,7 +510,6 @@ type
     AristoApiProfBeGetTuvFn             = "be/getTuv"
     AristoApiProfBeGetLstFn             = "be/getLst"
     AristoApiProfBePutVtxFn             = "be/putVtx"
-    AristoApiProfBePutKeyFn             = "be/putKey"
     AristoApiProfBePutTuvFn             = "be/putTuv"
     AristoApiProfBePutLstFn             = "be/putLst"
     AristoApiProfBePutEndFn             = "be/putEnd"
@@ -557,8 +556,6 @@ when AutoValidateApiHooks:
 
     doAssert not api.partAccountTwig.isNil
     doAssert not api.partStorageTwig.isNil
-    doAssert not api.partUntwigGeneric.isNil
-    doAssert not api.partUntwigGenericOk.isNil
     doAssert not api.partUntwigPath.isNil
     doAssert not api.partUntwigPathOk.isNil
 
@@ -910,12 +907,6 @@ func init*(
           be.putVtxFn(a, b, c)
     data.list[AristoApiProfBePutVtxFn.ord].masked = true
 
-    beDup.putKeyFn =
-      proc(a: PutHdlRef; b: RootedVertexID, c: HashKey) =
-        AristoApiProfBePutKeyFn.profileRunner:
-          be.putKeyFn(a, b, c)
-    data.list[AristoApiProfBePutKeyFn.ord].masked = true
-
     beDup.putTuvFn =
       proc(a: PutHdlRef; b: VertexID) =
         AristoApiProfBePutTuvFn.profileRunner:

nimbus/db/aristo/aristo_blobify.nim

@@ -157,21 +157,22 @@ proc blobifyTo*(pyl: LeafPayload, data: var seq[byte]) =
     data &= pyl.stoData.blobify().data
     data &= [0x20.byte]
 
-proc blobifyTo*(vtx: VertexRef; data: var seq[byte]): Result[void,AristoError] =
+proc blobifyTo*(vtx: VertexRef; key: HashKey, data: var seq[byte]): Result[void,AristoError] =
   ## This function serialises the vertex argument to a database record.
   ## Contrary to RLP based serialisation, these records aim to align on
   ## fixed byte boundaries.
   ## ::
   ##   Branch:
+  ##     <HashKey>      -- optional hash key
   ##     [VertexID, ..] -- list of up to 16 child vertices lookup keys
   ##     seq[byte]      -- hex encoded partial path (non-empty for extension nodes)
   ##     uint64         -- lengths of each child vertex, each taking 4 bits
-  ##     0x80 + xx      -- marker(2) + pathSegmentLen(6)
+  ##     0x80 + xx      -- marker(0/2) + pathSegmentLen(6)
   ##
   ##   Leaf:
   ##     seq[byte]      -- opaque leaf data payload (might be zero length)
   ##     seq[byte]      -- hex encoded partial path (at least one byte)
-  ##     0xc0 + yy      -- marker(2) + partialPathLen(6)
+  ##     0xc0 + yy      -- marker(3) + partialPathLen(6)
   ##
   ## For a branch record, the bytes of the `access` array indicate the position
   ## of the Patricia Trie vertex reference. So the `vertexID` with index `n` has
@@ -182,6 +183,13 @@ proc blobifyTo*(vtx: VertexRef; data: var seq[byte]): Result[void,AristoError] =
     return err(BlobifyNilVertex)
   case vtx.vType:
   of Branch:
+    let code = if key.isValid:
+      data.add byte(key.len)
+      data.add key.data()
+      # TODO using 0 here for legacy reasons - a bit flag would be easier
+      0'u8 shl 6
+    else:
+      2'u8 shl 6
     var
       lens = 0u64
       pos = data.len
@@ -205,7 +213,7 @@ proc blobifyTo*(vtx: VertexRef; data: var seq[byte]): Result[void,AristoError] =
 
     data &= pSegm.data()
     data &= lens.toBytesBE
-    data &= [0x80u8 or psLen]
+    data &= [code or psLen]
 
   of Leaf:
     let
@@ -215,14 +223,14 @@ proc blobifyTo*(vtx: VertexRef; data: var seq[byte]): Result[void,AristoError] =
       return err(BlobifyLeafPathOverflow)
     vtx.lData.blobifyTo(data)
     data &= pSegm.data()
-    data &= [0xC0u8 or psLen]
+    data &= [(3'u8 shl 6) or psLen]
 
   ok()
 
-proc blobify*(vtx: VertexRef): seq[byte] =
+proc blobify*(vtx: VertexRef, key: HashKey): seq[byte] =
   ## Variant of `blobify()`
   result = newSeqOfCap[byte](128)
-  if vtx.blobifyTo(result).isErr:
+  if vtx.blobifyTo(key, result).isErr:
     result.setLen(0) # blobify only fails on invalid verticies
 
 proc blobifyTo*(lSst: SavedState; data: var seq[byte]): Result[void,AristoError] =
@@ -287,7 +295,7 @@ proc deblobifyType*(record: openArray[byte]; T: type VertexRef):
     return err(DeblobVtxTooShort)
 
   ok case record[^1] shr 6:
-  of 2: Branch
+  of 0, 2: Branch
   of 3: Leaf
   else:
     return err(DeblobUnknown)
@@ -300,16 +308,20 @@ proc deblobify*(
   ## argument `vtx` can be `nil`.
   if record.len < 3:                                  # minimum `Leaf` record
     return err(DeblobVtxTooShort)
-
-  ok case record[^1] shr 6:
-  of 2: # `Branch` vertex
-    if record.len < 11:                               # at least two edges
+  let kind = record[^1] shr 6
+  let start = if kind == 0:
+    int(record[0] + 1)
+  else:
+    0
+  ok case kind:
+  of 0, 2: # `Branch` vertex
+    if record.len - start < 11:                               # at least two edges
       return err(DeblobBranchTooShort)
     let
       aInx = record.len - 9
       aIny = record.len - 2
     var
-      offs = 0
+      offs = start
       lens = uint64.fromBytesBE record.toOpenArray(aInx, aIny)  # bitmap
       vtxList: array[16,VertexID]
       n = 0
@@ -346,12 +358,18 @@ proc deblobify*(
       vType: Leaf,
       pfx:  pathSegment)
 
-    ? record.toOpenArray(0, pLen - 1).deblobify(vtx.lData)
+    ? record.toOpenArray(start, pLen - 1).deblobify(vtx.lData)
     vtx
 
   else:
     return err(DeblobUnknown)
 
+proc deblobify*(record: openArray[byte], T: type HashKey): Opt[HashKey] =
+  if record.len > 1 and ((record[^1] shr 6) == 0) and (int(record[0]) + 1) < record.len:
+    HashKey.fromBytes(record.toOpenArray(1, int(record[0])))
+  else:
+    Opt.none(HashKey)
+
 proc deblobify*(
     data: openArray[byte];
     T: type SavedState;