Simplify the slashing db import/export CLI

Author: zah

CHANGELOG.md

@@ -6,10 +6,10 @@ other clients and bringing further performance optimizations.
 
 **New features:**
 
-* A new `slashingProtection` sub-command offering import and export options
-  for the EIP-3076 slashing protection interchange format. Please see the
-  the prepared [migration guides](https://nimbus.guide/migrate.html) outlining
-  the safest way to migrate to Nimbus from other clients.
+* A new `slashingdb` sub-command offering import and export options for the
+  EIP-3076 slashing protection interchange format. Please see the the prepared
+  [migration guides](https://nimbus.guide/migrate.html) outlining the safest
+  way to migrate to Nimbus from other clients.
 
 * Pruning of the slashing protection database and transition to more optimal
   queries *->* significant reduction of disk and CPU usage on nodes running

beacon_chain/conf.nim

@@ -76,9 +76,8 @@ type
     file
 
   SlashProtCmd* = enum
-    exportAll = "Export the whole validator slashing protection DB to json."
-    `import` = "Import the slashing protection json to the node."
-    `export` = "Export specified validators slashing protection data to json"
+    `import` = "Import a EIP-3076 slashing protection interchange file"
+    `export` = "Export a EIP-3076 slashing protection interchange file"
     # migrateAll = "Export and remove the whole validator slashing protection DB."
     # migrate = "Export and remove specified validators from Nimbus."
 
@@ -496,18 +495,20 @@ type
           name: "url" }: Uri
 
     of slashingdb:
-      # TODO: when this is not set, confutils crashes
-      interchangeFile* {.
-        desc: "Path to the slashing interchange file"
-        name: "interchange" .}: Option[string]
-
       case slashingdbCmd* {.command.}: SlashProtCmd
-      of SlashProtCmd.exportAll, SlashProtCmd.`import`:
-        discard
+      of SlashProtCmd.`import`:
+        importedInterchangeFile* {.
+          desc: "EIP-3076 slashing protection interchange file to import"
+          argument .}: InputFile
       of SlashProtCmd.`export`:
-        validators* {.
-          argument
-          desc: "One or more validators to export".}: seq[string]
+        exportedValidators* {.
+          desc: "Limit the export to specific validators " &
+                "(specified as numeric indices or public keys)"
+          abbr: "v"
+          name: "validator" }: seq[string]
+        exportedInterchangeFile* {.
+          desc: "EIP-3076 slashing protection interchange file to export"
+          argument }: OutFile
 
   ValidatorClientConf* = object
     logLevel* {.

beacon_chain/nimbus_beacon_node.nim

@@ -1936,42 +1936,17 @@ proc doSlashingExport(conf: BeaconNodeConf) {.raises: [IOError, Defect].}=
   # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312
   let db = SlashingProtectionDB.loadUnchecked(dir, filetrunc, readOnly = false)
 
-  let interchange = block:
-    if conf.interchangeFile.isSome():
-      string(conf.interchangeFile.unsafeGet())
-    else:
-      conf.validatorsDir() & "/" & "interchange-" & "" & ".json"
-
-  db.exportSlashingInterchange(interchange)
+  let interchange = conf.exportedInterchangeFile.string
+  db.exportSlashingInterchange(interchange, conf.exportedValidators)
   echo "Export finished: '", dir/filetrunc & ".sqlite3" , "' into '", interchange, "'"
 
-proc doSlashingPartialExport(conf: BeaconNodeConf) {.raises: [IOError, Defect].} =
-  let
-    dir = conf.validatorsDir()
-    filetrunc = SlashingDbName
-  # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312
-  let db = SlashingProtectionDB.loadUnchecked(dir, filetrunc, readOnly = false)
-
-  let interchange = block:
-    if conf.interchangeFile.isSome():
-      string(conf.interchangeFile.unsafeGet())
-    else:
-      conf.validatorsDir() & "/" & "interchange-" & "" & ".json"
-
-  db.exportPartialSlashingInterchange(conf.validators, interchange)
-  echo "Partial export finished: '", dir/filetrunc/".sqlite3" , "' into '", interchange, "'"
-
 proc doSlashingImport(conf: BeaconNodeConf) {.raises: [SerializationError, IOError, Defect].} =
   let
     dir = conf.validatorsDir()
     filetrunc = SlashingDbName
   # TODO: Make it read-only https://github.com/status-im/nim-eth/issues/312
 
-  let interchange = block:
-    if conf.interchangeFile.isSome():
-      string(conf.interchangeFile.unsafeGet())
-    else:
-      conf.validatorsDir() & "/" & "interchange-" & "" & ".json"
+  let interchange = conf.importedInterchangeFile.string
 
   var spdir: SPDIR
   try:
@@ -2003,10 +1978,8 @@ proc doSlashingImport(conf: BeaconNodeConf) {.raises: [SerializationError, IOErr
 proc doSlashingInterchange(conf: BeaconNodeConf) {.raises: [Defect, CatchableError].} =
   doAssert conf.cmd == slashingdb
   case conf.slashingdbCmd
-  of SlashProtCmd.exportAll:
-    conf.doSlashingExport()
   of SlashProtCmd.`export`:
-    conf.doSlashingPartialExport()
+    conf.doSlashingExport()
   of SlashProtCmd.`import`:
     conf.doSlashingImport()
 

beacon_chain/validators/slashing_protection.nim

@@ -9,10 +9,11 @@
 
 import
   # stdlib
-  std/os,
+  std/[os, algorithm, sequtils],
   # Status
   eth/db/[kvstore, kvstore_sqlite3],
-  stew/results, chronicles,
+  stew/[results, byteutils],
+  chronicles,
   # Internal
   ../spec/[datatypes, digest, crypto],
   ./slashing_protection_common,
@@ -167,13 +168,6 @@ proc loadUnchecked*(
   except:
     result.disagreementBehavior = kChooseV1
 
-  try:
-    result.db_v1.fromRawDB(kvstore result.db_v2.getRawDBHandle())
-    result.modes.incl(kCompleteArchiveV1)
-  except:
-    doAssert result.modes.card > 0, "Couldn't open the DB."
-    result.disagreementBehavior = kChooseV2
-
 proc close*(db: SlashingProtectionDB) =
   ## Close a slashing protection database
   db.db_v2.close()
@@ -307,3 +301,24 @@ proc inclSPDIR*(db: SlashingProtectionDB, spdir: SPDIR): SlashingImportStatus
 proc toSPDIR*(db: SlashingProtectionDB): SPDIR
              {.raises: [IOError, Defect].} =
   db.db_v2.toSPDIR()
+
+proc exportSlashingInterchange*(
+       db: SlashingProtectionDB,
+       path: string,
+       validatorsWhiteList: seq[string] = @[],
+       prettify = true) {.raises: [Defect, IOError].} =
+  ## Export a database to the Slashing Protection Database Interchange Format
+  # We could modify toSPDIR to do the filtering directly
+  # but this is not a performance sensitive operation.
+  # so it's better to keep it simple.
+  var spdir = db.toSPDIR()
+
+  if validatorsWhiteList.len > 0:
+    # O(a log b) with b the number of validators to keep
+    #        and a the total number of validators in DB
+    let validators = validatorsWhiteList.sorted()
+    spdir.data.keepItIf(validators.binarySearch("0x" & it.pubkey.PubKeyBytes.toHex()) != -1)
+
+  Json.saveFile(path, spdir, prettify)
+  echo "Exported slashing protection DB to '", path, "'"
+

beacon_chain/validators/slashing_protection_common.nim

@@ -222,32 +222,6 @@ proc readValue*(r: var JsonReader, a: var (SlotString or EpochString))
   except ValueError:
     raiseUnexpectedValue(r, "Integer in a string expected")
 
-proc exportSlashingInterchange*(
-       db: auto,
-       path: string, prettify = true) {.raises: [Defect, IOError].} =
-  ## Export a database to the Slashing Protection Database Interchange Format
-  let spdir = db.toSPDIR()
-  Json.saveFile(path, spdir, prettify)
-  echo "Exported slashing protection DB to '", path, "'"
-
-proc exportPartialSlashingInterchange*(
-       db: SlashingProtectionDB_Concept,
-       validators: seq[string],
-       path: string, prettify = true) {.raises: [Defect, IOError].} =
-  ## Export a database to the Slashing Protection Database Interchange Format
-  # We could modify toSPDIR to do the filtering directly
-  # but this is not a performance sensitive operation.
-  # so it's better to keep it simple.
-  var spdir = db.toSPDIR()
-
-  # O(a log b) with b the number of validators to keep
-  #        and a the total number of validators in DB
-  let validators = validators.sorted()
-  spdir.data.keepItIf(validators.binarySearch("0x" & it.pubkey.PubKeyBytes.toHex()) != -1)
-
-  Json.saveFile(path, spdir, prettify)
-  echo "Exported slashing protection DB to '", path, "'"
-
 proc importSlashingInterchange*(
        db: auto,
        path: string): SlashingImportStatus {.raises: [Defect, IOError, SerializationError].} =

docs/slashing_interchange.md

@@ -2,9 +2,9 @@
 
 Importing and exporting validators is available via the following commands:
 
-- `path/to/nimbus_beacon_node slashingdb import --interchange=infile.json`
-- `path/to/nimbus_beacon_node slashingdb exportAll --interchange=outfile.json`
-- `path/to/nimbus_beacon_node slashingdb export --interchange=outfile.json --validators=0xAAAA...AAA --validators=0xBBBB...BBBB --validators=0xCCCC...CCCC`
+- `path/to/nimbus_beacon_node slashingdb import infile.json`
+- `path/to/nimbus_beacon_node slashingdb export outfile.json`
+- `path/to/nimbus_beacon_node slashingdb export outfile.json --validator=0xAAAA...AAA --validator=0xBBBB...BBBB --validator=0xCCCC...CCCC`
 
 ## Importing new validators
 
@@ -13,7 +13,7 @@ Importing and exporting validators is available via the following commands:
 The default command for import into the database is:
 
 ```
-build/nimbus_beacon_node slashingdb import --interchange=interchange.json
+build/nimbus_beacon_node slashingdb import interchange.json
 ```
 
 ### With specified validators folder
@@ -22,23 +22,23 @@ The validators folder contains the valdiators setup.
 By default it is `path/to/datadir/validators`
 
 ```
-build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --validators-dir=path/to/validatorsdir/
+build/nimbus_beacon_node slashingdb export interchange.json --validators-dir=path/to/validatorsdir/
 ```
 
 ### With the data-dir folder
 
 The data-dir contains the beacon node setup.
 
 ```
-build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --data-dir=path/to/datadir/
+build/nimbus_beacon_node slashingdb export interchange.json --data-dir=path/to/datadir/
 ```
 
 ## Exporting all validators
 
 The default command for exporting the database is:
 
 ```
-build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json
+build/nimbus_beacon_node slashingdb export interchange.json
 ```
 
 On success you will have a message similar to:
@@ -54,22 +54,22 @@ The validators folder contains the valdiators setup.
 By default it is `path/to/datadir/validators`
 
 ```
-build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --validators-dir=path/to/validatorsdir/
+build/nimbus_beacon_node slashingdb export interchange.json --validators-dir=path/to/validatorsdir/
 ```
 
 ### With the data-dir folder
 
 The data-dir contains the beacon node setup.
 
 ```
-build/nimbus_beacon_node slashingdb exportAll --interchange=interchange.json --data-dir=path/to/datadir/
+build/nimbus_beacon_node slashingdb export interchange.json --data-dir=path/to/datadir/
 ```
 
 ## Partial exports
 
 Partial export can be done by specifying the public keys of the relevant validators.
-The `--validators` command can be specified multiple time, once per validator.
+The `--validator` command can be specified multiple time, once per validator.
 
 ```
-build/nimbus_beacon_node slashingdb export --interchange=interchange.json --validators=0xb5da853a51d935da6f3bd46934c719fcca1bbf0b493264d3d9e7c35a1023b73c703b56d598edf0239663820af36ec615
+build/nimbus_beacon_node slashingdb export interchange.json --validator=0xb5da853a51d935da6f3bd46934c719fcca1bbf0b493264d3d9e7c35a1023b73c703b56d598edf0239663820af36ec615
 ```