Improve the compatibility with various Web3Signer versions and configurations (#3640)

* Some Web3Signer versions insist replying with text/plain messages
* When reading blocks, the Web3Signer uses upper-case fork identifiers
  instead of lower-case identifies like the Beacon API.

Author: zah

beacon_chain/spec/eth2_apis/eth2_rest_serialization.nim

@@ -665,9 +665,9 @@ proc readValue*(
       reader, "Expected a valid hex string with " & $value.len() & " bytes")
 
 ## ForkedBeaconBlock
-proc readValue*(reader: var JsonReader[RestJson],
-                value: var ForkedBeaconBlock) {.
-     raises: [IOError, SerializationError, Defect].} =
+proc readValue*[BlockType: Web3SignerForkedBeaconBlock|ForkedBeaconBlock](
+    reader: var JsonReader[RestJson],
+    value: var BlockType) {.raises: [IOError, SerializationError, Defect].} =
   var
     version: Option[BeaconBlockFork]
     data: Option[JsonString]
@@ -680,17 +680,17 @@ proc readValue*(reader: var JsonReader[RestJson],
                                     "ForkedBeaconBlock")
       let vres = reader.readValue(string)
       case vres
-      of "phase0":
+      of "PHASE0", "phase0":
         version = some(BeaconBlockFork.Phase0)
-      of "altair":
+      of "ALTAIR", "altair":
         version = some(BeaconBlockFork.Altair)
-      of "bellatrix":
+      of "BELLATRIX", "bellatrix":
         version = some(BeaconBlockFork.Bellatrix)
       else:
         reader.raiseUnexpectedValue("Incorrect version field value")
-    of "data":
+    of "block", "block_header", "data":
       if data.isSome():
-        reader.raiseUnexpectedField("Multiple data fields found",
+        reader.raiseUnexpectedField("Multiple block or block_header fields found",
                                     "ForkedBeaconBlock")
       data = some(reader.readValue(JsonString))
     else:
@@ -711,7 +711,7 @@ proc readValue*(reader: var JsonReader[RestJson],
         none[phase0.BeaconBlock]()
     if res.isNone():
       reader.raiseUnexpectedValue("Incorrect phase0 block format")
-    value = ForkedBeaconBlock.init(res.get())
+    value = ForkedBeaconBlock.init(res.get()).BlockType
   of BeaconBlockFork.Altair:
     let res =
       try:
@@ -721,7 +721,7 @@ proc readValue*(reader: var JsonReader[RestJson],
         none[altair.BeaconBlock]()
     if res.isNone():
       reader.raiseUnexpectedValue("Incorrect altair block format")
-    value = ForkedBeaconBlock.init(res.get())
+    value = ForkedBeaconBlock.init(res.get()).BlockType
   of BeaconBlockFork.Bellatrix:
     let res =
       try:
@@ -731,20 +731,29 @@ proc readValue*(reader: var JsonReader[RestJson],
         none[bellatrix.BeaconBlock]()
     if res.isNone():
       reader.raiseUnexpectedValue("Incorrect bellatrix block format")
-    value = ForkedBeaconBlock.init(res.get())
+    value = ForkedBeaconBlock.init(res.get()).BlockType
+
+
+proc writeValue*[BlockType: Web3SignerForkedBeaconBlock|ForkedBeaconBlock](
+    writer: var JsonWriter[RestJson],
+    value: BlockType) {.raises: [IOError, Defect].} =
+
+  template forkIdentifier(id: string): auto =
+    when BlockType is ForkedBeaconBlock:
+      id
+    else:
+      (static toUpperAscii id)
 
-proc writeValue*(writer: var JsonWriter[RestJson], value: ForkedBeaconBlock) {.
-     raises: [IOError, Defect].} =
   writer.beginRecord()
   case value.kind
   of BeaconBlockFork.Phase0:
-    writer.writeField("version", "phase0")
+    writer.writeField("version", forkIdentifier "phase0")
     writer.writeField("data", value.phase0Data)
   of BeaconBlockFork.Altair:
-    writer.writeField("version", "altair")
+    writer.writeField("version", forkIdentifier "altair")
     writer.writeField("data", value.altairData)
   of BeaconBlockFork.Bellatrix:
-    writer.writeField("version", "bellatrix")
+    writer.writeField("version", forkIdentifier "bellatrix")
     writer.writeField("data", value.bellatrixData)
   writer.endRecord()
 
@@ -1452,7 +1461,7 @@ proc readValue*(reader: var JsonReader[RestJson],
         reader.raiseUnexpectedValue("Field `fork_info` is missing")
       let data =
         block:
-          let res = decodeJsonString(ForkedBeaconBlock, data.get(), true)
+          let res = decodeJsonString(Web3SignerForkedBeaconBlock, data.get(), true)
           if res.isErr():
             reader.raiseUnexpectedValue(
               "Incorrect field `beacon_block` format")

beacon_chain/spec/eth2_apis/rest_remote_signer_calls.nim

@@ -12,7 +12,7 @@ import
   nimcrypto/utils as ncrutils,
   serialization, json_serialization,
   json_serialization/std/[options, net, sets],
-  stew/[results, base10],
+  stew/[results, base10, byteutils],
   "."/[rest_types, eth2_rest_serialization]
 
 export chronos, httpclient, client, rest_types, eth2_rest_serialization, results
@@ -58,16 +58,19 @@ declareHistogram nbc_remote_signer_time,
    buckets = delayBuckets
 
 proc getUpcheck*(): RestResponse[Web3SignerStatusResponse] {.
-     rest, endpoint: "/upcheck", meth: MethodGet.}
+     rest, endpoint: "/upcheck",
+     meth: MethodGet, accept: "application/json" .}
   ## https://consensys.github.io/web3signer/web3signer-eth2.html#tag/Server-Status
 
 proc getKeys*(): RestResponse[Web3SignerKeysResponse] {.
-     rest, endpoint: "/api/v1/eth2/publicKeys", meth: MethodGet.}
+     rest, endpoint: "/api/v1/eth2/publicKeys",
+     meth: MethodGet, accept: "application/json" .}
   ## https://consensys.github.io/web3signer/web3signer-eth2.html#tag/Public-Key
 
 proc signDataPlain*(identifier: ValidatorPubKey,
                     body: Web3SignerRequest): RestPlainResponse {.
-     rest, endpoint: "/api/v1/eth2/sign/{identifier}", meth: MethodPost.}
+     rest, endpoint: "/api/v1/eth2/sign/{identifier}",
+     meth: MethodPost, accept: "application/json" .}
   # https://consensys.github.io/web3signer/web3signer-eth2.html#tag/Signing
 
 proc signData*(client: RestClientRef, identifier: ValidatorPubKey,
@@ -77,7 +80,8 @@ proc signData*(client: RestClientRef, identifier: ValidatorPubKey,
   inc(nbc_remote_signer_requests)
   let response =
     try:
-      await client.signDataPlain(identifier, body)
+      await client.signDataPlain(identifier, body,
+                                 restAcceptType = "application/json")
     except RestError as exc:
       let msg = "[" & $exc.name & "] " & $exc.msg
       debug "Error occured while generating signature",
@@ -101,19 +105,27 @@ proc signData*(client: RestClientRef, identifier: ValidatorPubKey,
     case response.status
     of 200:
       inc(nbc_remote_signer_200_responses)
-      let res = decodeBytes(Web3SignerSignatureResponse, response.data,
-                            response.contentType)
-      if res.isErr():
-        let msg = "Unable to decode remote signer response [" &
-                  $res.error() & "]"
-        inc(nbc_remote_signer_failures)
-        return Web3SignerDataResponse.err(msg)
-      let sig = res.get().signature.load()
-      if sig.isNone():
+      let sig = if response.contentType.contains("text/plain"):
+        let asStr = fromBytes(string, response.data)
+        let sigFromText = fromHex(ValidatorSig, asStr)
+        if sigFromText.isErr:
+          return Web3SignerDataResponse.err("Unable to decode signature from plain text")
+        sigFromText.get.load
+      else:
+        let res = decodeBytes(Web3SignerSignatureResponse, response.data,
+                              response.contentType)
+        if res.isErr:
+          let msg = "Unable to decode remote signer response [" & $res.error() & "]"
+          inc(nbc_remote_signer_failures)
+          return Web3SignerDataResponse.err(msg)
+        res.get.signature.load
+
+      if sig.isNone:
         let msg = "Remote signer returns invalid signature"
         inc(nbc_remote_signer_failures)
         return Web3SignerDataResponse.err(msg)
-      Web3SignerDataResponse.ok(sig.get())
+
+      Web3SignerDataResponse.ok(sig.get)
     of 400:
       inc(nbc_remote_signer_400_responses)
       let res = decodeBytes(Web3SignerErrorResponse, response.data,

beacon_chain/spec/eth2_apis/rest_types.nim

@@ -489,7 +489,7 @@ type
         serializedFieldName: "block".}: phase0.BeaconBlock
     of Web3SignerRequestKind.BlockV2:
       beaconBlock* {.
-        serializedFieldName: "beacon_block".}: ForkedBeaconBlock
+        serializedFieldName: "beacon_block".}: Web3SignerForkedBeaconBlock
     of Web3SignerRequestKind.Deposit:
       deposit*: Web3SignerDepositData
     of Web3SignerRequestKind.RandaoReveal:
@@ -658,7 +658,7 @@ func init*(t: typedesc[Web3SignerRequest], fork: Fork,
   )
 
 func init*(t: typedesc[Web3SignerRequest], fork: Fork,
-           genesis_validators_root: Eth2Digest, data: ForkedBeaconBlock,
+           genesis_validators_root: Eth2Digest, data: Web3SignerForkedBeaconBlock,
            signingRoot: Option[Eth2Digest] = none[Eth2Digest]()
           ): Web3SignerRequest =
   Web3SignerRequest(

beacon_chain/spec/forks.nim

@@ -107,6 +107,8 @@ type
     of BeaconBlockFork.Altair:    altairData*:    altair.BeaconBlock
     of BeaconBlockFork.Bellatrix: bellatrixData*: bellatrix.BeaconBlock
 
+  Web3SignerForkedBeaconBlock* {.borrow: `.`} = distinct ForkedBeaconBlock
+
   ForkedTrustedBeaconBlock* = object
     case kind*: BeaconBlockFork
     of BeaconBlockFork.Phase0:    phase0Data*:     phase0.TrustedBeaconBlock
@@ -345,8 +347,8 @@ template asTrusted*(x: ForkedSignedBeaconBlock): ForkedTrustedSignedBeaconBlock
   isomorphicCast[ForkedTrustedSignedBeaconBlock](x)
 
 template withBlck*(
-    x: ForkedBeaconBlock | ForkedSignedBeaconBlock |
-       ForkedTrustedSignedBeaconBlock,
+    x: ForkedBeaconBlock | Web3SignerForkedBeaconBlock |
+    ForkedSignedBeaconBlock | ForkedTrustedSignedBeaconBlock,
     body: untyped): untyped =
   case x.kind
   of BeaconBlockFork.Phase0:
@@ -368,6 +370,8 @@ func proposer_index*(x: ForkedBeaconBlock): uint64 =
 func hash_tree_root*(x: ForkedBeaconBlock): Eth2Digest =
   withBlck(x): hash_tree_root(blck)
 
+func hash_tree_root*(x: Web3SignerForkedBeaconBlock): Eth2Digest {.borrow.}
+
 template getForkedBlockField*(x: ForkedSignedBeaconBlock | ForkedTrustedSignedBeaconBlock, y: untyped): untyped =
   # unsafeAddr avoids a copy of the field in some cases
   (case x.kind

beacon_chain/validators/validator_pool.nim

@@ -205,7 +205,7 @@ proc signWithRemoteValidator*(v: AttachedValidator, fork: Fork,
                               genesis_validators_root: Eth2Digest,
                               blck: ForkedBeaconBlock): Future[SignatureResult]
                              {.async.} =
-  let request = Web3SignerRequest.init(fork, genesis_validators_root, blck)
+  let request = Web3SignerRequest.init(fork, genesis_validators_root, blck.Web3SignerForkedBeaconBlock)
   debug "Signing block proposal using remote signer",
         validator = shortLog(v)
   return await v.signData(request)