adding codex multicodecs and hashes

Author: dryajov

libp2p/cid.nim

@@ -61,7 +61,8 @@ const
     multiCodec("ed25519-pub"),
     multiCodec("codex-manifest"),
     multiCodec("codex-block"),
-    multiCodec("codex-root")
+    multiCodec("codex-slot-root"),
+    multiCodec("codex-proving-root")
   ]
 
 proc initCidCodeTable(): Table[int, MultiCodec] {.compileTime.} =

libp2p/multibase.nim

@@ -16,6 +16,7 @@
 {.push raises: [].}
 
 import tables
+import sequtils
 import stew/[base32, base58, base64, results]
 
 type
@@ -338,6 +339,7 @@ proc initMultiBaseNameTable(): Table[string, MBCodec] {.compileTime.} =
 const
   CodeMultiBases = initMultiBaseCodeTable()
   NameMultiBases = initMultiBaseNameTable()
+  MultibaseList* = MultiBaseCodecs.mapIt( it.name )
 
 proc encodedLength*(mbtype: typedesc[MultiBase], encoding: string,
                     length: int): int =

libp2p/multicodec.nim

@@ -177,8 +177,9 @@ const MultiCodecList = [
   ("skein1024-1008", 0xB3DE), ("skein1024-1016", 0xB3DF),
   ("skein1024-1024", 0xB3E0),
   # poseidon2
-  ("poseidon2-alt_bn_128-sponge-r2", 0xCD10),  # bn128 rate 2 sponge
-  ("poseidon2-alt_bn_128-mekle-2kb", 0xCD11),  # bn128 2kb compress & merkleize
+  ("poseidon2-alt_bn_128-sponge-r2", 0xCD10),       # bn128 rate 2 sponge
+  ("poseidon2-alt_bn_128-mekle-2kb", 0xCD11),       # bn128 2kb compress & merkleize
+  ("poseidon2-alt_bn_128-keyed-compress", 0xCD12),  # bn128 keyed compress
   # multiaddrs
   ("ip4", 0x04),
   ("ip6", 0x29),
@@ -240,6 +241,8 @@ const MultiCodecList = [
   ("codex-manifest", 0xCD01),
   ("codex-block", 0xCD02),
   ("codex-root", 0xCD03),
+  ("codex-slot-root", 0xCD04),
+  ("codex-proving-root", 0xCD05),
 ]
 
 type

libp2p/multihash.nim

@@ -24,6 +24,7 @@
 {.push raises: [].}
 
 import tables
+import sequtils
 import nimcrypto/[sha, sha2, keccak, blake2, hash, utils]
 import poseidon2
 import varint, vbuffer, multicodec, multibase
@@ -180,9 +181,14 @@ proc shake_256hash(data: openArray[byte], output: var openArray[byte]) =
     discard sctx.output(addr output[0], uint(len(output)))
     sctx.clear()
 
-proc poseidon2hash(data: openArray[byte], output: var openArray[byte]) =
+proc poseidon2_sponge_rate2(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
-    var digest = poseidon2.merkleRoot(data).toBytes()
+    var digest = poseidon2.Sponge.digest(data).toBytes()
+    copyMem(addr output[0], addr digest[0], uint(len(output)))
+
+proc poseidon2_merkle_2kb_sponge(data: openArray[byte], output: var openArray[byte]) =
+  if len(output) > 0:
+    var digest = poseidon2.SpongeMerkle.digest(data, 2048).toBytes()
     copyMem(addr output[0], addr digest[0], uint(len(output)))
 
 const
@@ -322,7 +328,8 @@ const
     MHash(mcodec: multiCodec("blake2s-240"), size: 30, coder: blake2Shash),
     MHash(mcodec: multiCodec("blake2s-248"), size: 31, coder: blake2Shash),
     MHash(mcodec: multiCodec("blake2s-256"), size: 32, coder: blake2Shash),
-    MHash(mcodec: multiCodec("poseidon2-alt_bn_128-a2-cdx1"), size: 32, coder: poseidon2hash)
+    MHash(mcodec: multiCodec("poseidon2-alt_bn_128-sponge-r2"), size: 32, coder: poseidon2_sponge_rate2),
+    MHash(mcodec: multiCodec("poseidon2-alt_bn_128-mekle-2kb"), size: 32, coder: poseidon2_merkle_2kb_sponge)
   ]
 
 proc initMultiHashCodeTable(): Table[MultiCodec, MHash] {.compileTime.} =
@@ -331,6 +338,7 @@ proc initMultiHashCodeTable(): Table[MultiCodec, MHash] {.compileTime.} =
 
 const
   CodeHashes = initMultiHashCodeTable()
+  MultiHashCodecsList* = HashesList.mapIt( it.mcodec )
 
 proc digestImplWithHash(hash: MHash, data: openArray[byte]): MultiHash =
   var buffer: array[MaxHashSize, byte]

tests/testcid.nim

@@ -62,3 +62,29 @@ suite "Content identifier CID test suite":
       cid1 != cid5
       cid2 != cid4
       cid3 != cid6
+
+  test "Check all cids and hashes":
+    var msg = cast[seq[byte]]("Hello World!")
+    for cidCodec in ContentIdsList:
+      for mhashCodec in MultiHashCodecsList:
+        let
+          cid = Cid.init(
+            CidVersion.CIDv1,
+            cidCodec,
+            MultiHash.digest($mhashCodec, msg).get()).get()
+        check:
+          cid.mcodec == cidCodec
+          cid.mhash().get().mcodec == mhashCodec
+
+  test "Check all cids and hashes base encode":
+    var msg = cast[seq[byte]]("Hello World!")
+    for cidCodec in ContentIdsList:
+      for mhashCodec in MultiHashCodecsList:
+        let
+          cid = Cid.init(
+            CidVersion.CIDv1,
+            cidCodec,
+            MultiHash.digest($mhashCodec, msg).get()).get()
+        check:
+          cid.mcodec == cidCodec
+          cid.mhash().get().mcodec == mhashCodec

tests/testmultihash.nim

@@ -14,7 +14,7 @@ import nimcrypto/utils
 import ../libp2p/multihash
 
 const
-  RustTestVectors = [
+  TestVectors = [
     [
       "sha1",
       "beep boop",
@@ -82,18 +82,19 @@ const
       """1D403EE2B40047B8060F68C67242175660F4174D0AF5C01D47168EC20ED619B0
          B7C42181F40AA1046F39E2EF9EFC6910782A998E0013D172458957957FAC9405
          B67D"""
+    ],
+    [
+      "poseidon2-alt_bn_128-sponge-r2",
+      "hello world",
+      """909A0320823F7FB71C0998153E73AC734AE4870518F5FE324BD2484B68B565C288CF1E1E"""
+    ],
+    [
+      "poseidon2-alt_bn_128-mekle-2kb",
+      "hello world",
+      """919A0320D9A6AE0CBF28C5E9CBE28D7231D3A4DEDF8B3826B0F8C3C002CA95C21253E614"""
     ]
   ]
 
-const CodexTestVectors = [
-  [
-    "poseidon2-alt_bn_128-a2-cdx1",
-    "hello world",
-    # TODO: https://github.com/codex-storage/nim-poseidon2/issues/5#issuecomment-1787447258
-    "8180B406 20 68656C6C6F20776F726C64000000000000000000000000000000000000000000"
-  ]
-]
-
 suite "MultiHash test suite":
 
   template checkTestVector(vector) =
@@ -106,10 +107,6 @@ suite "MultiHash test suite":
       hex(mh1) == hex(mh2)
       mh1 == mh2
 
-  test "rust-multihash test vectors":
-    for item in RustTestVectors:
-      checkTestVector(item)
-
-  test "codex test vectors":
-    for item in CodexTestVectors:
+  test "multihash test vectors":
+    for item in TestVectors:
       checkTestVector(item)