Implement p2p key codec as per the (new) spec

ntninja · ntninja · commit 0f29ab3371b0 · 2020-08-24T21:49:37.000+02:00
diff --git a/multiaddr/codecs/cid.py b/multiaddr/codecs/cid.py
@@ -0,0 +1,132 @@
+import base58
+import cid
+
+from . import LENGTH_PREFIXED_VAR_SIZE
+
+
+SIZE = LENGTH_PREFIXED_VAR_SIZE
+IS_PATH = False
+
+
+# Spec: https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#string-representation
+CIDv0_PREFIX_TO_LENGTH = {
+    # base58btc prefixes for valid lengths 1 – 42 with the identity “hash” function
+    '12': [5, 12, 19, 23, 30, 41, 52, 56],
+    '13': [9, 16, 34, 45],
+    '14': [27, 38, 49, 60],
+    '15': [3, 6, 20],
+    '16': [3, 6, 13, 20, 31, 42, 53],
+    '17': [3, 13, 42],
+    '18': [3],
+    '19': [3, 24, 57],
+    '1A': [24, 35, 46],
+    '1B': [35],
+    '1D': [17],
+    '1E': [10, 17],
+    '1F': [10],
+    '1G': [10, 28, 50],
+    '1H': [28, 39],
+    '1P': [21],
+    '1Q': [21],
+    '1R': [21, 54],
+    '1S': [54],
+    '1T': [7, 32, 43],
+    '1U': [7, 32, 43],
+    '1V': [7],
+    '1W': [7, 14],
+    '1X': [7, 14],
+    '1Y': [7, 14],
+    '1Z': [7, 14],
+    '1f': [4],
+    '1g': [4, 58],
+    '1h': [4, 25, 58],
+    '1i': [4, 25],
+    '1j': [4, 25],
+    '1k': [4, 25, 47],
+    '1m': [4, 36, 47],
+    '1n': [4, 36],
+    '1o': [4, 36],
+    '1p': [4],
+    '1q': [4],
+    '1r': [4],
+    '1s': [4],
+    '1t': [4],
+    '1u': [4],
+    '1v': [4],
+    '1w': [4],
+    '1x': [4],
+    '1y': [4],
+    '1z': [4, 18],
+
+    # base58btc prefix for length 42 with the sha256 hash function
+    'Qm': [46],
+}
+
+PROTO_NAME_TO_CIDv1_CODEC = {
+    # The “p2p” multiaddr protocol requires all keys to use the “libp2p-key” multicodec
+    "p2p": "libp2p-key",
+}
+
+
+def to_bytes(proto, string):
+    expected_codec = PROTO_NAME_TO_CIDv1_CODEC.get(proto.name)
+
+    if len(string) in CIDv0_PREFIX_TO_LENGTH.get(string[0:2], ()):  # CIDv0
+        # Upgrade the wire (binary) representation of any received CIDv0 string
+        # to CIDv1 if we can determine which multicodec value to use
+        if expected_codec:
+            return cid.make_cid(1, expected_codec, base58.b58decode(string)).buffer
+
+        return base58.b58decode(string)
+    else:  # CIDv1+
+        parsed = cid.from_string(string)
+
+        # Ensure CID has correct codec for protocol
+        if expected_codec and parsed.codec != expected_codec:
+            raise ValueError("“{0}” multiaddr CIDs must use the “{1}” multicodec"
+                             .format(proto.name, expected_codec))
+
+        return parsed.buffer
+
+
+def _is_binary_cidv0_multihash(buf):
+    if buf.startswith(b"\x12\x20") and len(buf) == 34:  # SHA2-256
+        return True
+
+    if (buf[0] == 0x00 and buf[1] in range(43)) and len(buf) == (buf[1] + 2):  # Identity hash
+        return True
+
+    return False
+
+
+def to_string(proto, buf):
+    expected_codec = PROTO_NAME_TO_CIDv1_CODEC.get(proto.name)
+
+    if _is_binary_cidv0_multihash(buf):  # CIDv0
+        if not expected_codec:
+            # Simply encode as base58btc as there is nothing better to do
+            return base58.b58encode(buf).decode('ascii')
+
+        # “Implementations SHOULD display peer IDs using the first (raw
+        #  base58btc encoded multihash) format until the second format is
+        #  widely supported.”
+        #
+        # In the future the following line should instead convert the multihash
+        # to CIDv1 and with the `expected_codec` and wrap it in base32:
+        #   return cid.make_cid(1, expected_codec, buf).encode("base32").decode("ascii")
+        return base58.b58encode(buf).decode("ascii")
+    else:  # CIDv1+
+        parsed = cid.from_bytes(buf)
+
+        # Ensure CID has correct codec for protocol
+        if expected_codec and parsed.codec != expected_codec:
+            raise ValueError("“{0}” multiaddr CIDs must use the “{1}” multicodec"
+                             .format(proto.name, expected_codec))
+
+        # “Implementations SHOULD display peer IDs using the first (raw
+        #  base58btc encoded multihash) format until the second format is
+        #  widely supported.”
+        if expected_codec and _is_binary_cidv0_multihash(parsed.multihash):
+            return base58.b58encode(parsed.multihash).decode("ascii")
+
+        return parsed.encode("base32").decode("ascii")
diff --git a/multiaddr/codecs/p2p.py b/multiaddr/codecs/p2p.py
diff --git a/multiaddr/protocols.py b/multiaddr/protocols.py
@@ -132,7 +132,7 @@ def __repr__(self):
     Protocol(P_SCTP, 'sctp', 'uint16be'),
     Protocol(P_UDT, 'udt', None),
     Protocol(P_UTP, 'utp', None),
-    Protocol(P_P2P, 'p2p', 'p2p'),
+    Protocol(P_P2P, 'p2p', 'cid'),
     Protocol(P_ONION, 'onion', 'onion'),
     Protocol(P_ONION3, 'onion3', 'onion3'),
     Protocol(P_QUIC, 'quic', None),
diff --git a/requirements.txt b/requirements.txt
@@ -2,3 +2,5 @@ varint
 base58
 netaddr
 idna
+py-cid
+py-multicodec >= 0.2.0
diff --git a/setup.py b/setup.py
@@ -54,6 +54,8 @@
         'varint',
         'base58',
         'netaddr',
+        'py-cid',
+        'py-multicodec >= 0.2.0',
     ],
     test_suite='tests',
     tests_require=[
diff --git a/tests/test_multiaddr.py b/tests/test_multiaddr.py
@@ -100,7 +100,8 @@ def test_invalid(addr_str):
      "/unix/Überrschung!/大柱",
      "/unix/stdio",
      "/ip4/1.2.3.4/tcp/80/unix/a/b/c/d/e/f",
-     "/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234/unix/stdio",
+     "/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC"
+     "/tcp/1234/unix/stdio",
      "/ip4/127.0.0.1/tcp/9090/http/p2p-webrtc-direct",
      "/dns/example.com",
      "/dns4/موقع.وزارة-الاتصالات.مصر"])  # nopep8
@@ -174,8 +175,9 @@ def test_invalid_protocols_with_string(proto_string):
      ("/ip4/0.0.0.0", 0, ("/ip4/0.0.0.0",)),
      ("/ip6/::1", 1, ("/ip6/::1",)),
      ("/onion/timaq4ygg2iegci7:80/http", 0, ("/onion/timaq4ygg2iegci7:80/http",)),
-     ("/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234", 1,
-      ("/ip4/127.0.0.1", "/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234",)),
+     ("/ip4/127.0.0.1/p2p/bafzbeigvf25ytwc3akrijfecaotc74udrhcxzh2cx3we5qqnw5vgrei4bm/tcp/1234", 1,
+      ("/ip4/127.0.0.1",
+       "/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234")),
      ("/ip4/1.2.3.4/tcp/80/unix/a/b/c/d/e/f", -1,
       ("/ip4/1.2.3.4", "/tcp/80", "/unix/a/b/c/d/e/f"))])
 def test_split(proto_string, maxsplit, expected):
@@ -188,7 +190,8 @@ def test_split(proto_string, maxsplit, expected):
      ((b"\x04\x00\x00\x00\x00",), "/ip4/0.0.0.0"),
      (("/ip6/::1",), "/ip6/::1"),
      (("/onion/timaq4ygg2iegci7:80/http",), "/onion/timaq4ygg2iegci7:80/http"),
-     ((b"\x04\x7F\x00\x00\x01", "/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234",),
+     ((b"\x04\x7F\x00\x00\x01",
+       "/p2p/bafzbeigvf25ytwc3akrijfecaotc74udrhcxzh2cx3we5qqnw5vgrei4bm/tcp/1234",),
       "/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234"),
      (("/ip4/1.2.3.4", "/tcp/80", "/unix/a/b/c/d/e/f"), "/ip4/1.2.3.4/tcp/80/unix/a/b/c/d/e/f")])
 def test_join(proto_parts, expected):
@@ -223,7 +226,7 @@ def assert_value_for_proto(multi, proto, expected):
 def test_get_value():
     ma = Multiaddr(
         "/ip4/127.0.0.1/utp/tcp/5555/udp/1234/utp/"
-        "p2p/QmbHVEEepCi7rn7VL7Exxpd2Ci9NNB6ifvqwhsrbRMgQFP")
+        "p2p/bafzbeigalb34xlqdtvyklzqa5ibmn6pssqsdskc4ty2e4jxy2kamquh22y")
 
     assert_value_for_proto(ma, P_IP4, "127.0.0.1")
     assert_value_for_proto(ma, P_UTP, None)
@@ -273,7 +276,7 @@ def test_get_value():
 def test_views():
     ma = Multiaddr(
         "/ip4/127.0.0.1/utp/tcp/5555/udp/1234/utp/"
-        "p2p/QmbHVEEepCi7rn7VL7Exxpd2Ci9NNB6ifvqwhsrbRMgQFP")
+        "p2p/bafzbeigalb34xlqdtvyklzqa5ibmn6pssqsdskc4ty2e4jxy2kamquh22y")
 
     for idx, (proto1, proto2, item, value) in enumerate(zip(ma, ma.keys(), ma.items(), ma.values())):  # noqa: E501
         assert (proto1, value) == (proto2, value) == item
diff --git a/tests/test_transforms.py b/tests/test_transforms.py
@@ -17,10 +17,11 @@
 from multiaddr.protocols import _names_to_protocols
 from multiaddr.protocols import Protocol
 
-# These test values were generated by running them
-# through the go implementation of multiaddr.
-# https://github.com/multiformats/go-multiaddr
-ADDR_BYTES_MAP_STR_TEST_DATA = [
+# These test values were generated by running them through the go implementation
+# of multiaddr (https://github.com/multiformats/go-multiaddr)
+#
+# All values are bijective.
+ADDR_BYTES_STR_TEST_DATA = [
     (_names_to_protocols['ip4'], b'\x0a\x0b\x0c\x0d', '10.11.12.13'),
     (_names_to_protocols['ip6'],
      b'\x1a\xa1\x2b\xb2\x3c\xc3\x4d\xd4\x5e\xe5\x6f\xf6\x7a\xb7\x8a\xc8',
@@ -30,22 +31,38 @@
      b'\x9a\x18\x08\x73\x06\x36\x90\x43\x09\x1f\x04\xd2',
      'timaq4ygg2iegci7:1234'),
     (_names_to_protocols['p2p'],
-     b'\x12\x20\xd5\x2e\xbb\x89\xd8\x5b\x02\xa2\x84\x94\x82\x03\xa6\x2f\xf2'
-     b'\x83\x89\xc5\x7c\x9f\x42\xbe\xec\x4e\xc2\x0d\xb7\x6a\x68\x91\x1c\x0b',
+     b'\x01\x72\x12\x20\xd5\x2e\xbb\x89\xd8\x5b\x02\xa2\x84\x94\x82\x03\xa6\x2f'
+     b'\xf2\x83\x89\xc5\x7c\x9f\x42\xbe\xec\x4e\xc2\x0d\xb7\x6a\x68\x91\x1c\x0b',
      'QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC'),
 
     # Additional test data
     (_names_to_protocols['dns4'],
      b'\xd9\x85\xd9\x88\xd9\x82\xd8\xb9.\xd9\x88\xd8\xb2\xd8\xa7\xd8\xb1\xd8\xa9'
      b'-\xd8\xa7\xd9\x84\xd8\xa7\xd8\xaa\xd8\xb5\xd8\xa7\xd9\x84\xd8\xa7\xd8\xaa'
      b'.\xd9\x85\xd8\xb5\xd8\xb1',
-     # Explicitly mark this as unicode to force the text to be LTR in editors
+     # Explicitly mark this as unicode, as the “u” forces the text to be displayed LTR in editors
      u'موقع.وزارة-الاتصالات.مصر'),
     (_names_to_protocols['dns4'],
      b'fu\xc3\x9fball.example',
      'fußball.example'),  # This will fail if IDNA-2003/NamePrep is used
 ]
 
+ADDR_BYTES_FROM_STR_TEST_DATA = [
+    # New CIDv1 string to new CIDv1 binary format mapping (non-bijective)
+    (_names_to_protocols['p2p'],
+     b'\x01\x72\x12\x20\xd5\x2e\xbb\x89\xd8\x5b\x02\xa2\x84\x94\x82\x03\xa6\x2f'
+     b'\xf2\x83\x89\xc5\x7c\x9f\x42\xbe\xec\x4e\xc2\x0d\xb7\x6a\x68\x91\x1c\x0b',
+     'bafzbeigvf25ytwc3akrijfecaotc74udrhcxzh2cx3we5qqnw5vgrei4bm'),
+]
+
+ADDR_BYTES_TO_STR_TEST_DATA = [
+    # Old CIDv0 binary to old CIDv0 string format mapping (non-bijective)
+    (_names_to_protocols['p2p'],
+     b'\x12\x20\xd5\x2e\xbb\x89\xd8\x5b\x02\xa2\x84\x94\x82\x03\xa6\x2f\xf2'
+     b'\x83\x89\xc5\x7c\x9f\x42\xbe\xec\x4e\xc2\x0d\xb7\x6a\x68\x91\x1c\x0b',
+     'QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC'),
+]
+
 BYTES_MAP_STR_TEST_DATA = [
     ("/ip4/127.0.0.1/udp/1234", b'\x04\x7f\x00\x00\x01\x91\x02\x04\xd2'),
     ("/ip4/127.0.0.1/tcp/4321", b'\x04\x7f\x00\x00\x01\x06\x10\xe1'),
@@ -57,7 +74,7 @@
 @pytest.mark.parametrize("codec_name, buf, expected", [
     (None, b'\x01\x02\x03', (0, 0)),
     ('ip4', b'\x01\x02\x03', (4, 0)),
-    ('p2p', b'\x40\x50\x60\x51', (64, 1)),
+    ('cid', b'\x40\x50\x60\x51', (64, 1)),
 ])
 def test_size_for_addr(codec_name, buf, expected):
     buf_io = io.BytesIO(buf)
@@ -76,13 +93,14 @@ def test_bytes_iter(buf, expected):
     assert list((proto, val) for _, proto, _, val in bytes_iter(buf)) == expected
 
 
-@pytest.mark.parametrize("proto, buf, expected", ADDR_BYTES_MAP_STR_TEST_DATA)
+@pytest.mark.parametrize("proto, buf, expected",
+                         ADDR_BYTES_STR_TEST_DATA + ADDR_BYTES_TO_STR_TEST_DATA)
 def test_codec_to_string(proto, buf, expected):
     assert codec_by_name(proto.codec).to_string(proto, buf) == expected
 
 
 @pytest.mark.parametrize("proto, expected, string",
-                         ADDR_BYTES_MAP_STR_TEST_DATA)
+                         ADDR_BYTES_STR_TEST_DATA + ADDR_BYTES_FROM_STR_TEST_DATA)
 def test_codec_to_bytes(proto, string, expected):
     assert codec_by_name(proto.codec).to_bytes(proto, string) == expected
 
@@ -149,6 +167,8 @@ def test_bytes_to_string_value_error(protocol_extension, bytes):
     (_names_to_protocols['onion'], 'timaq4ygg2iegci7:0'),
     (_names_to_protocols['onion'], 'timaq4ygg2iegci7:71234'),
     (_names_to_protocols['p2p'], '15230d52ebb89d85b02a284948203a'),
+    (_names_to_protocols['p2p'],  # CID type != "libp2p-key":
+     'bafyaajaiaejcbrrv5vds2whn3c464rsb5r2vpxeanneinzlijenlac77cju2pptf'),
     (_names_to_protocols['ip6zone'], ""),
 ])
 def test_codec_to_bytes_value_error(proto, address):
@@ -160,10 +180,45 @@ def test_codec_to_bytes_value_error(proto, address):
 
 @pytest.mark.parametrize("proto, buf", [
     (_names_to_protocols['tcp'], b'\xff\xff\xff\xff'),
+    (_names_to_protocols['p2p'],  # CID type != "libp2p-key":
+     b"\x01\x70\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41"
+     b"\xec\x75\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65"),
     (_names_to_protocols['ip6zone'], b""),
 ])
 def test_codec_to_string_value_error(proto, buf):
     # Codecs themselves may raise any exception type – it will then be converted
     # to `BinaryParseError` by a higher level
     with pytest.raises(Exception):
         codec_by_name(proto.codec).to_string(proto, buf)
+
+
+@pytest.mark.parametrize("proto, string, expected", [
+    (_names_to_protocols["p2p"],  # This one gets autoconverted to CIDv1
+     "12D3KooWPA6ax6t3jqTyGq73Zm1RmwppYqxaXzrtarfcTWGp5Wzx",
+     b"\x01\x72\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41"
+     b"\xec\x75\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65"),
+    (_names_to_protocols["ip6"],  # Others do not
+     "12D3KooWPA6ax6t3jqTyGq73Zm1RmwppYqxaXzrtarfcTWGp5Wzx",
+     b"\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41\xec\x75"
+     b"\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65"),
+])
+def test_cid_autoconvert_to_bytes(proto, string, expected):
+    assert codec_by_name("cid").to_bytes(proto, string) == expected
+
+
+@pytest.mark.parametrize("proto, buf, expected", [
+    (_names_to_protocols["p2p"],  # This one gets autoconverted to CIDv0
+     b"\x01\x72\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41"
+     b"\xec\x75\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65",
+     "12D3KooWPA6ax6t3jqTyGq73Zm1RmwppYqxaXzrtarfcTWGp5Wzx"),
+    (_names_to_protocols["ip6"],  # Others do not
+     b"\x01\x72\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41"
+     b"\xec\x75\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65",
+     "bafzaajaiaejcbrrv5vds2whn3c464rsb5r2vpxeanneinzlijenlac77cju2pptf"),
+    (_names_to_protocols["ip6"],  # (Needed to put identity conversion test somewhere)
+     b"\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41\xec\x75"
+     b"\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65",
+     "12D3KooWPA6ax6t3jqTyGq73Zm1RmwppYqxaXzrtarfcTWGp5Wzx"),
+])
+def test_cid_autoconvert_to_string(proto, buf, expected):
+    assert codec_by_name("cid").to_string(proto, buf) == expected

-Original file line number
+Diff line change
 base58
 netaddr
 idna
 +py-cid
 +py-multicodec >= 0.2.0