Centralize adding and removing of varint length in codec.py

Author: ntninja

multiaddr/codec.py

@@ -37,7 +37,10 @@ def string_to_bytes(string):
     for proto, codec, value in string_iter(string):
         bs.append(varint.encode(proto.code))
         if value is not None:
-            bs.append(codec.to_bytes(proto, value))
+            buf = codec.to_bytes(proto, value)
+            if codec.SIZE == LENGTH_PREFIXED_VAR_SIZE:
+                bs.append(varint.encode(len(buf)))
+            bs.append(buf)
     return b''.join(bs)
 
 
@@ -56,10 +59,9 @@ def bytes_to_string(buf):
 
 def size_for_addr(codec, buf):
     if codec.SIZE >= 0:
-        return codec.SIZE // 8
+        return codec.SIZE // 8, 0
     else:
-        size, num_bytes_read = read_varint_code(buf)
-        return size + num_bytes_read
+        return read_varint_code(buf)
 
 
 def string_iter(string):
@@ -102,7 +104,7 @@ def bytes_iter(buf):
 			codec = find_codec_by_name(proto.codec)
 		except ImportError as exc:
 			six.raise_from(ValueError("failed to parse %s addr: unknown" % proto.name), exc)
-		size = size_for_addr(codec, buf[num_bytes_read:])
-		length = size + num_bytes_read
-		yield proto, codec, buf[num_bytes_read:length]
+		size, num_bytes_read2 = size_for_addr(codec, buf[num_bytes_read:])
+		length = size + num_bytes_read2 + num_bytes_read
+		yield proto, codec, buf[(length - size):length]
 		buf = buf[length:]

multiaddr/codecs/fspath.py

@@ -2,10 +2,8 @@
 import os
 
 import six
-import varint
 
 from ..codec import LENGTH_PREFIXED_VAR_SIZE
-from ..protocols import read_varint_code
 
 
 SIZE = LENGTH_PREFIXED_VAR_SIZE
@@ -30,11 +28,8 @@ def fsdecode(path):
 
 
 def to_bytes(proto, string):
-	bytes = fsencode(string)
-	size = varint.encode(len(bytes))
-	return b''.join([size, bytes])
+	return fsencode(string)
 
 
 def to_string(proto, buf):
-	size, num_bytes_read = read_varint_code(buf)
-	return fsdecode(buf[num_bytes_read:])
+	return fsdecode(buf)

multiaddr/codecs/idna.py

@@ -1,22 +1,17 @@
 from __future__ import absolute_import
 
 import idna
-import varint
 
 from ..codec import LENGTH_PREFIXED_VAR_SIZE
-from ..protocols import read_varint_code
 
 
 SIZE = LENGTH_PREFIXED_VAR_SIZE
 IS_PATH = False
 
 
 def to_bytes(proto, string):
-	bytes = idna.encode(string, uts46=True)
-	size = varint.encode(len(bytes))
-	return b''.join([size, bytes])
+	return idna.encode(string, uts46=True)
 
 
 def to_string(proto, buf):
-	size, num_bytes_read = read_varint_code(buf)
-	return idna.decode(buf[num_bytes_read:])
+	return idna.decode(buf)

multiaddr/codecs/p2p.py

@@ -2,10 +2,8 @@
 
 import base58
 import six
-import varint
 
 from ..codec import LENGTH_PREFIXED_VAR_SIZE
-from ..protocols import read_varint_code
 
 
 SIZE = LENGTH_PREFIXED_VAR_SIZE
@@ -20,15 +18,10 @@ def to_bytes(proto, string):
 		mm = base58.b58decode(string)
 	except Exception as ex:
 		raise ValueError("failed to parse p2p addr: %s %s" % (string, str(ex)))
-	size = varint.encode(len(mm))
 	if len(mm) < 5:
 		raise ValueError("invalid P2P multihash: %s" % mm)
-	return b''.join([size, mm])
+	return mm
 
 
 def to_string(proto, buf):
-	size, num_bytes_read = read_varint_code(buf)
-	buf = buf[num_bytes_read:]
-	if len(buf) != size:
-		raise ValueError("inconsistent lengths")
 	return base58.b58encode(buf).decode('ascii')

tests/test_codec.py

@@ -26,16 +26,16 @@
      b'\x9a\x18\x08\x73\x06\x36\x90\x43\x09\x1f\x04\xd2',
      'timaq4ygg2iegci7:1234'),
     (_names_to_protocols['p2p'],
-     b'\x22\x12\x20\xd5\x2e\xbb\x89\xd8\x5b\x02\xa2\x84\x94\x82\x03\xa6\x2f\xf2'
+     b'\x12\x20\xd5\x2e\xbb\x89\xd8\x5b\x02\xa2\x84\x94\x82\x03\xa6\x2f\xf2'
      b'\x83\x89\xc5\x7c\x9f\x42\xbe\xec\x4e\xc2\x0d\xb7\x6a\x68\x91\x1c\x0b',
      'QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC'),
 
 # Additional test data
     (_names_to_protocols['dns4'],
-     b'\x30xn--4gbrim.xn----ymcbaaajlc6dj7bxne2c.xn--wgbh1c',
+     b'xn--4gbrim.xn----ymcbaaajlc6dj7bxne2c.xn--wgbh1c',
      u'موقع.وزارة-الاتصالات.مصر'),  # Explicietly mark this as unicode to force the text to be LTR in editors
     (_names_to_protocols['dns4'],
-     b'\x16xn--fuball-cta.example',
+     b'xn--fuball-cta.example',
      u'fußball.example'),  # This will fail if IDNA-2003/NamePrep is used
 ]
 
@@ -48,9 +48,9 @@
 
 
 @pytest.mark.parametrize("codec_name, buf, expected", [
-    (None, b'\x01\x02\x03', 0),
-    ('ip4', b'\x01\x02\x03', 4),
-    ('p2p', b'\x40\x50\x60\x51', 65),
+    (None, b'\x01\x02\x03', (0, 0)),
+    ('ip4', b'\x01\x02\x03', (4, 0)),
+    ('p2p', b'\x40\x50\x60\x51', (64, 1)),
 ])
 def test_size_for_addr(codec_name, buf, expected):
     assert size_for_addr(find_codec_by_name(codec_name), buf) == expected
@@ -147,7 +147,6 @@ def test_codec_to_bytes_value_error(proto, address):
 
 
 @pytest.mark.parametrize("proto, buf", [
-    (_names_to_protocols['p2p'], b'\x15\x23\x0d\x52\xeb\xb8\x9d\x85\xb0\x2a\x28\x49\x48\x20\x3a'),
     (_names_to_protocols['tcp'], b'\xff\xff\xff\xff')
 ])
 def test_codec_to_string_value_error(proto, buf):