feat: add length-prefixed protobuf support to identify protocol

acul71 · acul71 · commit 8ec67289da9f · 2025-07-13T15:55:37.000+02:00
diff --git a/examples/identify/identify.py b/examples/identify/identify.py
@@ -8,9 +8,9 @@
 from libp2p import (
     new_host,
 )
-from libp2p.identity.identify.identify import ID as IDENTIFY_PROTOCOL_ID
-from libp2p.identity.identify.pb.identify_pb2 import (
-    Identify,
+from libp2p.identity.identify.identify import (
+    ID as IDENTIFY_PROTOCOL_ID,
+    parse_identify_response,
 )
 from libp2p.peer.peerinfo import (
     info_from_p2p_addr,
@@ -84,11 +84,18 @@ async def run(port: int, destination: str) -> None:
 
             try:
                 print("Starting identify protocol...")
-                response = await stream.read()
+
+                # Read the complete response (could be either format)
+                # Read a larger chunk to get all the data before stream closes
+                response = await stream.read(8192)  # Read enough data in one go
+
                 await stream.close()
-                identify_msg = Identify()
-                identify_msg.ParseFromString(response)
+
+                # Parse the response using the robust protocol-level function
+                # This handles both old and new formats automatically
+                identify_msg = parse_identify_response(response)
                 print_identify_response(identify_msg)
+
             except Exception as e:
                 print(f"Identify protocol error: {e}")
 
diff --git a/libp2p/host/defaults.py b/libp2p/host/defaults.py
@@ -26,5 +26,8 @@
 
 def get_default_protocols(host: IHost) -> "OrderedDict[TProtocol, StreamHandlerFn]":
     return OrderedDict(
-        ((IdentifyID, identify_handler_for(host)), (PingID, handle_ping))
+        (
+            (IdentifyID, identify_handler_for(host, use_varint_format=False)),
+            (PingID, handle_ping),
+        )
     )
diff --git a/libp2p/identity/identify/identify.py b/libp2p/identity/identify/identify.py
@@ -16,7 +16,9 @@
     StreamClosed,
 )
 from libp2p.utils import (
+    decode_varint_with_size,
     get_agent_version,
+    varint,
 )
 
 from .pb.identify_pb2 import (
@@ -72,7 +74,47 @@ def _mk_identify_protobuf(
     )
 
 
-def identify_handler_for(host: IHost) -> StreamHandlerFn:
+def parse_identify_response(response: bytes) -> Identify:
+    """
+    Parse identify response that could be either:
+    - Old format: raw protobuf
+    - New format: length-prefixed protobuf
+
+    This function provides backward and forward compatibility.
+    """
+    # Try new format first: length-prefixed protobuf
+    if len(response) >= 1:
+        length, varint_size = decode_varint_with_size(response)
+        if varint_size > 0 and length > 0 and varint_size + length <= len(response):
+            protobuf_data = response[varint_size : varint_size + length]
+            try:
+                identify_response = Identify()
+                identify_response.ParseFromString(protobuf_data)
+                # Sanity check: must have agent_version (protocol_version is optional)
+                if identify_response.agent_version:
+                    logger.debug(
+                        "Parsed length-prefixed identify response (new format)"
+                    )
+                    return identify_response
+            except Exception:
+                pass  # Fall through to old format
+
+    # Fall back to old format: raw protobuf
+    try:
+        identify_response = Identify()
+        identify_response.ParseFromString(response)
+        logger.debug("Parsed raw protobuf identify response (old format)")
+        return identify_response
+    except Exception as e:
+        logger.error(f"Failed to parse identify response: {e}")
+        logger.error(f"Response length: {len(response)}")
+        logger.error(f"Response hex: {response.hex()}")
+        raise
+
+
+def identify_handler_for(
+    host: IHost, use_varint_format: bool = False
+) -> StreamHandlerFn:
     async def handle_identify(stream: INetStream) -> None:
         # get observed address from ``stream``
         peer_id = (
@@ -100,7 +142,21 @@ async def handle_identify(stream: INetStream) -> None:
         response = protobuf.SerializeToString()
 
         try:
-            await stream.write(response)
+            if use_varint_format:
+                # Send length-prefixed protobuf message (new format)
+                await stream.write(varint.encode_uvarint(len(response)))
+                await stream.write(response)
+                logger.debug(
+                    "Sent new format (length-prefixed) identify response to %s",
+                    peer_id,
+                )
+            else:
+                # Send raw protobuf message (old format for backward compatibility)
+                await stream.write(response)
+                logger.debug(
+                    "Sent old format (raw protobuf) identify response to %s",
+                    peer_id,
+                )
         except StreamClosed:
             logger.debug("Fail to respond to %s request: stream closed", ID)
         else:
diff --git a/libp2p/identity/identify_push/identify_push.py b/libp2p/identity/identify_push/identify_push.py
@@ -25,6 +25,10 @@
 )
 from libp2p.utils import (
     get_agent_version,
+    varint,
+)
+from libp2p.utils.varint import (
+    decode_varint_from_bytes,
 )
 
 from ..identify.identify import (
@@ -55,8 +59,29 @@ async def handle_identify_push(stream: INetStream) -> None:
         peer_id = stream.muxed_conn.peer_id
 
         try:
-            # Read the identify message from the stream
-            data = await stream.read()
+            # Read length-prefixed identify message from the stream
+            # First read the varint length prefix
+            length_bytes = b""
+            while True:
+                b = await stream.read(1)
+                if not b:
+                    break
+                length_bytes += b
+                if b[0] & 0x80 == 0:
+                    break
+
+            if not length_bytes:
+                logger.warning("No length prefix received from peer %s", peer_id)
+                return
+
+            msg_length = decode_varint_from_bytes(length_bytes)
+
+            # Read the protobuf message
+            data = await stream.read(msg_length)
+            if len(data) != msg_length:
+                logger.warning("Incomplete message received from peer %s", peer_id)
+                return
+
             identify_msg = Identify()
             identify_msg.ParseFromString(data)
 
@@ -159,7 +184,8 @@ async def push_identify_to_peer(
             identify_msg = _mk_identify_protobuf(host, observed_multiaddr)
             response = identify_msg.SerializeToString()
 
-            # Send the identify message
+            # Send length-prefixed identify message
+            await stream.write(varint.encode_uvarint(len(response)))
             await stream.write(response)
 
             # Close the stream
diff --git a/libp2p/utils/__init__.py b/libp2p/utils/__init__.py
@@ -7,6 +7,8 @@
     encode_varint_prefixed,
     read_delim,
     read_varint_prefixed_bytes,
+    decode_varint_from_bytes,
+    decode_varint_with_size,
 )
 from libp2p.utils.version import (
     get_agent_version,
@@ -20,4 +22,6 @@
     "get_agent_version",
     "read_delim",
     "read_varint_prefixed_bytes",
+    "decode_varint_from_bytes",
+    "decode_varint_with_size",
 ]
diff --git a/libp2p/utils/varint.py b/libp2p/utils/varint.py
@@ -39,6 +39,30 @@ def encode_uvarint(number: int) -> bytes:
     return buf
 
 
+def decode_varint_from_bytes(data: bytes) -> int:
+    """
+    Decode a varint from bytes and return the value.
+
+    This is a synchronous version of decode_uvarint_from_stream for already-read bytes.
+    """
+    res = 0
+    for shift in itertools.count(0, 7):
+        if shift > SHIFT_64_BIT_MAX:
+            raise ParseError("Integer is too large...")
+
+        if not data:
+            raise ParseError("Unexpected end of data")
+
+        value = data[0]
+        data = data[1:]
+
+        res += (value & LOW_MASK) << shift
+
+        if not value & HIGH_MASK:
+            break
+    return res
+
+
 async def decode_uvarint_from_stream(reader: Reader) -> int:
     """https://en.wikipedia.org/wiki/LEB128."""
     res = 0
@@ -56,6 +80,33 @@ async def decode_uvarint_from_stream(reader: Reader) -> int:
     return res
 
 
+def decode_varint_with_size(data: bytes) -> tuple[int, int]:
+    """
+    Decode a varint from bytes and return (value, bytes_consumed).
+    Returns (0, 0) if the data doesn't start with a valid varint.
+    """
+    try:
+        # Calculate how many bytes the varint consumes
+        varint_size = 0
+        for i, byte in enumerate(data):
+            varint_size += 1
+            if (byte & 0x80) == 0:
+                break
+
+        if varint_size == 0:
+            return 0, 0
+
+        # Extract just the varint bytes
+        varint_bytes = data[:varint_size]
+
+        # Decode the varint
+        value = decode_varint_from_bytes(varint_bytes)
+
+        return value, varint_size
+    except Exception:
+        return 0, 0
+
+
 def encode_varint_prefixed(msg_bytes: bytes) -> bytes:
     varint_len = encode_uvarint(len(msg_bytes))
     return varint_len + msg_bytes
diff --git a/tests/core/identity/identify/test_identify.py b/tests/core/identity/identify/test_identify.py
@@ -11,9 +11,7 @@
     PROTOCOL_VERSION,
     _mk_identify_protobuf,
     _multiaddr_to_bytes,
-)
-from libp2p.identity.identify.pb.identify_pb2 import (
-    Identify,
+    parse_identify_response,
 )
 from tests.utils.factories import (
     host_pair_factory,
@@ -29,14 +27,18 @@ async def test_identify_protocol(security_protocol):
         host_b,
     ):
         # Here, host_b is the requester and host_a is the responder.
-        # observed_addr represent host_b’s address as observed by host_a
-        # (i.e., the address from which host_b’s request was received).
+        # observed_addr represent host_b's address as observed by host_a
+        # (i.e., the address from which host_b's request was received).
         stream = await host_b.new_stream(host_a.get_id(), (ID,))
-        response = await stream.read()
+
+        # Read the response (could be either format)
+        # Read a larger chunk to get all the data before stream closes
+        response = await stream.read(8192)  # Read enough data in one go
+
         await stream.close()
 
-        identify_response = Identify()
-        identify_response.ParseFromString(response)
+        # Parse the response (handles both old and new formats)
+        identify_response = parse_identify_response(response)
 
         logger.debug("host_a: %s", host_a.get_addrs())
         logger.debug("host_b: %s", host_b.get_addrs())
@@ -62,8 +64,9 @@ async def test_identify_protocol(security_protocol):
 
         logger.debug("observed_addr: %s", Multiaddr(identify_response.observed_addr))
         logger.debug("host_b.get_addrs()[0]: %s", host_b.get_addrs()[0])
-        logger.debug("cleaned_addr= %s", cleaned_addr)
-        assert identify_response.observed_addr == _multiaddr_to_bytes(cleaned_addr)
+
+        # The observed address should match the cleaned address
+        assert Multiaddr(identify_response.observed_addr) == cleaned_addr
 
         # Check protocols
         assert set(identify_response.protocols) == set(host_a.get_mux().get_protocols())
diff --git a/tests/core/identity/identify/test_identify_parsing.py b/tests/core/identity/identify/test_identify_parsing.py
diff --git a/tests/core/utils/test_varint.py b/tests/core/utils/test_varint.py

Original file line number	Diff line number	Diff line change
`@@ -26,5 +26,8 @@`
`26`	`26`
`27`	`27`	`def get_default_protocols(host: IHost) -> "OrderedDict[TProtocol, StreamHandlerFn]":`
`28`	`28`	`return OrderedDict(`
`29`		`- ((IdentifyID, identify_handler_for(host)), (PingID, handle_ping))`
	`29`	`+ (`
	`30`	`+ (IdentifyID, identify_handler_for(host, use_varint_format=False)),`
	`31`	`+ (PingID, handle_ping),`
	`32`	`+ )`
`30`	`33`	`)`