Merge branch 'main' of https://github.com/Khwahish29/py-libp2p into replace-async-with-anyio

Author: mystical-prog

Makefile

@@ -124,7 +124,7 @@ ifndef bump
 endif
 
 check-git:
-	# require that upstream is configured for ethereum/py-libp2p
+	# require that upstream is configured for libp2p/py-libp2p
 	@if ! git remote -v | grep "upstream[[:space:]][email protected]:libp2p/py-libp2p.git (push)\|upstream[[:space:]]https://github.com/libp2p/py-libp2p (push)"; then \
 		echo "Error: You must have a remote named 'upstream' that points to 'py-libp2p'"; \
 		exit 1; \

docs/contributing.rst

@@ -3,7 +3,7 @@ Contributing
 
 Thank you for your interest in contributing! We welcome all contributions no matter
 their size. Please read along to learn how to get started. If you get stuck, feel free
-to ask for help in `Ethereum Python Discord server <https://discord.gg/GHryRvPB84>`_.
+to ask for help in `Libp2p Discover Server <https://discord.gg/GK8TxRNh2s>`_.
 
 Setting the stage
 ~~~~~~~~~~~~~~~~~
@@ -294,7 +294,7 @@ do not pass the CI build yet won't get reviewed unless explicitly requested.
 
 If the pull request introduces changes that should be reflected in the release notes,
 please add a newsfragment file as explained
-`here <https://github.com/ethereum/py-libp2p/blob/main/newsfragments/README.md>`_.
+`here <https://github.com/libp2p/py-libp2p/tree/main/newsfragments>`_.
 
 If possible, the change to the release notes file should be included in the commit that
 introduces the feature or bugfix.

docs/getting_started.rst

@@ -94,14 +94,106 @@ For Python, you can use the bootstrap list to connect to known peers:
 Debugging
 ---------
 
-When running libp2p you may want to see what things are happening behind the scenes. You can enable debug logging by setting the appropriate log level:
+When running libp2p you may want to see what things are happening behind the scenes. You can enable debug logging using the `LIBP2P_DEBUG` environment variable. This allows for fine-grained control over which modules log at which levels.
 
-.. code:: python
+Basic Usage
+~~~~~~~~~~~
+
+To enable debug logging for all modules:
+
+.. code:: bash
+
+    # Method 1: Using export (persists for the shell session)
+    # On Unix-like systems (Linux, macOS):
+    export LIBP2P_DEBUG=DEBUG
+    # On Windows (Command Prompt):
+    set LIBP2P_DEBUG=DEBUG
+    # On Windows (PowerShell):
+    $env:LIBP2P_DEBUG="DEBUG"
+
+    # Method 2: Setting for a single command
+    # On Unix-like systems:
+    LIBP2P_DEBUG=DEBUG python your_script.py
+    # On Windows (Command Prompt):
+    set LIBP2P_DEBUG=DEBUG && python your_script.py
+    # On Windows (PowerShell):
+    $env:LIBP2P_DEBUG="DEBUG"; python your_script.py
+
+    # Method 3: Redirect logs to /dev/null to suppress console output
+    # On Unix-like systems:
+    LIBP2P_DEBUG=DEBUG python your_script.py 2>/dev/null
+    # On Windows (Command Prompt):
+    set LIBP2P_DEBUG=DEBUG && python your_script.py >nul 2>&1
+    # On Windows (PowerShell):
+    $env:LIBP2P_DEBUG="DEBUG"; python your_script.py *> $null
+
+To enable debug logging for specific modules:
+
+.. code:: bash
+
+    # Debug logging for identity module only
+    # On Unix-like systems:
+    export LIBP2P_DEBUG=identity.identify:DEBUG
+    # On Windows (Command Prompt):
+    set LIBP2P_DEBUG=identity.identify:DEBUG
+    # On Windows (PowerShell):
+    $env:LIBP2P_DEBUG="identity.identify:DEBUG"
+
+    # Multiple modules with different levels
+    # On Unix-like systems:
+    export LIBP2P_DEBUG=identity.identify:DEBUG,transport:INFO
+    # On Windows (Command Prompt):
+    set LIBP2P_DEBUG=identity.identify:DEBUG,transport:INFO
+    # On Windows (PowerShell):
+    $env:LIBP2P_DEBUG="identity.identify:DEBUG,transport:INFO"
+
+Log Files
+~~~~~~~~~
+
+By default, logs are written to a file in the system's temporary directory with a timestamp and unique identifier. The file path is printed to stderr when logging starts.
+
+When no custom log file is specified:
+    - Logs are written to both a default file and to stderr (console output)
+    - The default file path is printed to stderr when logging starts
+
+When a custom log file is specified:
+    - Logs are written only to the specified file
+    - No console output is generated
+
+To specify a custom log file location:
+
+.. code:: bash
+
+    # Method 1: Using export (persists for the shell session)
+    # On Unix-like systems:
+    export LIBP2P_DEBUG_FILE=/path/to/your/logfile.log
+    # On Windows (Command Prompt):
+    set LIBP2P_DEBUG_FILE=C:\path\to\your\logfile.log
+    # On Windows (PowerShell):
+    $env:LIBP2P_DEBUG_FILE="C:\path\to\your\logfile.log"
+
+    # Method 2: Setting for a single command
+    # On Unix-like systems:
+    LIBP2P_DEBUG=DEBUG LIBP2P_DEBUG_FILE=/path/to/your/logfile.log python your_script.py
+    # On Windows (Command Prompt):
+    set LIBP2P_DEBUG=DEBUG && set LIBP2P_DEBUG_FILE=C:\path\to\your\logfile.log && python your_script.py
+    # On Windows (PowerShell):
+    $env:LIBP2P_DEBUG="DEBUG"; $env:LIBP2P_DEBUG_FILE="C:\path\to\your\logfile.log"; python your_script.py
+
+Log Format
+~~~~~~~~~~
+
+Log messages follow this format:
+
+.. code:: text
+
+    timestamp - module_name - level - message
+
+Example output:
 
-    import logging
+.. code:: text
 
-    # Set debug level for libp2p
-    logging.getLogger('libp2p').setLevel(logging.DEBUG)
+    2024-01-01 12:00:00,123 - libp2p.identity.identify - DEBUG - Starting identify protocol
 
 What's Next
 -----------

docs/libp2p.rst

@@ -19,6 +19,7 @@ Subpackages
    libp2p.stream_muxer
    libp2p.tools
    libp2p.transport
+   libp2p.utils
 
 Submodules
 ----------

docs/libp2p.utils.rst

@@ -0,0 +1,29 @@
+libp2p.utils package
+====================
+
+Submodules
+----------
+
+libp2p.utils.logging module
+---------------------------
+
+.. automodule:: libp2p.utils.logging
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+libp2p.utils.varint module
+--------------------------
+
+.. automodule:: libp2p.utils.varint
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+libp2p.utils.version module
+---------------------------
+
+.. automodule:: libp2p.utils.version
+   :members:
+   :undoc-members:
+   :show-inheritance:

examples/pubsub/pubsub.py

@@ -136,6 +136,7 @@ async def run(topic: str, destination: Optional[str], port: Optional[int]) -> No
         degree=3,  # Number of peers to maintain in mesh
         degree_low=2,  # Lower bound for mesh peers
         degree_high=4,  # Upper bound for mesh peers
+        direct_peers=None,  # Direct peers
         time_to_live=60,  # TTL for message cache in seconds
         gossip_window=2,  # Smaller window for faster gossip
         gossip_history=5,  # Keep more history

libp2p/__init__.py

@@ -47,6 +47,12 @@
 from libp2p.transport.upgrader import (
     TransportUpgrader,
 )
+from libp2p.utils.logging import (
+    setup_logging,
+)
+
+# Initialize logging configuration
+setup_logging()
 
 
 def generate_new_rsa_identity() -> KeyPair:

libp2p/peer/peerstore.py

@@ -4,6 +4,7 @@
 from collections.abc import (
     Sequence,
 )
+import sys
 from typing import (
     Any,
 )
@@ -32,6 +33,8 @@
     PeerInfo,
 )
 
+PERMANENT_ADDR_TTL = sys.maxsize
+
 
 class PeerStore(IPeerStore):
     peer_data_map: dict[ID, PeerData]

libp2p/pubsub/gossipsub.py

@@ -29,6 +29,12 @@
 from libp2p.peer.id import (
     ID,
 )
+from libp2p.peer.peerinfo import (
+    PeerInfo,
+)
+from libp2p.peer.peerstore import (
+    PERMANENT_ADDR_TTL,
+)
 from libp2p.pubsub import (
     floodsub,
 )
@@ -82,17 +88,24 @@ class GossipSub(IPubsubRouter, Service):
     heartbeat_initial_delay: float
     heartbeat_interval: int
 
+    direct_peers: dict[ID, PeerInfo]
+    direct_connect_initial_delay: float
+    direct_connect_interval: int
+
     def __init__(
         self,
         protocols: Sequence[TProtocol],
         degree: int,
         degree_low: int,
         degree_high: int,
+        direct_peers: Sequence[PeerInfo] = None,
         time_to_live: int = 60,
         gossip_window: int = 3,
         gossip_history: int = 5,
         heartbeat_initial_delay: float = 0.1,
         heartbeat_interval: int = 120,
+        direct_connect_initial_delay: float = 0.1,
+        direct_connect_interval: int = 300,
     ) -> None:
         self.protocols = list(protocols)
         self.pubsub = None
@@ -119,10 +132,19 @@ def __init__(
         self.heartbeat_initial_delay = heartbeat_initial_delay
         self.heartbeat_interval = heartbeat_interval
 
+        # Create direct peers
+        self.direct_peers = dict()
+        for direct_peer in direct_peers or []:
+            self.direct_peers[direct_peer.peer_id] = direct_peer
+        self.direct_connect_interval = direct_connect_interval
+        self.direct_connect_initial_delay = direct_connect_initial_delay
+
     async def run(self) -> None:
         if self.pubsub is None:
             raise NoPubsubAttached
         self.manager.run_daemon_task(self.heartbeat)
+        if len(self.direct_peers) > 0:
+            self.manager.run_daemon_task(self.direct_connect_heartbeat)
         await self.manager.wait_finished()
 
     # Interface functions
@@ -142,6 +164,12 @@ def attach(self, pubsub: Pubsub) -> None:
         """
         self.pubsub = pubsub
 
+        if len(self.direct_peers) > 0:
+            for pi in self.direct_peers:
+                self.pubsub.host.get_peerstore().add_addrs(
+                    pi, self.direct_peers[pi].addrs, PERMANENT_ADDR_TTL
+                )
+
         logger.debug("attached to pusub")
 
     def add_peer(self, peer_id: ID, protocol_id: TProtocol) -> None:
@@ -241,6 +269,10 @@ def _get_peers_to_send(
             if topic not in self.pubsub.peer_topics:
                 continue
 
+            # direct peers
+            _direct_peers: set[ID] = {_peer for _peer in self.direct_peers}
+            send_to.update(_direct_peers)
+
             # floodsub peers
             floodsub_peers: set[ID] = {
                 peer_id
@@ -425,6 +457,24 @@ async def heartbeat(self) -> None:
 
             await trio.sleep(self.heartbeat_interval)
 
+    async def direct_connect_heartbeat(self) -> None:
+        """
+        Connect to direct peers.
+        """
+        await trio.sleep(self.direct_connect_initial_delay)
+        while True:
+            for direct_peer in self.direct_peers:
+                if direct_peer not in self.pubsub.peers:
+                    try:
+                        await self.pubsub.host.connect(self.direct_peers[direct_peer])
+                    except Exception as e:
+                        logger.debug(
+                            "failed to connect to a direct peer %s: %s",
+                            direct_peer,
+                            e,
+                        )
+            await trio.sleep(self.direct_connect_interval)
+
     def mesh_heartbeat(
         self,
     ) -> tuple[DefaultDict[ID, list[str]], DefaultDict[ID, list[str]]]:
@@ -654,6 +704,14 @@ async def handle_graft(
 
         # Add peer to mesh for topic
         if topic in self.mesh:
+            for direct_peer in self.direct_peers:
+                if direct_peer == sender_peer_id:
+                    logger.warning(
+                        "GRAFT: ignoring request from direct peer %s", sender_peer_id
+                    )
+                    await self.emit_prune(topic, sender_peer_id)
+                    return
+
             if sender_peer_id not in self.mesh[topic]:
                 self.mesh[topic].add(sender_peer_id)
         else:

libp2p/tools/constants.py

@@ -1,9 +1,15 @@
+from collections.abc import (
+    Sequence,
+)
 from typing import (
     NamedTuple,
 )
 
 import multiaddr
 
+from libp2p.peer.peerinfo import (
+    PeerInfo,
+)
 from libp2p.pubsub import (
     floodsub,
     gossipsub,
@@ -26,11 +32,14 @@ class GossipsubParams(NamedTuple):
     degree: int = 10
     degree_low: int = 9
     degree_high: int = 11
+    direct_peers: Sequence[PeerInfo] = None
     time_to_live: int = 30
     gossip_window: int = 3
     gossip_history: int = 5
     heartbeat_initial_delay: float = 0.1
     heartbeat_interval: float = 0.5
+    direct_connect_initial_delay: float = 0.1
+    direct_connect_interval: int = 300
 
 
 GOSSIPSUB_PARAMS = GossipsubParams()