Merge pull request #2506 from opentensor/release/8.4.3

ibraheem-abe · web-flow · commit f26db805cdc1 · 2024-12-02T14:34:50.000-08:00
Release/8.4.3
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,16 @@
 # Changelog
 
+## 8.4.3 /2024-12-02
+
+## What's Changed
+
+* Fix logging config parsing by @ibraheem-opentensor in https://github.com/opentensor/bittensor/pull/2500
+* Improve  `submit_extrinsic` util by @thewhaleking in https://github.com/opentensor/bittensor/pull/2502
+* Backmerge master to staging for 843 by @ibraheem-opentensor in https://github.com/opentensor/bittensor/pull/2505
+* WS ensure_connected socket catch by @thewhaleking in https://github.com/opentensor/bittensor/pull/2507
+
+**Full Changelog**: https://github.com/opentensor/bittensor/compare/v8.4.2...v8.4.3
+
 ## 8.4.2 /2024-11-28
 
 ## What's Changed
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-8.4.2
+8.4.3
diff --git a/bittensor/core/extrinsics/commit_weights.py b/bittensor/core/extrinsics/commit_weights.py
@@ -71,7 +71,7 @@ def do_commit_weights(
         keypair=wallet.hotkey,
     )
     response = submit_extrinsic(
-        substrate=self.substrate,
+        subtensor=self,
         extrinsic=extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,
@@ -184,7 +184,7 @@ def do_reveal_weights(
         keypair=wallet.hotkey,
     )
     response = submit_extrinsic(
-        substrate=self.substrate,
+        subtensor=self,
         extrinsic=extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,
diff --git a/bittensor/core/extrinsics/registration.py b/bittensor/core/extrinsics/registration.py
@@ -66,7 +66,7 @@ def _do_pow_register(
     )
     extrinsic = self.substrate.create_signed_extrinsic(call=call, keypair=wallet.hotkey)
     response = submit_extrinsic(
-        substrate=self.substrate,
+        self,
         extrinsic=extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,
@@ -297,7 +297,7 @@ def _do_burned_register(
         call=call, keypair=wallet.coldkey
     )
     response = submit_extrinsic(
-        self.substrate,
+        self,
         extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,
diff --git a/bittensor/core/extrinsics/root.py b/bittensor/core/extrinsics/root.py
@@ -33,7 +33,7 @@ def _do_root_register(
         call=call, keypair=wallet.coldkey
     )
     response = submit_extrinsic(
-        self.substrate,
+        self,
         extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,
@@ -159,7 +159,7 @@ def _do_set_root_weights(
         era={"period": period},
     )
     response = submit_extrinsic(
-        self.substrate,
+        self,
         extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,
diff --git a/bittensor/core/extrinsics/serving.py b/bittensor/core/extrinsics/serving.py
@@ -75,7 +75,7 @@ def do_serve_axon(
     )
     extrinsic = self.substrate.create_signed_extrinsic(call=call, keypair=wallet.hotkey)
     response = submit_extrinsic(
-        substrate=self.substrate,
+        self,
         extrinsic=extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,
@@ -289,7 +289,7 @@ def publish_metadata(
 
         extrinsic = substrate.create_signed_extrinsic(call=call, keypair=wallet.hotkey)
         response = submit_extrinsic(
-            substrate,
+            self,
             extrinsic,
             wait_for_inclusion=wait_for_inclusion,
             wait_for_finalization=wait_for_finalization,
diff --git a/bittensor/core/extrinsics/set_weights.py b/bittensor/core/extrinsics/set_weights.py
@@ -83,7 +83,7 @@ def do_set_weights(
         era={"period": period},
     )
     response = submit_extrinsic(
-        substrate=self.substrate,
+        self,
         extrinsic=extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,
diff --git a/bittensor/core/extrinsics/transfer.py b/bittensor/core/extrinsics/transfer.py
@@ -70,7 +70,7 @@ def do_transfer(
         call=call, keypair=wallet.coldkey
     )
     response = submit_extrinsic(
-        substrate=self.substrate,
+        self,
         extrinsic=extrinsic,
         wait_for_inclusion=wait_for_inclusion,
         wait_for_finalization=wait_for_finalization,
diff --git a/bittensor/core/extrinsics/utils.py b/bittensor/core/extrinsics/utils.py
@@ -5,13 +5,14 @@
 import threading
 from typing import TYPE_CHECKING
 
-from substrateinterface.exceptions import SubstrateRequestException, ExtrinsicNotFound
+from substrateinterface.exceptions import SubstrateRequestException
 
 from bittensor.utils.btlogging import logging
 from bittensor.utils import format_error_message
 
 if TYPE_CHECKING:
-    from substrateinterface import SubstrateInterface, ExtrinsicReceipt
+    from bittensor.core.subtensor import Subtensor
+    from substrateinterface import ExtrinsicReceipt
     from scalecodec.types import GenericExtrinsic
 
 try:
@@ -26,7 +27,7 @@
 
 
 def submit_extrinsic(
-    substrate: "SubstrateInterface",
+    subtensor: "Subtensor",
     extrinsic: "GenericExtrinsic",
     wait_for_inclusion: bool,
     wait_for_finalization: bool,
@@ -39,7 +40,7 @@ def submit_extrinsic(
     it logs the error and re-raises the exception.
 
     Args:
-        substrate (substrateinterface.SubstrateInterface): The substrate interface instance used to interact with the blockchain.
+        subtensor: The Subtensor instance used to interact with the blockchain.
         extrinsic (scalecodec.types.GenericExtrinsic): The extrinsic to be submitted to the blockchain.
         wait_for_inclusion (bool): Whether to wait for the extrinsic to be included in a block.
         wait_for_finalization (bool): Whether to wait for the extrinsic to be finalized on the blockchain.
@@ -51,20 +52,22 @@ def submit_extrinsic(
         SubstrateRequestException: If the submission of the extrinsic fails, the error is logged and re-raised.
     """
     extrinsic_hash = extrinsic.extrinsic_hash
-    starting_block = substrate.get_block()
+    starting_block = subtensor.substrate.get_block()
 
     timeout = EXTRINSIC_SUBMISSION_TIMEOUT
     event = threading.Event()
 
     def submit():
         try:
-            response_ = substrate.submit_extrinsic(
+            response_ = subtensor.substrate.submit_extrinsic(
                 extrinsic,
                 wait_for_inclusion=wait_for_inclusion,
                 wait_for_finalization=wait_for_finalization,
             )
         except SubstrateRequestException as e:
-            logging.error(format_error_message(e.args[0], substrate=substrate))
+            logging.error(
+                format_error_message(e.args[0], substrate=subtensor.substrate)
+            )
             # Re-raise the exception for retrying of the extrinsic call. If we remove the retry logic,
             # the raise will need to be removed.
             raise
@@ -76,28 +79,10 @@ def submit():
         response = None
         future = executor.submit(submit)
         if not event.wait(timeout):
-            logging.error("Timed out waiting for extrinsic submission.")
-            after_timeout_block = substrate.get_block()
-
-            for block_num in range(
-                starting_block["header"]["number"],
-                after_timeout_block["header"]["number"] + 1,
-            ):
-                block_hash = substrate.get_block_hash(block_num)
-                try:
-                    response = substrate.retrieve_extrinsic_by_hash(
-                        block_hash, f"0x{extrinsic_hash.hex()}"
-                    )
-                except ExtrinsicNotFound:
-                    continue
-                if response:
-                    break
-            if response is None:
-                logging.error(
-                    f"Extrinsic '0x{extrinsic_hash.hex()}' not submitted. "
-                    f"Initially attempted to submit at block {starting_block['header']['number']}."
-                )
-                raise SubstrateRequestException
+            logging.error("Timed out waiting for extrinsic submission. Reconnecting.")
+            # force reconnection of the websocket
+            subtensor._get_substrate(force=True)
+            raise SubstrateRequestException
 
         else:
             response = future.result()
diff --git a/bittensor/core/settings.py b/bittensor/core/settings.py
@@ -15,7 +15,7 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-__version__ = "8.4.2"
+__version__ = "8.4.3"
 
 import os
 import re
diff --git a/bittensor/core/subtensor.py b/bittensor/core/subtensor.py
@@ -212,11 +212,21 @@ def close(self):
         if self.substrate:
             self.substrate.close()
 
-    def _get_substrate(self):
-        """Establishes a connection to the Substrate node using configured parameters."""
+    def _get_substrate(self, force: bool = False):
+        """
+        Establishes a connection to the Substrate node using configured parameters.
+
+        Args:
+            force: forces a reconnection if this flag is set
+
+        """
         try:
             # Set up params.
-            if self.websocket is None or self.websocket.close_code is not None:
+            if force and self.websocket:
+                logging.debug("Closing websocket connection")
+                self.websocket.close()
+
+            if force or self.websocket is None or self.websocket.close_code is not None:
                 self.websocket = ws_client.connect(
                     self.chain_endpoint,
                     open_timeout=self._connection_timeout,
@@ -229,7 +239,7 @@ def _get_substrate(self):
                 websocket=self.websocket,
             )
             if self.log_verbose:
-                logging.debug(
+                logging.info(
                     f"Connected to {self.network} network and {self.chain_endpoint}."
                 )
 
diff --git a/bittensor/utils/btlogging/loggingmachine.py b/bittensor/utils/btlogging/loggingmachine.py
@@ -23,7 +23,6 @@
 
 import argparse
 import atexit
-import copy
 import logging as stdlogging
 import multiprocessing as mp
 import os
@@ -182,7 +181,8 @@ def _extract_logging_config(self, config: "Config") -> dict:
         Returns:
             (dict): btlogging's config from Bittensor config or Bittensor config.
         """
-        if hasattr(config, "logging"):
+        # This is to handle nature of DefaultMunch
+        if getattr(config, "logging", None):
             return config.logging
         else:
             return config
@@ -213,14 +213,14 @@ def set_config(self, config: "Config"):
         Args:
             config (bittensor.core.config.Config): Bittensor config instance.
         """
-        self._config = config
-        if config.logging_dir and config.record_log:
+        self._config = self._extract_logging_config(config)
+        if self._config.logging_dir and self._config.record_log:
             expanded_dir = os.path.expanduser(config.logging_dir)
             logfile = os.path.abspath(os.path.join(expanded_dir, DEFAULT_LOG_FILE_NAME))
             self._enable_file_logging(logfile)
-        if config.trace:
+        if self._config.trace:
             self.enable_trace()
-        elif config.debug:
+        elif self._config.debug:
             self.enable_debug()
 
     def _create_and_start_listener(self, handlers):
@@ -618,7 +618,7 @@ def __call__(
         logging_dir: str = None,
     ):
         if config is not None:
-            cfg = copy.deepcopy(config)
+            cfg = self._extract_logging_config(config)
             if debug is not None:
                 cfg.debug = debug
             elif trace is not None:
diff --git a/bittensor/utils/networking.py b/bittensor/utils/networking.py
@@ -168,10 +168,11 @@ def ensure_connected(func):
     def is_connected(substrate) -> bool:
         """Check if the substrate connection is active."""
         sock = substrate.websocket.socket
-        return (
-            sock is not None
-            and sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) == 0
-        )
+        try:
+            sock_opt = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+            return sock is not None and sock_opt == 0
+        except (OSError, AttributeError):
+            return False
 
     @retry(
         exceptions=ConnectionRefusedError,
diff --git a/tests/e2e_tests/test_commit_weights.py b/tests/e2e_tests/test_commit_weights.py
@@ -6,6 +6,7 @@
 from bittensor.core.subtensor import Subtensor
 from bittensor.utils.balance import Balance
 from bittensor.utils.weight_utils import convert_weights_and_uids_for_emit
+from bittensor.core.extrinsics import utils
 from tests.e2e_tests.utils.chain_interactions import (
     add_stake,
     register_subnet,
@@ -32,6 +33,7 @@ async def test_commit_and_reveal_weights(local_chain):
         AssertionError: If any of the checks or verifications fail
     """
     netuid = 1
+    utils.EXTRINSIC_SUBMISSION_TIMEOUT = 12  # handle fast blocks
     print("Testing test_commit_and_reveal_weights")
     # Register root as Alice
     keypair, alice_wallet = setup_wallet("//Alice")
diff --git a/tests/unit_tests/extrinsics/test_utils.py b/tests/unit_tests/extrinsics/test_utils.py

Original file line number	Diff line number	Diff line change
`@@ -83,7 +83,7 @@ def do_set_weights(`
`83`	`83`	`era={"period": period},`
`84`	`84`	`)`
`85`	`85`	`response = submit_extrinsic(`
`86`		`- substrate=self.substrate,`
	`86`	`+ self,`
`87`	`87`	`extrinsic=extrinsic,`
`88`	`88`	`wait_for_inclusion=wait_for_inclusion,`
`89`	`89`	`wait_for_finalization=wait_for_finalization,`
Original file line number	Diff line number	Diff line change
`@@ -70,7 +70,7 @@ def do_transfer(`
`70`	`70`	`call=call, keypair=wallet.coldkey`
`71`	`71`	`)`
`72`	`72`	`response = submit_extrinsic(`
`73`		`- substrate=self.substrate,`
	`73`	`+ self,`
`74`	`74`	`extrinsic=extrinsic,`
`75`	`75`	`wait_for_inclusion=wait_for_inclusion,`
`76`	`76`	`wait_for_finalization=wait_for_finalization,`