Merge pull request #290 from bancorprotocol/288-add-read-only-setting

288 add read only setting

Author: mikewcasale

fastlane_bot/events/async_event_update_utils.py

@@ -148,7 +148,7 @@ def get_pool_info(
     tkn1: Dict[str, Any],
     pool_data_keys: frozenset,
 ) -> Dict[str, Any]:
-    fee_raw = eval(pool["fee"])
+    fee_raw = pool["fee"]
     pool_info = {
         "exchange_name": pool["exchange_name"],
         "address": pool["address"],
@@ -184,32 +184,35 @@ def get_pool_info(
 
     return pool_info
 
-def sanitize_token_symbol(token_symbol: str, token_address: str) -> str:
+def sanitize_token_symbol(token_symbol: str, token_address: str, read_only: bool) -> str:
     """
     This function ensures token symbols are compatible with the bot's data structures.
     If a symbol is not compatible with Dataframes or CSVs, this function will return the token's address.
 
     :param token_symbol: the token's symbol
     :param token_address: the token's address
+    :param read_only: bool indicating whether the bot is running in read_only mode
 
     returns: str
     """
     sanitization_path = os.path.normpath("fastlane_bot/data/data_sanitization_center/sanitary_data.csv")
     try:
-        token_pd = pd.DataFrame([{"symbol": token_symbol}], columns=["symbol"])
-        token_pd.to_csv(sanitization_path)
+        if not read_only:
+            token_pd = pd.DataFrame([{"symbol": token_symbol}], columns=["symbol"])
+            token_pd.to_csv(sanitization_path)
         return token_symbol
     except Exception:
         return token_address
 
 
 def add_token_info(
-    pool_info: Dict[str, Any], tkn0: Dict[str, Any], tkn1: Dict[str, Any]
+    pool_info: Dict[str, Any], tkn0: Dict[str, Any], tkn1: Dict[str, Any], read_only: bool
 ) -> Dict[str, Any]:
+    print(f"called add_token_info")
     tkn0_symbol = tkn0["symbol"].replace("/", "_").replace("-", "_")
     tkn1_symbol = tkn1["symbol"].replace("/", "_").replace("-", "_")
-    tkn0_symbol = sanitize_token_symbol(token_symbol=tkn0_symbol, token_address=tkn0["address"])
-    tkn1_symbol = sanitize_token_symbol(token_symbol=tkn1_symbol, token_address=tkn1["address"])
+    tkn0_symbol = sanitize_token_symbol(token_symbol=tkn0_symbol, token_address=tkn0["address"], read_only=read_only)
+    tkn1_symbol = sanitize_token_symbol(token_symbol=tkn1_symbol, token_address=tkn1["address"], read_only=read_only)
     tkn0["symbol"] = tkn0_symbol
     tkn1["symbol"] = tkn1_symbol
 
@@ -310,30 +313,45 @@ def process_contract_chunks(
     subset: List[str],
     func: Callable,
     df_combined: pd.DataFrame = None,
+    read_only: bool = False,
 ) -> pd.DataFrame:
+    lst = []
     # write chunks to csv
     for idx, chunk in enumerate(chunks):
         loop = asyncio.get_event_loop()
         df = loop.run_until_complete(func(chunk))
-        df.to_csv(f"{dirname}/{base_filename}{idx}.csv", index=False)
+        if not read_only:
+            df.to_csv(f"{dirname}/{base_filename}{idx}.csv", index=False)
+        else:
+            lst.append(df)
 
-    # concatenate and deduplicate
     filepaths = glob(f"{dirname}/*.csv")
-    if filepaths:
-        df_orig = df_combined.copy() if df_combined is not None else None
-        df_combined = pd.concat([pd.read_csv(filepath) for filepath in filepaths])
-        df_combined = (
-            pd.concat([df_orig, df_combined]) if df_orig is not None else df_combined
-        )
-        df_combined = df_combined.drop_duplicates(subset=subset)
-        df_combined.to_csv(filename, index=False)
 
-    # clear temp dir
-    for filepath in filepaths:
-        try:
-            os.remove(filepath)
-        except Exception as e:
-            cfg.logger.error(f"Failed to remove {filepath} {e}??? This is spooky...")
+    if not read_only:
+        # concatenate and deduplicate
+
+        if filepaths:
+            df_orig = df_combined.copy() if df_combined is not None else None
+            df_combined = pd.concat([pd.read_csv(filepath) for filepath in filepaths])
+            df_combined = (
+                pd.concat([df_orig, df_combined]) if df_orig is not None else df_combined
+            )
+            df_combined = df_combined.drop_duplicates(subset=subset)
+            df_combined.to_csv(filename, index=False)
+            # clear temp dir
+            for filepath in filepaths:
+                try:
+                    os.remove(filepath)
+                except Exception as e:
+                    cfg.logger.error(f"Failed to remove {filepath} {e}??? This is spooky...")
+    else:
+        if lst:
+            dfs = pd.concat(lst)
+            dfs = dfs.drop_duplicates(subset=subset)
+            if df_combined is not None:
+                df_combined = pd.concat([df_combined, dfs])
+            else:
+                df_combined = dfs
 
     return df_combined
 
@@ -392,6 +410,7 @@ def async_update_pools_from_contracts(mgr: Any, current_block: int, logging_path
         filename="tokens_and_fee_df.csv",
         subset=["exchange_name", "address", "cid", "tkn0_address", "tkn1_address"],
         func=main_get_tokens_and_fee,
+        read_only=mgr.read_only,
     )
 
     contracts, tokens_df = get_token_contracts(mgr, tokens_and_fee_df)
@@ -405,16 +424,18 @@ def async_update_pools_from_contracts(mgr: Any, current_block: int, logging_path
         df_combined=pd.read_csv(
             f"fastlane_bot/data/blockchain_data/{mgr.blockchain}/tokens.csv"
         ),
+        read_only=mgr.read_only,
     )
     tokens_df["symbol"] = (
         tokens_df["symbol"]
         .str.replace(" ", "_")
         .str.replace("/", "_")
         .str.replace("-", "_")
     )
-    tokens_df.to_csv(
-        f"fastlane_bot/data/blockchain_data/{mgr.blockchain}/tokens.csv", index=False
-    )
+    if not mgr.read_only:
+        tokens_df.to_csv(
+            f"fastlane_bot/data/blockchain_data/{mgr.blockchain}/tokens.csv", index=False
+        )
     tokens_df["address"] = tokens_df["address"].apply(
         lambda x: Web3.to_checksum_address(x)
     )
@@ -440,44 +461,6 @@ def async_update_pools_from_contracts(mgr: Any, current_block: int, logging_path
         ]
     )
 
-    # def correct_tkn(tkn_address, keyname):
-    #     try:
-    #         return tokens_df[tokens_df["address"] == tkn_address][keyname].values[0]
-    #     except IndexError:
-    #         return np.nan
-    #
-    # static_pool_data = new_pool_data_df.copy()
-    # static_pool_data["tkn0_address"] = static_pool_data["tkn0_address"].apply(
-    #     lambda x: Web3.to_checksum_address(x)
-    # )
-    # static_pool_data["tkn1_address"] = static_pool_data["tkn1_address"].apply(
-    #     lambda x: Web3.to_checksum_address(x)
-    # )
-    # static_pool_data["tkn0_decimals"] = static_pool_data["tkn0_address"].apply(
-    #     lambda x: correct_tkn(x, "decimals")
-    # )
-    # static_pool_data["tkn1_decimals"] = static_pool_data["tkn1_address"].apply(
-    #     lambda x: correct_tkn(x, "decimals")
-    # )
-    # static_pool_data["tkn0_key"] = static_pool_data["tkn0_address"].apply(
-    #     lambda x: correct_tkn(x, "key")
-    # )
-    # static_pool_data["tkn1_key"] = static_pool_data["tkn1_address"].apply(
-    #     lambda x: correct_tkn(x, "key")
-    # )
-    # static_pool_data["tkn0_symbol"] = static_pool_data["tkn0_address"].apply(
-    #     lambda x: correct_tkn(x, "symbol")
-    # )
-    # static_pool_data["tkn1_symbol"] = static_pool_data["tkn1_address"].apply(
-    #     lambda x: correct_tkn(x, "symbol")
-    # )
-    # static_pool_data["pair_name"] = (
-    #     static_pool_data["tkn0_key"] + "/" + static_pool_data["tkn1_key"]
-    # )
-    #
-    # new_pool_data_df = static_pool_data.copy()
-    # del static_pool_data
-
     new_pool_data_df["descr"] = (
         new_pool_data_df["exchange_name"]
         + " "
@@ -502,9 +485,6 @@ def async_update_pools_from_contracts(mgr: Any, current_block: int, logging_path
     )
 
     duplicate_new_pool_ct = len(duplicate_cid_rows)
-    # assert len(mgr.pools_to_add_from_contracts) == (
-    #     len(new_pool_data_df) + duplicate_new_pool_ct
-    # )
 
     all_pools_df = (
         pd.DataFrame(mgr.pool_data)

fastlane_bot/events/exceptions.py

@@ -0,0 +1,7 @@
+class ReadOnlyException(Exception):
+    def __init__(self, filepath):
+        self.filepath = filepath
+
+    def __str__(self):
+        return (f"tokens.csv does not exist at {self.filepath}. Please run the bot without the `read_only` flag to "
+                f"create this file.")

fastlane_bot/events/managers/base.py

@@ -53,6 +53,8 @@ class BaseManager:
         The tokens mapping.
     SUPPORTED_EXCHANGES : Dict[str, Any]
         The supported exchanges.
+    read_only : bool
+        Whether the bot is running in read only mode.
     """
 
     web3: Web3
@@ -99,6 +101,7 @@ class BaseManager:
     static_pools: Dict[str, List[str]] = field(default_factory=dict)
 
     prefix_path: str = ""
+    read_only: bool = False
 
     def __post_init__(self):
         initialized_exchanges = []

fastlane_bot/events/managers/contracts.py

@@ -328,7 +328,7 @@ def get_token_info_from_contract(
                 contract=contract,
                 addr=addr,
                 token_data=token_data,
-                tokens_filepath=tokens_filepath,
+                tokens_filepath=tokens_filepath
             )
         except self.FailedToGetTokenDetailsException as e:
             self.cfg.logger.debug(
@@ -420,14 +420,15 @@ def _get_and_save_token_info_from_contract(
             except FileExistsError:
                 pass
 
-        collision_safety = str(random.randrange(1, 1000))
-        ts = datetime.now().strftime("%d-%H-%M-%S-%f")
-        ts += collision_safety
-        row.to_csv(
-            os.path.normpath(
-                f"{self.prefix_path}fastlane_bot/data/blockchain_data/{self.cfg.NETWORK}/token_detail/{ts}.csv"
-            ),
-            index=False,
-        )
+        if not self.read_only:
+            collision_safety = str(random.randrange(1, 1000))
+            ts = datetime.now().strftime("%d-%H-%M-%S-%f")
+            ts += collision_safety
+            row.to_csv(
+                os.path.normpath(
+                    f"{self.prefix_path}fastlane_bot/data/blockchain_data/{self.cfg.NETWORK}/token_detail/{ts}.csv"
+                ),
+                index=False,
+            )
 
         return (symbol, decimals)

fastlane_bot/events/utils.py

@@ -26,6 +26,7 @@
 from fastlane_bot import Config
 from fastlane_bot.bot import CarbonBot
 from fastlane_bot.config.multiprovider import MultiProviderContractWrapper
+from fastlane_bot.events.exceptions import ReadOnlyException
 from fastlane_bot.events.interface import QueryInterface
 from fastlane_bot.events.managers.manager import Manager
 
@@ -242,6 +243,7 @@ def get_static_data(
     exchanges: List[str],
     blockchain: str,
     static_pool_data_filename: str,
+    read_only: bool = False,
 ) -> Tuple[pd.DataFrame, pd.DataFrame, Dict[str, str], Dict[str, str]]:
     """
     Helper function to get static pool data, tokens, and Uniswap v2 event mappings.
@@ -256,6 +258,8 @@ def get_static_data(
         The name of the blockchain being used
     static_pool_data_filename : str
         The filename of the static pool data CSV file.
+    read_only : bool, optional
+        Whether to run the bot in read-only mode, by default False
 
     Returns
     -------
@@ -288,9 +292,13 @@ def get_static_data(
     )
 
     tokens_filepath = os.path.join(base_path, "tokens.csv")
-    if not os.path.exists(tokens_filepath):
+    if not os.path.exists(tokens_filepath) and not read_only:
         df = pd.DataFrame(columns=["address", "symbol", "decimals"])
         df.to_csv(tokens_filepath)
+    elif not os.path.exists(tokens_filepath) and read_only:
+        raise ReadOnlyException(
+            f"Tokens file {tokens_filepath} does not exist. Please run the bot in non-read-only mode to create it."
+        )
     tokens = read_csv_file(tokens_filepath)
     tokens["address"] = tokens["address"].apply(lambda x: Web3.to_checksum_address(x))
     tokens = tokens.drop_duplicates(subset=["address"])
@@ -1921,18 +1929,22 @@ def handle_static_pools_update(mgr: Any):
             mgr.static_pools[attr_name] = exchange_pools
 
 
-def handle_tokens_csv(mgr, prefix_path):
+def handle_tokens_csv(mgr, prefix_path, read_only: bool = False):
     tokens_filepath = os.path.normpath(
         f"{prefix_path}fastlane_bot/data/blockchain_data/{mgr.cfg.NETWORK}/tokens.csv"
     )
+
     try:
         token_data = pd.read_csv(tokens_filepath)
     except Exception as e:
-        mgr.cfg.logger.info(
-            f"[events.utils.handle_tokens_csv] Error reading token data: {e}... creating new file"
-        )
-        token_data = pd.DataFrame(mgr.tokens)
-        token_data.to_csv(tokens_filepath, index=False)
+        if not read_only:
+            mgr.cfg.logger.info(
+                f"[events.utils.handle_tokens_csv] Error reading token data: {e}... creating new file"
+            )
+            token_data = pd.DataFrame(mgr.tokens)
+            token_data.to_csv(tokens_filepath, index=False)
+        else:
+            raise ReadOnlyException(tokens_filepath) from e
 
     extra_info = glob(
         os.path.normpath(
@@ -1945,19 +1957,22 @@ def handle_tokens_csv(mgr, prefix_path):
         )
         token_data = pd.concat([token_data, extra_info_df], ignore_index=True)
         token_data = token_data.drop_duplicates(subset=["address"])
-        token_data.to_csv(tokens_filepath, index=False)
-        mgr.tokens = token_data.to_dict(orient="records")
 
-        # delete all files in token_detail
-        for f in extra_info:
-            try:
-                os.remove(f)
-            except FileNotFoundError:
-                pass
+        if not read_only:
+            token_data.to_csv(tokens_filepath, index=False)
 
-        mgr.cfg.logger.info(
-            f"[events.utils.handle_tokens_csv] Updated token data with {len(extra_info)} new tokens"
-        )
+            # delete all files in token_detail
+            for f in extra_info:
+                try:
+                    os.remove(f)
+                except FileNotFoundError:
+                    pass
+
+    mgr.tokens = token_data.to_dict(orient="records")
+
+    mgr.cfg.logger.info(
+        f"[events.utils.handle_tokens_csv] Updated token data with {len(extra_info)} new tokens"
+    )
 
 
 def self_funding_warning_sequence(cfg):