add full requirements

.github/workflows/main.yml

@@ -12,10 +12,13 @@ jobs:
     uses: Drakkar-Software/.github/.github/workflows/python3_lint_workflow.yml@master
     with:
       project_main_package: octobot_trading
+      use_full_requirements: true
 
   tests:
     needs: lint
     uses: Drakkar-Software/.github/.github/workflows/python3_tests_workflow.yml@master
+    with:
+      use_full_requirements: true
     secrets:
       COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
       http_proxy: ${{ secrets.EXCHANGE_HTTP_PROXY }}
@@ -24,6 +27,8 @@ jobs:
     needs: tests
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
     uses: Drakkar-Software/.github/.github/workflows/python3_sdist_workflow.yml@master
+    with:
+      use_full_requirements: true
     secrets:
       PYPI_OFFICIAL_UPLOAD_URL: ${{ secrets.PYPI_OFFICIAL_UPLOAD_URL }}
       PYPI_USERNAME: __token__

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.232] - 2024-11-26
+### Added 
+[Requirements] [full] requirements installation
+[Exchanges] add preconfigured_exchange
+[ExchangeBuilder] add leave_rest_exchange_open
+[CCXT] add filtered_fetched_markets
+
 ## [2.4.231] - 2025-11-18
 ### Added
 [Exchanges] add ChannelSpecs and make force channels more flexible

Dockerfile

@@ -16,7 +16,7 @@ RUN apt-get update \
 
 # configuration and installation
 RUN pip3 install cython \
-    && pip3 install -r requirements.txt -r dev_requirements.txt
+    && pip3 install -r requirements.txt -r dev_requirements.txt -r full_requirements.txt
 
 # tests
 #RUN pytest tests

MANIFEST.in

@@ -4,5 +4,6 @@ include README.md
 include LICENSE
 include CHANGELOG.md
 include requirements.txt
+include full_requirements.txt
 
 global-exclude *.c

README.md

@@ -1,4 +1,4 @@
-# OctoBot-Trading [2.4.231](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
+# OctoBot-Trading [1.4.232](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/903b6b22bceb4661b608a86fea655f69)](https://app.codacy.com/gh/Drakkar-Software/OctoBot-Trading?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot-Trading&utm_campaign=Badge_Grade_Dashboard)
 [![PyPI](https://img.shields.io/pypi/v/OctoBot-Trading.svg)](https://pypi.python.org/pypi/OctoBot-Trading/)
 [![Coverage Status](https://coveralls.io/repos/github/Drakkar-Software/OctoBot-Trading/badge.svg?branch=master)](https://coveralls.io/github/Drakkar-Software/OctoBot-Trading?branch=master)

full_requirements.txt

@@ -0,0 +1,7 @@
+# Drakkar-Software requirements
+OctoBot-Backtesting[full]>=1.9, <1.10
+OctoBot-Commons[full]>=1.9.82, <1.10
+OctoBot-Tentacles-Manager[full]>=2.9, <2.10
+
+# Scripting requirements
+tinydb==4.5.2

octobot_trading/__init__.py

@@ -15,4 +15,4 @@
 #  License along with this library.
 
 PROJECT_NAME = "OctoBot-Trading"
-VERSION = "2.4.231"  # major.minor.revision
+VERSION = "1.4.232"  # major.minor.revision

octobot_trading/constants.py

@@ -95,6 +95,7 @@
 ENABLE_CCXT_VERBOSE = os_util.parse_boolean_environment_var("ENABLE_CCXT_VERBOSE", "False")
 ENABLE_CCXT_RATE_LIMIT = os_util.parse_boolean_environment_var("ENABLE_CCXT_RATE_LIMIT", "True")
 ENABLE_CCXT_REQUESTS_COUNTER = os_util.parse_boolean_environment_var("ENABLE_CCXT_REQUESTS_COUNTER", "False")
+FETCH_MIN_EXCHANGE_MARKETS = os_util.parse_boolean_environment_var("FETCH_MIN_EXCHANGE_MARKETS", "False")
 CCXT_DEFAULT_CACHE_LIMIT = int(os.getenv("CCXT_DEFAULT_CACHE_LIMIT", "1000"))  # 1000: default ccxt value
 CCXT_TRADES_CACHE_LIMIT = int(os.getenv("CCXT_TRADES_CACHE_LIMIT", str(CCXT_DEFAULT_CACHE_LIMIT)))
 CCXT_ORDERS_CACHE_LIMIT = int(os.getenv("CCXT_ORDERS_CACHE_LIMIT", str(CCXT_DEFAULT_CACHE_LIMIT)))

octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py

@@ -14,6 +14,7 @@
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library.
 import asyncio
+import contextlib
 try:
     from aiohttp_socks import ProxyConnectionError
 except ImportError:
@@ -172,6 +173,28 @@ def set_sandbox_mode(exchange_connector, is_sandboxed):
     return None
 
 
+@contextlib.contextmanager
+def filtered_fetched_markets(client, market_filter: typing.Callable[[dict], bool]):
+    origin_fetch_markets = client.fetch_markets
+
+    async def _filted_fetched_markets(*args, **kwargs):
+        all_markets = await origin_fetch_markets(*args, **kwargs)
+        filtered_markets = [
+            market 
+            for market in all_markets
+            if market_filter(market)
+        ]
+        commons_logging.get_logger(__name__).info(
+            f"Keeping {len(filtered_markets)} out of {len(all_markets)} fetched markets"
+        )
+        return filtered_markets
+    try:
+        client.fetch_markets = _filted_fetched_markets
+        yield
+    finally:
+        client.fetch_markets = origin_fetch_markets
+
+
 def load_markets_from_cache(client, authenticated_cache: bool, market_filter: typing.Union[None, typing.Callable[[dict], bool]] = None):
     client.set_markets(
         market

octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py

@@ -118,12 +118,21 @@ def load_user_inputs_from_class(cls, tentacles_setup_config, tentacle_config):
         # no user input in connector
         pass
 
-    async def _load_markets(self, client, reload: bool):
+    async def _load_markets(
+        self, 
+        client, 
+        reload: bool, 
+        market_filter: typing.Optional[typing.Callable[[dict], bool]] = None
+    ):
         """
         Override if necessary
         """
         try:
-            await client.load_markets(reload=reload)
+            if self.exchange_manager.exchange.FETCH_MIN_EXCHANGE_MARKETS and market_filter:
+                with ccxt_client_util.filtered_fetched_markets(client, market_filter):
+                    await client.load_markets(reload=reload)
+            else:
+                await client.load_markets(reload=reload)
         except Exception as err:
             # ensure this is not a proxy error, raise dedicated error if it is
             if proxy_error := ccxt_client_util.get_proxy_error_if_any(self, err):
@@ -135,7 +144,7 @@ async def _load_markets(self, client, reload: bool):
     async def load_symbol_markets(
         self,
         reload=False,
-        market_filter: typing.Union[None, typing.Callable[[dict], bool]] = None
+        market_filter: typing.Optional[typing.Callable[[dict], bool]] = None
     ):
         authenticated_cache = self.exchange_manager.exchange.requires_authentication_for_this_configuration_only()
         force_load_markets = reload
@@ -151,7 +160,7 @@ async def load_symbol_markets(
                 f"{' sandbox' if self.exchange_manager.is_sandboxed else ''} exchange markets ({reload=} {authenticated_cache=})"
             )
             try:
-                await self._load_markets(self.client, reload)
+                await self._load_markets(self.client, reload, market_filter=market_filter)
                 ccxt_client_util.set_markets_cache(self.client, authenticated_cache)
             except (
                 ccxt.AuthenticationError, ccxt.ArgumentsRequired, ccxt.static_dependencies.ecdsa.der.UnexpectedDER,
@@ -189,7 +198,7 @@ async def load_symbol_markets(
                     unauth_client = None
                     try:
                         unauth_client = self._client_factory(True)[0]
-                        await self._load_markets(unauth_client, reload)
+                        await self._load_markets(unauth_client, reload, market_filter=market_filter)
                         ccxt_client_util.set_markets_cache(unauth_client, False)
                         # apply markets to target client
                         ccxt_client_util.load_markets_from_cache(self.client, False, market_filter=market_filter)

octobot_trading/exchanges/connectors/ccxt/constants.py

@@ -17,3 +17,4 @@
 CCXT_INFO = "info"
 CCXT_OPTIONS = "options"
 CCXT_FEES = "fees"
+CCXT_FETCH_MARKETS = "fetchMarkets"

octobot_trading/exchanges/exchange_builder.py

@@ -249,6 +249,14 @@ def use_market_filter(self, market_filter: typing.Union[None, typing.Callable[[d
         self.exchange_manager.market_filter = market_filter
         return self
 
+    def set_rest_exchange(self, rest_exchange: typing.Optional["exchanges.RestExchange"]):
+        self.exchange_manager.preconfigured_exchange = rest_exchange
+        return self
+
+    def leave_rest_exchange_open(self, leave_rest_exchange_open: bool):
+        self.exchange_manager.leave_rest_exchange_open = leave_rest_exchange_open
+        return self
+
     def is_ignoring_config(self, ignore_config=True):
         self.exchange_manager.ignore_config = ignore_config
         return self

octobot_trading/exchanges/exchange_factory.py

@@ -54,9 +54,14 @@ async def create_real_exchange(exchange_manager, exchange_config_by_exchange: ty
     :param exchange_manager: the related exchange manager
     :param exchange_config_by_exchange: optional exchange configurations
     """
-    await _create_rest_exchange(exchange_manager, exchange_config_by_exchange)
+    if exchange_manager.preconfigured_exchange:
+        exchange_manager.exchange = exchange_manager.preconfigured_exchange
+        exchange_manager.exchange.exchange_manager = exchange_manager
+    else:
+        await _create_rest_exchange(exchange_manager, exchange_config_by_exchange)
     try:
-        await exchange_manager.exchange.initialize()
+        if exchange_manager.preconfigured_exchange is None:
+            await exchange_manager.exchange.initialize()
         _create_exchange_backend(exchange_manager)
         if exchange_manager.exchange_only:
             return

octobot_trading/exchanges/exchange_manager.py

@@ -73,6 +73,8 @@ def __init__(self, config, exchange_class_string):
 
         self.trader: exchanges.Trader = None # type: ignore
         self.exchange: exchanges.RestExchange = None # type: ignore
+        self.preconfigured_exchange: typing.Optional[exchanges.RestExchange] = None
+        self.leave_rest_exchange_open: bool = False
         self.exchange_backend: trading_backend.exchanges.Exchange = None # type: ignore
         self.is_broker_enabled: bool = False
         self.trading_modes: list = []
@@ -131,7 +133,7 @@ async def stop(self, warning_on_missing_elements=True, enable_logs=True):
         # stop exchange channels
         if enable_logs:
             self.logger.debug(f"Stopping exchange channels for exchange_id: {self.id} ...")
-        if self.exchange is not None:
+        if self.exchange is not None and not self.leave_rest_exchange_open:
             try:
                 exchange_channel.get_exchange_channels(self.id)
                 await exchange_channel.stop_exchange_channels(self, should_warn=warning_on_missing_elements)

octobot_trading/exchanges/types/rest_exchange.py

@@ -147,6 +147,10 @@ class RestExchange(abstract_exchange.AbstractExchange):
     # set when the exchange can allow users to pay fees in a custom currency (ex: BNB on binance)
     LOCAL_FEES_CURRENCIES: typing.List[str] = []
 
+    # Set False in case this exchange's markets should never be filtered as soon as they are fetched
+    # Therefore overriding the env var value for this exchange
+    FETCH_MIN_EXCHANGE_MARKETS = constants.FETCH_MIN_EXCHANGE_MARKETS
+
     DEFAULT_CONNECTOR_CLASS = ccxt_connector.CCXTConnector
 
     def __init__(

octobot_trading/exchanges/util/exchange_util.py

@@ -221,7 +221,9 @@ async def get_local_exchange_manager(
     is_sandboxed: bool, ignore_config=False, builder=None, use_cached_markets=True,
     is_broker_enabled: bool = False, exchange_config_by_exchange: typing.Optional[dict[str, dict]] = None,
     disable_unauth_retry: bool = False,
-    market_filter: typing.Union[None, typing.Callable[[dict], bool]] = None
+    market_filter: typing.Union[None, typing.Callable[[dict], bool]] = None,
+    rest_exchange: typing.Optional[exchanges_types.RestExchange] = None,
+    leave_rest_exchange_open: bool = False,
 ):
     exchange_type = exchange_config.get(common_constants.CONFIG_EXCHANGE_TYPE, get_default_exchange_type(exchange_name))
     builder = builder or exchange_builder.ExchangeBuilder(
@@ -239,6 +241,8 @@ async def get_local_exchange_manager(
         .is_broker_enabled(is_broker_enabled) \
         .use_cached_markets(use_cached_markets) \
         .use_market_filter(market_filter) \
+        .set_rest_exchange(rest_exchange) \
+        .leave_rest_exchange_open(leave_rest_exchange_open) \
         .is_ignoring_config(ignore_config) \
         .disable_trading_mode() \
         .build()

octobot_trading/modes/abstract_trading_mode.py

@@ -114,6 +114,9 @@ def __init__(self, config, exchange_manager):
         self.is_health_check_enabled: bool = False
         self._last_health_check_time: float = 0
 
+        # Pending bot logs to be inserted after execution
+        self.pending_bot_logs: list["octobot.community.BotLogData"] = []
+
     # Used to know the current state of the trading mode.
     # Overwrite in subclasses
     def get_current_state(self) -> tuple:

octobot_trading/util/test_tools/exchange_data.py

@@ -102,7 +102,7 @@ class PortfolioDetails(octobot_commons.dataclasses.FlexibleDataclass, octobot_co
     initial_value: float = 0    # value of the portfolio content (not the full_content)
     content: dict = dataclasses.field(default_factory=dict) # might be a subset of global_content
     full_content: dict = dataclasses.field(default_factory=dict)  # full exchange portfolio
-    asset_values: dict = dataclasses.field(default_factory=dict)    # unitary value of each asset in reference market
+    asset_values: dict[str, float] = dataclasses.field(default_factory=dict)    # unitary value of each asset in reference market
 
 
 @dataclasses.dataclass

octobot_trading/util/test_tools/exchanges_test_tools.py

@@ -88,11 +88,11 @@ async def _get_symbol_prices(exchange_manager, symbol, parsed_tf, limit):
     return await exchange_manager.exchange.get_symbol_prices(symbol, parsed_tf, limit=limit)
 
 
-async def _update_ohlcv(
-    exchange_manager, symbol: str, time_frame: str, exchange_data: exchange_data_import.ExchangeData,
+async def fetch_ohlcv(
+    exchange_manager, symbol: str, time_frame: str,
     history_size=1, start_time=0, end_time=0, close_price_only=False,
     include_latest_candle=True
-):
+) -> exchange_data_import.MarketDetails:
     parsed_tf = common_enums.TimeFrames(time_frame)
     if start_time == 0:
         ohlcvs = await _get_symbol_prices(exchange_manager, symbol, parsed_tf, history_size)
@@ -105,7 +105,7 @@ async def _update_ohlcv(
             ohlcvs.extend(ohlcv)
     if not include_latest_candle:
         ohlcvs = ohlcvs[:-1]
-    details = exchange_data_import.MarketDetails(
+    return exchange_data_import.MarketDetails(
         symbol=symbol,
         time_frame=time_frame,
         close=[ohlcv[common_enums.PriceIndexes.IND_PRICE_CLOSE.value] for ohlcv in ohlcvs],
@@ -115,7 +115,18 @@ async def _update_ohlcv(
         volume=[ohlcv[common_enums.PriceIndexes.IND_PRICE_VOL.value] for ohlcv in ohlcvs] if not close_price_only else [],
         time=[ohlcv[common_enums.PriceIndexes.IND_PRICE_TIME.value] for ohlcv in ohlcvs],
     )
-    exchange_data.markets.append(details)
+
+
+async def _update_ohlcv(
+    exchange_manager, symbol: str, time_frame: str, exchange_data: exchange_data_import.ExchangeData,
+    history_size=1, start_time=0, end_time=0, close_price_only=False,
+    include_latest_candle=True
+):
+    market = await fetch_ohlcv(
+        exchange_manager, symbol, time_frame, history_size, 
+        start_time, end_time, close_price_only, include_latest_candle
+    )
+    exchange_data.markets.append(market)
 
 
 async def add_symbols_details(
@@ -232,7 +243,7 @@ async def _get_open_orders(exchange_manager, symbol: str, open_orders: list, ign
 
 async def get_open_orders(
     exchange_manager,
-    exchange_data: exchange_data_import.ExchangeData,
+    exchange_data: typing.Optional[exchange_data_import.ExchangeData],
     symbols: list = None,
     ignore_unsupported_orders: bool = True,
 ) -> list:
@@ -312,7 +323,7 @@ async def _get_trades(exchange_manager, symbol: str, trades: list):
 
 async def get_trades(
     exchange_manager,
-    exchange_data: exchange_data_import.ExchangeData,
+    exchange_data: typing.Optional[exchange_data_import.ExchangeData],
     symbols: list = None
 ) -> list:
     trades = []
@@ -408,7 +419,7 @@ async def wait_for_other_status(order: personal_data.Order, timeout) -> personal
 )
 async def get_positions(
     exchange_manager,
-    exchange_data: exchange_data_import.ExchangeData,
+    exchange_data: typing.Optional[exchange_data_import.ExchangeData],
     symbols: list = None
 ) -> list[dict]:
     symbols = symbols or [market.symbol for market in exchange_data.markets]

requirements.txt

@@ -15,8 +15,5 @@ cryptography # Never specify a version (managed by https://github.com/Drakkar-So
 # OrderBook requirement
 sortedcontainers==2.4.0
 
-# Scripting requirements
-tinydb==4.5.2
-
 # Caching
 cachetools>=5.5.0, <6

setup.py

@@ -26,6 +26,9 @@
     DESCRIPTION = f.read()
 
 REQUIRED = open('requirements.txt').readlines()
+EXTRAS_REQUIRED = {
+    'full': open('full_requirements.txt').readlines(),
+}
 REQUIRES_PYTHON = '>=3.8'
 
 setup(
@@ -44,6 +47,7 @@
     zip_safe=False,
     data_files=[],
     install_requires=REQUIRED,
+    extras_require=EXTRAS_REQUIRED,
     python_requires=REQUIRES_PYTHON,
     classifiers=[
         'Development Status :: 5 - Production/Stable',

tests/exchanges/connectors/ccxt/test_ccxt_connector.py

@@ -55,7 +55,7 @@ def __init__(self):
             self.set_markets_calls = []
             self.urls = {}
 
-        async def load_markets(self, reload=False):
+        async def load_markets(self, reload=False, market_filter=None):
             pass
 
         def set_markets(self, markets):
@@ -83,7 +83,7 @@ def __init__(self):
             self.set_markets_calls = []
             self.urls = {}
 
-        async def load_markets(self, reload=False):
+        async def load_markets(self, reload=False, market_filter=None):
             pass
 
         def set_markets(self, markets):
@@ -121,7 +121,7 @@ def __init__(self):
             self.set_markets_calls = []
             self.urls = {}
 
-        async def load_markets(self, reload=False):
+        async def load_markets(self, reload=False, market_filter=None):
             pass
 
         def set_markets(self, markets):