diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3006e5354..cd3f3880a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,10 +12,13 @@ jobs: uses: Drakkar-Software/.github/.github/workflows/python3_lint_workflow.yml@master with: project_main_package: octobot_trading + use_full_requirements: true tests: needs: lint uses: Drakkar-Software/.github/.github/workflows/python3_tests_workflow.yml@master + with: + use_full_requirements: true secrets: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} http_proxy: ${{ secrets.EXCHANGE_HTTP_PROXY }} @@ -24,6 +27,8 @@ jobs: needs: tests if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') uses: Drakkar-Software/.github/.github/workflows/python3_sdist_workflow.yml@master + with: + use_full_requirements: true secrets: PYPI_OFFICIAL_UPLOAD_URL: ${{ secrets.PYPI_OFFICIAL_UPLOAD_URL }} PYPI_USERNAME: __token__ diff --git a/CHANGELOG.md b/CHANGELOG.md index 887f21d35..d254878b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.4.232] - 2024-11-26 +### Added +[Requirements] [full] requirements installation +[Exchanges] add preconfigured_exchange +[ExchangeBuilder] add leave_rest_exchange_open +[CCXT] add filtered_fetched_markets + ## [2.4.231] - 2025-11-18 ### Added [Exchanges] add ChannelSpecs and make force channels more flexible diff --git a/Dockerfile b/Dockerfile index cfb125fb0..21715d890 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update \ # configuration and installation RUN pip3 install cython \ - && pip3 install -r requirements.txt -r dev_requirements.txt + && pip3 install -r requirements.txt -r dev_requirements.txt -r full_requirements.txt # tests #RUN pytest tests diff --git a/MANIFEST.in b/MANIFEST.in index b2aa816eb..3162bb365 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,5 +4,6 @@ include README.md include LICENSE include CHANGELOG.md include requirements.txt +include full_requirements.txt global-exclude *.c diff --git a/README.md b/README.md index 624d6f3e6..9f177f52b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# OctoBot-Trading [2.4.231](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md) +# OctoBot-Trading [1.4.232](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/903b6b22bceb4661b608a86fea655f69)](https://app.codacy.com/gh/Drakkar-Software/OctoBot-Trading?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot-Trading&utm_campaign=Badge_Grade_Dashboard) [![PyPI](https://img.shields.io/pypi/v/OctoBot-Trading.svg)](https://pypi.python.org/pypi/OctoBot-Trading/) [![Coverage Status](https://coveralls.io/repos/github/Drakkar-Software/OctoBot-Trading/badge.svg?branch=master)](https://coveralls.io/github/Drakkar-Software/OctoBot-Trading?branch=master) diff --git a/full_requirements.txt b/full_requirements.txt new file mode 100644 index 000000000..4547783ed --- /dev/null +++ b/full_requirements.txt @@ -0,0 +1,7 @@ +# Drakkar-Software requirements +OctoBot-Backtesting[full]>=1.9, <1.10 +OctoBot-Commons[full]>=1.9.82, <1.10 +OctoBot-Tentacles-Manager[full]>=2.9, <2.10 + +# Scripting requirements +tinydb==4.5.2 diff --git a/octobot_trading/__init__.py b/octobot_trading/__init__.py index 5c9ac7650..81be17f35 100644 --- a/octobot_trading/__init__.py +++ b/octobot_trading/__init__.py @@ -15,4 +15,4 @@ # License along with this library. PROJECT_NAME = "OctoBot-Trading" -VERSION = "2.4.231" # major.minor.revision +VERSION = "1.4.232" # major.minor.revision diff --git a/octobot_trading/constants.py b/octobot_trading/constants.py index 080dfb123..5def29216 100644 --- a/octobot_trading/constants.py +++ b/octobot_trading/constants.py @@ -95,6 +95,7 @@ ENABLE_CCXT_VERBOSE = os_util.parse_boolean_environment_var("ENABLE_CCXT_VERBOSE", "False") ENABLE_CCXT_RATE_LIMIT = os_util.parse_boolean_environment_var("ENABLE_CCXT_RATE_LIMIT", "True") ENABLE_CCXT_REQUESTS_COUNTER = os_util.parse_boolean_environment_var("ENABLE_CCXT_REQUESTS_COUNTER", "False") +FETCH_MIN_EXCHANGE_MARKETS = os_util.parse_boolean_environment_var("FETCH_MIN_EXCHANGE_MARKETS", "False") CCXT_DEFAULT_CACHE_LIMIT = int(os.getenv("CCXT_DEFAULT_CACHE_LIMIT", "1000")) # 1000: default ccxt value CCXT_TRADES_CACHE_LIMIT = int(os.getenv("CCXT_TRADES_CACHE_LIMIT", str(CCXT_DEFAULT_CACHE_LIMIT))) CCXT_ORDERS_CACHE_LIMIT = int(os.getenv("CCXT_ORDERS_CACHE_LIMIT", str(CCXT_DEFAULT_CACHE_LIMIT))) diff --git a/octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py b/octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py index 4d60fdbfb..ba5ac8f45 100644 --- a/octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py +++ b/octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. import asyncio +import contextlib try: from aiohttp_socks import ProxyConnectionError except ImportError: @@ -172,6 +173,28 @@ def set_sandbox_mode(exchange_connector, is_sandboxed): return None +@contextlib.contextmanager +def filtered_fetched_markets(client, market_filter: typing.Callable[[dict], bool]): + origin_fetch_markets = client.fetch_markets + + async def _filted_fetched_markets(*args, **kwargs): + all_markets = await origin_fetch_markets(*args, **kwargs) + filtered_markets = [ + market + for market in all_markets + if market_filter(market) + ] + commons_logging.get_logger(__name__).info( + f"Keeping {len(filtered_markets)} out of {len(all_markets)} fetched markets" + ) + return filtered_markets + try: + client.fetch_markets = _filted_fetched_markets + yield + finally: + client.fetch_markets = origin_fetch_markets + + def load_markets_from_cache(client, authenticated_cache: bool, market_filter: typing.Union[None, typing.Callable[[dict], bool]] = None): client.set_markets( market diff --git a/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py b/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py index 4eba1384c..6d968ba78 100644 --- a/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py +++ b/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py @@ -118,12 +118,21 @@ def load_user_inputs_from_class(cls, tentacles_setup_config, tentacle_config): # no user input in connector pass - async def _load_markets(self, client, reload: bool): + async def _load_markets( + self, + client, + reload: bool, + market_filter: typing.Optional[typing.Callable[[dict], bool]] = None + ): """ Override if necessary """ try: - await client.load_markets(reload=reload) + if self.exchange_manager.exchange.FETCH_MIN_EXCHANGE_MARKETS and market_filter: + with ccxt_client_util.filtered_fetched_markets(client, market_filter): + await client.load_markets(reload=reload) + else: + await client.load_markets(reload=reload) except Exception as err: # ensure this is not a proxy error, raise dedicated error if it is if proxy_error := ccxt_client_util.get_proxy_error_if_any(self, err): @@ -135,7 +144,7 @@ async def _load_markets(self, client, reload: bool): async def load_symbol_markets( self, reload=False, - market_filter: typing.Union[None, typing.Callable[[dict], bool]] = None + market_filter: typing.Optional[typing.Callable[[dict], bool]] = None ): authenticated_cache = self.exchange_manager.exchange.requires_authentication_for_this_configuration_only() force_load_markets = reload @@ -151,7 +160,7 @@ async def load_symbol_markets( f"{' sandbox' if self.exchange_manager.is_sandboxed else ''} exchange markets ({reload=} {authenticated_cache=})" ) try: - await self._load_markets(self.client, reload) + await self._load_markets(self.client, reload, market_filter=market_filter) ccxt_client_util.set_markets_cache(self.client, authenticated_cache) except ( ccxt.AuthenticationError, ccxt.ArgumentsRequired, ccxt.static_dependencies.ecdsa.der.UnexpectedDER, @@ -189,7 +198,7 @@ async def load_symbol_markets( unauth_client = None try: unauth_client = self._client_factory(True)[0] - await self._load_markets(unauth_client, reload) + await self._load_markets(unauth_client, reload, market_filter=market_filter) ccxt_client_util.set_markets_cache(unauth_client, False) # apply markets to target client ccxt_client_util.load_markets_from_cache(self.client, False, market_filter=market_filter) diff --git a/octobot_trading/exchanges/connectors/ccxt/constants.py b/octobot_trading/exchanges/connectors/ccxt/constants.py index c92a1d664..2d1697ff4 100644 --- a/octobot_trading/exchanges/connectors/ccxt/constants.py +++ b/octobot_trading/exchanges/connectors/ccxt/constants.py @@ -17,3 +17,4 @@ CCXT_INFO = "info" CCXT_OPTIONS = "options" CCXT_FEES = "fees" +CCXT_FETCH_MARKETS = "fetchMarkets" diff --git a/octobot_trading/exchanges/exchange_builder.py b/octobot_trading/exchanges/exchange_builder.py index 2a1794da6..a31b7d50a 100644 --- a/octobot_trading/exchanges/exchange_builder.py +++ b/octobot_trading/exchanges/exchange_builder.py @@ -249,6 +249,14 @@ def use_market_filter(self, market_filter: typing.Union[None, typing.Callable[[d self.exchange_manager.market_filter = market_filter return self + def set_rest_exchange(self, rest_exchange: typing.Optional["exchanges.RestExchange"]): + self.exchange_manager.preconfigured_exchange = rest_exchange + return self + + def leave_rest_exchange_open(self, leave_rest_exchange_open: bool): + self.exchange_manager.leave_rest_exchange_open = leave_rest_exchange_open + return self + def is_ignoring_config(self, ignore_config=True): self.exchange_manager.ignore_config = ignore_config return self diff --git a/octobot_trading/exchanges/exchange_factory.py b/octobot_trading/exchanges/exchange_factory.py index bd7b27ed7..486e95d94 100644 --- a/octobot_trading/exchanges/exchange_factory.py +++ b/octobot_trading/exchanges/exchange_factory.py @@ -54,9 +54,14 @@ async def create_real_exchange(exchange_manager, exchange_config_by_exchange: ty :param exchange_manager: the related exchange manager :param exchange_config_by_exchange: optional exchange configurations """ - await _create_rest_exchange(exchange_manager, exchange_config_by_exchange) + if exchange_manager.preconfigured_exchange: + exchange_manager.exchange = exchange_manager.preconfigured_exchange + exchange_manager.exchange.exchange_manager = exchange_manager + else: + await _create_rest_exchange(exchange_manager, exchange_config_by_exchange) try: - await exchange_manager.exchange.initialize() + if exchange_manager.preconfigured_exchange is None: + await exchange_manager.exchange.initialize() _create_exchange_backend(exchange_manager) if exchange_manager.exchange_only: return diff --git a/octobot_trading/exchanges/exchange_manager.py b/octobot_trading/exchanges/exchange_manager.py index 1bd1c5c47..b87277df7 100644 --- a/octobot_trading/exchanges/exchange_manager.py +++ b/octobot_trading/exchanges/exchange_manager.py @@ -73,6 +73,8 @@ def __init__(self, config, exchange_class_string): self.trader: exchanges.Trader = None # type: ignore self.exchange: exchanges.RestExchange = None # type: ignore + self.preconfigured_exchange: typing.Optional[exchanges.RestExchange] = None + self.leave_rest_exchange_open: bool = False self.exchange_backend: trading_backend.exchanges.Exchange = None # type: ignore self.is_broker_enabled: bool = False self.trading_modes: list = [] @@ -131,7 +133,7 @@ async def stop(self, warning_on_missing_elements=True, enable_logs=True): # stop exchange channels if enable_logs: self.logger.debug(f"Stopping exchange channels for exchange_id: {self.id} ...") - if self.exchange is not None: + if self.exchange is not None and not self.leave_rest_exchange_open: try: exchange_channel.get_exchange_channels(self.id) await exchange_channel.stop_exchange_channels(self, should_warn=warning_on_missing_elements) diff --git a/octobot_trading/exchanges/types/rest_exchange.py b/octobot_trading/exchanges/types/rest_exchange.py index f4334e2e7..6c7ed51cf 100644 --- a/octobot_trading/exchanges/types/rest_exchange.py +++ b/octobot_trading/exchanges/types/rest_exchange.py @@ -147,6 +147,10 @@ class RestExchange(abstract_exchange.AbstractExchange): # set when the exchange can allow users to pay fees in a custom currency (ex: BNB on binance) LOCAL_FEES_CURRENCIES: typing.List[str] = [] + # Set False in case this exchange's markets should never be filtered as soon as they are fetched + # Therefore overriding the env var value for this exchange + FETCH_MIN_EXCHANGE_MARKETS = constants.FETCH_MIN_EXCHANGE_MARKETS + DEFAULT_CONNECTOR_CLASS = ccxt_connector.CCXTConnector def __init__( diff --git a/octobot_trading/exchanges/util/exchange_util.py b/octobot_trading/exchanges/util/exchange_util.py index c9e541e95..7f7449f56 100644 --- a/octobot_trading/exchanges/util/exchange_util.py +++ b/octobot_trading/exchanges/util/exchange_util.py @@ -221,7 +221,9 @@ async def get_local_exchange_manager( is_sandboxed: bool, ignore_config=False, builder=None, use_cached_markets=True, is_broker_enabled: bool = False, exchange_config_by_exchange: typing.Optional[dict[str, dict]] = None, disable_unauth_retry: bool = False, - market_filter: typing.Union[None, typing.Callable[[dict], bool]] = None + market_filter: typing.Union[None, typing.Callable[[dict], bool]] = None, + rest_exchange: typing.Optional[exchanges_types.RestExchange] = None, + leave_rest_exchange_open: bool = False, ): exchange_type = exchange_config.get(common_constants.CONFIG_EXCHANGE_TYPE, get_default_exchange_type(exchange_name)) builder = builder or exchange_builder.ExchangeBuilder( @@ -239,6 +241,8 @@ async def get_local_exchange_manager( .is_broker_enabled(is_broker_enabled) \ .use_cached_markets(use_cached_markets) \ .use_market_filter(market_filter) \ + .set_rest_exchange(rest_exchange) \ + .leave_rest_exchange_open(leave_rest_exchange_open) \ .is_ignoring_config(ignore_config) \ .disable_trading_mode() \ .build() diff --git a/octobot_trading/modes/abstract_trading_mode.py b/octobot_trading/modes/abstract_trading_mode.py index 90a69edf8..6526876ad 100644 --- a/octobot_trading/modes/abstract_trading_mode.py +++ b/octobot_trading/modes/abstract_trading_mode.py @@ -114,6 +114,9 @@ def __init__(self, config, exchange_manager): self.is_health_check_enabled: bool = False self._last_health_check_time: float = 0 + # Pending bot logs to be inserted after execution + self.pending_bot_logs: list["octobot.community.BotLogData"] = [] + # Used to know the current state of the trading mode. # Overwrite in subclasses def get_current_state(self) -> tuple: diff --git a/octobot_trading/util/test_tools/exchange_data.py b/octobot_trading/util/test_tools/exchange_data.py index f11399897..33f4ba5c5 100644 --- a/octobot_trading/util/test_tools/exchange_data.py +++ b/octobot_trading/util/test_tools/exchange_data.py @@ -102,7 +102,7 @@ class PortfolioDetails(octobot_commons.dataclasses.FlexibleDataclass, octobot_co initial_value: float = 0 # value of the portfolio content (not the full_content) content: dict = dataclasses.field(default_factory=dict) # might be a subset of global_content full_content: dict = dataclasses.field(default_factory=dict) # full exchange portfolio - asset_values: dict = dataclasses.field(default_factory=dict) # unitary value of each asset in reference market + asset_values: dict[str, float] = dataclasses.field(default_factory=dict) # unitary value of each asset in reference market @dataclasses.dataclass diff --git a/octobot_trading/util/test_tools/exchanges_test_tools.py b/octobot_trading/util/test_tools/exchanges_test_tools.py index 05b6fe565..254670818 100644 --- a/octobot_trading/util/test_tools/exchanges_test_tools.py +++ b/octobot_trading/util/test_tools/exchanges_test_tools.py @@ -88,11 +88,11 @@ async def _get_symbol_prices(exchange_manager, symbol, parsed_tf, limit): return await exchange_manager.exchange.get_symbol_prices(symbol, parsed_tf, limit=limit) -async def _update_ohlcv( - exchange_manager, symbol: str, time_frame: str, exchange_data: exchange_data_import.ExchangeData, +async def fetch_ohlcv( + exchange_manager, symbol: str, time_frame: str, history_size=1, start_time=0, end_time=0, close_price_only=False, include_latest_candle=True -): +) -> exchange_data_import.MarketDetails: parsed_tf = common_enums.TimeFrames(time_frame) if start_time == 0: ohlcvs = await _get_symbol_prices(exchange_manager, symbol, parsed_tf, history_size) @@ -105,7 +105,7 @@ async def _update_ohlcv( ohlcvs.extend(ohlcv) if not include_latest_candle: ohlcvs = ohlcvs[:-1] - details = exchange_data_import.MarketDetails( + return exchange_data_import.MarketDetails( symbol=symbol, time_frame=time_frame, close=[ohlcv[common_enums.PriceIndexes.IND_PRICE_CLOSE.value] for ohlcv in ohlcvs], @@ -115,7 +115,18 @@ async def _update_ohlcv( volume=[ohlcv[common_enums.PriceIndexes.IND_PRICE_VOL.value] for ohlcv in ohlcvs] if not close_price_only else [], time=[ohlcv[common_enums.PriceIndexes.IND_PRICE_TIME.value] for ohlcv in ohlcvs], ) - exchange_data.markets.append(details) + + +async def _update_ohlcv( + exchange_manager, symbol: str, time_frame: str, exchange_data: exchange_data_import.ExchangeData, + history_size=1, start_time=0, end_time=0, close_price_only=False, + include_latest_candle=True +): + market = await fetch_ohlcv( + exchange_manager, symbol, time_frame, history_size, + start_time, end_time, close_price_only, include_latest_candle + ) + exchange_data.markets.append(market) async def add_symbols_details( @@ -232,7 +243,7 @@ async def _get_open_orders(exchange_manager, symbol: str, open_orders: list, ign async def get_open_orders( exchange_manager, - exchange_data: exchange_data_import.ExchangeData, + exchange_data: typing.Optional[exchange_data_import.ExchangeData], symbols: list = None, ignore_unsupported_orders: bool = True, ) -> list: @@ -312,7 +323,7 @@ async def _get_trades(exchange_manager, symbol: str, trades: list): async def get_trades( exchange_manager, - exchange_data: exchange_data_import.ExchangeData, + exchange_data: typing.Optional[exchange_data_import.ExchangeData], symbols: list = None ) -> list: trades = [] @@ -408,7 +419,7 @@ async def wait_for_other_status(order: personal_data.Order, timeout) -> personal ) async def get_positions( exchange_manager, - exchange_data: exchange_data_import.ExchangeData, + exchange_data: typing.Optional[exchange_data_import.ExchangeData], symbols: list = None ) -> list[dict]: symbols = symbols or [market.symbol for market in exchange_data.markets] diff --git a/requirements.txt b/requirements.txt index a819a0cff..eeca19a68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,5 @@ cryptography # Never specify a version (managed by https://github.com/Drakkar-So # OrderBook requirement sortedcontainers==2.4.0 -# Scripting requirements -tinydb==4.5.2 - # Caching cachetools>=5.5.0, <6 diff --git a/setup.py b/setup.py index dc44b25c5..2aeb7d400 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,9 @@ DESCRIPTION = f.read() REQUIRED = open('requirements.txt').readlines() +EXTRAS_REQUIRED = { + 'full': open('full_requirements.txt').readlines(), +} REQUIRES_PYTHON = '>=3.8' setup( @@ -44,6 +47,7 @@ zip_safe=False, data_files=[], install_requires=REQUIRED, + extras_require=EXTRAS_REQUIRED, python_requires=REQUIRES_PYTHON, classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/tests/exchanges/connectors/ccxt/test_ccxt_connector.py b/tests/exchanges/connectors/ccxt/test_ccxt_connector.py index 29cf375f0..06b1a2d25 100644 --- a/tests/exchanges/connectors/ccxt/test_ccxt_connector.py +++ b/tests/exchanges/connectors/ccxt/test_ccxt_connector.py @@ -55,7 +55,7 @@ def __init__(self): self.set_markets_calls = [] self.urls = {} - async def load_markets(self, reload=False): + async def load_markets(self, reload=False, market_filter=None): pass def set_markets(self, markets): @@ -83,7 +83,7 @@ def __init__(self): self.set_markets_calls = [] self.urls = {} - async def load_markets(self, reload=False): + async def load_markets(self, reload=False, market_filter=None): pass def set_markets(self, markets): @@ -121,7 +121,7 @@ def __init__(self): self.set_markets_calls = [] self.urls = {} - async def load_markets(self, reload=False): + async def load_markets(self, reload=False, market_filter=None): pass def set_markets(self, markets):