Merge pull request #2052 from Drakkar-Software/dev

master merge

Author: GuillaumeDSM

.github/workflows/docker.yml

@@ -89,6 +89,14 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Login to Scaleway
+        if: github.event_name == 'push'
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ secrets.SCALEWAY_DOCKER_REGISTRY }}
+          username: nologin
+          password: ${{ secrets.SCALEWAY_PRIVATE_KEY }}
+
       - name: Build latest
         if: github.event_name != 'push'
         uses: docker/build-push-action@master
@@ -162,6 +170,20 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
 
+      - name: Build and push on tag on scaleway
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        uses: docker/build-push-action@master
+        with:
+          context: .
+          file: ./Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          platforms: linux/amd64
+          push: true
+          tags: |
+            ${{ secrets.SCALEWAY_DOCKER_REGISTRY }}/${{ env.IMAGE }}:${{ env.STABLE }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+
       - name: Image digest
         run: echo ${{ steps.docker_build_and_psuh.outputs.digest }}
 

CHANGELOG.md

@@ -6,6 +6,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 *It is strongly advised to perform an update of your tentacles after updating OctoBot. (start.py tentacles --install --all)*
 
+## [0.4.11] - 2022-10-12
+### Added
+- User inputs system
+- Phemex exchange
+- Run storage
+### Updated
+- Configuration for each tentacle
+- Community bot system instead of devices
+### Fixed
+- Exchange API issues (Bybit, Kucoin)
+
 ## [0.4.10] - 2022-09-13
 ### Updated
 - Beta tentacles

Dockerfile

@@ -39,5 +39,5 @@ VOLUME /octobot/tentacles
 VOLUME /octobot/user
 EXPOSE 5001
 
-HEALTHCHECK --interval=1m --timeout=30s --retries=3 CMD curl --fail http://localhost:5001 || exit 1
+HEALTHCHECK --interval=1m --timeout=30s --retries=3 CMD curl -sS http://127.0.0.1:5001 || exit 1
 ENTRYPOINT ["./docker-entrypoint.sh"]

README.md

@@ -1,4 +1,4 @@
-# OctoBot [0.4.10](https://octobot.click/gh-changelog)
+# OctoBot [0.4.11](https://octobot.click/gh-changelog)
 [![PyPI](https://img.shields.io/pypi/v/OctoBot.svg)](https://octobot.click/gh-pypi)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/e07fb190156d4efb8e7d07aaa5eff2e1)](https://app.codacy.com/gh/Drakkar-Software/OctoBot?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot&utm_campaign=Badge_Grade_Dashboard)[![Downloads](https://pepy.tech/badge/octobot/month)](https://pepy.tech/project/octobot)
 [![Dockerhub](https://img.shields.io/docker/pulls/drakkarsoftware/octobot.svg)](https://octobot.click/gh-dockerhub)

octobot/api/backtesting.py

@@ -26,15 +26,22 @@ def create_independent_backtesting(config,
                                    start_timestamp=None,
                                    end_timestamp=None,
                                    enable_logs=True,
-                                   stop_when_finished=False) -> backtesting.IndependentBacktesting:
-    return backtesting.IndependentBacktesting(config, tentacles_setup_config, data_files,
-                                              data_file_path,
-                                              run_on_common_part_only=run_on_common_part_only,
-                                              join_backtesting_timeout=join_backtesting_timeout,
-                                              start_timestamp=start_timestamp,
-                                              end_timestamp=end_timestamp,
-                                              enable_logs=enable_logs,
-                                              stop_when_finished=stop_when_finished)
+                                   stop_when_finished=False,
+                                   enforce_total_databases_max_size_after_run=True,
+                                   enable_storage=True) \
+        -> backtesting.IndependentBacktesting:
+    return backtesting.IndependentBacktesting(
+        config, tentacles_setup_config, data_files,
+        data_file_path,
+        run_on_common_part_only=run_on_common_part_only,
+        join_backtesting_timeout=join_backtesting_timeout,
+        start_timestamp=start_timestamp,
+        end_timestamp=end_timestamp,
+        enable_logs=enable_logs,
+        stop_when_finished=stop_when_finished,
+        enforce_total_databases_max_size_after_run=enforce_total_databases_max_size_after_run,
+        enable_storage=enable_storage,
+    )
 
 
 async def initialize_and_run_independent_backtesting(independent_backtesting, log_errors=True) -> None:

octobot/backtesting/independent_backtesting.pxd

@@ -43,6 +43,7 @@ cdef class IndependentBacktesting:
     cdef public object post_backtesting_task
     cdef public object previous_log_level
     cdef public object stopped_event
+    cdef public bint enforce_total_databases_max_size_after_run
 
     cdef public octobot_backtesting.OctoBotBacktesting octobot_backtesting
 

octobot/backtesting/independent_backtesting.py

@@ -39,6 +39,8 @@
 import octobot_trading.api as trading_api
 import octobot_trading.enums as trading_enums
 
+import octobot.storage as storage
+
 
 class IndependentBacktesting:
     def __init__(self, config,
@@ -50,7 +52,9 @@ def __init__(self, config,
                  start_timestamp=None,
                  end_timestamp=None,
                  enable_logs=True,
-                 stop_when_finished=False):
+                 stop_when_finished=False,
+                 enforce_total_databases_max_size_after_run=True,
+                 enable_storage=True):
         self.octobot_origin_config = config
         self.tentacles_setup_config = tentacles_setup_config
         self.backtesting_config = {}
@@ -72,14 +76,16 @@ def __init__(self, config,
         self.enable_logs = enable_logs
         self.stop_when_finished = stop_when_finished
         self.previous_log_level = commons_logging.get_global_logger_level()
+        self.enforce_total_databases_max_size_after_run = enforce_total_databases_max_size_after_run
         self.octobot_backtesting = backtesting.OctoBotBacktesting(self.backtesting_config,
                                                                   self.tentacles_setup_config,
                                                                   self.symbols_to_create_exchange_classes,
                                                                   self.backtesting_files,
                                                                   run_on_common_part_only,
                                                                   start_timestamp=start_timestamp,
                                                                   end_timestamp=end_timestamp,
-                                                                  enable_logs=self.enable_logs)
+                                                                  enable_logs=self.enable_logs,
+                                                                  enable_storage=enable_storage)
 
     async def initialize_and_run(self, log_errors=True):
         try:
@@ -162,11 +168,16 @@ async def _post_backtesting_end_callback(self):
         else:
             # stop backtesting importers to release database files
             await self.octobot_backtesting.stop_importers()
+        if self.enforce_total_databases_max_size_after_run:
+            try:
+                await storage.enforce_total_databases_max_size()
+            except Exception as e:
+                self.logger.exception(e, True, f"Error when enforcing max run databases size: {e}")
 
     @staticmethod
     def _get_market_delta(symbol, exchange_manager, min_timeframe):
         market_data = trading_api.get_symbol_historical_candles(
-            trading_api.get_symbol_data(exchange_manager, symbol.legacy_symbol()), min_timeframe)
+            trading_api.get_symbol_data(exchange_manager, str(symbol)), min_timeframe)
         market_begin = market_data[enums.PriceIndexes.IND_PRICE_CLOSE.value][0]
         market_end = market_data[enums.PriceIndexes.IND_PRICE_CLOSE.value][-1]
 
@@ -355,8 +366,9 @@ async def _generate_backtesting_id_if_missing(self):
                 optimization_campaign.OptimizationCampaign.get_campaign_name(self.tentacles_setup_config)
             )
             run_dbs_identifier.backtesting_id = await run_dbs_identifier.generate_new_backtesting_id()
-            # initialize to lock the backtesting id
-            await run_dbs_identifier.initialize()
+            if self.octobot_backtesting.enable_storage:
+                # initialize to lock the backtesting id
+                await run_dbs_identifier.initialize()
             self.backtesting_config[common_constants.CONFIG_BACKTESTING_ID] = run_dbs_identifier.backtesting_id
 
     def _find_reference_market_and_update_contract_type(self):
@@ -412,7 +424,7 @@ def _add_config_default_backtesting_values(self):
     def _add_crypto_currencies_config(self):
         for symbols in self.symbols_to_create_exchange_classes.values():
             for symbol in symbols:
-                symbol_id = symbol.legacy_symbol()
+                symbol_id = str(symbol)
                 if symbol_id not in self.backtesting_config[common_constants.CONFIG_CRYPTO_CURRENCIES]:
                     self.backtesting_config[common_constants.CONFIG_CRYPTO_CURRENCIES][symbol_id] = {
                         common_constants.CONFIG_CRYPTO_PAIRS: []

octobot/backtesting/octobot_backtesting.pxd

@@ -32,11 +32,13 @@ cdef class OctoBotBacktesting:
     cdef public list backtesting_files
     cdef public object backtesting
     cdef public bint run_on_common_part_only
+    cdef public object start_time
     cdef public object start_timestamp
     cdef public object end_timestamp
     cdef public bint enable_logs
     cdef public dict exchange_type_by_exchange
     cdef public object futures_contract_type
+    cdef public bint enable_storage
 
     cpdef object memory_leak_checkup(self, list to_check_elements)
     cpdef object check_remaining_objects(self)

octobot/backtesting/octobot_backtesting.py

@@ -13,12 +13,17 @@
 #
 #  You should have received a copy of the GNU General Public
 #  License along with OctoBot. If not, see <https://www.gnu.org/licenses/>.
+import json
 import uuid
 import gc
 import sys
 import asyncio
+import time
 
-import octobot_commons.logging as logging
+import octobot_commons.logging as commons_logging
+import octobot_commons.configuration as commons_configuration
+import octobot_commons.databases as commons_databases
+import octobot_commons.constants as commons_constants
 
 import octobot_backtesting.api as backtesting_api
 import octobot_backtesting.importers as importers
@@ -32,10 +37,9 @@
 import octobot_trading.api as trading_api
 import octobot_trading.enums as trading_enums
 
-import octobot_commons.databases as databases
-import octobot_commons.constants as commons_constants
-
 import octobot.logger as logger
+import octobot.storage as storage
+import octobot.databases_util as databases_util
 
 
 class OctoBotBacktesting:
@@ -47,8 +51,9 @@ def __init__(self, backtesting_config,
                  run_on_common_part_only,
                  start_timestamp=None,
                  end_timestamp=None,
-                 enable_logs=True):
-        self.logger = logging.get_logger(self.__class__.__name__)
+                 enable_logs=True,
+                 enable_storage=True):
+        self.logger = commons_logging.get_logger(self.__class__.__name__)
         self.backtesting_config = backtesting_config
         self.tentacles_setup_config = tentacles_setup_config
         self.bot_id = str(uuid.uuid4())
@@ -60,14 +65,26 @@ def __init__(self, backtesting_config,
         self.backtesting_files = backtesting_files
         self.backtesting = None
         self.run_on_common_part_only = run_on_common_part_only
+        self.start_time = None
         self.start_timestamp = start_timestamp
         self.end_timestamp = end_timestamp
         self.enable_logs = enable_logs
         self.exchange_type_by_exchange = {}
         self.futures_contract_type = trading_enums.FutureContractType.LINEAR_PERPETUAL
+        self.enable_storage = enable_storage
 
     async def initialize_and_run(self):
         self.logger.info(f"Starting on {self.backtesting_files} with {self.symbols_to_create_exchange_classes}")
+        self.start_time = time.time()
+        await commons_databases.init_bot_storage(
+            self.bot_id,
+            databases_util.get_run_databases_identifier(
+                self.backtesting_config,
+                self.tentacles_setup_config,
+                enable_storage=self.enable_storage,
+            ),
+            False
+        )
         await self._init_evaluators()
         await self._init_service_feeds()
         await self._init_exchanges()
@@ -91,16 +108,27 @@ async def stop(self, memory_check=False, should_raise=False):
             if self.backtesting is None:
                 self.logger.warning("No backtesting to stop, there was probably an issue when starting the backtesting")
             else:
+                exchange_managers = trading_api.get_exchange_managers_from_exchange_ids(self.exchange_manager_ids)
+                if exchange_managers and self.enable_storage:
+                    try:
+                        for exchange_manager in exchange_managers:
+                            await trading_api.store_history_in_run_storage(exchange_manager)
+                    except Exception as e:
+                        self.logger.exception(e, True, f"Error when saving exchange historical data: {e}")
+                    try:
+                        await self._store_metadata(exchange_managers)
+                        self.logger.info(f"Stored backtesting run metadata")
+                    except Exception as e:
+                        self.logger.exception(e, True, f"Error when saving run metadata: {e}")
                 await backtesting_api.stop_backtesting(self.backtesting)
             try:
-                for exchange_manager in trading_api.get_exchange_managers_from_exchange_ids(self.exchange_manager_ids):
-                    exchange_managers.append(exchange_manager)
+                for exchange_manager in exchange_managers:
                     await trading_api.stop_exchange(exchange_manager)
             except KeyError:
                 # exchange managers are not added in global exchange list when an exception occurred
                 pass
             # close run databases
-            await trading_api.close_bot_storage(self.bot_id)
+            await commons_databases.close_bot_storage(self.bot_id)
             # stop evaluators
             for evaluators in self.evaluators:
                 # evaluators by type
@@ -109,7 +137,7 @@ async def stop(self, memory_check=False, should_raise=False):
                     if evaluator is not None:
                         await evaluator_api.stop_evaluator(evaluator)
             # close all caches (if caches have to be used later on, they can always be re-opened)
-            await databases.CacheManager().close_cache(commons_constants.UNPROVIDED_CACHE_IDENTIFIER)
+            await commons_databases.CacheManager().close_cache(commons_constants.UNPROVIDED_CACHE_IDENTIFIER)
             try:
                 await evaluator_api.stop_all_evaluator_channels(self.matrix_id)
             except KeyError:
@@ -170,6 +198,10 @@ def check_remaining_objects(self):
         }
         for obj in gc.get_objects():
             if isinstance(obj, to_watch_objects):
+                if isinstance(obj, exchanges.ExchangeManager) and not obj.is_initialized:
+                    # Ignore exchange managers that have not been initialized
+                    # and are irrelevant. Pytest fixtures can also retain references failing tests
+                    continue
                 objects_references[type(obj)][1].append(obj)
                 objects_references[type(obj)] = (objects_references[type(obj)][0] + 1,
                                                  objects_references[type(obj)][1])
@@ -186,6 +218,24 @@ def check_remaining_objects(self):
                 f"[Dev oriented error: no effect on backtesting result, please report if you see it]: {errors}"
             )
 
+    async def _store_metadata(self, exchange_managers):
+        run_db = commons_databases.RunDatabasesProvider.instance().get_run_db(self.bot_id)
+        await run_db.flush()
+        user_inputs = await commons_configuration.get_user_inputs(run_db)
+        await storage.store_run_metadata(
+            self.bot_id,
+            exchange_managers,
+            self.start_time,
+            user_inputs=user_inputs,
+        )
+        metadata = await storage.store_backtesting_run_metadata(
+            exchange_managers,
+            self.start_time,
+            user_inputs,
+            commons_databases.RunDatabasesProvider.instance().get_run_databases_identifier(self.bot_id),
+        )
+        self.logger.info(f"Backtesting metadata:\n{json.dumps(metadata, indent=4)}")
+
     async def _init_evaluators(self):
         self.matrix_id = await evaluator_api.initialize_evaluators(self.backtesting_config, self.tentacles_setup_config)
         await evaluator_api.create_evaluator_channels(self.matrix_id, is_backtesting=True)
@@ -223,7 +273,6 @@ async def _init_exchanges(self):
                                                                         data_files=self.backtesting_files)
         # modify_backtesting_channels before creating exchanges as they require the current backtesting time to
         # initialize
-        trading_api.init_bot_storage(self.bot_id, self.backtesting_config, self.tentacles_setup_config)
         await backtesting_api.adapt_backtesting_channels(self.backtesting,
                                                          self.backtesting_config,
                                                          importers.ExchangeDataImporter,
@@ -241,7 +290,8 @@ async def _init_exchanges(self):
                 .is_simulated() \
                 .is_rest_only() \
                 .is_backtesting(self.backtesting) \
-                .is_future(is_future, self.futures_contract_type)
+                .is_future(is_future, self.futures_contract_type) \
+                .enable_storage(self.enable_storage)
             try:
                 await exchange_builder.build()
             finally:
@@ -260,5 +310,8 @@ async def start_exchange_loggers(self):
 def _get_remaining_object_error(obj, expected, actual):
     error = f"too many remaining {obj.__name__} instances: expected: {expected} actual {actual[0]}"
     for i in range(len(actual[1])):
-        error += f"{sys.getrefcount(actual[1][i])} references on {actual[1][i]}"
+        debug_info = ""
+        if isinstance(actual[1][i], exchanges.ExchangeManager):
+            debug_info = f" ({actual[1][i].debug_info})"
+        error += f"{sys.getrefcount(actual[1][i])} references on {actual[1][i]} {debug_info}"
         return error

octobot/cli.py

@@ -13,10 +13,8 @@
 #
 #  You should have received a copy of the GNU General Public
 #  License along with OctoBot. If not, see <https://www.gnu.org/licenses/>.
-# prevents distutils_patch.py:26: UserWarning: Distutils was imported before Setuptools. This usage is discouraged
-# and may exhibit undesirable behaviors or errors. Please use Setuptools' objects directly or at least import Setuptools
-# first
-from distutils.version import LooseVersion
+
+import packaging.version as packaging_version
 
 import argparse
 import os
@@ -401,7 +399,7 @@ def main(args=None):
     try:
         from octobot_tentacles_manager import VERSION
 
-        if LooseVersion(VERSION) < MIN_TENTACLE_MANAGER_VERSION:
+        if packaging_version.Version(VERSION) < packaging_version.Version(MIN_TENTACLE_MANAGER_VERSION):
             print("OctoBot requires OctoBot-Tentacles-Manager in a minimum version of " + MIN_TENTACLE_MANAGER_VERSION +
                   " you can install and update OctoBot-Tentacles-Manager using the following command: "
                   "python3 -m pip install -U OctoBot-Tentacles-Manager", file=sys.stderr)