Merge pull request freqtrade#11154 from freqtrade/feat/zip_backtest

Zip backtest results

Author: xmatthias

freqtrade/commands/analyze_commands.py

@@ -1,63 +1,23 @@
 import logging
-from pathlib import Path
 from typing import Any
 
 from freqtrade.enums import RunMode
-from freqtrade.exceptions import ConfigurationError, OperationalException
 
 
 logger = logging.getLogger(__name__)
 
 
-def setup_analyze_configuration(args: dict[str, Any], method: RunMode) -> dict[str, Any]:
-    """
-    Prepare the configuration for the entry/exit reason analysis module
-    :param args: Cli args from Arguments()
-    :param method: Bot running mode
-    :return: Configuration
-    """
-    from freqtrade.configuration import setup_utils_configuration
-
-    config = setup_utils_configuration(args, method)
-
-    no_unlimited_runmodes = {
-        RunMode.BACKTEST: "backtesting",
-    }
-    if method in no_unlimited_runmodes.keys():
-        from freqtrade.data.btanalysis import get_latest_backtest_filename
-
-        if "exportfilename" in config:
-            if config["exportfilename"].is_dir():
-                btfile = Path(get_latest_backtest_filename(config["exportfilename"]))
-                signals_file = f"{config['exportfilename']}/{btfile.stem}_signals.pkl"
-            else:
-                if config["exportfilename"].exists():
-                    btfile = Path(config["exportfilename"])
-                    signals_file = f"{btfile.parent}/{btfile.stem}_signals.pkl"
-                else:
-                    raise ConfigurationError(f"{config['exportfilename']} does not exist.")
-        else:
-            raise ConfigurationError("exportfilename not in config.")
-
-        if not Path(signals_file).exists():
-            raise OperationalException(
-                f"Cannot find latest backtest signals file: {signals_file}."
-                "Run backtesting with `--export signals`."
-            )
-
-    return config
-
-
 def start_analysis_entries_exits(args: dict[str, Any]) -> None:
     """
     Start analysis script
     :param args: Cli args from Arguments()
     :return: None
     """
+    from freqtrade.configuration import setup_utils_configuration
     from freqtrade.data.entryexitanalysis import process_entry_exit_reasons
 
     # Initialize configuration
-    config = setup_analyze_configuration(args, RunMode.BACKTEST)
+    config = setup_utils_configuration(args, RunMode.BACKTEST)
 
     logger.info("Starting freqtrade in analysis mode")
 

freqtrade/data/btanalysis.py

@@ -3,8 +3,10 @@
 """
 
 import logging
+import zipfile
 from copy import copy
 from datetime import datetime, timezone
+from io import BytesIO, StringIO
 from pathlib import Path
 from typing import Any, Literal
 
@@ -165,8 +167,16 @@ def load_backtest_stats(filename: Path | str) -> BacktestResultType:
     if not filename.is_file():
         raise ValueError(f"File {filename} does not exist.")
     logger.info(f"Loading backtest result from {filename}")
-    with filename.open() as file:
-        data = json_load(file)
+
+    if filename.suffix == ".zip":
+        data = json_load(
+            StringIO(
+                load_file_from_zip(filename, filename.with_suffix(".json").name).decode("utf-8")
+            )
+        )
+    else:
+        with filename.open() as file:
+            data = json_load(file)
 
     # Legacy list format does not contain metadata.
     if isinstance(data, dict):
@@ -194,8 +204,10 @@ def load_and_merge_backtest_result(strategy_name: str, filename: Path, results:
 
 
 def _get_backtest_files(dirname: Path) -> list[Path]:
-    # Weird glob expression here avoids including .meta.json files.
-    return list(reversed(sorted(dirname.glob("backtest-result-*-[0-9][0-9].json"))))
+    # Get both json and zip files separately and combine the results
+    json_files = dirname.glob("backtest-result-*-[0-9][0-9]*.json")
+    zip_files = dirname.glob("backtest-result-*-[0-9][0-9]*.zip")
+    return list(reversed(sorted(list(json_files) + list(zip_files))))
 
 
 def _extract_backtest_result(filename: Path) -> list[BacktestHistoryEntryType]:
@@ -267,7 +279,11 @@ def get_backtest_market_change(filename: Path, include_ts: bool = True) -> pd.Da
     """
     Read backtest market change file.
     """
-    df = pd.read_feather(filename)
+    if filename.suffix == ".zip":
+        data = load_file_from_zip(filename, f"{filename.stem}_market_change.feather")
+        df = pd.read_feather(BytesIO(data))
+    else:
+        df = pd.read_feather(filename)
     if include_ts:
         df.loc[:, "__date_ts"] = df.loc[:, "date"].astype(np.int64) // 1000 // 1000
     return df
@@ -388,6 +404,93 @@ def load_backtest_data(filename: Path | str, strategy: str | None = None) -> pd.
     return df
 
 
+def load_file_from_zip(zip_path: Path, filename: str) -> bytes:
+    """
+    Load a file from a zip file
+    :param zip_path: Path to the zip file
+    :param filename: Name of the file to load
+    :return: Bytes of the file
+    :raises: ValueError if loading goes wrong.
+    """
+    try:
+        with zipfile.ZipFile(zip_path) as zipf:
+            try:
+                with zipf.open(filename) as file:
+                    return file.read()
+            except KeyError:
+                logger.error(f"File {filename} not found in zip: {zip_path}")
+                raise ValueError(f"File {filename} not found in zip: {zip_path}") from None
+    except FileNotFoundError:
+        raise ValueError(f"Zip file {zip_path} not found.")
+    except zipfile.BadZipFile:
+        logger.error(f"Bad zip file: {zip_path}.")
+        raise ValueError(f"Bad zip file: {zip_path}.") from None
+
+
+def load_backtest_analysis_data(backtest_dir: Path, name: str):
+    """
+    Load backtest analysis data either from a pickle file or from within a zip file
+    :param backtest_dir: Directory containing backtest results
+    :param name: Name of the analysis data to load (signals, rejected, exited)
+    :return: Analysis data
+    """
+    import joblib
+
+    if backtest_dir.is_dir():
+        lbf = Path(get_latest_backtest_filename(backtest_dir))
+        zip_path = backtest_dir / lbf
+    else:
+        zip_path = backtest_dir
+
+    if zip_path.suffix == ".zip":
+        # Load from zip file
+        analysis_name = f"{zip_path.stem}_{name}.pkl"
+        data = load_file_from_zip(zip_path, analysis_name)
+        if not data:
+            return None
+        loaded_data = joblib.load(BytesIO(data))
+
+        logger.info(f"Loaded {name} candles from zip: {str(zip_path)}:{analysis_name}")
+        return loaded_data
+
+    else:
+        # Load from separate pickle file
+        if backtest_dir.is_dir():
+            scpf = Path(backtest_dir, f"{zip_path.stem}_{name}.pkl")
+        else:
+            scpf = Path(backtest_dir.parent / f"{backtest_dir.stem}_{name}.pkl")
+
+        try:
+            with scpf.open("rb") as scp:
+                loaded_data = joblib.load(scp)
+                logger.info(f"Loaded {name} candles: {str(scpf)}")
+                return loaded_data
+        except Exception:
+            logger.exception(f"Cannot load {name} data from pickled results.")
+            return None
+
+
+def load_rejected_signals(backtest_dir: Path):
+    """
+    Load rejected signals from backtest directory
+    """
+    return load_backtest_analysis_data(backtest_dir, "rejected")
+
+
+def load_signal_candles(backtest_dir: Path):
+    """
+    Load signal candles from backtest directory
+    """
+    return load_backtest_analysis_data(backtest_dir, "signals")
+
+
+def load_exit_signal_candles(backtest_dir: Path) -> dict[str, dict[str, pd.DataFrame]]:
+    """
+    Load exit signal candles from backtest directory
+    """
+    return load_backtest_analysis_data(backtest_dir, "exited")
+
+
 def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataFrame:
     """
     Find overlapping trades by expanding each trade once per period it was open

freqtrade/data/entryexitanalysis.py

@@ -1,56 +1,25 @@
 import logging
 from pathlib import Path
 
-import joblib
 import pandas as pd
 
 from freqtrade.configuration import TimeRange
 from freqtrade.constants import Config
 from freqtrade.data.btanalysis import (
     BT_DATA_COLUMNS,
-    get_latest_backtest_filename,
     load_backtest_data,
     load_backtest_stats,
+    load_exit_signal_candles,
+    load_rejected_signals,
+    load_signal_candles,
 )
-from freqtrade.exceptions import OperationalException
+from freqtrade.exceptions import ConfigurationError, OperationalException
 from freqtrade.util import print_df_rich_table
 
 
 logger = logging.getLogger(__name__)
 
 
-def _load_backtest_analysis_data(backtest_dir: Path, name: str):
-    if backtest_dir.is_dir():
-        scpf = Path(
-            backtest_dir,
-            Path(get_latest_backtest_filename(backtest_dir)).stem + "_" + name + ".pkl",
-        )
-    else:
-        scpf = Path(backtest_dir.parent / f"{backtest_dir.stem}_{name}.pkl")
-
-    try:
-        with scpf.open("rb") as scp:
-            loaded_data = joblib.load(scp)
-            logger.info(f"Loaded {name} candles: {str(scpf)}")
-    except Exception as e:
-        logger.error(f"Cannot load {name} data from pickled results: ", e)
-        return None
-
-    return loaded_data
-
-
-def _load_rejected_signals(backtest_dir: Path):
-    return _load_backtest_analysis_data(backtest_dir, "rejected")
-
-
-def _load_signal_candles(backtest_dir: Path):
-    return _load_backtest_analysis_data(backtest_dir, "signals")
-
-
-def _load_exit_signal_candles(backtest_dir: Path) -> dict[str, dict[str, pd.DataFrame]]:
-    return _load_backtest_analysis_data(backtest_dir, "exited")
-
-
 def _process_candles_and_indicators(
     pairlist, strategy_name, trades, signal_candles, date_col: str = "open_date"
 ):
@@ -374,19 +343,21 @@ def process_entry_exit_reasons(config: Config):
         timerange = TimeRange.parse_timerange(
             None if config.get("timerange") is None else str(config.get("timerange"))
         )
-
-        backtest_stats = load_backtest_stats(config["exportfilename"])
+        try:
+            backtest_stats = load_backtest_stats(config["exportfilename"])
+        except ValueError as e:
+            raise ConfigurationError(e) from e
 
         for strategy_name, results in backtest_stats["strategy"].items():
             trades = load_backtest_data(config["exportfilename"], strategy_name)
 
             if trades is not None and not trades.empty:
-                signal_candles = _load_signal_candles(config["exportfilename"])
-                exit_signals = _load_exit_signal_candles(config["exportfilename"])
+                signal_candles = load_signal_candles(config["exportfilename"])
+                exit_signals = load_exit_signal_candles(config["exportfilename"])
 
                 rej_df = None
                 if do_rejected:
-                    rejected_signals_dict = _load_rejected_signals(config["exportfilename"])
+                    rejected_signals_dict = load_rejected_signals(config["exportfilename"])
                     rej_df = prepare_results(
                         rejected_signals_dict,
                         strategy_name,

freqtrade/misc.py

@@ -19,6 +19,15 @@
 logger = logging.getLogger(__name__)
 
 
+def dump_json_to_file(file_obj: TextIO, data: Any) -> None:
+    """
+    Dump JSON data into a file object
+    :param file_obj: File object to write to
+    :param data: JSON Data to save
+    """
+    rapidjson.dump(data, file_obj, default=str, number_mode=rapidjson.NM_NATIVE)
+
+
 def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = True) -> None:
     """
     Dump JSON data into a file
@@ -35,32 +44,16 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool =
             logger.info(f'dumping json to "{filename}"')
 
         with gzip.open(filename, "wt", encoding="utf-8") as fpz:
-            rapidjson.dump(data, fpz, default=str, number_mode=rapidjson.NM_NATIVE)
+            dump_json_to_file(fpz, data)
     else:
         if log:
             logger.info(f'dumping json to "{filename}"')
         with filename.open("w") as fp:
-            rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE)
+            dump_json_to_file(fp, data)
 
     logger.debug(f'done json to "{filename}"')
 
 
-def file_dump_joblib(filename: Path, data: Any, log: bool = True) -> None:
-    """
-    Dump object data into a file
-    :param filename: file to create
-    :param data: Object data to save
-    :return:
-    """
-    import joblib
-
-    if log:
-        logger.info(f'dumping joblib to "{filename}"')
-    with filename.open("wb") as fp:
-        joblib.dump(data, fp)
-    logger.debug(f'done joblib dump to "{filename}"')
-
-
 def json_load(datafile: TextIO) -> Any:
     """
     load data with rapidjson

freqtrade/optimize/backtest_caching.py

@@ -40,4 +40,4 @@ def get_strategy_run_id(strategy) -> str:
 def get_backtest_metadata_filename(filename: Path | str) -> Path:
     """Return metadata filename for specified backtest results file."""
     filename = Path(filename)
-    return filename.parent / Path(f"{filename.stem}.meta{filename.suffix}")
+    return filename.parent / Path(f"{filename.stem}.meta.json")