Fix issues #544 and #364: Add --extra-docker-config-file and --docker…

lean/commands/backtest.py

@@ -278,6 +278,15 @@ def _migrate_csharp_csproj(project_dir: Path) -> None:
               default="{}",
               help="Extra docker configuration as a JSON string. "
                    "For more information https://docker-py.readthedocs.io/en/stable/containers.html")
+@option("--extra-docker-config-file",
+              type=PathParameter(exists=True, file_okay=True, dir_okay=False),
+              help="Path to a JSON file with extra docker configuration. "
+                   "This is recommended over --extra-docker-config on Windows to avoid shell quote issues.")
+@option("--docker-timeout",
+              type=int,
+              help="Timeout in seconds for Docker operations (default: 60). "
+                   "Increase this for slow connections or large image pulls. "
+                   "Can also be set via DOCKER_CLIENT_TIMEOUT environment variable.")
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -298,6 +307,8 @@ def backtest(project: Path,
              addon_module: Optional[List[str]],
              extra_config: Optional[Tuple[str, str]],
              extra_docker_config: Optional[str],
+             extra_docker_config_file: Optional[Path],
+             docker_timeout: Optional[int],
              no_update: bool,
              parameter: List[Tuple[str, str]],
              **kwargs) -> None:
@@ -316,9 +327,14 @@ def backtest(project: Path,
     Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
     """
     from datetime import datetime
-    from json import loads
+    from lean.components.util.json_parser import load_json_from_file_or_string
 
     logger = container.logger
+
+    # Set Docker timeout if specified
+    if docker_timeout is not None:
+        container.docker_manager.set_timeout(docker_timeout)
+
     project_manager = container.project_manager
     algorithm_file = project_manager.find_algorithm_file(Path(project))
     lean_config_manager = container.lean_config_manager
@@ -402,6 +418,12 @@ def backtest(project: Path,
         # Override existing parameters if any are provided via --parameter
         lean_config["parameters"] = lean_config_manager.get_parameters(parameter)
 
+    # Parse extra Docker configuration from string or file
+    parsed_extra_docker_config = load_json_from_file_or_string(
+        json_string=extra_docker_config if extra_docker_config != "{}" else None,
+        json_file=extra_docker_config_file
+    )
+
     lean_runner = container.lean_runner
     lean_runner.run_lean(lean_config,
                          environment_name,
@@ -411,5 +433,5 @@ def backtest(project: Path,
                          debugging_method,
                          release,
                          detach,
-                         loads(extra_docker_config),
+                         parsed_extra_docker_config,
                          paths_to_mount)

lean/commands/live/deploy.py

@@ -110,6 +110,15 @@ def _get_history_provider_name(data_provider_live_names: [str]) -> [str]:
               default="{}",
               help="Extra docker configuration as a JSON string. "
                    "For more information https://docker-py.readthedocs.io/en/stable/containers.html")
+@option("--extra-docker-config-file",
+              type=PathParameter(exists=True, file_okay=True, dir_okay=False),
+              help="Path to a JSON file with extra docker configuration. "
+                   "This is recommended over --extra-docker-config on Windows to avoid shell quote issues.")
+@option("--docker-timeout",
+              type=int,
+              help="Timeout in seconds for Docker operations (default: 60). "
+                   "Increase this for slow connections or large image pulls. "
+                   "Can also be set via DOCKER_CLIENT_TIMEOUT environment variable.")
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -131,6 +140,8 @@ def deploy(project: Path,
            addon_module: Optional[List[str]],
            extra_config: Optional[Tuple[str, str]],
            extra_docker_config: Optional[str],
+           extra_docker_config_file: Optional[Path],
+           docker_timeout: Optional[int],
            no_update: bool,
            **kwargs) -> None:
     """Start live trading a project locally using Docker.
@@ -155,10 +166,14 @@ def deploy(project: Path,
     """
     from copy import copy
     from datetime import datetime
-    from json import loads
+    from lean.components.util.json_parser import load_json_from_file_or_string
 
     logger = container.logger
-
+
+    # Set Docker timeout if specified
+    if docker_timeout is not None:
+        container.docker_manager.set_timeout(docker_timeout)
+
     project_manager = container.project_manager
     algorithm_file = project_manager.find_algorithm_file(Path(project))
 
@@ -323,5 +338,8 @@ def deploy(project: Path,
                          None,
                          release,
                          detach,
-                         loads(extra_docker_config),
+                         load_json_from_file_or_string(
+                             json_string=extra_docker_config if extra_docker_config != "{}" else None,
+                             json_file=extra_docker_config_file
+                         ),
                          paths_to_mount)

lean/commands/optimize.py

@@ -139,6 +139,15 @@ def get_filename_timestamp(path: Path) -> datetime:
               default="{}",
               help="Extra docker configuration as a JSON string. "
                    "For more information https://docker-py.readthedocs.io/en/stable/containers.html")
+@option("--extra-docker-config-file",
+              type=PathParameter(exists=True, file_okay=True, dir_okay=False),
+              help="Path to a JSON file with extra docker configuration. "
+                   "This is recommended over --extra-docker-config on Windows to avoid shell quote issues.")
+@option("--docker-timeout",
+              type=int,
+              help="Timeout in seconds for Docker operations (default: 60). "
+                   "Increase this for slow connections or large image pulls. "
+                   "Can also be set via DOCKER_CLIENT_TIMEOUT environment variable.")
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -164,6 +173,8 @@ def optimize(project: Path,
              addon_module: Optional[List[str]],
              extra_config: Optional[Tuple[str, str]],
              extra_docker_config: Optional[str],
+             extra_docker_config_file: Optional[Path],
+             docker_timeout: Optional[int],
              no_update: bool,
              **kwargs) -> None:
     """Optimize a project's parameters locally using Docker.
@@ -207,8 +218,13 @@ def optimize(project: Path,
     from docker.types import Mount
     from re import findall, search
     from os import cpu_count
+    from lean.components.util.json_parser import load_json_from_file_or_string
     from math import floor
 
+    # Set Docker timeout if specified
+    if docker_timeout is not None:
+        container.docker_manager.set_timeout(docker_timeout)
+
     should_detach = detach and not estimate
     environment_name = "backtesting"
     project_manager = container.project_manager
@@ -343,7 +359,12 @@ def optimize(project: Path,
     )
 
     # Add known additional run options from the extra docker config
-    LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config))
+    # Parse extra Docker configuration from string or file
+    parsed_extra_docker_config = load_json_from_file_or_string(
+        json_string=extra_docker_config if extra_docker_config != "{}" else None,
+        json_file=extra_docker_config_file
+    )
+    LeanRunner.parse_extra_docker_config(run_options, parsed_extra_docker_config)
 
     project_manager.copy_code(algorithm_file.parent, output / "code")
 

lean/commands/research.py

@@ -71,6 +71,15 @@ def _check_docker_output(chunk: str, port: int) -> None:
               default="{}",
               help="Extra docker configuration as a JSON string. "
                    "For more information https://docker-py.readthedocs.io/en/stable/containers.html")
+@option("--extra-docker-config-file",
+              type=PathParameter(exists=True, file_okay=True, dir_okay=False),
+              help="Path to a JSON file with extra docker configuration. "
+                   "This is recommended over --extra-docker-config on Windows to avoid shell quote issues.")
+@option("--docker-timeout",
+              type=int,
+              help="Timeout in seconds for Docker operations (default: 60). "
+                   "Increase this for slow connections or large image pulls. "
+                   "Can also be set via DOCKER_CLIENT_TIMEOUT environment variable.")
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -86,6 +95,8 @@ def research(project: Path,
              update: bool,
              extra_config: Optional[Tuple[str, str]],
              extra_docker_config: Optional[str],
+             extra_docker_config_file: Optional[Path],
+             docker_timeout: Optional[int],
              no_update: bool,
              **kwargs) -> None:
     """Run a Jupyter Lab environment locally using Docker.
@@ -96,10 +107,14 @@ def research(project: Path,
     """
     from docker.types import Mount
     from docker.errors import APIError
-    from json import loads
+    from lean.components.util.json_parser import load_json_from_file_or_string
 
     logger = container.logger
-
+
+    # Set Docker timeout if specified
+    if docker_timeout is not None:
+        container.docker_manager.set_timeout(docker_timeout)
+
     project_manager = container.project_manager
     algorithm_file = project_manager.find_algorithm_file(project, not_throw = True)
 
@@ -195,7 +210,12 @@ def research(project: Path,
     run_options["commands"].append("./start.sh")
 
     # Add known additional run options from the extra docker config
-    LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config))
+    # Parse extra Docker configuration from string or file
+    parsed_extra_docker_config = load_json_from_file_or_string(
+        json_string=extra_docker_config if extra_docker_config != "{}" else None,
+        json_file=extra_docker_config_file
+    )
+    LeanRunner.parse_extra_docker_config(run_options, parsed_extra_docker_config)
 
     try:
         container.docker_manager.run_image(research_image, **run_options)

lean/components/docker/docker_manager.py

@@ -27,16 +27,25 @@
 class DockerManager:
     """The DockerManager contains methods to manage and run Docker images."""
 
-    def __init__(self, logger: Logger, temp_manager: TempManager, platform_manager: PlatformManager) -> None:
+    def __init__(self, logger: Logger, temp_manager: TempManager, platform_manager: PlatformManager, timeout: int = 60) -> None:
         """Creates a new DockerManager instance.
 
         :param logger: the logger to use when printing messages
         :param temp_manager: the TempManager instance used when creating temporary directories
         :param platform_manager: the PlatformManager used when checking which operating system is in use
+        :param timeout: the timeout in seconds for Docker client operations (default: 60)
         """
         self._logger = logger
         self._temp_manager = temp_manager
         self._platform_manager = platform_manager
+        self._timeout = timeout
+
+    def set_timeout(self, timeout: int) -> None:
+        """Set the timeout for Docker client operations.
+
+        :param timeout: The timeout in seconds for Docker client operations
+        """
+        self._timeout = timeout
 
     def get_image_labels(self, image: str) -> str:
         docker_image = self._get_docker_client().images.get(image)
@@ -570,7 +579,16 @@ def _get_docker_client(self):
 
         try:
             from docker import from_env
-            docker_client = from_env()
+            from os import environ
+
+            # Check for environment variable override
+            try:
+                timeout = int(environ.get("DOCKER_CLIENT_TIMEOUT", self._timeout))
+            except ValueError:
+                # Fall back to instance timeout on invalid value
+                timeout = self._timeout
+
+            docker_client = from_env(timeout=timeout)
         except Exception:
             raise error
 

lean/components/util/json_parser.py

@@ -0,0 +1,94 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+from typing import Dict, Any, Optional
+from json import loads, JSONDecodeError
+
+
+def parse_json_safely(json_string: str) -> Dict[str, Any]:
+    """
+    Attempts to parse a JSON string with multiple fallback strategies.
+
+    This function is designed to handle JSON strings that may have been
+    mangled by Windows shells (PowerShell/CMD) which strip or escape quotes.
+
+    :param json_string: The JSON string to parse
+    :return: Parsed dictionary
+    :raises ValueError: If all parsing attempts fail
+    """
+    if not json_string or json_string.strip() == "":
+        return {}
+
+    # Try standard JSON parsing first
+    try:
+        return loads(json_string)
+    except JSONDecodeError as e:
+        original_error = str(e)
+
+    # Try fixing common Windows shell issues
+    # Try single quotes to double quotes (common Windows PowerShell issue)
+    try:
+        return loads(json_string.replace("'", '"'))
+    except JSONDecodeError:
+        pass
+
+    # If all attempts fail, provide helpful error message
+    raise ValueError(
+        f"Failed to parse JSON configuration. Original error: {original_error}\n"
+        f"Input: {json_string}\n\n"
+        f"On Windows, JSON strings may be mangled by the shell. Consider using --extra-docker-config-file instead.\n"
+        f"Example: Create a file 'docker-config.json' with your configuration and use:\n"
+        f"  --extra-docker-config-file docker-config.json"
+    )
+
+
+def load_json_from_file_or_string(
+    json_string: Optional[str] = None,
+    json_file: Optional[Path] = None
+) -> Dict[str, Any]:
+    """
+    Loads JSON configuration from either a string or a file.
+
+    If both json_file and json_string are provided, json_file takes precedence.
+
+    :param json_string: JSON string to parse (optional)
+    :param json_file: Path to JSON file (optional)
+    :return: Parsed dictionary, or empty dict if both parameters are None
+    :raises ValueError: If parsing fails or if file doesn't exist
+    """
+    # Validate that both parameters aren't provided (though we allow it, file takes precedence)
+    if json_file is not None and json_string is not None:
+        # Log a warning would be ideal, but we'll prioritize file as documented
+        pass
+
+    if json_file is not None:
+        if not json_file.exists():
+            raise ValueError(f"Configuration file not found: {json_file}")
+
+        try:
+            with open(json_file, 'r', encoding='utf-8') as f:
+                content = f.read()
+                return loads(content)
+        except JSONDecodeError as e:
+            raise ValueError(
+                f"Failed to parse JSON from file {json_file}: {e}\n"
+                f"Please ensure the file contains valid JSON."
+            )
+        except Exception as e:
+            raise ValueError(f"Failed to read file {json_file}: {e}")
+
+    if json_string is not None:
+        return parse_json_safely(json_string)
+
+    return {}

lean/container.py

@@ -102,7 +102,10 @@ def initialize(self,
 
         self.docker_manager = docker_manager
         if not self.docker_manager:
-            self.docker_manager = DockerManager(self.logger, self.temp_manager, self.platform_manager)
+            from os import environ
+            # Get timeout from environment variable, default to 60 seconds
+            timeout = int(environ.get("DOCKER_CLIENT_TIMEOUT", 60))
+            self.docker_manager = DockerManager(self.logger, self.temp_manager, self.platform_manager, timeout)
 
         self.project_manager = ProjectManager(self.logger,
                                               self.project_config_manager,