Improve remote execution api

examples/behavior_launcher.py

@@ -83,8 +83,6 @@ def main():
         LauncherCliArgs,
         cli_args=[
             "--debug-mode",
-            "--temp-dir",
-            "./local/.temp",
             "--allow-dirty",
             "--skip-hardware-validation",
             "--data-dir",

src/clabe/apps/__init__.py

@@ -6,12 +6,12 @@
     ExecutableApp,
     Executor,
     OutputParser,
+    StdCommand,
     identity_parser,
 )
 from ._bonsai import AindBehaviorServicesBonsaiApp, BonsaiApp
 from ._curriculum import CurriculumApp, CurriculumSettings, CurriculumSuggestion
 from ._python_script import PythonScriptApp
-from .rpc_executors import RpcExecutor
 
 __all__ = [
     "BonsaiApp",
@@ -29,5 +29,5 @@
     "OutputParser",
     "PythonScriptApp",
     "ExecutableApp",
-    "RpcExecutor",
+    "StdCommand",
 ]

src/clabe/apps/_base.py

@@ -266,6 +266,13 @@ def _parse_output(self, result: CommandResult) -> TOutput:
         return self._output_parser(result)
 
 
+class StdCommand(Command[CommandResult]):
+    """Standard command that returns the raw CommandResult."""
+
+    def __init__(self, cmd: str) -> None:
+        super().__init__(cmd, identity_parser)
+
+
 def identity_parser(result: CommandResult) -> CommandResult:
     """Helper parser that returns the CommandResult as-is."""
     return result

src/clabe/apps/_bonsai.py

@@ -9,6 +9,7 @@
 import pydantic
 from aind_behavior_services import AindBehaviorRigModel, AindBehaviorSessionModel, AindBehaviorTaskLogicModel
 
+from ..constants import TMP_DIR
 from ._base import Command, CommandResult, ExecutableApp, identity_parser
 from ._executors import _DefaultExecutorMixin
 
@@ -178,7 +179,7 @@ def __init__(
         self,
         workflow: os.PathLike,
         *,
-        temp_directory: os.PathLike,
+        temp_directory: Optional[os.PathLike] = None,
         rig: Optional[AindBehaviorRigModel] = None,
         session: Optional[AindBehaviorSessionModel] = None,
         task_logic: Optional[AindBehaviorTaskLogicModel] = None,
@@ -231,6 +232,8 @@ def __init__(
             # -p:"TaskLogicPath"="/tmp/task_logic_temp.json"
             ```
         """
+        self._temp_directory = Path(temp_directory or TMP_DIR)
+
         additional_externalized_properties = kwargs.pop("additional_externalized_properties", {}) or {}
         if rig:
             additional_externalized_properties["RigPath"] = os.path.abspath(self._save_temp_model(model=rig))
@@ -243,7 +246,6 @@ def __init__(
         super().__init__(
             workflow=workflow, additional_externalized_properties=additional_externalized_properties, **kwargs
         )
-        self._temp_directory = Path(temp_directory)
 
     def _save_temp_model(self, model: pydantic.BaseModel) -> Path:
         """

src/clabe/apps/open_ephys.py

@@ -2,17 +2,29 @@
 import os
 from enum import Enum
 from pathlib import Path
-from typing import Any, Literal, Optional
+from typing import Any, Literal
 
 import requests
 from pydantic import BaseModel, Field
 
+from ..services import ServiceSettings
 from ._base import Command, CommandResult, ExecutableApp, identity_parser
 from ._executors import _DefaultExecutorMixin
 
 logger = logging.getLogger(__name__)
 
 
+class OpenEphysAppSettings(ServiceSettings):
+    """Settings for Open Ephys App."""
+
+    __yml_section__ = "open_ephys"
+
+    signal_chain: os.PathLike
+    executable: os.PathLike = Path("./.open_ephys/open_ephys.exe")
+    address: str = "localhost"
+    port: int = 37497
+
+
 class OpenEphysApp(ExecutableApp, _DefaultExecutorMixin):
     """
     A class to manage the execution of Open Ephys GUI.
@@ -29,10 +41,7 @@ class OpenEphysApp(ExecutableApp, _DefaultExecutorMixin):
 
     def __init__(
         self,
-        signal_chain: os.PathLike,
-        *,
-        executable: os.PathLike = Path("./.open_ephys/open_ephys.exe"),
-        client: Optional["_OpenEphysGuiClient"] = None,
+        settings: OpenEphysAppSettings,
         skip_validation: bool = False,
     ) -> None:
         """
@@ -51,9 +60,10 @@ def __init__(
             app.run()
             ```
         """
-        self.signal_chain = Path(signal_chain).resolve()
-        self.executable = Path(executable).resolve()
-        self._client = client or _OpenEphysGuiClient()
+        self.settings = settings
+        self.signal_chain = Path(self.settings.signal_chain).resolve()
+        self.executable = Path(self.settings.executable).resolve()
+        self._client = _OpenEphysGuiClient(host=self.settings.address, port=self.settings.port)
 
         if not skip_validation:
             self.validate()
@@ -79,6 +89,11 @@ def command(self) -> Command[CommandResult]:
         """Get the command to execute."""
         return self._command
 
+    @property
+    def client(self) -> "_OpenEphysGuiClient":
+        """Get the Open Ephys GUI client."""
+        return self._client
+
 
 class Status(str, Enum):
     """GUI acquisition/recording mode."""
@@ -233,7 +248,7 @@ def get_status(self) -> Status:
             Current status containing the GUI mode (IDLE, ACQUIRE, or RECORD).
         """
         data = self._get("/status")
-        return Status(**data)
+        return StatusResponse(**data).mode
 
     def set_status(self, mode: Status) -> Status:
         """Set the GUI's acquisition/recording status.
@@ -249,7 +264,7 @@ def set_status(self, mode: Status) -> Status:
         """
         request = StatusRequest(mode=mode)
         data = self._put("/status", request)
-        return Status(**data)
+        return StatusResponse(**data).mode
 
     def start_acquisition(self) -> Status:
         """Start data acquisition without recording.

src/clabe/cli.py

@@ -1,12 +1,12 @@
 from pydantic_settings import BaseSettings, CliApp, CliSubCommand
 
-from .rpc._server import _RpcServerStartCli
+from .xml_rpc._server import _XmlRpcServerStartCli
 
 
 class CliAppSettings(BaseSettings, cli_prog_name="clabe", cli_kebab_case=True):
     """CLI application settings."""
 
-    rpc_server: CliSubCommand[_RpcServerStartCli]
+    xml_rpc_server: CliSubCommand[_XmlRpcServerStartCli]
 
     def cli_cmd(self):
         """Run the selected subcommand."""

src/clabe/constants.py

@@ -0,0 +1,43 @@
+import enum
+import logging
+import os
+import typing as t
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+TMP_DIR = ".cache"
+
+
+PROGRAMDATA_DIR = os.environ.get("PROGRAMDATA", "C:/ProgramData")
+
+# The config files will be used in order, with the first one having the highest priority
+
+KNOWN_CONFIG_FILES: t.List[str] = [
+    "./local/clabe.yml",
+    "./clabe.yml",
+    str(Path(PROGRAMDATA_DIR) / "clabe.yml"),
+]
+
+
+for i, p in enumerate(KNOWN_CONFIG_FILES):
+    if Path(p).exists():
+        logger.debug(f"Found config file: {p} with rank priority {i}")
+
+
+class ByAnimalFiles(enum.StrEnum):
+    """
+    Enum for file types associated with animals in the experiment.
+
+    Defines the standard file types that can be associated with individual
+    animals/subjects in behavior experiments.
+
+    Example:
+        ```python
+        # Use the task logic file type
+        filename = f"{ByAnimalFiles.TASK_LOGIC}.json"
+        ```
+    """
+
+    TASK_LOGIC = "task_logic"
+    TRAINER_STATE = "trainer_state"

src/clabe/launcher/_base.py

@@ -16,6 +16,7 @@
 )
 
 from .. import __version__, logging_helper
+from ..constants import TMP_DIR
 from ..git_manager import GitRepository
 from ..ui import DefaultUIHelper, UiHelper
 from ..utils import abspath, format_datetime, utcnow
@@ -68,10 +69,14 @@ def __init__(
         """
         self._settings = settings
         self.ui_helper = ui_helper
-        self.temp_dir = abspath(settings.temp_dir) / format_datetime(utcnow())
-        self.temp_dir.mkdir(parents=True, exist_ok=True)
+        self.temp_dir = Path(TMP_DIR) / format_datetime(utcnow())
         self.computer_name = os.environ["COMPUTERNAME"]
 
+        repository_dir = Path(self.settings.repository_dir) if self.settings.repository_dir is not None else None
+        self.repository = GitRepository() if repository_dir is None else GitRepository(path=repository_dir)
+
+        self._ensure_directory_structure()
+
         # Solve logger
         if attached_logger:
             _logger = logging_helper.add_file_handler(attached_logger, self.temp_dir / "launcher.log")
@@ -84,11 +89,6 @@ def __init__(
 
         self._logger = _logger
 
-        repository_dir = Path(self.settings.repository_dir) if self.settings.repository_dir is not None else None
-        self.repository = GitRepository() if repository_dir is None else GitRepository(path=repository_dir)
-
-        self._ensure_directory_structure()
-
         self._session: Optional[AindBehaviorSessionModel] = None
         self._has_copied_logs = False
 
@@ -178,7 +178,7 @@ async def my_async_experiment(launcher: Launcher):
             try:
                 self.copy_logs()
             except ValueError as ve:  # In the case session_directory fails
-                logger.error("Failed to copy logs: %s", ve)  # we swallow the error
+                logger.error("Failed to copy logs from %s. Error: %s", self.temp_dir, ve)  # we swallow the error
                 self._exit(-1)
             else:
                 self._exit(_code)

src/clabe/launcher/_cli.py

@@ -1,5 +1,4 @@
 import os
-from pathlib import Path
 from typing import Optional
 
 from pydantic import Field
@@ -26,6 +25,3 @@ class LauncherCliArgs(ServiceSettings, cli_prog_name="clabe", cli_kebab_case=Tru
     skip_hardware_validation: CliImplicitFlag[bool] = Field(
         default=False, description="Whether to skip hardware validation"
     )
-    temp_dir: os.PathLike = Field(
-        default=Path("local/.temp"), description="The directory used for the launcher temp files"
-    )

src/clabe/pickers/default_behavior.py

@@ -11,9 +11,9 @@
 
 from .. import ui
 from .._typing import TRig, TSession, TTaskLogic
+from ..constants import ByAnimalFiles
 from ..launcher import Launcher
 from ..services import ServiceSettings
-from ..utils import ByAnimalFiles
 from ..utils.aind_auth import validate_aind_username
 
 logger = logging.getLogger(__name__)

src/clabe/services.py

@@ -4,7 +4,7 @@
 
 import pydantic_settings as ps
 
-from .utils import KNOWN_CONFIG_FILES
+from .constants import KNOWN_CONFIG_FILES
 
 logger = logging.getLogger(__name__)
 

src/clabe/utils/__init__.py

@@ -1,9 +1,5 @@
-"""Constants used throughout the CLABE package."""
-
-import enum
 import logging
 import os
-import typing as t
 from pathlib import Path
 
 from aind_behavior_services.utils import format_datetime, model_from_json_file, utcnow
@@ -15,23 +11,8 @@
     "format_datetime",
     "model_from_json_file",
     "utcnow",
-    "ByAnimalFiles",
-    "KNOWN_CONFIG_FILES",
-]
-
-PROGRAMDATA_DIR = os.environ.get("PROGRAMDATA", "C:/ProgramData")
-
-# The config files will be used in order, with the first one having the highest priority
-KNOWN_CONFIG_FILES: t.List[str] = [
-    "./local/clabe.yml",
-    "./clabe.yml",
-    str(Path(PROGRAMDATA_DIR) / "clabe.yml"),
 ]
 
-for i, p in enumerate(KNOWN_CONFIG_FILES):
-    if Path(p).exists():
-        logger.debug(f"Found config file: {p} with rank priority {i}")
-
 
 def abspath(path: os.PathLike) -> Path:
     """
@@ -44,21 +25,3 @@ def abspath(path: os.PathLike) -> Path:
         Path: The absolute path
     """
     return Path(path).resolve()
-
-
-class ByAnimalFiles(enum.StrEnum):
-    """
-    Enum for file types associated with animals in the experiment.
-
-    Defines the standard file types that can be associated with individual
-    animals/subjects in behavior experiments.
-
-    Example:
-        ```python
-        # Use the task logic file type
-        filename = f"{ByAnimalFiles.TASK_LOGIC}.json"
-        ```
-    """
-
-    TASK_LOGIC = "task_logic"
-    TRAINER_STATE = "trainer_state"

src/clabe/xml_rpc/__init__.py

@@ -0,0 +1,14 @@
+from ._client import XmlRpcClient, XmlRpcClientSettings
+from ._executor import XmlRpcExecutor
+from ._server import XmlRpcServer, XmlRpcServerSettings
+from .models import FileInfo, JobResult
+
+__all__ = [
+    "XmlRpcServerSettings",
+    "XmlRpcServer",
+    "XmlRpcClientSettings",
+    "XmlRpcClient",
+    "JobResult",
+    "FileInfo",
+    "XmlRpcExecutor",
+]