From 5e1aa1e91be424e8158cb76a95d1c997b5c1ecb3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 24 Jun 2025 18:16:59 -0400 Subject: [PATCH 1/6] Add uvx_galaxy engine type Skips cloning and run.sh to go directly for the galaxy entrypoint. Is much faster and cleaner to boot. Does not yet install conditional dependencies. --- planemo/engine/factory.py | 5 +- planemo/engine/galaxy.py | 20 +++ planemo/galaxy/config.py | 281 ++++++++++++++++++++++++++++++++++++++ planemo/galaxy/serve.py | 2 + planemo/options.py | 8 +- 5 files changed, 311 insertions(+), 5 deletions(-) diff --git a/planemo/engine/factory.py b/planemo/engine/factory.py index 7d57b7d3b..3d1ac1993 100644 --- a/planemo/engine/factory.py +++ b/planemo/engine/factory.py @@ -10,6 +10,7 @@ ExternalGalaxyEngine, LocalManagedGalaxyEngine, LocalManagedGalaxyEngineWithSingularityDB, + UvxManagedGalaxyEngine, ) from .toil import ToilEngine @@ -19,7 +20,7 @@ def is_galaxy_engine(**kwds): """Return True iff the engine configured is :class:`GalaxyEngine`.""" engine_type_str = kwds.get("engine", "galaxy") - return engine_type_str in ["galaxy", "docker_galaxy", "external_galaxy"] + return engine_type_str in ["galaxy", "docker_galaxy", "external_galaxy", "uvx_galaxy"] def build_engine(ctx, **kwds): @@ -34,6 +35,8 @@ def build_engine(ctx, **kwds): engine_type = DockerizedManagedGalaxyEngine elif engine_type_str == "external_galaxy": engine_type = ExternalGalaxyEngine + elif engine_type_str == "uvx_galaxy": + engine_type = UvxManagedGalaxyEngine elif engine_type_str == "cwltool": engine_type = CwlToolEngine elif engine_type_str == "toil": diff --git a/planemo/engine/galaxy.py b/planemo/engine/galaxy.py index c56862dab..3de5750f0 100644 --- a/planemo/engine/galaxy.py +++ b/planemo/engine/galaxy.py @@ -191,6 +191,26 @@ def _serve_kwds(self): return serve_kwds +class UvxManagedGalaxyEngine(LocalManagedGalaxyEngine): + """An :class:`Engine` implementation backed by Galaxy running via uvx. + + More information on Galaxy can be found at http://galaxyproject.org/. + """ + + def _serve_kwds(self): + serve_kwds = self._kwds.copy() + serve_kwds["uvx_galaxy"] = True + return serve_kwds + + @contextlib.contextmanager + def ensure_runnables_served(self, runnables): + """Ensure runnables are served using uvx Galaxy configuration.""" + with serve_daemon(self._ctx, runnables, **self._serve_kwds()) as config: + if "install_args_list" in self._serve_kwds(): + self.shed_install(config) + yield config + + class ExternalGalaxyEngine(GalaxyEngine): """An :class:`Engine` implementation backed by an external Galaxy instance.""" diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index 694f884b0..08693b291 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -158,6 +158,8 @@ def galaxy_config(ctx, runnables, **kwds): c = docker_galaxy_config elif kwds.get("external", False): c = external_galaxy_config + elif kwds.get("uvx_galaxy", False): + c = uvx_galaxy_config log_thread = None e = threading.Event() try: @@ -1421,7 +1423,286 @@ def _ensure_directory(path): os.makedirs(path) +class UvxGalaxyConfig(BaseManagedGalaxyConfig): + """A uvx-managed implementation of :class:`GalaxyConfig`.""" + + def __init__( + self, + ctx, + config_directory, + env, + test_data_dir, + port, + server_name, + master_api_key, + runnables, + kwds, + ): + super().__init__( + ctx, + config_directory, + env, + test_data_dir, + port, + server_name, + master_api_key, + runnables, + kwds, + ) + # Use config directory as placeholder for galaxy_root since uvx manages Galaxy + self.galaxy_root = config_directory + + @property + def host(self): + """Host for uvx Galaxy instance.""" + return self._kwds.get("host", "127.0.0.1") + + @property + def galaxy_config_file(self): + """Path to galaxy configuration file.""" + return self.env.get("GALAXY_CONFIG_FILE", os.path.join(self.config_directory, "galaxy.yml")) + + def kill(self): + """Kill uvx Galaxy process.""" + if self._ctx.verbose: + shell(["ps", "ax"]) + exists = os.path.exists(self.pid_file) + print(f"Killing pid file [{self.pid_file}]") + print(f"pid_file exists? [{exists}]") + if exists: + with open(self.pid_file) as f: + print(f"pid_file contents are [{f.read()}]") + + # Kill process using existing utility + kill_pid_file(self.pid_file) + + def startup_command(self, ctx, **kwds): + """Return a shell command used to startup this uvx Galaxy instance.""" + daemon = kwds.get("daemon", False) + uvx_cmd = self._build_uvx_command(**kwds) + + if daemon: + # Use shell background execution for daemon mode - return as single string for shell execution + return f"nohup {shell_join(uvx_cmd)} > {self.log_file} 2>&1 & echo $! > {self.pid_file}" + else: + # Direct foreground execution + return shell_join(uvx_cmd) + + def _build_uvx_command(self, **kwds): + """Build uvx galaxy command with appropriate flags.""" + cmd = ["uvx", "galaxy"] + + # Only pass config file - host and port are configured in galaxy.yml + cmd.extend(["-c", self.galaxy_config_file]) + + return cmd + + @property + def log_file(self): + """Log file used when planemo serves this uvx Galaxy instance.""" + file_name = f"{self.server_name}.log" + return os.path.join(self.config_directory, file_name) + + @property + def pid_file(self): + """PID file for uvx Galaxy process.""" + pid_file_name = f"{self.server_name}.pid" + return os.path.join(self.config_directory, pid_file_name) + + @property + def log_contents(self): + """Return contents of log file.""" + if not os.path.exists(self.log_file): + return "" + with open(self.log_file) as f: + return f.read() + + def cleanup(self): + """Clean up uvx Galaxy configuration.""" + shutil.rmtree(self.config_directory, CLEANUP_IGNORE_ERRORS) + + @property + def default_use_path_paste(self): + """Default path paste setting for uvx Galaxy.""" + return self.user_is_admin + + def _install_galaxy(self): + """Override to skip Galaxy installation - uvx manages this.""" + # No-op for uvx since it manages Galaxy installation + return True + + def _ensure_galaxy_repository_available(self): + """Override to skip repository cloning - not needed for uvx.""" + # No-op for uvx since no repository is needed + return True + + +@contextlib.contextmanager +def uvx_galaxy_config(ctx, runnables, for_tests=False, **kwds): + """Set up a ``UvxGalaxyConfig`` in an auto-cleaned context.""" + test_data_dir = _find_test_data(runnables, **kwds) + + with _config_directory(ctx, **kwds) as config_directory: + + def config_join(*args): + return os.path.join(config_directory, *args) + + server_name = "main" + + # Ensure dependency resolvers are configured + ensure_dependency_resolvers_conf_configured(ctx, kwds, os.path.join(config_directory, "resolvers_conf.xml")) + + # Handle basic galaxy configuration without installation + galaxy_root = config_directory # Use config directory as galaxy root for uvx + # Skip refgenie config for uvx since Galaxy is managed by uvx + + # Setup tool paths (but don't require galaxy_root) + all_tool_paths = _all_tool_paths(runnables, galaxy_root=None, extra_tools=kwds.get("extra_tools")) + kwds["all_in_one_handling"] = True + _handle_job_config_file(config_directory, server_name, test_data_dir, all_tool_paths, kwds) + _handle_file_sources(config_directory, test_data_dir, kwds) + + # Basic paths setup + file_path = kwds.get("file_path") or config_join("files") + _ensure_directory(file_path) + + tool_dependency_dir = kwds.get("tool_dependency_dir") or config_join("deps") + _ensure_directory(tool_dependency_dir) + + shed_tool_conf = kwds.get("shed_tool_conf") or config_join("shed_tools_conf.xml") + empty_tool_conf = config_join("empty_tool_conf.xml") + tool_conf = config_join("tool_conf.xml") + shed_data_manager_config_file = config_join("shed_data_manager_conf.xml") + + shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools") + _ensure_directory(shed_tool_path) + + sheds_config_path = _configure_sheds_config_file(ctx, config_directory, runnables, **kwds) + + database_location = config_join("galaxy.sqlite") + master_api_key = _get_master_api_key(kwds) + dependency_dir = os.path.join(config_directory, "deps") + _ensure_directory(dependency_dir) + port = _get_port(kwds) + + # Template args for file generation + # Use fallback for tool shed URL if none configured + shed_target_url = tool_shed_url(ctx, **kwds) or MAIN_TOOLSHED_URL + + template_args = dict( + shed_tool_path=shed_tool_path, + shed_tool_conf=shed_tool_conf, + shed_data_manager_config_file=shed_data_manager_config_file, + test_data_dir=test_data_dir, + shed_target_url=shed_target_url, + dependency_dir=dependency_dir, + file_path=file_path, + temp_directory=config_directory, + ) + + # Galaxy properties + properties = _shared_galaxy_properties(config_directory, kwds, for_tests=for_tests) + properties.update( + dict( + server_name=server_name, + host=kwds.get("host", "127.0.0.1"), + port=str(port), + enable_celery_tasks="true", + ftp_upload_dir_template="${ftp_upload_dir}", + ftp_upload_purge="false", + ftp_upload_dir=test_data_dir or os.path.abspath("."), + ftp_upload_site="Test Data", + check_upload_content="false", + tool_dependency_dir=dependency_dir, + file_path=file_path, + new_file_path="${temp_directory}/tmp", + tool_config_file=f"{tool_conf},{shed_tool_conf}", + tool_sheds_config_file=sheds_config_path, + manage_dependency_relationships="false", + job_working_directory="${temp_directory}/job_working_directory", + template_cache_path="${temp_directory}/compiled_templates", + citation_cache_type="file", + citation_cache_data_dir="${temp_directory}/citations/data", + citation_cache_lock_dir="${temp_directory}/citations/lock", + database_auto_migrate="true", + enable_beta_tool_formats="true", + id_secret="${id_secret}", + log_level="DEBUG" if ctx.verbose else "INFO", + debug="true" if ctx.verbose else "false", + watch_tools="auto", + default_job_shell="/bin/bash", + integrated_tool_panel_config=("${temp_directory}/integrated_tool_panel_conf.xml"), + migrated_tools_config=empty_tool_conf, + test_data_dir=test_data_dir, + shed_data_manager_config_file=shed_data_manager_config_file, + outputs_to_working_directory="true", + object_store_store_by="uuid", + ) + ) + + _handle_container_resolution(ctx, kwds, properties) + properties["database_connection"] = _database_connection(database_location, **kwds) + + if kwds.get("mulled_containers", False): + properties["mulled_channels"] = kwds.get("conda_ensure_channels", "") + + _handle_kwd_overrides(properties, kwds) + + # Build environment + env = _build_env_for_galaxy(properties, template_args) + env["PLANEMO"] = "1" + env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1" + + # Write configuration files (but skip Galaxy installation) + # Assume uvx Galaxy is modern (>= 22.01) and write YAML config directly + env["GALAXY_CONFIG_FILE"] = config_join("galaxy.yml") + env["GRAVITY_STATE_DIR"] = config_join("gravity") + with NamedTemporaryFile(suffix=".sock", delete=True) as nt: + env["SUPERVISORD_SOCKET"] = nt.name + write_file( + env["GALAXY_CONFIG_FILE"], + json.dumps( + { + "galaxy": properties, + "gravity": { + "galaxy_root": galaxy_root, + "gunicorn": { + "bind": f"{kwds.get('host', 'localhost')}:{port}", + "preload": False, + }, + "gx-it-proxy": { + "enable": False, + }, + }, + }, + indent=2, + ), + ) + + # Write tool configurations + tool_definition = _tool_conf_entry_for(all_tool_paths) + write_file(tool_conf, _sub(TOOL_CONF_TEMPLATE, dict(tool_definition=tool_definition))) + + shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args) + write_file(shed_tool_conf, shed_tool_conf_contents, force=False) + write_file(shed_data_manager_config_file, SHED_DATA_MANAGER_CONF_TEMPLATE) + + yield UvxGalaxyConfig( + ctx, + config_directory, + env, + test_data_dir, + port, + server_name, + master_api_key, + runnables, + kwds, + ) + + __all__ = ( "DATABASE_LOCATION_TEMPLATE", "galaxy_config", + "UvxGalaxyConfig", + "uvx_galaxy_config", ) diff --git a/planemo/galaxy/serve.py b/planemo/galaxy/serve.py index 58e352b40..b33f35639 100644 --- a/planemo/galaxy/serve.py +++ b/planemo/galaxy/serve.py @@ -31,6 +31,8 @@ def _serve(ctx, runnables, **kwds): engine = kwds.get("engine", "galaxy") if engine == "docker_galaxy": kwds["dockerize"] = True + elif engine == "uvx_galaxy": + kwds["uvx_galaxy"] = True daemon = kwds.get("daemon", False) if daemon: diff --git a/planemo/options.py b/planemo/options.py index 5044a7b74..08edea551 100644 --- a/planemo/options.py +++ b/planemo/options.py @@ -70,13 +70,13 @@ def run_engine_option(): """Annotate click command as consume the --engine option.""" return planemo_option( "--engine", - type=click.Choice(["galaxy", "docker_galaxy", "cwltool", "toil", "external_galaxy"]), + type=click.Choice(["galaxy", "docker_galaxy", "cwltool", "toil", "external_galaxy", "uvx_galaxy"]), default=None, use_global_config=True, help=( "Select an engine to run or test artifacts such as tools " "and workflows. Defaults to a local Galaxy, but running Galaxy within " - "a Docker container or the CWL reference implementation 'cwltool' and " + "a Docker container, via uvx, or the CWL reference implementation 'cwltool' and " "'toil' be selected." ), ) @@ -100,14 +100,14 @@ def serve_engine_option(): """ return planemo_option( "--engine", - type=click.Choice(["galaxy", "docker_galaxy", "external_galaxy"]), + type=click.Choice(["galaxy", "docker_galaxy", "external_galaxy", "uvx_galaxy"]), default="galaxy", use_global_config=True, use_env_var=True, help=( "Select an engine to serve artifacts such as tools " "and workflows. Defaults to a local Galaxy, but running Galaxy within " - "a Docker container." + "a Docker container or via uvx is also supported." ), ) From 232cc85b506a35755c49723509512c29650efacd Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 10 Jul 2025 12:30:37 +0200 Subject: [PATCH 2/6] Install extra deps --- planemo/galaxy/config.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py index 08693b291..4a6072b01 100644 --- a/planemo/galaxy/config.py +++ b/planemo/galaxy/config.py @@ -9,6 +9,7 @@ import random import shlex import shutil +import subprocess import threading import time from string import Template @@ -1490,7 +1491,22 @@ def startup_command(self, ctx, **kwds): def _build_uvx_command(self, **kwds): """Build uvx galaxy command with appropriate flags.""" - cmd = ["uvx", "galaxy"] + with NamedTemporaryFile("wb", prefix="planemo_galaxy_extra_deps", suffix=".txt", delete=False) as extra_deps: + p = subprocess.run( + [ + "uvx", + "--from", + "galaxy", + "galaxy-dependencies", + "--freeze", + "--config_file", + self.galaxy_config_file, + ], + stdout=extra_deps, + ) + p.check_returncode() + + cmd = ["uvx", "--with-requirements", extra_deps.name, "galaxy"] # Only pass config file - host and port are configured in galaxy.yml cmd.extend(["-c", self.galaxy_config_file]) From a865ba12232c109bf2eec9a9bee31e6e2bfc13c5 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 10 Jul 2025 12:34:15 +0200 Subject: [PATCH 3/6] Fix url bug --- planemo/shed/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/planemo/shed/__init__.py b/planemo/shed/__init__.py index 1082267fa..e3b8b76f7 100644 --- a/planemo/shed/__init__.py +++ b/planemo/shed/__init__.py @@ -792,7 +792,7 @@ def shed_repo_type(config, name): def _shed_config_to_url(shed_config): url = shed_config["url"] - if not url.startswith("http"): + if url and not url.startswith("http"): message = f"Invalid shed url specified [{url}]. Please specify a valid HTTP address or one of {list(SHED_SHORT_NAMES.keys())}" raise ValueError(message) return url From 03d1ce949fd89e7f412b8e785ba7068450248116 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 10 Jul 2025 15:27:17 +0200 Subject: [PATCH 4/6] Use constants from ``signal`` --- planemo/io.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/planemo/io.py b/planemo/io.py index 72f2dbed1..e59d30466 100644 --- a/planemo/io.py +++ b/planemo/io.py @@ -5,6 +5,7 @@ import fnmatch import os import shutil +import signal import subprocess import sys import tempfile @@ -228,18 +229,19 @@ def kill_posix(pid: int): def _check_pid(): try: - os.kill(pid, 0) + os.kill(pid, signal.SIGTERM) return True except OSError: return False if _check_pid(): - for sig in [15, 9]: + for sig in [signal.SIGTERM, signal.SIGKILL]: try: # gunicorn (unlike paste), seem to require killing process # group os.killpg(os.getpgid(pid), sig) - except OSError: + except OSError as e: + print(f"Failed to kill process group for pid {pid} with signal {sig}: {e}") return time.sleep(1) if not _check_pid(): From 2594853741ec5d6886fe42b3b9b8c1d7ea603925 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 10 Jul 2025 15:54:22 +0200 Subject: [PATCH 5/6] Kill instance when leaving ensure_runnables_served scope --- planemo/engine/galaxy.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/planemo/engine/galaxy.py b/planemo/engine/galaxy.py index 3de5750f0..fd7f07bf5 100644 --- a/planemo/engine/galaxy.py +++ b/planemo/engine/galaxy.py @@ -145,7 +145,10 @@ def ensure_runnables_served(self, runnables): with serve_daemon(self._ctx, runnables, **self._serve_kwds()) as config: if "install_args_list" in self._serve_kwds(): self.shed_install(config) - yield config + try: + yield config + finally: + config.kill() def shed_install(self, config): kwds = self._serve_kwds() @@ -208,7 +211,10 @@ def ensure_runnables_served(self, runnables): with serve_daemon(self._ctx, runnables, **self._serve_kwds()) as config: if "install_args_list" in self._serve_kwds(): self.shed_install(config) - yield config + try: + yield config + finally: + config.kill() class ExternalGalaxyEngine(GalaxyEngine): From 41f37c5da795454774f37a9f0ee22ae298c16580 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 11 Jul 2025 16:11:22 +0200 Subject: [PATCH 6/6] Drop unnecessary implementation --- planemo/engine/galaxy.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/planemo/engine/galaxy.py b/planemo/engine/galaxy.py index fd7f07bf5..c43f4ea0d 100644 --- a/planemo/engine/galaxy.py +++ b/planemo/engine/galaxy.py @@ -205,17 +205,6 @@ def _serve_kwds(self): serve_kwds["uvx_galaxy"] = True return serve_kwds - @contextlib.contextmanager - def ensure_runnables_served(self, runnables): - """Ensure runnables are served using uvx Galaxy configuration.""" - with serve_daemon(self._ctx, runnables, **self._serve_kwds()) as config: - if "install_args_list" in self._serve_kwds(): - self.shed_install(config) - try: - yield config - finally: - config.kill() - class ExternalGalaxyEngine(GalaxyEngine): """An :class:`Engine` implementation backed by an external Galaxy instance."""