Merge pull request #21163 from jmchilton/lazy_data_tables

Fix various bugs/hacks with tool_data_table handling.

Author: mvdbeek

lib/galaxy/app.py

@@ -266,9 +266,13 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi
     container_finder: containers.ContainerFinder
     install_model: ModelMapping
     object_store: BaseObjectStore
+    _tool_data_tables: Optional[BaseToolDataTableManager]
+    _genome_builds: Optional[GenomeBuilds]
 
     def __init__(self, fsmon=False, **kwargs) -> None:
         super().__init__()
+        self._genome_builds = None
+        self._tool_data_tables = None
         self.haltables = [
             ("object store", self._shutdown_object_store),
             ("database connection", self._shutdown_model),
@@ -314,7 +318,15 @@ def configure_fluent_log(self):
             self.trace_logger = None
 
     def _configure_genome_builds(self, data_table_name="__dbkeys__", load_old_style=True):
-        self.genome_builds = GenomeBuilds(self, data_table_name=data_table_name, load_old_style=load_old_style)
+        self._genome_builds = GenomeBuilds(self, data_table_name=data_table_name, load_old_style=load_old_style)
+
+    # lazy initialize genome_builds so tool_data_tables is also lazy
+    @property
+    def genome_builds(self) -> GenomeBuilds:
+        if self._genome_builds is None:
+            self._configure_genome_builds()
+        assert self._genome_builds is not None
+        return self._genome_builds
 
     def wait_for_toolbox_reload(self, old_toolbox):
         timer = ExecutionTimer()
@@ -412,22 +424,30 @@ def _set_enabled_container_types(self):
 
     def _configure_tool_data_tables(self, from_shed_config):
         # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path.
-        self.tool_data_tables: BaseToolDataTableManager = ToolDataTableManager(
+        tool_data_tables: BaseToolDataTableManager = ToolDataTableManager(
             tool_data_path=self.config.tool_data_path,
             config_filename=self.config.tool_data_table_config_path,
             other_config_dict=self.config,
         )
         # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables.
         try:
-            self.tool_data_tables.load_from_config_file(
+            tool_data_tables.load_from_config_file(
                 config_filename=self.config.shed_tool_data_table_config,
-                tool_data_path=self.tool_data_tables.tool_data_path,
+                tool_data_path=tool_data_tables.tool_data_path,
                 from_shed_config=from_shed_config,
             )
         except OSError as exc:
             # Missing shed_tool_data_table_config is okay if it's the default
             if exc.errno != errno.ENOENT or self.config.is_set("shed_tool_data_table_config"):
                 raise
+        self._tool_data_tables = tool_data_tables
+
+    @property
+    def tool_data_tables(self) -> BaseToolDataTableManager:
+        if self._tool_data_tables is None:
+            self._configure_tool_data_tables(from_shed_config=False)
+        assert self._tool_data_tables is not None
+        return self._tool_data_tables
 
     def _configure_datatypes_registry(self, use_display_applications=True, use_converters=True):
         # Create an empty datatypes registry.
@@ -746,10 +766,9 @@ def __init__(self, **kwargs) -> None:
         )
         self.api_keys_manager = self._register_singleton(ApiKeyManager)
 
-        # Tool Data Tables
+        # Setup lazy variables
         self._configure_tool_data_tables(from_shed_config=False)
-        # Load dbkey / genome build manager
-        self._configure_genome_builds(data_table_name="__dbkeys__", load_old_style=True)
+        self._configure_genome_builds()
 
         # Genomes
         self.genomes = self._register_singleton(Genomes)

lib/galaxy/celery/tasks.py

@@ -71,11 +71,6 @@
 log = get_logger(__name__)
 
 
-@lru_cache
-def setup_data_table_manager(app):
-    app._configure_tool_data_tables(from_shed_config=False)
-
-
 @lru_cache
 def cached_create_tool_from_representation(app: MinimalManagerApp, raw_tool_source: str):
     return create_tool_from_representation(app=app, raw_tool_source=raw_tool_source, tool_source_class="XmlToolSource")
@@ -448,7 +443,6 @@ def import_data_bundle(
     tool_data_file_path: Optional[str] = None,
     task_user_id: Optional[int] = None,
 ):
-    setup_data_table_manager(app)
     if src == "uri":
         assert uri
         tool_data_import_manager.import_data_bundle_by_uri(config, uri, tool_data_file_path=tool_data_file_path)

lib/galaxy/queue_worker.py

@@ -204,7 +204,6 @@ def reload_data_managers(app, **kwargs):
     reload_timer = util.ExecutionTimer()
 
     log.debug("Executing data managers reload on '%s'", app.config.server_name)
-    app._configure_tool_data_tables(from_shed_config=False)
     reload_tool_data_tables(app)
     reload_count = app.data_managers._reload_count + 1
     app.data_managers = DataManagers(app, None, reload_count)

lib/galaxy/structured_app.py

@@ -80,7 +80,10 @@ class BasicSharedApp(Container):
     auth_manager: AuthManager
     security_agent: Any
     quota_agent: QuotaAgent
-    tool_data_tables: "ToolDataTableManager"
+
+    @property
+    def tool_data_tables(self) -> "ToolDataTableManager":
+        raise NotImplementedError()
 
     @property
     def toolbox(self) -> "ToolBox":
@@ -95,10 +98,13 @@ class MinimalToolApp(Protocol):
     config: Any
     datatypes_registry: Registry
     object_store: BaseObjectStore
-    tool_data_tables: "ToolDataTableManager"
     file_sources: ConfiguredFileSources
     security: IdEncodingHelper
 
+    @property
+    def tool_data_tables(self) -> "ToolDataTableManager":
+        raise NotImplementedError()
+
 
 class MinimalApp(BasicSharedApp):
     is_webapp: bool  # is_webapp will be set to true when building WSGI app

lib/galaxy/tool_shed/galaxy_install/client.py

@@ -56,6 +56,8 @@ class InstallationTarget(HasToolBox, Protocol[ToolBoxType]):
     config: Any
     installed_repository_manager: "InstalledRepositoryManager"
     _toolbox_lock: threading.RLock
-    tool_data_tables: ToolDataTableManager
+
+    @property
+    def tool_data_tables(self) -> ToolDataTableManager: ...
 
     def wait_for_toolbox_reload(self, old_toolbox: ToolBoxType) -> None: ...

lib/galaxy/tool_shed/unittest_utils/__init__.py

@@ -222,7 +222,7 @@ def __init__(
         config.shed_tools_dir = str(tool_root_dir)
         self.watchers = Watchers(self)
         self.reload_toolbox()
-        self.tool_data_tables = ToolDataTableManager(
+        self._tool_data_tables = ToolDataTableManager(
             tool_data_path=self.config.tool_data_path,
             config_filename=self.config.shed_tool_data_table_config,
             other_config_dict=self.config,
@@ -231,6 +231,10 @@ def __init__(
         dependency_dir.mkdir()
         self.installed_repository_manager = InstalledRepositoryManager(self)
 
+    @property
+    def tool_data_tables(self) -> ToolDataTableManager:
+        return self._tool_data_tables
+
     @property
     def tool_dependency_dir(self) -> Optional[str]:
         return None

lib/galaxy/tool_util/parser/output_actions.py

@@ -27,7 +27,8 @@ class ToolOutputActionAppConfig(Protocol):
 
 
 class ToolOutputActionApp(Protocol):
-    tool_data_tables: Any  # TODO after refactor "ToolDataTableManager"
+    @property
+    def tool_data_tables(self) -> Any: ...  # TODO after refactor "ToolDataTableManager"
 
     @property
     def config(self) -> ToolOutputActionAppConfig: ...  # https://github.com/python/mypy/issues/7041

lib/galaxy/tools/remote_tool_eval.py

@@ -66,11 +66,15 @@ def __init__(
         self.datatypes_registry = datatypes_registry
         self.object_store = object_store
         self.genome_builds = GenomeBuilds(self)
-        self.tool_data_tables = tool_data_table_manager
+        self._tool_data_tables = tool_data_table_manager
         self.file_sources = file_sources
         self.biotools_metadata_source = None
         self.security = None  # type: ignore[assignment]
 
+    @property
+    def tool_data_tables(self) -> ToolDataTableManager:
+        return self._tool_data_tables
+
 
 def main(TMPDIR, WORKING_DIRECTORY, IMPORT_STORE_DIRECTORY) -> None:
     metadata_params = get_metadata_params(WORKING_DIRECTORY)

lib/tool_shed/structured_app.py

@@ -3,7 +3,6 @@
 from galaxy.structured_app import BasicSharedApp
 
 if TYPE_CHECKING:
-    from galaxy.tools.data import ToolDataTableManager
     from tool_shed.managers.model_cache import ModelCache
     from tool_shed.repository_registry import RegistryInterface
     from tool_shed.repository_types.registry import Registry as RepositoryTypesRegistry
@@ -19,4 +18,3 @@ class ToolShedApp(BasicSharedApp):
     hgweb_config_manager: "HgWebConfigManager"
     security_agent: "CommunityRBACAgent"
     model_cache: "ModelCache"
-    tool_data_tables: "ToolDataTableManager"

lib/tool_shed/webapp/app.py

@@ -88,8 +88,8 @@ def __init__(self, **kwd) -> None:
         self.tag_handler = CommunityTagHandler(self.model.context)
         # Initialize the Tool Shed tool data tables.  Never pass a configuration file here
         # because the Tool Shed should always have an empty dictionary!
-        self.tool_data_tables = galaxy.tools.data.ToolDataTableManager(self.config.tool_data_path)
-        self.genome_builds = GenomeBuilds(self)
+        self._tool_data_tables = galaxy.tools.data.ToolDataTableManager(self.config.tool_data_path)
+        self._genome_builds = GenomeBuilds(self)
         self.auth_manager = self._register_singleton(auth.AuthManager, auth.AuthManager(self.config))
         # Citation manager needed to load tools.
         self.citations_manager = self._register_singleton(CitationsManager, CitationsManager(self))
@@ -115,6 +115,14 @@ def __init__(self, **kwd) -> None:
         self.server_starttime = int(time.time())
         log.debug("Tool shed hgweb.config file is: %s", self.hgweb_config_manager.hgweb_config)
 
+    @property
+    def tool_data_tables(self) -> galaxy.tools.data.ToolDataTableManager:
+        return self._tool_data_tables
+
+    @property
+    def genome_builds(self) -> GenomeBuilds:
+        return self._genome_builds
+
 
 # Global instance of the universe app.
 app: Optional[ToolShedApp] = None