Merge pull request #418 from nsoranzo/ci_improvements

CI improvements and type annotations

Author: mvdbeek

.github/workflows/deploy.yaml

@@ -5,9 +5,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
       - uses: actions/setup-python@v6
         with:
-          python-version: '3.13'
+          python-version: '3.14'
+          cache: pip
       - name: Install requirements
         run: |
           python3 -m pip install build twine

.github/workflows/galaxy_framework.yaml

@@ -28,23 +28,21 @@ jobs:
       - uses: actions/checkout@v5
         with:
           path: 'pulsar'
+          persist-credentials: false
       - name: Checkout tools repo
         uses: actions/checkout@v5
         with:
           repository: galaxyproject/galaxy
           ref: ${{ matrix.galaxy-branch }}
           path: galaxy
+          persist-credentials: false
       - uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
+          cache: pip
       - name: Build Pulsar wheel and replace Galaxy's requirements.
         run: python tools/replace_galaxy_requirements_for_ci.py
         working-directory: 'pulsar'
-      - name: Cache pip dir
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy/requirements.txt') }}
       - name: Run tests
         run: ./run_tests.sh --framework
         working-directory: 'galaxy'

.github/workflows/pulsar.yaml

@@ -6,29 +6,38 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        tox-env: [py37-lint, py37-docs]
+        python-version: ['3.7']
+        tox-env: [lint, docs]
+        include:
+          - python-version: '3.14'
+            tox-env: lint
     steps:
     - uses: actions/checkout@v5
+      with:
+        persist-credentials: false
     - uses: actions/setup-python@v6
       with:
-        python-version: '3.7'
+        python-version: ${{ matrix.python-version }}
+        cache: pip
     - name: Install tox
       run: pip install tox
     - name: Run lint
       run: tox -e ${{ matrix.tox-env }}
   mypy:
     name: MyPy
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        include:
-        - tox-env: py311-mypy
-          python: '3.11'
+        tox-env: [mypy]
+        python-version: ['3.14']
     steps:
     - uses: actions/checkout@v5
+      with:
+        persist-credentials: false
     - uses: actions/setup-python@v6
       with:
-        python-version: ${{ matrix.python }}
+        python-version: ${{ matrix.python-version }}
+        cache: pip
     - name: Install tox
       run: pip install tox
     - name: Setup pycurl
@@ -40,17 +49,11 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
+        tox-env: [test-ci, test-unit]
+        python-version: ['3.7', '3.11']
         include:
-        - tox-env: py37-test-ci
-          python: '3.7'
-        - tox-env: py37-test-unit
-          python: '3.7'
-        - tox-env: py311-test-ci
-          python: '3.11'
-        - tox-env: py311-test-unit
-          python: '3.11'
-        - tox-env: py39-install_wheel
-          python: '3.9'
+        - tox-env: install_wheel
+          python-version: '3.9'
     services:
       job-files:
         image: galaxy/simple-job-files:latest
@@ -62,9 +65,12 @@ jobs:
           - '/tmp:/tmp:rw'
     steps:
     - uses: actions/checkout@v5
+      with:
+        persist-credentials: false
     - uses: actions/setup-python@v6
       with:
-        python-version: ${{ matrix.python }}
+        python-version: ${{ matrix.python-version }}
+        cache: pip
     - name: Install tox
       run: pip install tox
     - name: Run setup
@@ -84,13 +90,16 @@ jobs:
   #   strategy:
   #     matrix:
   #       include:
-  #       - tox-env: py37-test-funnel
-  #         python: 3.7
+  #       - tox-env: test-funnel
+  #         python-version: 3.7
   #   steps:
   #   - uses: actions/checkout@v5
+  #     with:
+  #       persist-credentials: false
   #   - uses: actions/setup-python@v6
   #     with:
-  #       python-version: ${{ matrix.python }}
+  #       python-version: ${{ matrix.python-version }}
+  #       cache: pip
   #   - name: Set up Go 1.x
   #     uses: actions/setup-go@v2
   #     with:

dev-requirements.txt

@@ -23,7 +23,7 @@ sphinxcontrib-apidoc
 pyflakes
 flake8
 
-mypy<=1.0.1  # https://github.com/pydantic/pydantic/issues/5192
+mypy
 types-paramiko
 types-setuptools
 types-PyYAML

mypy.ini

@@ -1,4 +1,5 @@
 [mypy]
+plugins = pydantic.mypy
 show_error_codes = True
 pretty = True
 

pulsar/client/staging/__init__.py

@@ -7,11 +7,18 @@
     exists,
     join,
 )
+from typing import (
+    TYPE_CHECKING,
+    Union,
+)
 
 from galaxy.util.bunch import Bunch
 
 from ..util import PathHelper
 
+if TYPE_CHECKING:
+    from galaxy.tool_util.parser.interface import RequiredFiles
+
 COMMAND_VERSION_FILENAME = "COMMAND_VERSION"
 DEFAULT_DYNAMIC_COLLECTION_PATTERN = [
     "|".join(
@@ -51,6 +58,8 @@ class ClientJobDescription:
     command_line : str
         The local command line to execute, this will be rewritten for
         the remote server.
+    tool :
+        The Galaxy tool to execute.
     config_files : list
         List of Galaxy 'configfile's produced for this job. These will
         be rewritten and sent to remote server.
@@ -60,9 +69,6 @@ class ClientJobDescription:
     client_outputs : ClientOutputs
         Description of outputs produced by job (at least output files along
         with optional version string and working directory outputs.
-    tool_dir : str
-        Directory containing tool to execute (if a wrapper is used, it will
-        be transferred to remote server).
     working_directory : str
         Local path created by Galaxy for running this job (job_wrapper.tool_working_directory).
     metadata_directory : str
@@ -72,8 +78,6 @@ class ClientJobDescription:
         tool dependency context for remote depenency resolution.
     env: list
         List of dict object describing environment variables to populate.
-    version_file : str
-        Path to version file expected on the client server
     arbitrary_files : dict()
         Additional non-input, non-tool, non-config, non-working directory files
         to transfer before staging job. This is most likely data indices but
@@ -99,7 +103,7 @@ def __init__(
         env=[],
         arbitrary_files=None,
         job_directory_files=None,
-        tool_directory_required_files=None,
+        tool_directory_required_files: Union["RequiredFiles", None] = None,
         rewrite_paths=True,
         touch_outputs=None,
         container=None,

pulsar/client/staging/up.py

@@ -14,7 +14,10 @@
     escape,
     findall,
 )
-from typing import Optional
+from typing import (
+    Optional,
+    TYPE_CHECKING,
+)
 
 from ..action_mapper import (
     FileActionMapper,
@@ -32,10 +35,13 @@
     PathHelper,
 )
 
+if TYPE_CHECKING:
+    from ..staging import ClientJobDescription
+
 log = getLogger(__name__)
 
 
-def submit_job(client, client_job_description, job_config=None):
+def submit_job(client, client_job_description: "ClientJobDescription", job_config=None):
     """
     """
     file_stager = FileStager(client, client_job_description, job_config)
@@ -99,7 +105,7 @@ class FileStager:
         Description of client view of job to stage and execute remotely.
     """
 
-    def __init__(self, client, client_job_description, job_config):
+    def __init__(self, client, client_job_description: "ClientJobDescription", job_config):
         """
         """
         self.client = client

pulsar/tools/toolbox.py

@@ -4,6 +4,10 @@
     dirname,
     join,
 )
+from typing import (
+    List,
+    Union,
+)
 from xml.etree import ElementTree
 
 from pulsar.tools.validator import ExpressionValidator
@@ -19,8 +23,8 @@ class ToolBox:
     to support simple, non-toolshed based tool setups.
     """
 
-    def __init__(self, path_string):
-        self.tool_configs = []
+    def __init__(self, path_string) -> None:
+        self.tool_configs: List[Union[SimpleToolConfig, ToolShedToolConfig]] = []
         paths = [path.strip() for path in path_string.split(",")]
         for path in paths:
             toolbox_tree = ElementTree.parse(path)
@@ -41,17 +45,17 @@ def __load_tools_from_els(self, toolbox_root, tool_path):
             except Exception:
                 log.exception('Failed to load tool.')
 
-    def get_tool(self, id):
+    def get_tool(self, id) -> Union["SimpleToolConfig", "ToolShedToolConfig"]:
         # Need to handle multiple tools per id someday, but
         # starting simple.
         tools = self.__find_tools_by_id(id)
         if not tools:
             raise KeyError("Failed to find tool with id '%s'" % id)
         if len(tools) > 1:
-            log.warn("Found multiple tools with id '%s', returning first." % id)
+            log.warning("Found multiple tools with id '%s', returning first.", id)
         return tools[0]
 
-    def __find_tools_by_id(self, id):
+    def __find_tools_by_id(self, id) -> List[Union["SimpleToolConfig", "ToolShedToolConfig"]]:
         return [tool for tool in self.tool_configs if tool.id == id]
 
 

pulsar/tools/validator.py

@@ -11,7 +11,7 @@
 class ExpressionValidator:
 
     def __init__(self, xml_el):
-        if type(xml_el) == str:
+        if isinstance(xml_el, str):
             xml_el = fromstring(xml_el)
         self.xml_el = xml_el
 

pulsar/util/pastescript/serve.py

@@ -258,7 +258,8 @@ def pad(self, s, length, dir='left'):
         else:
             return ' ' * (length - len(s)) + s
 
-    def _standard_parser(cls, verbose=True,
+    @classmethod
+    def standard_parser(cls, verbose=True,
                         interactive=False,
                         no_interactive=False,
                         simulate=False,
@@ -308,8 +309,6 @@ class MyCommand(Command):
                               help="Overwrite files (warnings will be emitted for non-matching files otherwise)")
         return parser
 
-    standard_parser = classmethod(_standard_parser)
-
     def quote_first_command_arg(self, arg):
         """
         There's a bug in Windows when running an executable that's