Fixes to type annotations and test configurations

docs/developing.rst

@@ -12,11 +12,6 @@ are fine.
 
 ::
 
-    [distutils]
-    index-servers =
-        pypi
-        test
-
     [pypi]
     username:<username>
     password:<password>

pyproject.toml

@@ -15,10 +15,12 @@ reverse_relative = true
 skip_gitignore = true
 
 [tool.ruff]
+target-version = "py310"
+
+[tool.ruff.lint]
 # Enable: pycodestyle errors (E), Pyflakes (F), flake8-bugbear (B),
 # flake8-logging-format (G) and pyupgrade (UP)
 select = ["E", "F", "B", "G", "UP"]
-target-version = "py310"
 # Exceptions:
 # B008 Do not perform function calls in argument defaults (for FastAPI Depends and Body)
 # B9 flake8-bugbear opinionated warnings

requirements.txt

@@ -4,7 +4,8 @@ bioblend>=0.10.0
 Jinja2
 galaxy-tool-util>=20.9.1
 galaxy-util>=20.9.0
+packaging
 pysam
 rich
 setuptools
-pydantic>=2.0
+pydantic>=2.0

scripts/commit_version.py

@@ -13,7 +13,7 @@ def main(argv):
     source_dir = argv[1]
     version = argv[2]
     history_path = os.path.join(PROJECT_DIRECTORY, "HISTORY.rst")
-    with open(history_path, "r") as f:
+    with open(history_path) as f:
         history = f.read()
     today = datetime.datetime.today()
     today_str = today.strftime("%Y-%m-%d")
@@ -22,21 +22,12 @@ def main(argv):
         f.write(history)
 
     source_mod_path = os.path.join(PROJECT_DIRECTORY, source_dir, "__init__.py")
-    with open(source_mod_path, "r") as f:
+    with open(source_mod_path) as f:
         mod = f.read()
-    mod = re.sub("__version__ = '[\d\.]*\.dev0'", "__version__ = '%s'" % version, mod)
+    mod = re.sub(r'__version__ = "[\d\.]*\.dev0"', f'__version__ = "{version}"', mod)
     with open(source_mod_path, "w") as f:
         mod = f.write(mod)
-    shell(
-        [
-            "git",
-            "commit",
-            "-m",
-            "Version %s" % version,
-            "HISTORY.rst",
-            "%s/__init__.py" % source_dir,
-        ]
-    )
+    shell(["git", "commit", "-m", f"Version {version}", "HISTORY.rst", f"{source_dir}/__init__.py"])
     shell(["git", "tag", version])
 
 

scripts/new_version.py

@@ -4,18 +4,19 @@
 import re
 import subprocess
 import sys
-from distutils.version import StrictVersion
+
+from packaging.version import Version
 
 PROJECT_DIRECTORY = os.path.join(os.path.dirname(__file__), "..")
 
 
 def main(argv):
     source_dir = argv[1]
-    old_version = StrictVersion(argv[2])
+    old_version = Version(argv[2])
     dot_at = 1
     if len(argv) > 3:
         dot_at = int(argv[3])
-    old_version_tuple = old_version.version
+    old_version_tuple = old_version.release
     new_version_tuple = list(old_version_tuple)
     for i in range(len(new_version_tuple)):
         if i < dot_at:
@@ -27,7 +28,7 @@ def main(argv):
     new_version = ".".join(map(str, new_version_tuple))
 
     history_path = os.path.join(PROJECT_DIRECTORY, "HISTORY.rst")
-    with open(history_path, "r") as f:
+    with open(history_path) as f:
         history = f.read()
 
     def extend(from_str, line):
@@ -47,21 +48,12 @@ def extend(from_str, line):
         f.write(history)
 
     source_mod_path = os.path.join(PROJECT_DIRECTORY, source_dir, "__init__.py")
-    with open(source_mod_path, "r") as f:
+    with open(source_mod_path) as f:
         mod = f.read()
-    mod = re.sub("__version__ = '[\d\.]+'", "__version__ = '%s.dev0'" % new_version, mod, 1)
+    mod = re.sub(r'__version__ = "[\d\.]+"', f"__version__ = '{new_version}.dev0'", mod, 1)
     with open(source_mod_path, "w") as f:
         mod = f.write(mod)
-    shell(
-        [
-            "git",
-            "commit",
-            "-m",
-            "Starting work on %s" % new_version,
-            "HISTORY.rst",
-            "%s/__init__.py" % source_dir,
-        ]
-    )
+    shell(["git", "commit", "-m", f"Starting work on {new_version}", "HISTORY.rst", f"{source_dir}/__init__.py"])
 
 
 def shell(cmds, **kwds):

scripts/print_version_for_release.py

@@ -3,15 +3,16 @@
 import ast
 import re
 import sys
-from distutils.version import LooseVersion
+
+from packaging.version import Version
 
 source_dir = sys.argv[1]
 
 _version_re = re.compile(r"__version__\s+=\s+(.*)")
 
-with open("%s/__init__.py" % source_dir, "rb") as f:
-    version = str(ast.literal_eval(_version_re.search(f.read().decode("utf-8")).group(1)))
+with open(f"{source_dir}/__init__.py") as f:
+    version = str(ast.literal_eval(_version_re.search(f.read()).group(1)))
 
+version_obj = Version(version)
 # Strip .devN
-version_tuple = LooseVersion(version).version[0:3]
-print(".".join(map(str, version_tuple)))
+print(version_obj.base_version)

src/ephemeris/_idc_split_data_manager_genomes.py

@@ -22,8 +22,6 @@
 from galaxy.util import safe_makedirs
 from pydantic import (
     BaseModel,
-    Extra,
-    RootModel,
 )
 
 from . import get_galaxy_connection
@@ -95,15 +93,6 @@ class RunDataManagers(BaseModel):
     data_managers: list[RunDataManager]
 
 
-class DataManager(BaseModel, extra=Extra.forbid):
-    tags: list[str]
-    tool_id: str
-
-
-class DataManagers(RootModel):
-    root: dict[str, DataManager]
-
-
 class Genome(BaseModel):
     pass
 
@@ -119,11 +108,15 @@ def ucsc_description_for_build(requested_build: str) -> str:
     tree = ElementTree.fromstring(text)
 
     for dsn in tree:
-        build = dsn.find("SOURCE").attrib["id"]
-        if build != requested_build:
+        SOURCE_el = dsn.find("SOURCE")
+        if SOURCE_el is None or SOURCE_el.attrib["id"] != requested_build:
             continue
 
-        description = dsn.find("DESCRIPTION").text.replace(" - Genome at UCSC", "").replace(" Genome at UCSC", "")
+        DESCRIPTION_el = dsn.find("DESCRIPTION")
+        assert DESCRIPTION_el is not None
+        description = DESCRIPTION_el.text
+        assert description is not None
+        description = description.replace(" - Genome at UCSC", "").replace(" Genome at UCSC", "")
 
         fields = description.split(" ")
         temp = fields[0]

src/ephemeris/get_tool_list_from_galaxy.py

@@ -2,11 +2,11 @@
 """Tool to extract a tool list from galaxy."""
 
 from argparse import ArgumentParser
-from distutils.version import StrictVersion
 
 import yaml
 from bioblend.galaxy.tools import ToolClient
 from bioblend.galaxy.toolshed import ToolShedClient
+from packaging.version import Version
 
 from . import get_galaxy_connection
 from .common_parser import (
@@ -296,7 +296,7 @@ def get_repo_from_tool(tool):
 
 def check_galaxy_version(gi):
     version = gi.config.get_version()
-    if StrictVersion(version["version_major"]) < StrictVersion("16.04"):
+    if Version(version["version_major"]) < Version("16.04"):
         raise Exception("This script needs galaxy version 16.04 or newer")
 
 

src/ephemeris/run_data_managers.py

@@ -29,7 +29,10 @@
 import logging
 import time
 from collections import namedtuple
-from typing import Literal
+from typing import (
+    Any,
+    Literal,
+)
 
 from bioblend.galaxy import GalaxyInstance
 from bioblend.galaxy.tool_data import ToolDataClient
@@ -127,10 +130,10 @@ def __init__(self, galaxy_instance: GalaxyInstance, configuration):
         self.data_managers = self.config.get("data_managers")
         self.genomes = self.config.get("genomes", "")
         self.source_tables = DEFAULT_SOURCE_TABLES
-        self.fetch_jobs = []
-        self.skipped_fetch_jobs = []
-        self.index_jobs = []
-        self.skipped_index_jobs = []
+        self.fetch_jobs: list[dict[str, Any]] = []
+        self.skipped_fetch_jobs: list[dict[str, Any]] = []
+        self.index_jobs: list[dict[str, Any]] = []
+        self.skipped_index_jobs: list[dict[str, Any]] = []
 
     def initiate_job_lists(self):
         """
@@ -151,7 +154,7 @@ def initiate_job_lists(self):
                 self.index_jobs.extend(jobs)
                 self.skipped_index_jobs.extend(skipped_jobs)
 
-    def get_dm_jobs(self, dm):
+    def get_dm_jobs(self, dm) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
         """Gets the job entries for a single dm. Puts entries that already present in skipped_job_list.
         :returns job_list, skipped_job_list"""
         job_list = []

tox.ini

@@ -1,40 +1,33 @@
 # TODO: implement doc linting
 [tox]
-envlist = py{310}-lint, py{310}-pytest, py{310}
+envlist = lint, pytest, mypy
 source_dir = src/ephemeris
 test_dir = tests
 
-[testenv:py310-lint]
+[testenv]
 commands =
-  black --check {[tox]source_dir} {[tox]test_dir}
-  isort --check-only {[tox]source_dir} {[tox]test_dir}
-  ruff check {[tox]source_dir} {[tox]test_dir}
-  flake8 {[tox]source_dir} {[tox]test_dir}
-skip_install = True
-deps =
-    black
-    isort
-    ruff
-    flake8
-    flake8-import-order
-
-[testenv:py310-pytest]
-deps =
-    -r requirements.txt
-    -r dev-requirements.txt
-allowlist_externals =
-    sed
-commands =
-    pytest -v --cov={envsitepackagesdir}/ephemeris --cov-report xml {[tox]test_dir}
+    lint: black --check {[tox]source_dir} {[tox]test_dir}
+    lint: isort --check-only {[tox]source_dir} {[tox]test_dir}
+    lint: ruff check {[tox]source_dir} {[tox]test_dir}
+    lint: flake8 {[tox]source_dir} {[tox]test_dir}
+    mypy: mypy {[tox]source_dir}
+    pytest: pytest -v --cov={envsitepackagesdir}/ephemeris --cov-report xml {[tox]test_dir}
     # Replace the installed package directory by the source directory.
     # This is needed for codacy to understand which files have coverage testing
     # Unfortunately this has to run in the tox env to have access to envsitepackagesdir
-    sed -i 's|{envsitepackagesdir}|src|' coverage.xml
-
-[testenv:mypy]
-commands =
-  mypy -p ephemeris
+    pytest: sed -i 's|{envsitepackagesdir}|src|' coverage.xml
 deps =
-  mypy
-  types-PyYAML
-  types-requests
+    lint: black
+    lint: isort
+    lint: ruff
+    lint: flake8
+    lint: flake8-import-order
+    mypy: mypy
+    mypy: types-PyYAML
+    mypy: types-requests
+    mypy,pytest: -r requirements.txt
+    pytest: -r dev-requirements.txt
+allowlist_externals =
+    pytest: sed
+skip_install =
+    lint,mypy: True