Include Singularity and Containers classes into config setup, and update related components

Author: karlnyr

microSALT/cli.py

@@ -211,6 +211,8 @@ def analyse(
         threshold=cfg.threshold,
         pubmlst=cfg.pubmlst,
         pasteur=cfg.pasteur,
+        singularity=cfg.singularity,
+        containers=cfg.containers,
         sampleinfo=sampleinfo,
         run_settings=run_settings,
     )
@@ -221,6 +223,8 @@ def analyse(
         threshold=cfg.threshold,
         pubmlst=cfg.pubmlst,
         pasteur=cfg.pasteur,
+        singularity=cfg.singularity,
+        containers=cfg.containers,
         sampleinfo=sampleinfo,
         force=force_update,
     )
@@ -318,6 +322,8 @@ def finish(ctx, sampleinfo_file, input, track, dry, email, skip_update, report,
         threshold=cfg.threshold,
         pubmlst=cfg.pubmlst,
         pasteur=cfg.pasteur,
+        singularity=cfg.singularity,
+        containers=cfg.containers,
         sampleinfo=sampleinfo,
     )
     try:
@@ -346,6 +352,8 @@ def finish(ctx, sampleinfo_file, input, track, dry, email, skip_update, report,
         config_path=cfg.config_path,
         pubmlst=cfg.pubmlst,
         pasteur=cfg.pasteur,
+        singularity=cfg.singularity,
+        containers=cfg.containers,
         sampleinfo=sampleinfo,
         input=input,
     )
@@ -376,7 +384,8 @@ def add(ctx, organism, force):
     cfg = ctx.obj["config"]
     referee = Referencer(
         log=logger, folders=cfg.folders, threshold=cfg.threshold,
-        pubmlst=cfg.pubmlst, pasteur=cfg.pasteur, force=force,
+        pubmlst=cfg.pubmlst, pasteur=cfg.pasteur,
+        singularity=cfg.singularity, containers=cfg.containers, force=force,
     )
     try:
         referee.add_pubmlst(organism)
@@ -386,7 +395,8 @@ def add(ctx, organism, force):
     click.echo("INFO - Checking versions of all references..")
     referee = Referencer(
         log=logger, folders=cfg.folders, threshold=cfg.threshold,
-        pubmlst=cfg.pubmlst, pasteur=cfg.pasteur, force=force,
+        pubmlst=cfg.pubmlst, pasteur=cfg.pasteur,
+        singularity=cfg.singularity, containers=cfg.containers, force=force,
     )
     referee.update_refs()
 
@@ -399,6 +409,7 @@ def observe(ctx):
     refe = Referencer(
         log=logger, folders=cfg.folders, threshold=cfg.threshold,
         pubmlst=cfg.pubmlst, pasteur=cfg.pasteur,
+        singularity=cfg.singularity, containers=cfg.containers,
     )
     click.echo("INFO - Currently stored organisms:")
     for org in sorted(refe.existing_organisms()):
@@ -500,6 +511,7 @@ def review(ctx, type, customer, skip_update, email, output):
     ext_refs = Referencer(
         log=logger, folders=cfg.folders, threshold=cfg.threshold,
         pubmlst=cfg.pubmlst, pasteur=cfg.pasteur,
+        singularity=cfg.singularity, containers=cfg.containers,
     )
     if not skip_update:
         ext_refs.update_refs()
@@ -521,7 +533,8 @@ def update_refs(ctx, force_update: bool):
     cfg = ctx.obj["config"]
     ext_refs = Referencer(
         log=logger, folders=cfg.folders, threshold=cfg.threshold,
-        pubmlst=cfg.pubmlst, pasteur=cfg.pasteur, force=force_update,
+        pubmlst=cfg.pubmlst, pasteur=cfg.pasteur,
+        singularity=cfg.singularity, containers=cfg.containers, force=force_update,
     )
     ext_refs.update_refs()
     done()
@@ -535,7 +548,8 @@ def update_from_static(ctx, force_update: bool):
     cfg = ctx.obj["config"]
     ext_refs = Referencer(
         log=logger, folders=cfg.folders, threshold=cfg.threshold,
-        pubmlst=cfg.pubmlst, pasteur=cfg.pasteur, force=force_update,
+        pubmlst=cfg.pubmlst, pasteur=cfg.pasteur,
+        singularity=cfg.singularity, containers=cfg.containers, force=force_update,
     )
     ext_refs.fetch_external()
     done()
@@ -551,7 +565,8 @@ def update_organism(ctx, external: bool, force_update: bool, organism: str):
     cfg = ctx.obj["config"]
     ext_refs = Referencer(
         log=logger, folders=cfg.folders, threshold=cfg.threshold,
-        pubmlst=cfg.pubmlst, pasteur=cfg.pasteur, force=force_update,
+        pubmlst=cfg.pubmlst, pasteur=cfg.pasteur,
+        singularity=cfg.singularity, containers=cfg.containers, force=force_update,
     )
     ext_refs.update_organism(external=external, organism=organism)
     done()
@@ -572,6 +587,7 @@ def overwrite(ctx, sample_name, force):
     ext_refs = Referencer(
         log=logger, folders=cfg.folders, threshold=cfg.threshold,
         pubmlst=cfg.pubmlst, pasteur=cfg.pasteur,
+        singularity=cfg.singularity, containers=cfg.containers,
     )
     ext_refs.resync(type="overwrite", sample=sample_name, ignore=force)
     done()

microSALT/config.py

@@ -76,6 +76,22 @@ class PasteurCredentials(BIGSdbCredentials):
     pass
 
 
+class Singularity(BaseModel):
+    binary: str = "/usr/bin/singularity"
+    bind_paths: list[str] = []
+    trimmomatic_adapters: str = "/opt/conda/share/trimmomatic/adapters/"
+
+
+class Containers(BaseModel):
+    skesa: str = ""
+    blast: str = ""
+    bwa: str = ""
+    samtools: str = ""
+    picard: str = ""
+    trimmomatic: str = ""
+    quast: str = ""
+
+
 class MicroSALTConfig(BaseModel):
     slurm_header: SlurmHeader
     regex: Regex
@@ -84,6 +100,8 @@ class MicroSALTConfig(BaseModel):
     threshold: Threshold
     pubmlst: PubMLSTCredentials = PubMLSTCredentials()
     pasteur: PasteurCredentials = PasteurCredentials()
+    singularity: Singularity = Singularity()
+    containers: Containers = Containers()
     # Runtime fields set by the CLI, not from the JSON file
     dry: bool = False
     config_path: str = ""

microSALT/utils/job_creator.py

@@ -17,13 +17,13 @@
 import yaml
 
 from microSALT import __version__
-from microSALT.config import Folders, Threshold, SlurmHeader, Regex, PubMLSTCredentials, PasteurCredentials
+from microSALT.config import Folders, Threshold, SlurmHeader, Regex, PubMLSTCredentials, PasteurCredentials, Singularity, Containers
 from microSALT.store.db_manipulator import DB_Manipulator
 from microSALT.utils.referencer import Referencer
 
 
 class Job_Creator:
-    def __init__(self, log, folders: Folders, slurm_header: SlurmHeader, regex: Regex, dry: bool, config_path: str, threshold: Threshold, pubmlst: PubMLSTCredentials, pasteur: PasteurCredentials, sampleinfo={}, run_settings={}):
+    def __init__(self, log, folders: Folders, slurm_header: SlurmHeader, regex: Regex, dry: bool, config_path: str, threshold: Threshold, pubmlst: PubMLSTCredentials, pasteur: PasteurCredentials, singularity: Singularity, containers: Containers, sampleinfo={}, run_settings={}):
         self.folders = folders
         self.slurm_header = slurm_header
         self.regex = regex
@@ -32,6 +32,8 @@ def __init__(self, log, folders: Folders, slurm_header: SlurmHeader, regex: Rege
         self.threshold = threshold
         self.pubmlst = pubmlst
         self.pasteur = pasteur
+        self.singularity = singularity
+        self.containers = containers
         self.logger = log
         self.batchfile = "/tmp/batchfile.sbatch"
 
@@ -85,7 +87,7 @@ def __init__(self, log, folders: Folders, slurm_header: SlurmHeader, regex: Rege
             self.finishdir = f"{folders.results}/{self.name}_{self.now}"
         self.db_pusher = DB_Manipulator(log=log, folders=folders, threshold=threshold)
         self.concat_files = dict()
-        self.ref_resolver = Referencer(log=log, folders=folders, threshold=threshold, pubmlst=pubmlst, pasteur=pasteur)
+        self.ref_resolver = Referencer(log=log, folders=folders, threshold=threshold, pubmlst=pubmlst, pasteur=pasteur, singularity=singularity, containers=containers)
 
     def get_sbatch(self):
         """Returns sbatchfile, slightly superflous"""
@@ -95,11 +97,11 @@ def get_headerargs(self):
         headerline = f"-A {self.slurm_header.project} -p {self.slurm_header.type} -n {self.slurm_header.threads} -t {self.slurm_header.time} -J {self.slurm_header.job_prefix}_{self.name} --qos {self.slurm_header.qos} --output {self.finishdir}/slurm_{self.name}.log"
         return headerline
 
-    def _singularity_exec(self, tool, command):
+    def _singularity_exec(self, tool: str, command: str) -> str:
         """Return command wrapped with singularity exec for the given tool container."""
-        sif = self.config["containers"][tool]
-        binary = self.config["singularity"]["binary"]
-        bind_list = list(self.config["singularity"].get("bind_paths", []))
+        sif = getattr(self.containers, tool)
+        binary = self.singularity.binary
+        bind_list = list(self.singularity.bind_paths)
         if self.finishdir and self.finishdir not in bind_list:
             bind_list.append(self.finishdir)
         bind = f"--bind {','.join(bind_list)}" if bind_list else ""
@@ -619,6 +621,8 @@ def project_job(self, single_sample=False):
                         threshold=self.threshold,
                         pubmlst=self.pubmlst,
                         pasteur=self.pasteur,
+                        singularity=self.singularity,
+                        containers=self.containers,
                         sampleinfo=local_sampleinfo,
                         run_settings=sample_settings,
                     )

microSALT/utils/referencer.py

@@ -12,19 +12,21 @@
 
 from Bio import Entrez
 
-from microSALT.config import Folders, Threshold, PubMLSTCredentials, PasteurCredentials
+from microSALT.config import Folders, Threshold, PubMLSTCredentials, PasteurCredentials, Singularity, Containers
 from microSALT.store.db_manipulator import DB_Manipulator
 from microSALT.utils.pubmlst.client import BaseClient, get_client
 from microSALT.utils.pubmlst.exceptions import InvalidURLError, PubMLSTError
 from microSALT.utils.pubmlst.helpers import get_service_by_url
 
 
 class Referencer:
-    def __init__(self, log, folders: Folders, threshold: Threshold, pubmlst: PubMLSTCredentials, pasteur: PasteurCredentials, sampleinfo={}, force=False):
+    def __init__(self, log, folders: Folders, threshold: Threshold, pubmlst: PubMLSTCredentials, pasteur: PasteurCredentials, singularity: Singularity = None, containers: Containers = None, sampleinfo={}, force=False):
         self.folders = folders
         self.threshold = threshold
         self.pubmlst = pubmlst
         self.pasteur = pasteur
+        self.singularity = singularity or Singularity()
+        self.containers = containers or Containers()
         self.logger = log
         self.db_access = DB_Manipulator(log=log, folders=folders, threshold=threshold)
         self.updated = list()
@@ -55,11 +57,11 @@ def set_client(self, service: str, database: str = None):
         """Set the client for PubMLST API interactions."""
         self.client: BaseClient = get_client(service, database, self.folders, self.pubmlst, self.pasteur)
 
-    def _singularity_exec(self, tool, command):
+    def _singularity_exec(self, tool: str, command: str) -> str:
         """Return command wrapped with singularity exec for the given tool container."""
-        sif = self.config['containers'][tool]
-        binary = self.config['singularity']['binary']
-        bind_list = self.config['singularity'].get('bind_paths', [])
+        sif = getattr(self.containers, tool)
+        binary = self.singularity.binary
+        bind_list = list(self.singularity.bind_paths)
         bind = f"--bind {','.join(bind_list)}" if bind_list else ""
         return f"{binary} exec {bind} {sif} {command}"
 

microSALT/utils/scraper.py

@@ -10,15 +10,15 @@
 import sys
 import time
 
-from microSALT.config import Folders, Threshold, SlurmHeader, Regex, PubMLSTCredentials, PasteurCredentials
+from microSALT.config import Folders, Threshold, SlurmHeader, Regex, PubMLSTCredentials, PasteurCredentials, Singularity, Containers
 from microSALT.store.db_manipulator import DB_Manipulator
 from microSALT.utils.referencer import Referencer
 from microSALT.utils.job_creator import Job_Creator
 
 
 # TODO: Rewrite so samples use seperate objects
 class Scraper:
-    def __init__(self, log, folders: Folders, threshold: Threshold, slurm_header: SlurmHeader, regex: Regex, dry: bool, config_path: str, pubmlst: PubMLSTCredentials, pasteur: PasteurCredentials, sampleinfo={}, input=""):
+    def __init__(self, log, folders: Folders, threshold: Threshold, slurm_header: SlurmHeader, regex: Regex, dry: bool, config_path: str, pubmlst: PubMLSTCredentials, pasteur: PasteurCredentials, singularity: Singularity = None, containers: Containers = None, sampleinfo={}, input=""):
         self.folders = folders
         self.threshold = threshold
         self.slurm_header = slurm_header
@@ -27,10 +27,12 @@ def __init__(self, log, folders: Folders, threshold: Threshold, slurm_header: Sl
         self.config_path = config_path
         self.pubmlst = pubmlst
         self.pasteur = pasteur
+        self.singularity = singularity or Singularity()
+        self.containers = containers or Containers()
         self.logger = log
         self.db_pusher = DB_Manipulator(log=log, folders=folders, threshold=threshold)
-        self.referencer = Referencer(log=log, folders=folders, threshold=threshold, pubmlst=pubmlst, pasteur=pasteur)
-        self.job_fallback = Job_Creator(log=log, folders=folders, slurm_header=slurm_header, regex=regex, dry=dry, config_path=config_path, threshold=threshold, pubmlst=pubmlst, pasteur=pasteur, sampleinfo=sampleinfo)
+        self.referencer = Referencer(log=log, folders=folders, threshold=threshold, pubmlst=pubmlst, pasteur=pasteur, singularity=self.singularity, containers=self.containers)
+        self.job_fallback = Job_Creator(log=log, folders=folders, slurm_header=slurm_header, regex=regex, dry=dry, config_path=config_path, threshold=threshold, pubmlst=pubmlst, pasteur=pasteur, singularity=self.singularity, containers=self.containers, sampleinfo=sampleinfo)
         self.infolder = os.path.abspath(input)
         self.sampledir = ""
 

tests/conftest.py

@@ -5,12 +5,14 @@
 from importlib.resources import files as resource_files
 
 from microSALT.config import (
+    Containers,
     Database,
     Folders,
     MicroSALTConfig,
     PasteurCredentials,
     PubMLSTCredentials,
     Regex,
+    Singularity,
     SlurmHeader,
     Threshold,
 )
@@ -73,6 +75,8 @@ def config(tmp_path_factory: pytest.TempPathFactory) -> MicroSALTConfig:
         threshold=Threshold(),
         pubmlst=PubMLSTCredentials(),
         pasteur=PasteurCredentials(),
+        singularity=Singularity(),
+        containers=Containers(),
     )
     cfg.folders.expec = str(
         resource_files("microSALT").joinpath("unique_references", "ExPEC.fsa")

tests/utils/conftest.py

@@ -121,6 +121,8 @@ def scraper(config: MicroSALTConfig, logger: logging.Logger, testdata: list[dict
         config_path=config.config_path,
         pubmlst=config.pubmlst,
         pasteur=config.pasteur,
+        singularity=config.singularity,
+        containers=config.containers,
         sampleinfo=testdata[0],
     )
 
@@ -133,6 +135,8 @@ def init_references(config: MicroSALTConfig, logger: logging.Logger, testdata: l
         threshold=config.threshold,
         pubmlst=config.pubmlst,
         pasteur=config.pasteur,
+        singularity=config.singularity,
+        containers=config.containers,
         sampleinfo=testdata,
     )
     ref_obj.identify_new(testdata[0].get("CG_ID_project"), project=True)
@@ -170,6 +174,8 @@ def blast_scraper_context(
         config_path=config.config_path,
         pubmlst=config.pubmlst,
         pasteur=config.pasteur,
+        singularity=config.singularity,
+        containers=config.containers,
         sampleinfo=testdata[0],
     )