remove adapters from folder

Author: karlnyr

configExample.json

@@ -34,9 +34,7 @@
     "_comment": "Download path for NCBI genomes, for alignment usage",
     "genomes": "/tmp/MLST/references/genomes",
     "_comment": "Credentials",
-    "credentials": "/tmp/MLST/credentials",
-    "_comment": "Trimmomatic adapter FASTA files directory",
-    "adapters": "/path/to/trimmomatic/adapters/"
+    "credentials": "/tmp/MLST/credentials"
   },
   "_comment": "Database/Flask configuration",
   "database": {
@@ -90,12 +88,13 @@
   },
   "_comment": "Absolute paths to pre-built Singularity SIF images on the cluster",
   "containers": {
-    "skesa":       "/fs1/resources/containers/skesa_2.5.1.sif",
-    "blast":       "/fs1/resources/containers/blast_2.12.0.sif",
-    "bwa":         "/fs1/resources/containers/bwa_0.7.17.sif",
-    "samtools":    "/fs1/resources/containers/samtools_1.13.sif",
-    "picard":      "/fs1/resources/containers/picard_2.20.3.sif",
+    "skesa": "/fs1/resources/containers/skesa_2.5.1.sif",
+    "blast": "/fs1/resources/containers/blast_2.12.0.sif",
+    "bwa": "/fs1/resources/containers/bwa_0.7.17.sif",
+    "samtools": "/fs1/resources/containers/samtools_1.13.sif",
+    "picard": "/fs1/resources/containers/picard_2.20.3.sif",
     "trimmomatic": "/fs1/resources/containers/trimmomatic_0.39.sif",
-    "quast":       "/fs1/resources/containers/quast_5.3.0.sif"
+    "quast": "/fs1/resources/containers/quast_5.3.0.sif"
   }
-}
+}
+

microSALT/cli.py

@@ -16,7 +16,11 @@
 from microSALT import __version__, logging_levels, setup_logger
 from microSALT.config import MicroSALTConfig, load_config
 from microSALT.exc.exceptions import RefUpdateLockError
-from microSALT.store.database import create_tables, get_scoped_session_registry, initialize_database
+from microSALT.store.database import (
+    create_tables,
+    get_scoped_session_registry,
+    initialize_database,
+)
 from microSALT.utils.job_creator import Job_Creator
 from microSALT.utils.referencer import Referencer
 from microSALT.utils.reporter import Reporter
@@ -110,7 +114,6 @@ def _ensure_directories(config: MicroSALTConfig) -> None:
         config.folders.resistances,
         config.folders.genomes,
         config.folders.credentials,
-        config.folders.adapters,
     ]
     for path in folder_paths:
         p = pathlib.Path(os.path.expandvars(os.path.expanduser(path)))

microSALT/config.py

@@ -30,7 +30,6 @@ class Folders(BaseModel):
     resistances: str
     genomes: str
     credentials: str
-    adapters: str
     expec: str = ""  # filled in after construction
 
 
@@ -69,9 +68,11 @@ class BIGSdbCredentials(BaseModel):
     client_id: str = ""
     client_secret: str = ""
 
+
 class PubMLSTCredentials(BIGSdbCredentials):
     pass
 
+
 class PasteurCredentials(BIGSdbCredentials):
     pass
 

microSALT/utils/job_creator.py

@@ -11,21 +11,45 @@
 import shutil
 import subprocess
 import sys
-from importlib.metadata import entry_points
 import time
 from datetime import datetime
+from importlib.metadata import entry_points
 from pathlib import Path
 
 import yaml
 
 from microSALT import __version__
-from microSALT.config import Folders, Threshold, SlurmHeader, Regex, PubMLSTCredentials, PasteurCredentials, Singularity, Containers
+from microSALT.config import (
+    Containers,
+    Folders,
+    PasteurCredentials,
+    PubMLSTCredentials,
+    Regex,
+    Singularity,
+    SlurmHeader,
+    Threshold,
+)
 from microSALT.store.db_manipulator import DB_Manipulator
 from microSALT.utils.referencer import Referencer
 
 
 class Job_Creator:
-    def __init__(self, log, folders: Folders, slurm_header: SlurmHeader, regex: Regex, dry: bool, config_path: str, threshold: Threshold, pubmlst: PubMLSTCredentials, pasteur: PasteurCredentials, singularity: Singularity, containers: Containers, sampleinfo={}, run_settings={}):
+    def __init__(
+        self,
+        log,
+        folders: Folders,
+        slurm_header: SlurmHeader,
+        regex: Regex,
+        dry: bool,
+        config_path: str,
+        threshold: Threshold,
+        pubmlst: PubMLSTCredentials,
+        pasteur: PasteurCredentials,
+        singularity: Singularity,
+        containers: Containers,
+        sampleinfo={},
+        run_settings={},
+    ):
         self.folders = folders
         self.slurm_header = slurm_header
         self.regex = regex
@@ -89,7 +113,15 @@ def __init__(self, log, folders: Folders, slurm_header: SlurmHeader, regex: Rege
             self.finishdir = f"{folders.results}/{self.name}_{self.now}"
         self.db_pusher = DB_Manipulator(log=log, folders=folders, threshold=threshold)
         self.concat_files = dict()
-        self.ref_resolver = Referencer(log=log, folders=folders, threshold=threshold, pubmlst=pubmlst, pasteur=pasteur, singularity=singularity, containers=containers)
+        self.ref_resolver = Referencer(
+            log=log,
+            folders=folders,
+            threshold=threshold,
+            pubmlst=pubmlst,
+            pasteur=pasteur,
+            singularity=singularity,
+            containers=containers,
+        )
 
     def get_sbatch(self):
         """Returns sbatchfile, slightly superflous"""
@@ -141,15 +173,13 @@ def verify_fastq(self):
                     else:
                         pairno = 2 - 1 % int(file_match[1])  # 1->2, 2->1
                         # Construct mate name
-                        pairname = f"{file_match.string[:file_match.end(1) - 1]}{pairno}{file_match.string[file_match.end(1):file_match.end()]}"
+                        pairname = f"{file_match.string[: file_match.end(1) - 1]}{pairno}{file_match.string[file_match.end(1) : file_match.end()]}"
                     if pairname in files:
                         files.pop(files.index(pairname))
                         verified_files.append(file_match[0])
                         verified_files.append(pairname)
                 else:
-                    raise Exception(
-                        f"Some fastq files have no mate in directory {self.indir}."
-                    )
+                    raise Exception(f"Some fastq files have no mate in directory {self.indir}.")
         if verified_files == []:
             raise Exception(
                 f"No files in directory {self.indir} match file_pattern '{self.regex.file_pattern}'."
@@ -163,9 +193,7 @@ def verify_fastq(self):
                 if bsize > 1000:
                     self.logger.warning(f"Input fastq {vfile} exceeds 1000MB")
             except Exception:
-                self.logger.warning(
-                    f"Unable to verify size of input file {self.indir}/{vfile}"
-                )
+                self.logger.warning(f"Unable to verify size of input file {self.indir}/{vfile}")
 
         # Warn about invalid fastq files
         for vfile in verified_files:
@@ -198,9 +226,7 @@ def create_assemblysection(self):
             f"--contigs_out {contigs_file_raw} "
             f"--reads {self.concat_files['f']},{self.concat_files['r']}"
         )
-        batchfile.write(
-            f"mkdir -p {assembly_dir} &" f"{self._singularity_exec('skesa', skesa_cmd)}\n"
-        )
+        batchfile.write(f"mkdir -p {assembly_dir} &{self._singularity_exec('skesa', skesa_cmd)}\n")
 
         # Convert sequence naming in Skesa output into Spades format in the contigs fasta file:
         # ----------------------------------------------
@@ -318,13 +344,10 @@ def create_variantsection(self):
             + "\n"
         )
         batchfile.write(
-            self._singularity_exec("samtools", f"samtools index {outbase}.bam_sort_rmdup")
-            + "\n"
+            self._singularity_exec("samtools", f"samtools index {outbase}.bam_sort_rmdup") + "\n"
         )
         batchfile.write(
-            self._singularity_exec(
-                "samtools", f"samtools idxstats {outbase}.bam_sort_rmdup"
-            )
+            self._singularity_exec("samtools", f"samtools idxstats {outbase}.bam_sort_rmdup")
             + f" &> {outbase}.stats.ref\n"
         )
         # Removal of temp aligment files
@@ -365,13 +388,6 @@ def create_preprocsection(self):
         forward = list()
         reverse = list()
 
-        for root, dirs, files in os.walk(self.folders.adapters):
-            if "NexteraPE-PE.fa" not in files:
-                self.logger.error(
-                    "Adapters folder at {} does not contain NexteraPE-PE.fa. Review paths.yml"
-                )
-            else:
-                break
         trimdir = f"{self.finishdir}/trimmed"
         files = self.verify_fastq()
         batchfile = open(self.batchfile, "a+")
@@ -406,7 +422,7 @@ def create_preprocsection(self):
                         f"trimmomatic PE -threads {self.slurm_header.threads}"
                         f" -phred33 {self.concat_files.get('f')} {self.concat_files.get('r')}"
                         f" {fp} {fu} {rp} {ru}"
-                        f" ILLUMINACLIP:{self.folders.adapters}/NexteraPE-PE.fa:2:30:10"
+                        f" ILLUMINACLIP:{self.singularity.trimmomatic_adapters}/NexteraPE-PE.fa:2:30:10"
                         " LEADING:3 TRAILING:3 SLIDINGWINDOW:4:15 MINLEN:36"
                     ),
                 )
@@ -579,9 +595,7 @@ def project_job(self, single_sample=False):
             else:
                 self.create_project(self.name)
         except Exception:
-            self.logger.error(
-                f"LIMS interaction failed. Unable to read/write project {self.name}"
-            )
+            self.logger.error(f"LIMS interaction failed. Unable to read/write project {self.name}")
         # Writes the job creation sbatch
         if single_sample:
             try:
@@ -648,7 +662,9 @@ def project_job(self, single_sample=False):
 
     def _write_mailjob(self, mailfile: str, report: str, custom_conf: str) -> None:
         """Write the mailjob.sh script that runs `microsalt utils finish` after all jobs complete."""
-        _ep = next(ep for ep in entry_points(group="console_scripts") if ep.value == "microSALT.cli:root")
+        _ep = next(
+            ep for ep in entry_points(group="console_scripts") if ep.value == "microSALT.cli:root"
+        )
         microsalt_bin = Path(sys.executable).parent / _ep.name
         with open(mailfile, "w+") as mb:
             mb.write("#!/usr/bin/env bash\n\n")
@@ -788,9 +804,7 @@ def sample_job(self):
             except Exception:
                 self.logger.error(f"Unable to access LIMS info for sample {self.name}")
         except Exception as e:
-            self.logger.error(
-                f"Unable to create job for sample {self.name}\nSource: {e!s}"
-            )
+            self.logger.error(f"Unable to create job for sample {self.name}\nSource: {e!s}")
             shutil.rmtree(self.finishdir, ignore_errors=True)
             raise
 
@@ -824,9 +838,7 @@ def snp_job(self):
         batchfile = open(self.batchfile, "a+")
         batchfile.close()
 
-        headerline = (
-            f"-A {self.slurm_header.project} -p {self.slurm_header.type} -n 1 -t 24:00:00 -J {self.slurm_header.job_prefix}_{self.name} --qos {self.slurm_header.qos} --output {self.finishdir}/slurm_{self.name}.log"
-        )
+        headerline = f"-A {self.slurm_header.project} -p {self.slurm_header.type} -n 1 -t 24:00:00 -J {self.slurm_header.job_prefix}_{self.name} --qos {self.slurm_header.qos} --output {self.finishdir}/slurm_{self.name}.log"
         outfile = self.get_sbatch()
         bash_cmd = f"sbatch {headerline} {outfile}"
         samproc = subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE)

scripts/compare_typing_reports.py

@@ -139,7 +139,6 @@ def parse_report(path: Path) -> Report:
     # Each sample detail section has an overview table containing "CG Prov ID".
     # We walk all tables and use context to pair MLST/resistance tables with
     # the sample they belong to.
-    current_cg_id: str | None = None
     current_detail: SampleDetail | None = None
 
     for table in soup.find_all("table"):
@@ -258,6 +257,11 @@ def compare_reports(r1: Report, r2: Report) -> int:
         # MLST allele comparison
         d1 = r1.details.get(cg_id)
         d2 = r2.details.get(cg_id)
+        if d1 is None or d2 is None:
+            missing = "A" if d1 is None else "B"
+            sample_diffs.append(
+                f"  {_YELLOW}WARNING: per-sample detail section missing in report {missing}{_RESET}"
+            )
         if d1 and d2:
             loci1 = {m.loci: m.allele for m in d1.mlst}
             loci2 = {m.loci: m.allele for m in d2.mlst}

tests/conftest.py

@@ -1,9 +1,11 @@
 import json
 import logging
 import pathlib
-import pytest
 from importlib.resources import files as resource_files
 
+import pytest
+
+from microSALT import setup_logger
 from microSALT.config import (
     Containers,
     Database,
@@ -16,7 +18,6 @@
     SlurmHeader,
     Threshold,
 )
-from microSALT import setup_logger
 from microSALT.store.database import initialize_database
 from microSALT.store.db_manipulator import DB_Manipulator
 
@@ -65,7 +66,6 @@ def config(tmp_path_factory: pytest.TempPathFactory) -> MicroSALTConfig:
             resistances=str(resistances),
             genomes=str(genomes),
             credentials=str(credentials),
-            adapters="/path/to/trimmomatic/adapters/",
         ),
         database=Database(
             SQLALCHEMY_DATABASE_URI=f"sqlite:///{db_path}",
@@ -78,9 +78,7 @@ def config(tmp_path_factory: pytest.TempPathFactory) -> MicroSALTConfig:
         singularity=Singularity(),
         containers=Containers(),
     )
-    cfg.folders.expec = str(
-        resource_files("microSALT").joinpath("unique_references", "ExPEC.fsa")
-    )
+    cfg.folders.expec = str(resource_files("microSALT").joinpath("unique_references", "ExPEC.fsa"))
     cfg.config_path = str(base / "config.json")
 
     setup_logger(logging_level="INFO")

tests/test_cli.py

@@ -58,7 +58,6 @@ def setup_config(tmp_path: Path) -> MicroSALTConfig:
             resistances=str(base / "resistances"),
             genomes=str(base / "genomes"),
             credentials=str(base / "credentials"),
-            adapters=str(base / "adapters"),
         ),
         database=Database(SQLALCHEMY_DATABASE_URI=f"sqlite:///{base / 'microsalt.db'}"),
         threshold=Threshold(),

tests/test_config.py

@@ -4,13 +4,13 @@
 from pydantic import ValidationError
 
 from microSALT.config import (
-    MicroSALTConfig,
-    load_config,
+    Database,
     Folders,
+    MicroSALTConfig,
     Regex,
     SlurmHeader,
     Threshold,
-    Database,
+    load_config,
 )
 
 CONFIGEXAMPLE = str(pathlib.Path(__file__).parent.parent / "configExample.json")
@@ -53,7 +53,6 @@ def test_folders_fields():
     assert cfg.folders.resistances
     assert cfg.folders.genomes
     assert cfg.folders.credentials
-    assert cfg.folders.adapters
 
 
 def test_expec_path_injected():