Merge pull request #242 from Clinical-Genomics/crud-separation

Crud separation

Author: karlnyr

microSALT/exc/exceptions.py

@@ -7,4 +7,11 @@ def __init__(self, message: str = ""):
 
 class RefUpdateLockError(MicroSALTError):
     """Raised when an operation is attempted while a reference update is in progress."""
+
+    pass
+
+
+class JobCreationError(MicroSALTError):
+    """Raised when there is an error creating a job."""
+
     pass

microSALT/store/db_manipulator.py

@@ -28,7 +28,7 @@ class ProfileTable:
     """
 
     def __init__(self, prefix: str, metadata, profiles_path: str, log):
-        self.tables: dict[str, Table] = dict()
+        self.tables: dict[str, Table] = {}
         self.prefix = prefix
         self.metadata = metadata
         self.profiles_path = profiles_path

microSALT/store/models.py

@@ -127,7 +127,6 @@ class Projects(Base):
 
     CG_ID_project = Column(String(32), primary_key=True, nullable=False)
     Customer_ID_project = Column(String(32))
-    date_ordered = Column(DateTime)
     Customer_ID = Column(String(32))
 
 

microSALT/store/orm_models.py

@@ -2,7 +2,6 @@
 By: Isak Sylvin, @sylvinite"""
 
 #!/usr/bin/env python
-
 import glob
 import gzip
 import json
@@ -29,7 +28,9 @@
     SlurmHeader,
     Threshold,
 )
+from microSALT.exc.exceptions import JobCreationError
 from microSALT.store.db_manipulator import DB_Manipulator
+from microSALT.store.orm_models import Projects
 from microSALT.utils.referencer import Referencer
 
 
@@ -525,57 +526,68 @@ def create_snpsection(self):
 
     def create_collection(self):
         """Creates collection entry in database"""
-        if self.db_pusher.exists("Collections", {"ID_collection": self.name}):
-            self.db_pusher.purge_rec(name=self.name, type="Collections")
+        if self.db_pusher.read_exists("Collections", {"ID_collection": self.name}):
+            self.db_pusher.delete_collection(self.name)
             for sample in self.pool:
                 self.db_pusher.add_rec(
                     {"ID_collection": self.name, "CG_ID_sample": sample}, "Collections"
                 )
 
-        addedprojs = list()
+        addedprojs = []
         for sample in self.pool:
-            proj = re.search(r"(\w+)A(?:\w+)", sample).group(1)
-            if proj not in addedprojs:
-                self.create_project(proj)
-                addedprojs.append(proj)
+            lims_project = re.search(r"(\w+)A(?:\w+)", sample).group(1)
+            projects: Projects | None = self.db_pusher.read_report(lims_project)
+            if not projects and lims_project not in addedprojs:
+                self.create_project(lims_project)
+                addedprojs.append(lims_project)
 
-    def create_project(self, name):
+    def create_project(self, name: str):
         """Creates project in database"""
-        proj_col = dict()
-        proj_col["CG_ID_project"] = name
-        proj_col["Customer_ID_project"] = self.sample.get("Customer_ID_project")
-        proj_col["Customer_ID"] = self.sample.get("Customer_ID")
-        self.db_pusher.add_rec(proj_col, "Projects")
-        self.db_pusher.upd_rec({"CG_ID_project": name}, "Projects", proj_col)
-
-    def create_sample(self, name):
+        if not self.sample:
+            raise JobCreationError(
+                "No sample information provided. Cannot create project in database."
+            )
+        project_data: dict[str, str] = {
+            "CG_ID_project": name,
+            "Customer_ID_project": self.sample["Customer_ID_project"],
+            "Customer_ID": self.sample["Customer_ID"],
+        }
+        self.db_pusher.add_to_session(self.db_pusher.add_project(**project_data))
+        self.db_pusher.commit_session()
+
+    def create_sample(self):
         """Creates sample in database"""
+
         try:
-            sample_col = self.db_pusher.get_columns("Samples")
-            sample_col["CG_ID_sample"] = self.sample.get("CG_ID_sample")
-            sample_col["CG_ID_project"] = self.sample.get("CG_ID_project")
-            sample_col["Customer_ID_sample"] = self.sample.get("Customer_ID_sample")
-            sample_col["reference_genome"] = self.sample.get("reference")
-            sample_col["reference_length"] = self.sample.get("reference_length")
-            sample_col["date_analysis"] = self.dt
-            sample_col["organism"] = self.sample.get("organism")
-            sample_col["application_tag"] = self.sample.get("application_tag")
-            sample_col["priority"] = self.sample.get("priority")
-            sample_col["date_arrival"] = datetime.strptime(
-                self.sample.get("date_arrival"), "%Y-%m-%d %H:%M:%S"
-            )
-            sample_col["date_sequencing"] = datetime.strptime(
-                self.sample.get("date_sequencing"), "%Y-%m-%d %H:%M:%S"
-            )
-            sample_col["date_libprep"] = datetime.strptime(
-                self.sample.get("date_libprep"), "%Y-%m-%d %H:%M:%S"
-            )
-            sample_col["method_libprep"] = self.sample.get("method_libprep")
-            sample_col["method_sequencing"] = self.sample.get("method_sequencing")
-            # self.db_pusher.purge_rec(sample_col['CG_ID_sample'], 'sample')
-            self.db_pusher.add_rec(sample_col, "Samples")
-        except Exception:
-            self.logger.error(f"Unable to add sample {self.name} to database")
+            if not self.sample:
+                raise JobCreationError(
+                    "No sample information provided. Cannot create sample in database."
+                )
+            sample_data: dict[str, str | datetime] = {
+                "CG_ID_sample": self.sample["CG_ID_sample"],
+                "CG_ID_project": self.sample["CG_ID_project"],
+                "Customer_ID_sample": self.sample["Customer_ID_sample"],
+                "reference_genome": self.sample["reference"],
+                "reference_length": self.sample["reference_length"],
+                "date_analysis": self.dt,
+                "organism": self.sample["organism"],
+                "application_tag": self.sample["application_tag"],
+                "priority": self.sample["priority"],
+                "date_arrival": datetime.strptime(self.sample["date_arrival"], "%Y-%m-%d %H:%M:%S"),
+                "date_sequencing": datetime.strptime(
+                    self.sample["date_sequencing"], "%Y-%m-%d %H:%M:%S"
+                ),
+                "date_libprep": datetime.strptime(self.sample["date_libprep"], "%Y-%m-%d %H:%M:%S"),
+                "method_libprep": self.sample["method_libprep"],
+                "method_sequencing": self.sample["method_sequencing"],
+            }
+            self.db_pusher.add_to_session(self.db_pusher.add_sample(**sample_data))
+            self.db_pusher.commit_session()
+        except JobCreationError as e:
+            self.logger.error(f"Unable to add sample {self.name} to database: {e}")
+        except KeyError as e:
+            self.logger.error(f"Missing key {e} in sample information for sample {self.name}")
+            raise JobCreationError(f"Missing key {e} in sample information for sample {self.name}")
 
     def project_job(self, single_sample=False):
         if self.dry:
@@ -800,7 +812,7 @@ def sample_job(self):
             except Exception:
                 raise
             try:
-                self.create_sample(self.name)
+                self.create_sample()
             except Exception:
                 self.logger.error(f"Unable to access LIMS info for sample {self.name}")
         except Exception as e:

microSALT/utils/job_creator.py

@@ -179,7 +179,7 @@ def _find_entry_for_organism(
 
     def _should_update_external(self, organ: str, entry: ET.Element) -> dict | bool:
         """Determine if the external data for an organism should be updated."""
-        currver = self.db_access.get_version(f"profile_{organ}")
+        currver = self.db_access.read_version(f"profile_{organ}")
         st_link = entry.find("./mlst/database/profiles/url").text
         service = get_service_by_url(st_link)
         if service == "pasteur":
@@ -320,14 +320,14 @@ def resync(self, type="", sample="", ignore=False):
         """Manipulates samples that have an internal ST that differs from pubMLST ST"""
         if type == "list":
             # Add single sample support later
-            self.db_access.list_unresolved()
+            self.db_access.read_unresolved()
         elif type == "overwrite":
             if ignore:
-                self.db_access.rm_novel(sample=sample)
+                self.db_access.set_novel_ignored(sample=sample)
             else:
-                self.db_access.sync_novel(overwrite=True, sample=sample)
+                self.db_access.set_novel_st(overwrite=True, sample=sample)
         else:
-            self.db_access.sync_novel(overwrite=False, sample=sample)
+            self.db_access.set_novel_st(overwrite=False, sample=sample)
 
     def fetch_resistances(self, force=False):
         cwd = os.getcwd()
@@ -571,7 +571,7 @@ def download_pubmlst(self, organism, subtype_href, force=False):
         try:
             # Pull version
             extver = self.external_version(organism, subtype_href)
-            currver = self.db_access.get_version(f"profile_{organism}")
+            currver = self.db_access.read_version(f"profile_{organism}")
             if int(extver.replace("-", "")) <= int(currver.replace("-", "")) and not force:
                 self.logger.info(
                     f"Profile for {organism.replace('_', ' ').capitalize()} already at the latest version."
@@ -649,7 +649,7 @@ def fetch_pubmlst(self, force=False):
                         seqdef_url[name] = subtype["href"]
 
         for key, val in seqdef_url.items():
-            internal_ver = self.db_access.get_version(f"profile_{key}")
+            internal_ver = self.db_access.read_version(f"profile_{key}")
             external_ver = self.external_version(key, val)
 
             if (internal_ver < external_ver) or force:

microSALT/utils/referencer.py

@@ -104,7 +104,7 @@ def report(self, type="default", customer="all"):
                     copyfile(k, v)
 
     def gen_version(self, name):
-        self.db_pusher.get_report(name)
+        self.db_pusher.read_report(name)
         self.db_pusher.set_report(name)
 
     def gen_STtracker(self, customer="all", silent=False):
@@ -124,7 +124,7 @@ def gen_STtracker(self, customer="all", silent=False):
 
     def gen_qc(self, silent=False):
         try:
-            last_version = self.db_pusher.get_report(self.name).version
+            last_version = self.db_pusher.read_report(self.name).version
         except Exception:
             self.logger.error(f"Project {self.name} does not exist")
             sys.exit(-1)
@@ -147,7 +147,7 @@ def gen_qc(self, silent=False):
 
     def gen_typing(self, silent=False):
         try:
-            last_version = self.db_pusher.get_report(self.name).version
+            last_version = self.db_pusher.read_report(self.name).version
         except Exception:
             self.logger.error(f"Project {self.name} does not exist")
             sys.exit(-1)
@@ -269,7 +269,7 @@ def gen_motif(self, motif="resistance", silent=False):
     def gen_delivery(self):
         deliv = dict()
         deliv["files"] = list()
-        last_version = self.db_pusher.get_report(self.name).version
+        last_version = self.db_pusher.read_report(self.name).version
         output = f"{self.folders.reports}/deliverables/{self.sample.get('Customer_ID_project')}_deliverables.yaml"
         local = f"{self.output}/{self.sample.get('Customer_ID_project')}_deliverables.yaml"