Refactor to use 'microsalt' consistently across the codebase and improve database initialization

Author: karlnyr

.github/pull_request_template.md

@@ -25,10 +25,10 @@ _Test routine to verify the stability of the PR:_
 - _`conda activate S_microSALT`_
 - _(SITUATIONAL) `export MICROSALT_CONFIG=/home/proj/dropbox/microSALT.json`_
 - _Select a relevant subset of the following:_
-- _`microSALT analyse project MIC3109`_
-- _`microSALT analyse project MIC4107`_
-- _`microSALT analyse project MIC4109`_
-- _`microSALT analyse project ACC5551`_
+- _`microsalt analyse project MIC3109`_
+- _`microsalt analyse project MIC4107`_
+- _`microsalt analyse project MIC4109`_
+- _`microsalt analyse project ACC5551`_
 
 _Verify that the results for projects MIC3109, MIC4107, MIC4109 & ACC5551 are consistent with the results attached to AMSystem doc 1490, Microbial_WGS.xlsx_
 

.github/workflows/run_tests.yml

@@ -4,7 +4,7 @@ on: [push]
 
 
 env:
-  CONDA_PREFIX: /home/runner/work/microSALT/microSALT/tests
+  CONDA_PREFIX: /home/runner/work/microSALT/microsalt/tests
   COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
 
 jobs:
@@ -35,7 +35,6 @@ jobs:
 
     - name: Install Dependencies
       run: |
-        mkdir ~/.microSALT && cp configExample.json ~/.microSALT/config.json
         poetry install --no-interaction --all-extras
 
     - name: Lint with flake8

microSALT/server/app.py

@@ -8,8 +8,7 @@
 
 from flask import Flask
 from microsalt.server.views import bp
-from microsalt.store.database import initialize_database
-
+from microsalt.store.orm_models import db
 logger = logging.getLogger("main_logger")
 
 APP: Flask = None
@@ -18,10 +17,10 @@ def initialize_app(config: dict) -> None:
     global APP
     APP = Flask(__name__, template_folder="templates")
     APP.config.setdefault("SQLALCHEMY_DATABASE_URI", "sqlite:///:memory:")
-    APP.config.setdefault("SQLALCHEMY_BINDS", None)
+    APP.config.setdefault("SQLALCHEMY_BINDS", {})
     APP.config.setdefault("SQLALCHEMY_TRACK_MODIFICATIONS", False)
-    initialize_database(config)
     _setup_app_from_config(APP, config)
+    db.init_app(APP)
     APP.register_blueprint(bp)
 
 def get_app() -> Flask:
@@ -40,6 +39,8 @@ def create_path(path: str, logger: logging.Logger):
 def handle_config_entry(entry, value, logger):
 
     if isinstance(value, str) and "/" in value and entry not in ["genologics"]:
+        if os.path.abspath(value) == "/path":
+            logger.error(f"Path for {entry} is not set.")
         if not value.startswith("/"):
             sys.exit(-1)
         create_path(os.path.abspath(value), logger)

microSALT/server/views.py

@@ -13,9 +13,22 @@
 log = logging.getLogger("werkzeug")
 log.setLevel(logging.CRITICAL)
 
+def inspect_db_schema(session):
+    """Inspect the database schema and log the tables and columns."""
+    
+    from sqlalchemy import inspect
+    inspector = inspect(session.bind)
+    tables = inspector.get_table_names()
+    schema_info = {}
+    for table in tables:
+        columns = inspector.get_columns(table)
+        schema_info[table] = [column['name'] for column in columns]
+    return schema_info
+
 @bp.route("/")
 def start_page():
     session = get_session()
+    schema_info = inspect_db_schema(session)
     projects = session.query(Projects).all()
     return render_template("start_page.html", projects=projects)
 

microSALT/store/database.py

@@ -1,4 +1,4 @@
-from sqlalchemy import create_engine, MetaData
+from sqlalchemy import SingletonThreadPool, create_engine, MetaData
 from sqlalchemy.engine.base import Engine
 from sqlalchemy.orm import Session, scoped_session, sessionmaker
 
@@ -9,7 +9,7 @@
 def initialize_database(config: dict):
     global ENGINE, SESSION
     ENGINE = create_engine(
-        config["database"]["SQLALCHEMY_DATABASE_URI"], connect_args={"check_same_thread": False, "timeout": 15}
+        config["database"]["SQLALCHEMY_DATABASE_URI"], poolclass=SingletonThreadPool, connect_args={"check_same_thread": False, "timeout": 15}
     )
     session_factory = sessionmaker(bind=ENGINE)
     SESSION = scoped_session(session_factory)

microSALT/store/db_manipulator.py

@@ -37,7 +37,7 @@ def __init__(self, config, log):
         self.logger = log
         self.engine = get_engine()
         self.session = get_session()
-        self.metadata = MetaData(self.engine)
+        self.metadata = MetaData()
         self.profiles = Profiles(self.metadata, self.config, self.logger).tables
         self.novel = Novel(self.metadata, self.config, self.logger).tables
         # Turns off pymysql deprecation warnings until they can update their code
@@ -139,10 +139,8 @@ def add_rec(self, data_dict: Dict[str, str], tablename: str, force=False):
                         tablename
                     )
                 )
-            pk_values = list()
-            for item in pk_list:
-                pk_values.append(data_dict[item])
-            existing = self.session.query(table).get(pk_values)
+            pk_values = {item: data_dict[item] for item in pk_list}
+            existing = self.session.query(table).filter_by(**pk_values).first()
             # Add record
             if not existing or force:
                 newobj = table()

microSALT/store/models.py

@@ -4,11 +4,10 @@
 #!/usr/bin/env python
 
 import os
-from sqlalchemy import *
-
+from sqlalchemy import MetaData
 
 class Profiles:
-    def __init__(self, metadata, config, log):
+    def __init__(self, metadata: MetaData, config: dict, log):
         self.tables = dict()
         self.metadata = metadata
         self.config = config
@@ -24,7 +23,7 @@ def __init__(self, metadata, config, log):
                 )
             )
 
-    def add_table(self, file):
+    def add_table(self, file: str):
         try:
             with open(
                 "{}/{}".format(self.config["folders"]["profiles"], file), "r"
@@ -55,7 +54,7 @@ def add_table(self, file):
 
 
 class Novel:
-    def __init__(self, metadata, config, log):
+    def __init__(self, metadata: MetaData, config: dict, log):
         self.tables = dict()
         self.metadata = metadata
         self.config = config
@@ -71,7 +70,7 @@ def __init__(self, metadata, config, log):
                 )
             )
 
-    def add_table(self, file):
+    def add_table(self, file: str):
         try:
             with open(
                 "{}/{}".format(self.config["folders"]["profiles"], file), "r"
@@ -98,4 +97,4 @@ def add_table(self, file):
                 p = eval(header)
                 self.tables[file] = p
         except Exception as e:
-            self.logger.error("Unable to open profile file {}".format(file))
+            self.logger.error("Unable to open profile file {}".format(file))

microSALT/store/orm_models.py

@@ -5,7 +5,10 @@
 
 from flask_sqlalchemy import SQLAlchemy
 from sqlalchemy import *
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import relationship, DeclarativeBase
+
+class Base(DeclarativeBase):
+    pass
 
 db = SQLAlchemy()
 
@@ -56,9 +59,7 @@ class Seq_types(db.Model):
     __tablename__ = "seq_types"
     samples = relationship("Samples", back_populates="seq_types")
 
-    CG_ID_sample = db.Column(
-        db.String(15), ForeignKey("samples.CG_ID_sample"), primary_key=True
-    )
+    CG_ID_sample = db.Column(db.String(15), ForeignKey("samples.CG_ID_sample"), primary_key=True)
     loci = db.Column(db.String(10), primary_key=True)
     allele = db.Column(db.SmallInteger)
     contig_name = db.Column(db.String(20), primary_key=True)
@@ -78,9 +79,7 @@ class Resistances(db.Model):
     __tablename__ = "resistances"
     samples = relationship("Samples", back_populates="resistances")
 
-    CG_ID_sample = db.Column(
-        db.String(15), ForeignKey("samples.CG_ID_sample"), primary_key=True
-    )
+    CG_ID_sample = db.Column(db.String(15), ForeignKey("samples.CG_ID_sample"), primary_key=True)
     gene = db.Column(db.String(50), primary_key=True)
     instance = db.Column(db.String(30), primary_key=True)
     contig_name = db.Column(db.String(20), primary_key=True)
@@ -101,9 +100,7 @@ class Expacs(db.Model):
     __tablename__ = "expacs"
     samples = relationship("Samples", back_populates="expacs")
 
-    CG_ID_sample = db.Column(
-        db.String(15), ForeignKey("samples.CG_ID_sample"), primary_key=True
-    )
+    CG_ID_sample = db.Column(db.String(15), ForeignKey("samples.CG_ID_sample"), primary_key=True)
     gene = db.Column(db.String(50), primary_key=True)
     instance = db.Column(db.String(30), primary_key=True)
     contig_name = db.Column(db.String(20), primary_key=True)
@@ -143,9 +140,7 @@ class Reports(db.Model):
     __tablename__ = "reports"
     projects = relationship("Projects", back_populates="reports")
 
-    CG_ID_project = db.Column(
-        db.String(15), ForeignKey("projects.CG_ID_project"), primary_key=True
-    )
+    CG_ID_project = db.Column(db.String(15), ForeignKey("projects.CG_ID_project"), primary_key=True)
     steps_aggregate = db.Column(db.String(100))
     date = db.Column(db.DateTime)
     version = db.Column(db.Integer, default=1, primary_key=True)

microSALT/utils/reporter.py

@@ -663,7 +663,7 @@ def mail(self):
 
     def run_flask(self):
         """Target function for running Flask"""
-        self.exit_flag.clear()  # Reset the flag when starting
+        self.exit_flag.clear()
         while not self.exit_flag.is_set():
             try:
                 self.app.run(host="127.0.0.1", port=5001, debug=False, use_reloader=False)
@@ -680,18 +680,18 @@ def start_web(self):
             self.logger.info("Webserver already running.")
             return
 
-        self.exit_flag.clear()  # Ensure flag is reset before starting
+        self.exit_flag.clear()
         self.server = threading.Thread(target=self.run_flask, daemon=True)
         self.server.start()
         self.logger.info("Started webserver on http://127.0.0.1:5000/")
-        time.sleep(0.15)  # Allow time for server to initialize
+        time.sleep(0.15)
 
     def kill_flask(self):
         """Stops the Flask web server."""
         if self.server and self.server.is_alive():
             self.logger.info("Closing webserver on http://127.0.0.1:5000/")
-            self.exit_flag.set()  # Signal the Flask loop to exit
-            self.server.join(timeout=2)  # Wait for the thread to exit
+            self.exit_flag.set()
+            self.server.join(timeout=2)
             self.server = None
 
     def restart_web(self):

tests/conftest.py

@@ -1,17 +1,18 @@
 import json
 import pytest
-import tempfile
+import os
 import shutil
+import tempfile
 
 from flask import Flask
 from pathlib import Path
 from typing import Generator
 
-from microSALT.cli import initialize_logger
-from microSALT.server.app import initialize_app, get_app
-from microSALT.store.database import initialize_database, drop_all_tables
-from microSALT.store.db_manipulator import DB_Manipulator
-from microSALT.utils.reporter import Reporter
+from microsalt.cli import initialize_logger
+from microsalt.server.app import initialize_app, get_app
+from microsalt.store.database import initialize_database, drop_all_tables
+from microsalt.store.db_manipulator import DB_Manipulator
+from microsalt.utils.reporter import Reporter
 
 
 def unpack_db_json(fixture_dir, filename):
@@ -25,6 +26,7 @@ def unpack_db_json(fixture_dir, filename):
 def app(config) -> Generator[Flask, None, None]:
     initialize_app(config)
     yield get_app()
+    
 
 
 @pytest.fixture(scope="session")
@@ -53,6 +55,11 @@ def temp_dirs() -> Generator[dict, None, None]:
         "expec": tempfile.mkdtemp(),
     }
     yield temp_dirs
+    for path in temp_dirs.values():
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+        elif os.path.isfile(path):
+            os.remove(path)
 
 
 @pytest.fixture(scope="session")
@@ -253,18 +260,15 @@ def exp_config() -> dict:
     }
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
 def reporter(
     app: Flask, config, dbm, logger, sampleinfo_sample, temp_dirs
 ) -> Generator[Reporter, None, None]:
-    reporter = Reporter(
+    yield Reporter(
         app=app,
         config=config,
         dbm=dbm,
         log=logger,
         output=temp_dirs["reports"],
         sampleinfo=sampleinfo_sample,
     )
-    reporter.start_web()
-    yield reporter
-    reporter.kill_flask()