create db_handler and connect engine

Author: BradleySappington

src/wfi_reference_pipeline/config/config_access.py

@@ -81,7 +81,7 @@ def get_logging_config(config_file="config.yml"):
     Parameters
     ----------
     config_file : str, optional
-        The name of the configuration file to load, by default "crds_submission_config.yml".
+        The name of the configuration file to load, by default "config.yml".
 
     Returns
     -------
@@ -101,7 +101,7 @@ def get_data_files_config(config_file="config.yml"):
     Parameters
     ----------
     config_file : str, optional
-        The name of the configuration file to load, by default "crds_submission_config.yml".
+        The name of the configuration file to load, by default "config.yml".
 
     Returns
     -------
@@ -112,6 +112,24 @@ def get_data_files_config(config_file="config.yml"):
     _validate_config(settings, CONFIG_SCHEMA)
     return settings["data_files"]
 
+def get_db_config(config_file="config.yml"):
+    """Get configuration settings from config.yml for database
+    Validate that the settings are in the correct format before returning
+
+    Parameters
+    ----------
+    config_file : str, optional
+        The name of the configuration file to load, by default config.yml.
+
+    Returns
+    -------
+    dict
+        A dictionary containing the configuration settings.
+    """
+    settings = _get_config(config_file)
+    _validate_config(settings, CONFIG_SCHEMA)
+    return settings["database"]
+
 
 def get_pipelines_config(ref_type, config_file="pipelines_config.yml"):
     """Get configuration settings from pipelines_config.yml for any reference type
@@ -122,7 +140,7 @@ def get_pipelines_config(ref_type, config_file="pipelines_config.yml"):
     ref_type : CONSTANT
         The defined reference type from constants.py
     config_file : str, optional
-        The name of the configuration file to load, by default "crds_submission_config.yml".
+        The name of the configuration file to load, by default "pipelines_config.yml".
 
     Returns
     -------
@@ -152,7 +170,7 @@ def get_quality_control_config(ref_type, detector=None, config_file=None):
     ref_type : CONSTANT
         The defined reference type from constants.py
     config_file : str, optional
-        The name of the configuration file to load, by default "crds_submission_config.yml".
+        The name of the configuration file to load, by default None.
 
     Returns
     -------

src/wfi_reference_pipeline/config/example_config.yml

@@ -17,4 +17,9 @@ data_files:
   # crds_ready_dir: string location to store CRDS ready deliverable reference files
   ingest_dir: "/PATH/TO/DATAFILES/INGEST"
   prep_dir: "/PATH/TO/DATAFILES/PREPPED"
-  crds_ready_dir: "/PATH/TO/DATAFILES/CRDS_READY"
+  crds_ready_dir: "/PATH/TO/DATAFILES/CRDS_READY"
+
+database:
+  use_rtbdb: false
+  sql_server_str: "<SERVER_NAME.stsci.edu>"
+  sql_database_str: "<DATABASE_NAME"

src/wfi_reference_pipeline/pipelines/pipeline.py

@@ -2,8 +2,9 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
 
-from wfi_reference_pipeline.config.config_access import get_data_files_config
+from wfi_reference_pipeline.config.config_access import get_data_files_config, get_db_config
 from wfi_reference_pipeline.constants import REF_TYPE_DARK, WFI_DETECTORS
+from wfi_reference_pipeline.utilities.db_handler import DBHandler
 from wfi_reference_pipeline.utilities.file_handler import FileHandler
 from wfi_reference_pipeline.utilities.logging_functions import configure_logging
 from wfi_reference_pipeline.utilities.quality_control.dark_quality_control import (
@@ -52,6 +53,11 @@ def __init__(self, ref_type, detector):
         self.file_handler = FileHandler(
             self.ref_type, self.prep_path, self.pipeline_out_path
         )
+        self._db_config = get_db_config()
+        if self._db_config["use_rtbdb"]:
+            self.db_handler = DBHandler(self.ref_type, self._db_config["sql_server_str"], self._db_config["sql_database_str"])
+        else:
+            self.db_handler = None
 
     @abstractmethod
     def select_uncal_files(self):

src/wfi_reference_pipeline/utilities/db_handler.py

@@ -0,0 +1,60 @@
+import logging
+
+from rtb_db.utilities.login import connect_server
+from rtb_db.utilities.table_tools import ensure_connection_is_engine, table_names
+
+from wfi_reference_pipeline import constants
+
+
+class DBHandler:
+    """
+    Utility class to safely handle access to the rtbdb
+    This class is designed to ONLY be initialized in the Pipeline base class
+    This class should be initialized as part of a base pipeline initialization procedure AFTER config values have been read and stored
+    """
+
+    def __init__(self, ref_type, sql_server_str, sql_database_str):
+        if ref_type not in constants.WFI_REF_TYPES:
+            raise ValueError(
+                f"ref_type {ref_type} not valid, must be: {constants.WFI_REF_TYPES}"
+            )
+
+        self.ref_type = ref_type
+        self.sql_engine = None
+        self.sql_server_str = sql_server_str
+        self.sql_database_str = sql_database_str
+        self.connect()
+
+    def connect(self):
+        """Confirm if the user has access to the desired database. If so, establish a SQLalchemy connection.
+        Saves SQLalchemy engine to self.sql_engine and saves the db availability boolean to self.use_db.
+
+        Returns
+        -------
+        self.use_db
+            boolean to trigger if montor should attempt to use the database
+
+        """
+        try:
+            engine = connect_server(driver="FreeTDS",
+                                    server=self.sql_server_str,
+                                    port=1433,
+                                    tds_version=7.1,
+                                    database=self.sql_database_str)
+
+            engine = ensure_connection_is_engine(engine)
+            self.sql_engine = engine
+
+            # TODO - SAPP -> GET TABLE NAMES
+            
+            # ensure metrics table exists on the active database server
+            # assert set(self.sql_table_names).issubset(table_names(self.sql_engine)) # TODO figure this out, not currently storing sql_table_names
+
+
+        ### TODO: This is not actually catching if the engine exists or not...
+        ### add raise exception to rtb-db login.connect_sqlalchemy
+        except Exception as err:
+            logging.warning('Unable to connect to RTB database.')
+            logging.error(f"Received {err}")
+            return
+

src/wfi_reference_pipeline/utilities/schemas.py

@@ -25,6 +25,15 @@
             },
             "required": ["ingest_dir", "prep_dir", "crds_ready_dir"],
         },
+        "database": {
+            "type": "object",
+            "properties": {
+                "use_rtbdb": {"type": "boolean"},
+                "sql_server_str": {"type": "string"},
+                "sql_database_str": {"type": "string"},
+            },
+            "required": ["use_rtbdb", "sql_server_str", "sql_database_str"],
+        },
     },
     # List which entries are needed (all of them)
     "required": ["logging", "data_files"],