performance month is moved to global and local loading (#470)

Author: faridseifi

bulk-up/src/bulk_up/log_to_data.py

@@ -4,7 +4,6 @@
 
 import pandas as pd
 from rdflib import RDF, RDFS, Graph, URIRef
-from scaffold.utils.utils import get_performance_month
 
 module_path = ".."  # run this script from the bulk-up folder. This will add the directory above (which is precision-feedback-pipeline/) to system path to be able to import pipeline modules
 sys.path.append(module_path)
@@ -104,7 +103,8 @@ def generate_response(output_message, input_message):
 
 
 def add_signal_properties(row, output_message, input_message):
-    performance_month = get_performance_month(input_message)
+    performance_month = input_message["performance_month"]
+
     performance_df = pd.DataFrame(
         input_message["Performance_data"][1:],
         columns=input_message["Performance_data"][0],

scaffold/cli.py

@@ -120,29 +120,29 @@ def batch_csv(
         ),
     ] = False,
 ):
-    startup.startup(performance_data_path)
+    startup.startup(
+        performance_data_path=performance_data_path, performance_m=performance_month
+    )
 
     success_count = 0
     failure_count = 0
     for staff_number in (
         startup.performance_data["staff_number"].drop_duplicates().head(max_files)
     ):
         try:
-            context.from_global(staff_number, performance_month)
+            context.from_global(staff_number)
             try:
                 full_message = pipeline()
                 # full_message["message_instance_id"] = input_data["message_instance_id"]
-                full_message["performance_data"] = performance_month
+                full_message["performance_data"] = context.performance_month
             except Exception as e:
                 # e.detail["message_instance_id"] = input_data["message_instance_id"]
                 raise e
             if not stats_only:
                 directory = performance_data_path.parent / "messages"
                 os.makedirs(directory, exist_ok=True)
 
-                new_filename = (
-                    f"Provider_{staff_number} - message for {performance_month}.json"
-                )
+                new_filename = f"Provider_{staff_number} - message for {context.performance_month}.json"
                 output_path = directory / new_filename
 
                 output_path.write_bytes(

scaffold/context.py

@@ -2,7 +2,6 @@
 from rdflib import Graph
 
 from scaffold import startup
-from scaffold.utils.utils import get_performance_month
 
 preferences_dict = {}
 history_dict = {}
@@ -28,7 +27,11 @@ def from_req(req_info):
     except Exception:
         pass
 
-    performance_month = get_performance_month(req_info, performance_df["month"].max())
+    performance_month = startup.performance_month
+    if req_info["performance_month"]:
+        performance_month = req_info["performance_month"]
+    if not performance_month:
+        performance_month = performance_df["month"].max()
 
     staff_number = int(performance_df.at[0, "staff_number"])
 
@@ -48,7 +51,7 @@ def from_req(req_info):
     subject_graph += startup.base_graph
 
 
-def from_global(staff_num, perf_month):
+def from_global(staff_num):
     global \
         preferences_dict, \
         history_dict, \
@@ -58,28 +61,32 @@ def from_global(staff_num, perf_month):
         subject_graph
 
     staff_number = int(staff_num)
-    history_dict = {}
+
+    try:
+        performance_df = startup.performance_data[
+            startup.performance_data["staff_number"] == staff_number
+        ].reset_index(drop=True)
+    except Exception:
+        pass
+
+    performance_month = startup.performance_month
+    if not performance_month:
+        performance_month = performance_df["month"].max()
+
     preferences_dict = {}
-    performance_month = perf_month
     try:
         p = startup.preferences.loc[staff_number, "preferences"]
         preferences_dict = set_preferences(p)
     except Exception:
         return set_preferences({})
 
+    history_dict = {}
     try:
         staff_data = startup.history[startup.history["staff_number"] == staff_number]
         history_dict = staff_data.set_index("month")["history"].to_dict()
     except Exception:
         pass
 
-    try:
-        performance_df = startup.performance_data[
-            startup.performance_data["staff_number"] == staff_number
-        ].reset_index(drop=True)
-    except Exception:
-        pass
-
     subject_graph = Graph()
     subject_graph += startup.base_graph
 

scaffold/startup.py

@@ -27,13 +27,14 @@
     columns=["staff_number", "month", "history"], index=["staff_number"]
 )
 performance_data = pd.DataFrame()
+performance_month = ""
 
 # Set up request session as se, config to handle file URIs with FileAdapter
 se = requests.Session()
 se.mount("file://", FileAdapter())
 
 
-def startup(performance_data_path: pathlib.Path = None):
+def startup(performance_data_path: pathlib.Path = None, performance_m: str = ""):
     ## Log of instance configuration
     logger.debug("Startup configuration for this instance:")
     for attribute in dir(settings):
@@ -48,7 +49,8 @@ def startup(performance_data_path: pathlib.Path = None):
             default_preferences, \
             preferences, \
             history, \
-            performance_data
+            performance_data, \
+            performance_month
 
         mpm = load_mpm()
 
@@ -76,6 +78,12 @@ def startup(performance_data_path: pathlib.Path = None):
         if performance_data_path:
             performance_data = pd.read_csv(performance_data_path, parse_dates=["month"])
 
+        if settings.performance_month:
+            performance_month = settings.settings.performance_month
+
+        if performance_m:
+            performance_month = performance_m
+
     except Exception as e:
         print("Startup aborted, see traceback:")
         raise e

scaffold/utils/settings.py

@@ -54,7 +54,7 @@ def __init__(self):
             "outputs", cast=bool, default=False
         )  # Logging of intermediate files
         self.performance_month = config(
-            "performance_month", default=None
+            "performance_month", cast=str, default=""
         )  # performance_month for instance
         self.use_mi = config("use_mi", cast=bool, default=True)  # use mi
         self.use_preferences = config(

scaffold/utils/utils.py

@@ -143,13 +143,3 @@ def set_logger():
     logger.at_least = (
         lambda lvl: logger.level(lvl).no >= logger.level(settings.log_level).no
     )
-
-
-def get_performance_month(req_info, max_month):
-    performance_month = req_info["performance_month"]
-    if settings.performance_month:
-        performance_month = settings.settings.performance_month
-
-    if not performance_month:
-        performance_month = max_month
-    return performance_month