453 clean up main (#454)

* remove excessive timing and logging statements from main. some formatting

* api separated and simple cli for one json input added

* restructuring the app to meet Python package structure to be able to build and install the app and use the cli script

Author: faridseifi

.env.remote

@@ -1,7 +1,7 @@
 # required knowledgebase paths
-mpm=https://raw.githubusercontent.com/Display-Lab/knowledge-base/v1.4/prioritization_algorithms/motivational_potential_model.csv
-preferences=https://raw.githubusercontent.com/Display-Lab/knowledge-base/v1.4/preferences.json
-manifest=https://github.com/Display-Lab/knowledge-base/releases/download/v1.4/mpog_manifest.yaml
+mpm=https://raw.githubusercontent.com/Display-Lab/knowledge-base/1.7/prioritization_algorithms/motivational_potential_model.csv
+preferences=https://raw.githubusercontent.com/Display-Lab/knowledge-base/1.7/preferences.json
+manifest=https://raw.githubusercontent.com/Display-Lab/knowledge-base/refs/tags/1.7/mpog_manifest.yaml
 
 # defaults
 # log_level=WARNING

.gitignore

@@ -21,7 +21,7 @@ outputs/*
 python/.vscode/settings.json
 **/dist/
 **/__pycache__/
-.python-version
+
 ES.json
 venv/
 .venv/

.python-version

@@ -0,0 +1 @@
+3.13

Dockerfile

@@ -48,5 +48,5 @@ ENV PATH="/code/app/venv/bin:$PATH"
 
 # Start up main app
 EXPOSE 8080
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"]
+CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8080"]
 

Procfile

@@ -1 +1 @@
-web: gunicorn -w 1 -k uvicorn.workers.UvicornWorker main:app
+web: gunicorn -w 1 -k uvicorn.workers.UvicornWorker scaffold.api:app

README.md

@@ -79,10 +79,10 @@ manifest=file:///Users/bob/knowledge-base/mpog_local_manifest.yaml
 ...
 ```
 
-Run SCAFFOLD
+Run SCAFFOLD API
 
 ```zsh
-ENV_PATH=.env.local uvicorn main:app
+ENV_PATH=.env.local uvicorn scaffold.api:app
 # Expect to see a server start message like this "INFO:     Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit)"
 ```
 
@@ -92,6 +92,19 @@ You can use Postman or your favorite tool to send a message and check the result
 curl --data "@tests/test_cases/input_message.json" http://localhost:8000/createprecisionfeedback/
 ```
 
+Run SCAFFOLD CLI
+First install the python app. Then use the following command to run the pipeline on one input file
+
+```zsh
+ENV_PATH=/user/.../dev.env pipeline single '/path/to/input/file.json'
+```
+
+or use the following command to run the pipeline api
+
+```zsh
+ENV_PATH=/user/.../dev.env pipeline web
+```
+
 ## Environment variables
 
 ### Knowledge base settings
@@ -154,13 +167,13 @@ If the key is a relative path, it must end with a '/'. In that case the key is g
 ### examples
 
 ```zsh
- ENV_PATH=/user/.../dev.env log_level=INFO use_preferences=True use_coachiness=True use_mi=True generate_image=False uvicorn main:app --workers=5
+ ENV_PATH=/user/.../dev.env log_level=INFO use_preferences=True use_coachiness=True use_mi=True generate_image=False uvicorn api:app --workers=5
 ```
 
 
 for windows:
 ```psh
-$env:ENV_PATH=/user/.../dev.env; $env:log_level="INFO"; $env:use_preferences="True"; $env:use_coachiness="True"; $env:use_mi="True"; $env:generate_image="False"; uvicorn main:app --workers=5
+$env:ENV_PATH=/user/.../dev.env; $env:log_level="INFO"; $env:use_preferences="True"; $env:use_coachiness="True"; $env:use_mi="True"; $env:generate_image="False"; uvicorn api:app --workers=5
 ```
 
 > :point_right: `uvicorn` can be run with multiple workers. This is useful when testing with a client that can send multiple requests.

bulk-up/src/bulk_up/dataset_to_inputs.py

@@ -1,103 +1,131 @@
+import copy
 import json
 import os
-
-import pandas as pd
 import uuid
-import copy
 from datetime import datetime
+
+import pandas as pd
 from dateutil.relativedelta import relativedelta
 
 OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "outputs")
-INPUT_DIR = os.environ.get("INPUT_DIR", "/home/faridsei/dev/test/OBI/obi_cat2_dystocia_compliance.xlsx")
+INPUT_DIR = os.environ.get(
+    "INPUT_DIR", "/home/faridsei/dev/test/OBI/obi_cat2_dystocia_compliance.xlsx"
+)
 PERFORMANCE_MONTH = os.environ.get("PERFORMANCE_MONTH", None)
-QUARTERLY_DATA=os.environ.get("QUARTERLY_DATA", False)
+QUARTERLY_DATA = os.environ.get("QUARTERLY_DATA", False)
 INPUT_TEMPLATE = {
-  "@context": {
-    "@vocab": "http://schema.org/",
-    "slowmo": "http://example.com/slowmo#",
-    "csvw": "http://www.w3.org/ns/csvw#",
-    "dc": "http://purl.org/dc/terms/",
-    "psdo": "http://purl.obolibrary.org/obo/",
-    "slowmo:Measure": "http://example.com/slowmo#Measure",
-    "slowmo:IsAboutPerformer": "http://example.com/slowmo#IsAboutPerformer",
-    "slowmo:ColumnUse": "http://example.com/slowmo#ColumnUse",
-    "slowmo:IsAboutTemplate": "http://example.com/slowmo#IsAboutTemplate",
-    "slowmo:spek": "http://example.com/slowmo#spek",
-    "slowmo:IsAboutCausalPathway": "http://example.com/slowmo#IsAboutCausalPathway",
-    "slowmo:IsAboutOrganization": "http://example.com/slowmo#IsAboutOrganization",
-    "slowmo:IsAboutMeasure": "http://example.com/slowmo#IsAboutMeasure",
-    "slowmo:InputTable": "http://example.com/slowmo#InputTable",
-    "slowmo:WithComparator": "http://example.com/slowmo#WithComparator",
-    "has_part": "http://purl.obolibrary.org/obo/bfo#BFO_0000051",
-    "has_disposition": "http://purl.obolibrary.org/obo/RO_0000091"
-  },
-  "message_instance_id": "",
-  "performance_month":  "",
-  "Performance_data": [
-    [ "staff_number", "measure", "month", "passed_count", "flagged_count", "denominator", "peer_average_comparator", "peer_75th_percentile_benchmark", "peer_90th_percentile_benchmark", "MPOG_goal" ],
- 
-  ],
-  "History": {
-    
-  },
-  "Preferences": {}
+    "@context": {
+        "@vocab": "http://schema.org/",
+        "slowmo": "http://example.com/slowmo#",
+        "csvw": "http://www.w3.org/ns/csvw#",
+        "dc": "http://purl.org/dc/terms/",
+        "psdo": "http://purl.obolibrary.org/obo/",
+        "slowmo:Measure": "http://example.com/slowmo#Measure",
+        "slowmo:IsAboutPerformer": "http://example.com/slowmo#IsAboutPerformer",
+        "slowmo:ColumnUse": "http://example.com/slowmo#ColumnUse",
+        "slowmo:IsAboutTemplate": "http://example.com/slowmo#IsAboutTemplate",
+        "slowmo:spek": "http://example.com/slowmo#spek",
+        "slowmo:IsAboutCausalPathway": "http://example.com/slowmo#IsAboutCausalPathway",
+        "slowmo:IsAboutOrganization": "http://example.com/slowmo#IsAboutOrganization",
+        "slowmo:IsAboutMeasure": "http://example.com/slowmo#IsAboutMeasure",
+        "slowmo:InputTable": "http://example.com/slowmo#InputTable",
+        "slowmo:WithComparator": "http://example.com/slowmo#WithComparator",
+        "has_part": "http://purl.obolibrary.org/obo/bfo#BFO_0000051",
+        "has_disposition": "http://purl.obolibrary.org/obo/RO_0000091",
+    },
+    "message_instance_id": "",
+    "performance_month": "",
+    "Performance_data": [
+        [
+            "staff_number",
+            "measure",
+            "month",
+            "passed_count",
+            "flagged_count",
+            "denominator",
+            "peer_average_comparator",
+            "peer_75th_percentile_benchmark",
+            "peer_90th_percentile_benchmark",
+            "MPOG_goal",
+        ],
+    ],
+    "History": {},
+    "Preferences": {},
 }
 
-measure_name_to_id={
+measure_name_to_id = {
     "Cat II Compliance - 12 month rolling average": "CATII12",
     "Cat II Compliance - Monthly": "CATII",
     "Dystocia Compliance - 12 month rolling average": "DC12",
-    "Dystocia Compliance - Monthly": "DC"
-    
+    "Dystocia Compliance - Monthly": "DC",
 }
 
-sheet_name = "obi_cat2_dystocia_compliance" #"Sheet1"  # Change this to the name of the sheet in your .xlsx file
+sheet_name = "obi_cat2_dystocia_compliance"  # "Sheet1"  # Change this to the name of the sheet in your .xlsx file
 df = pd.read_excel(INPUT_DIR, sheet_name=sheet_name, engine="openpyxl")
-df['Time interval'] = pd.to_datetime(df['Time interval']).dt.strftime('%Y-%m-%d')
+df["Time interval"] = pd.to_datetime(df["Time interval"]).dt.strftime("%Y-%m-%d")
 
-unique_site_ids = df['site_id'].unique()
+unique_site_ids = df["site_id"].unique()
 if not PERFORMANCE_MONTH:
-  PERFORMANCE_MONTH = df['Time interval'].max()
-site_id= None
+    PERFORMANCE_MONTH = df["Time interval"].max()
+site_id = None
 for site_id in unique_site_ids:
-    site_rows = df[df['site_id'] == site_id]
+    site_rows = df[df["site_id"] == site_id]
     input_file = copy.deepcopy(INPUT_TEMPLATE)
     measure = ""
     Numerator = 0
-    Denominator = 0 
+    Denominator = 0
     for _, row in site_rows.iterrows():
         Numerator += row["Numerator"]
         Denominator += row["Denominator"]
         time_interval = row["Time interval"]
         if QUARTERLY_DATA:
-          
-          date1 = datetime.strptime(row["Time interval"], '%Y-%m-%d')
-          date2 = datetime.strptime(PERFORMANCE_MONTH, '%Y-%m-%d')  
-          year_diff = date2.year - date1.year
-          month_diff = date2.month - date1.month
+            date1 = datetime.strptime(row["Time interval"], "%Y-%m-%d")
+            date2 = datetime.strptime(PERFORMANCE_MONTH, "%Y-%m-%d")
+            year_diff = date2.year - date1.year
+            month_diff = date2.month - date1.month
 
-          # Total months difference
-          total_months = year_diff * 12 + month_diff
+            # Total months difference
+            total_months = year_diff * 12 + month_diff
 
-          # Check if the difference is a multiple of 4 then just continue to the next row (added up the Numerators and Denominators and checked the measure to be the same)
-          if total_months % 3 != 0:
-            
-            #make sure the quarter data is for the same measure
-            if measure == "":
-              measure = measure_name_to_id[row["Performance measure name"]]
-            if measure != measure_name_to_id[row["Performance measure name"]]:
-              raise ValueError("Sorry, wrong quarterly data issue for site "+ site_id + "meassure "+ measure + "date "+ row["Time interval"]) 
-            continue            
-          time_interval = (date2 - relativedelta(months=total_months / 3)).strftime('%Y-%m-%d')
+            # Check if the difference is a multiple of 4 then just continue to the next row (added up the Numerators and Denominators and checked the measure to be the same)
+            if total_months % 3 != 0:
+                # make sure the quarter data is for the same measure
+                if measure == "":
+                    measure = measure_name_to_id[row["Performance measure name"]]
+                if measure != measure_name_to_id[row["Performance measure name"]]:
+                    raise ValueError(
+                        "Sorry, wrong quarterly data issue for site "
+                        + site_id
+                        + "meassure "
+                        + measure
+                        + "date "
+                        + row["Time interval"]
+                    )
+                continue
+            time_interval = (date2 - relativedelta(months=total_months / 3)).strftime(
+                "%Y-%m-%d"
+            )
         # if not row["Performance level (monthly rate)"]:
         #   continue
         # Format the row and write it to the file
-        input_file["Performance_data"].append([int(site_id),measure_name_to_id[row["Performance measure name"]],time_interval,
-                                               Numerator,Denominator-Numerator,Denominator,0,0,0,row["Target"]*100  ])
+        input_file["Performance_data"].append(
+            [
+                int(site_id),
+                measure_name_to_id[row["Performance measure name"]],
+                time_interval,
+                Numerator,
+                Denominator - Numerator,
+                Denominator,
+                0,
+                0,
+                0,
+                row["Target"] * 100,
+            ]
+        )
         input_file["message_instance_id"] = str(uuid.uuid4())
         input_file["performance_month"] = PERFORMANCE_MONTH
-        
-        measure =""
+
+        measure = ""
         Numerator = 0
         Denominator = 0
 
@@ -106,4 +134,3 @@
     file_path = os.path.join(OUTPUT_DIR, output_file_name)
     with open(file_path, "w") as file:
         json.dump(input_file, file, indent=4)
-

bulk-up/src/bulk_up/history_extractor.py

@@ -32,10 +32,10 @@ def add_history(filename):
             for row in performance_data:
                 if row[1] not in passed_rate:
                     passed_rate[row[1]] = {}
-                if int(row[5])!=0:
+                if int(row[5]) != 0:
                     passed_rate[row[1]][row[2]] = int(row[3]) / int(row[5])
                 else:
-                    passed_rate[row[1]][row[2]]=0
+                    passed_rate[row[1]][row[2]] = 0
 
             with lock:
                 for key, value in data["History"].items():

bulk-up/src/bulk_up/log_to_data.py

@@ -32,9 +32,7 @@
 df = pd.read_excel(INPUT_DIR, sheet_name=SHEET_NAME, engine="openpyxl")
 response_df: pd.DataFrame = pd.DataFrame()
 
-graph: Graph = manifest_to_graph(
-    KNOWLEDGE_BASE_LOCAL_MANIFEST
-)
+graph: Graph = manifest_to_graph(KNOWLEDGE_BASE_LOCAL_MANIFEST)
 
 is_about_to_columns: dict = {
     PSDO.achievement_set: "represented set",