Merge branch 'main' into feature/documentation-updates

Author: Benjamin Pachev

dev-requirements.txt

@@ -1,4 +1,3 @@
--r requirements.txt
 ruff
 pytest
 pytest-env
@@ -12,4 +11,4 @@ coverage
 types-seaborn
 nbconvert
 nbdime
-types-requests
+types-requests

docs/preprocess.md

@@ -4,14 +4,16 @@
 ### Attributes Omitted
 - **UUID**
 - **Nodes**: NodesList have more specific information
-- **Preempted**: Status have more valid information
+- **Preempted**: Contains unreliable data. Use Status column instead (PREEMPT for
+    unfinished, COMPLETE/FAILED/etc. for finished preempted jobs).
 - **EndTime**: Can be calculated from StartTime and Elapsed
 
 ### Options for Including or Omitting Jobs
 - **Keeping CPU jobs:**
     - If `GPUType` is null, the value will be filled with `["cpu"]`
     - If `GPUs` is null or is 0, the value will be 0.
 - **Keeping jobs where the status is "Failed" or "Cancelled"**
+- **Keeping jobs where the QOS is customized (not normal, long, or short)**
 
 ### Records Omitted If:
 - `Elapsed` is less than the minimum threshold

notebooks/Efficiency Analysis.ipynb

@@ -96,6 +96,7 @@
     "\n",
     "from src.analysis import efficiency_analysis as ea\n",
     "from src.visualization import JobsWithMetricsVisualizer, UsersWithMetricsVisualizer\n",
+    "from src.utilities import load_and_preprocess_jobs\n",
     "\n",
     "# Automatically reload modules before executing code\n",
     "# This is useful for development to see changes without restarting the kernel.\n",
@@ -112,7 +113,7 @@
    "outputs": [],
    "source": [
     "# Load the jobs DataFrame from DuckDB\n",
-    "preprocessed_jobs_df = ea.load_preprocessed_jobs_dataframe_from_duckdb(\n",
+    "preprocessed_jobs_df = load_and_preprocess_jobs(\n",
     "    db_path=\"../data/slurm_data.db\",\n",
     "    table_name=\"Jobs\",\n",
     ")\n",
@@ -608,7 +609,11 @@
    ]
   }
  ],
- "metadata": {},
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
  "nbformat": 4,
  "nbformat_minor": 5
 }