Add PI group visualization code as a new class

Author: MisterArdavan

notebooks/Analysis/No VRAM Use Analysis.ipynb

@@ -224,9 +224,7 @@
     "# Plot top inefficient jobs by VRAM-hours, with VRAM-hours as labels\n",
     "jobs_with_metrics_visualizer = JobsWithMetricsVisualizer(inefficient_jobs_vram_hours.head(10))\n",
     "jobs_with_metrics_visualizer.visualize(\n",
-    "    column=\"vram_hours\",\n",
-    "    bar_label_columns=[\"vram_hours\", \"allocated_vram\"],\n",
-    "    figsize=(10, 6)\n",
+    "    column=\"vram_hours\", bar_label_columns=[\"vram_hours\", \"allocated_vram\"], figsize=(10, 6)\n",
     ")"
    ]
   },
@@ -294,7 +292,7 @@
     "\n",
     "# Histogram with logarithmic x-axis (focused range only)\n",
     "ax = sns.histplot(x_focus, bins=10**4, color=\"#1f77b4\")\n",
-    "ax.set_xscale('log')\n",
+    "ax.set_xscale(\"log\")\n",
     "ax.set_xlabel(\"VRAM-Hours (log scale)\")\n",
     "ax.set_ylabel(\"Count\")\n",
     "ax.set_title(\"VRAM-Hours Distribution\")\n",
@@ -375,7 +373,7 @@
     "    order=pi_accounts,\n",
     "    orient=\"h\",\n",
     "    palette=\"Blues_r\",\n",
-    "    hue=top_pi_accounts[\"pi_account\"].tolist()\n",
+    "    hue=top_pi_accounts[\"pi_account\"].tolist(),\n",
     ")\n",
     "\n",
     "# We'll replace the default tick labels with custom two-line labels placed OUTSIDE the left spine.\n",
@@ -414,10 +412,10 @@
     "# Y-axis label: place further left than custom tick labels.\n",
     "ax.set_ylabel(\"PI Account / Users\", rotation=90, labelpad=20)\n",
     "# Position the label using axes fraction (x< x_outside)\n",
-    "ax.yaxis.set_label_coords(x_outside - 0.30, 0.5) \n",
+    "ax.yaxis.set_label_coords(x_outside - 0.30, 0.5)\n",
     "\n",
     "# Hide y-axis tick labels (already blank) but keep small outward ticks if desired\n",
-    "ax.tick_params(axis='y', which='both', direction='out', length=4, pad=2)\n",
+    "ax.tick_params(axis=\"y\", which=\"both\", direction=\"out\", length=4, pad=2)\n",
     "# plt.subplots_adjust(left=0.7)\n",
     "\n",
     "plt.xlabel(\"VRAM-Hours\")\n",

notebooks/Analysis/Requested and Used VRAM.ipynb

@@ -102,8 +102,9 @@
     "\n",
     "from src.analysis import ResourceHoarding\n",
     "from src.analysis import efficiency_analysis as ea\n",
-    "from src.visualization import JobsWithMetricsVisualizer, UsersWithMetricsVisualizer\n",
-    "from src.config.enum_constants import ResourceHoardingDataFrameNameEnum"
+    "from src.visualization import JobsWithMetricsVisualizer, UsersWithMetricsVisualizer, PIGroupsWithMetricsVisualizer\n",
+    "from src.config.enum_constants import ResourceHoardingDataFrameNameEnum\n",
+    "from src.config.paths import PI_GROUPS_VISUALIZATION_DATA_DIR"
    ]
   },
   {
@@ -145,9 +146,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "analyzer = ResourceHoarding(\n",
-    "    jobs_df=preprocessed_jobs_df\n",
-    ")"
+    "analyzer = ResourceHoarding(jobs_df=preprocessed_jobs_df)"
    ]
   },
   {
@@ -158,8 +157,7 @@
    "outputs": [],
    "source": [
     "filtered_jobs = analyzer.filter_jobs_for_analysis(\n",
-    "    gpu_mem_usage_filter={\"min\": 0, \"inclusive\": False},\n",
-    "    requested_vram_filter={\"min\": 0, \"inclusive\": False}\n",
+    "    gpu_mem_usage_filter={\"min\": 0, \"inclusive\": False}, requested_vram_filter={\"min\": 0, \"inclusive\": False}\n",
     ")\n",
     "filtered_jobs"
    ]
@@ -241,9 +239,7 @@
     "# Plot top inefficient jobs by requested VRAM efficiency score, with VRAM-hours as labels\n",
     "jobs_with_metrics_visualizer = JobsWithMetricsVisualizer(inefficient_jobs_vram_hours.head(10))\n",
     "jobs_with_metrics_visualizer.visualize(\n",
-    "    column=\"requested_vram_efficiency_score\",\n",
-    "    bar_label_columns=[\"vram_hours\", \"allocated_vram\"],\n",
-    "    figsize=(10, 6)\n",
+    "    column=\"requested_vram_efficiency_score\", bar_label_columns=[\"vram_hours\", \"allocated_vram\"], figsize=(10, 6)\n",
     ")"
    ]
   },
@@ -381,7 +377,7 @@
     "    ascending=True,  # Sort by avg_requested_vram_efficiency_score in ascending order\n",
     "    filter_criteria={\n",
     "        \"avg_requested_vram_efficiency_score\": {\"max\": -10, \"inclusive\": True},  # score threshold\n",
-    "        \"job_count\": {\"min\": 5, \"inclusive\": True}  # minimum job count threshold\n",
+    "        \"job_count\": {\"min\": 5, \"inclusive\": True},  # minimum job count threshold\n",
     "    },\n",
     ")\n",
     "# Display top inefficient users by avg_requested_vram_efficiency_score\n",
@@ -416,6 +412,7 @@
     "        print(\"All scores are zero; histogram not informative.\")\n",
     "    else:\n",
     "        import numpy as np\n",
+    "\n",
     "        fig, ax = plt.subplots(figsize=(8, 5))\n",
     "\n",
     "        # Separate negatives (expected) from zeros\n",
@@ -452,7 +449,7 @@
     "            linthresh = min_abs\n",
     "        else:\n",
     "            linthresh = 1e-6  # fallback small threshold\n",
-    "        ax.set_xscale('symlog', linthresh=linthresh, linscale=1.0, base=10)\n",
+    "        ax.set_xscale(\"symlog\", linthresh=linthresh, linscale=1.0, base=10)\n",
     "\n",
     "        # Annotation: counts (negative & zero) and total\n",
     "        neg_count = (scores < 0).sum()\n",
@@ -461,7 +458,7 @@
     "        ax.text(\n",
     "            0.98,\n",
     "            0.95,\n",
-    "            (\"Counts:\\n\" f\"Negative: {neg_count}\\n\" f\"Zero: {zero_count}\\n\" f\"# of Users: {total}\"),\n",
+    "            (f\"Counts:\\nNegative: {neg_count}\\nZero: {zero_count}\\n# of Users: {total}\"),\n",
     "            transform=ax.transAxes,\n",
     "            ha=\"right\",\n",
     "            va=\"top\",\n",
@@ -477,7 +474,7 @@
     "        ax2.plot(mids, cdf, color=\"crimson\", marker=\"o\", linestyle=\"-\", linewidth=1, markersize=3)\n",
     "        ax2.set_ylim(0, 1)\n",
     "        ax2.set_ylabel(\"Cumulative Fraction\", color=\"crimson\")\n",
-    "        ax2.tick_params(axis='y', colors='crimson')\n",
+    "        ax2.tick_params(axis=\"y\", colors=\"crimson\")\n",
     "\n",
     "        # Ensure x-axis spans to (slightly) include zero for clarity\n",
     "        left, right = ax.get_xlim()\n",
@@ -491,7 +488,7 @@
     "# - We plot the actual (negative/zero) scores instead of absolute values.\n",
     "# - symlog x-scale provides a log-like compression for large negative magnitudes while keeping zero.\n",
     "# - linthresh picks the smallest non-zero magnitude so near-zero structure is visible.\n",
-    "# - CDF is computed over actual values to show accumulation from most negative toward zero.\n"
+    "# - CDF is computed over actual values to show accumulation from most negative toward zero."
    ]
   },
   {
@@ -620,8 +617,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from matplotlib.transforms import blended_transform_factory\n",
-    "\n",
     "inefficient_pis_avg_req_vram_eff_score = analyzer.sort_and_filter_records_with_metrics(\n",
     "    metrics_df_name_enum=ResourceHoardingDataFrameNameEnum.PI_GROUPS,\n",
     "    sorting_key=\"avg_requested_vram_efficiency_score\",\n",
@@ -635,120 +630,13 @@
     "print(\"\\nTop inefficient PI Groups by Avg Requested VRAM Efficiency Score:\")\n",
     "display(inefficient_pis_avg_req_vram_eff_score.head(10))\n",
     "\n",
-    "# Take top N\n",
-    "top_pi_accounts = inefficient_pis_avg_req_vram_eff_score.head(10)\n",
-    "pi_accounts = top_pi_accounts[\"pi_account\"].tolist()\n",
-    "user_counts = top_pi_accounts.get(\"user_count\", [\"-\"] * len(top_pi_accounts)).tolist()\n",
-    "\n",
-    "xmin = top_pi_accounts[\"avg_requested_vram_efficiency_score\"].min()\n",
-    "# If the minimum value is negative, we need to adjust the heights of the bars\n",
-    "# to ensure they start from zero for better visualization.\n",
-    "# This is particularly useful for metrics like allocated VRAM efficiency score.\n",
-    "if xmin < 0:\n",
-    "    col_heights = pd.Series(\n",
-    "        [abs(xmin)] * len(top_pi_accounts[\"avg_requested_vram_efficiency_score\"]),\n",
-    "        index=top_pi_accounts[\"avg_requested_vram_efficiency_score\"].index\n",
-    "    ) - abs(top_pi_accounts[\"avg_requested_vram_efficiency_score\"])\n",
-    "    print(f\"Minimum value for \\\"avg_requested_vram_efficiency_score\\\": {xmin}\")\n",
-    "else:\n",
-    "    col_heights = top_pi_accounts[\"avg_requested_vram_efficiency_score\"]\n",
-    "\n",
-    "plt.figure(figsize=(10, 6))\n",
-    "plot_df = pd.DataFrame({\n",
-    "    \"col_height\": col_heights.to_numpy(),\n",
-    "    \"job_hours\": top_pi_accounts[\"pi_acc_job_hours\"],\n",
-    "    \"pi_account\": top_pi_accounts[\"pi_account\"]\n",
-    "})\n",
-    "ax = sns.barplot(\n",
-    "    y=top_pi_accounts[\"pi_account\"].tolist(),\n",
-    "    x=col_heights.tolist(),\n",
-    "    order=top_pi_accounts[\"pi_account\"].tolist(),\n",
-    "    orient=\"h\",\n",
-    "    palette=\"Blues_r\",\n",
-    "    hue=top_pi_accounts[\"pi_account\"].tolist()\n",
-    ")\n",
-    "\n",
-    "# We'll replace the default tick labels with custom two-line labels placed OUTSIDE the left spine.\n",
-    "ax.set_yticks(range(len(pi_accounts)))\n",
-    "ax.set_yticklabels([])  # clear built-in labels\n",
-    "ax.set_ylabel(\"PI Account\")\n",
-    "\n",
-    "transform = blended_transform_factory(ax.transAxes, ax.transData)  # x in axes fraction, y in data coords\n",
-    "x_outside = -0.02  # negative x fraction places text just left of spine; adjust if needed\n",
-    "line_gap = 0.4  # vertical separation between the two lines\n",
-    "for y_pos, (pi, uc) in enumerate(zip(pi_accounts, user_counts, strict=True)):\n",
-    "    # First line (PI account) slightly above center\n",
-    "    ax.text(\n",
-    "        x_outside,\n",
-    "        y_pos - line_gap / 2,\n",
-    "        pi,\n",
-    "        ha=\"right\",\n",
-    "        va=\"center\",\n",
-    "        transform=transform,\n",
-    "        fontsize=10,\n",
-    "        clip_on=False,\n",
-    "    )\n",
-    "    # Second line (Users) slightly below center\n",
-    "    ax.text(\n",
-    "        x_outside,\n",
-    "        y_pos + line_gap / 2,\n",
-    "        f\"# of Users: {uc}\",\n",
-    "        ha=\"right\",\n",
-    "        va=\"center\",\n",
-    "        transform=transform,\n",
-    "        fontsize=9,\n",
-    "        color=\"dimgray\",\n",
-    "        clip_on=False,\n",
-    "    )\n",
-    "\n",
-    "# Y-axis label: place further left than custom tick labels.\n",
-    "ax.set_ylabel(\"PI Account\", rotation=90, labelpad=20)\n",
-    "# Position the label using axes fraction (x< x_outside)\n",
-    "ax.yaxis.set_label_coords(x_outside - 0.30, 0.5) \n",
-    "\n",
-    "# Hide y-axis tick labels (already blank) but keep small outward ticks if desired\n",
-    "ax.tick_params(axis='y', which='both', direction='out', length=4, pad=2)\n",
-    "plt.subplots_adjust(left=0.7)\n",
-    "\n",
-    "plt.xlabel(\"Average Requested VRAM Efficiency Score\")\n",
-    "plt.title(\"Top Inefficient PI Accounts by Average Requested VRAM Efficiency Score\")\n",
-    "\n",
-    "xmax = top_pi_accounts[\"avg_requested_vram_efficiency_score\"].max()\n",
-    "\n",
-    "# Set x-axis limit to 1.6 times the maximum value\n",
-    "# This ensures that the bars do not touch the right edge of the plot\n",
-    "xlim_multiplier = 1.6\n",
-    "xlim = abs(xmin) * xlim_multiplier if xmin < 0 else (xmax * xlim_multiplier if xmax > 0 else 1)\n",
-    "ax.set_xlim(0, xlim)\n",
-    "# If the minimum value is negative, we need to adjust the x-ticks accordingly\n",
-    "if xmin < 0:\n",
-    "    num_xticks = max(4, min(12, int(abs(xmin) // (xlim * 0.10)) + 1))\n",
-    "    xticks = np.linspace(xmin, 0, num=num_xticks)\n",
-    "    ax.set_xticks([abs(xmin) - abs(val) for val in xticks])\n",
-    "    ax.set_xticklabels([f\"{val:.2f}\" if -1 < val < 1 else f\"{val:.0f}\" for val in xticks], rotation=45)\n",
-    "for i, (vram_hours, pi_acc_job_hours, avg_req_vram_eff_score) in enumerate(\n",
-    "    zip(\n",
-    "        top_pi_accounts[\"pi_acc_vram_hours\"],\n",
-    "        top_pi_accounts[\"pi_acc_job_hours\"],\n",
-    "        col_heights,\n",
-    "        strict=True,\n",
-    "    )\n",
-    "):\n",
-    "    xpos = min(avg_req_vram_eff_score + xlim * 0.02, xlim * 0.92)\n",
-    "    ax.text(\n",
-    "        xpos,\n",
-    "        i,\n",
-    "        f\"VRAM-Hours: {vram_hours:.2f}\\nJob Hours: {pi_acc_job_hours:.2f}\",\n",
-    "        va=\"center\",\n",
-    "        ha=\"left\",\n",
-    "        fontsize=9,\n",
-    "        color=\"black\",\n",
-    "        clip_on=False,\n",
-    "    )\n",
-    "\n",
-    "ax.tick_params(axis='y', which='both', direction='out', length=4, pad=2)\n",
-    "plt.tight_layout()\n",
-    "plt.show()\n"
+    "pis_with_metrics_visualizer = PIGroupsWithMetricsVisualizer(inefficient_pis_avg_req_vram_eff_score.head(10))\n",
+    "pis_with_metrics_visualizer.visualize(\n",
+    "    output_dir_path=PI_GROUPS_VISUALIZATION_DATA_DIR,\n",
+    "    column=\"avg_requested_vram_efficiency_score\",\n",
+    "    bar_label_columns=[\"pi_acc_job_hours\", \"pi_acc_vram_hours\"],\n",
+    "    figsize=(10, 6),\n",
+    ")"
    ]
   }
  ],

src/config/paths.py

@@ -8,10 +8,11 @@
 PROJECT_ROOT = Path(__file__).parent.parent.parent
 
 # Data directory paths
-DATA_DIR = PROJECT_ROOT / "data"
-PREPROCESSING_DATA_DIR = DATA_DIR / "preprocessing"
-VISUALIZATION_DATA_DIR = DATA_DIR / "visualizations"
-REPORTS_DATA_DIR = DATA_DIR / "reports"
+DATA_DIR = (PROJECT_ROOT / "data").resolve()
+PREPROCESSING_DATA_DIR = (DATA_DIR / "preprocessing").resolve()
+VISUALIZATION_DATA_DIR = (DATA_DIR / "visualizations").resolve()
+PI_GROUPS_VISUALIZATION_DATA_DIR = (VISUALIZATION_DATA_DIR / "pi_groups").resolve()
+REPORTS_DATA_DIR = (DATA_DIR / "reports").resolve()
 
 # Specific file paths
 PREPROCESSING_ERRORS_LOG_FILE = PREPROCESSING_DATA_DIR / "preprocessing_errors.log"
@@ -20,4 +21,5 @@
 DATA_DIR.mkdir(exist_ok=True)
 PREPROCESSING_DATA_DIR.mkdir(exist_ok=True)
 VISUALIZATION_DATA_DIR.mkdir(exist_ok=True)
+PI_GROUPS_VISUALIZATION_DATA_DIR.mkdir(exist_ok=True)
 REPORTS_DATA_DIR.mkdir(exist_ok=True)

src/database/database_connection.py

@@ -42,7 +42,7 @@ def _connect(self, read_only: bool) -> duckdb.DuckDBPyConnection:
 
     def disconnect(self) -> None:
         """Close the connection to the DuckDB database."""
-        if hasattr(self, 'connection') and self.connection is not None:
+        if hasattr(self, "connection") and self.connection is not None:
             self.connection.close()
             self.connection = None
 

src/visualization/__init__.py

@@ -2,3 +2,4 @@
 from .columns import ColumnVisualizer as ColumnVisualizer
 from .efficiency_metrics import JobsWithMetricsVisualizer as JobsWithMetricsVisualizer
 from .efficiency_metrics import UsersWithMetricsVisualizer as UsersWithMetricsVisualizer
+from .efficiency_metrics import PIGroupsWithMetricsVisualizer as PIGroupsWithMetricsVisualizer