Remove methods in efficiency analysis that were dead code and were replaced by sort_and_filter

Author: MisterArdavan

notebooks/analysis/No VRAM Use Analysis.ipynb

@@ -108,7 +108,7 @@
    "source": [
     "# Load the jobs DataFrame from DuckDB\n",
     "preprocessed_jobs_df = ea.load_preprocessed_jobs_dataframe_from_duckdb(\n",
-    "    db_path=Path(project_root) / \"data/slurm_data_small.db\", table_name=\"Jobs\", anonymize=True\n",
+    "    db_path=Path(project_root) / \"data/slurm_data.db\", table_name=\"Jobs\", anonymize=True\n",
     ")\n",
     "display(preprocessed_jobs_df.head(10))\n",
     "print(preprocessed_jobs_df.shape)"
@@ -261,9 +261,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "inefficient_users_vram_hours = efficiency_analysis.find_inefficient_users_by_vram_hours(\n",
-    "    vram_hours_filter={\"min\": 200, \"inclusive\": True},  # VRAM-hours threshold for identifying inefficient users\n",
-    "    min_jobs=5,  # Minimum number of jobs to consider a user\n",
+    "inefficient_users_vram_hours = efficiency_analysis.sort_and_filter_records_with_metrics(\n",
+    "    metrics_df_name_enum=MetricsDataFrameNameEnum.USERS,\n",
+    "    sorting_key=\"vram_hours\",\n",
+    "    ascending=False,  # Sort by vram_hours in descending order\n",
+    "    filter_criteria={\n",
+    "        \"vram_hours\": {\"min\": 200, \"inclusive\": True},  # VRAM-hours threshold for identifying inefficient users\n",
+    "        \"job_count\": {\"min\": 5, \"inclusive\": True},  # Job count threshold for identifying inefficient users\n",
+    "    },\n",
     ")\n",
     "# Display top inefficient users by VRAM-hours\n",
     "print(\"\\nTop inefficient users by VRAM-hours:\")\n",

notebooks/analysis/Requested and Used VRAM.ipynb

@@ -231,7 +231,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "inefficient_jobs_vram_hours = analyzer.sort_and_filter_records_with_metrics(\n",
+    "inefficient_jobs_req_vram_score = analyzer.sort_and_filter_records_with_metrics(\n",
     "    metrics_df_name_enum=ResourceHoardingDataFrameNameEnum.JOBS,\n",
     "    sorting_key=\"requested_vram_efficiency_score\",\n",
     "    ascending=True,  # Sort by requested_vram_efficiency_score in ascending order\n",
@@ -240,7 +240,7 @@
     "    },\n",
     ")\n",
     "# Plot top inefficient jobs by requested VRAM efficiency score, with VRAM-hours as labels\n",
-    "jobs_with_metrics_visualizer = JobsWithMetricsVisualizer(inefficient_jobs_vram_hours.head(10))\n",
+    "jobs_with_metrics_visualizer = JobsWithMetricsVisualizer(inefficient_jobs_req_vram_score.head(10))\n",
     "jobs_with_metrics_visualizer.visualize(\n",
     "    output_dir_path=JOBS_VISUALIZATION_DATA_DIR,\n",
     "    column=\"requested_vram_efficiency_score\",\n",

notebooks/module_demos/Attribute Visualization.ipynb

@@ -133,10 +133,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "clean_jobs_df = Preprocess().preprocess_data(\n",
-    "    jobs_df,\n",
-    "    min_elapsed_seconds=600,\n",
-    "    anonymize=True)\n",
+    "clean_jobs_df = Preprocess().preprocess_data(jobs_df, min_elapsed_seconds=600, anonymize=True)\n",
     "display(clean_jobs_df)\n",
     "print(clean_jobs_df.shape)"
    ]

src/analysis/efficiency_analysis.py

@@ -507,105 +507,6 @@ def calculate_user_efficiency_metrics(self) -> pd.DataFrame:
         self.users_with_efficiency_metrics = users_w_efficiency_metrics
         return self.users_with_efficiency_metrics
 
-    def find_inefficient_users_by_alloc_vram_efficiency(
-        self, alloc_vram_efficiency_filter: int | float | dict | None, min_jobs: int = 5
-    ) -> pd.DataFrame:
-        """
-        Identify users with low expected allocated VRAM efficiency across their jobs compared to others
-
-        Args:
-            alloc_vram_efficiency_filter:
-                - int | float : select rows where expected_value_alloc_vram_efficiency == value
-                - dict with 'min'/'max' and required 'inclusive' (bool): select rows in the range
-            min_jobs (int): Minimum number of jobs a user must have to be included in the analysis
-
-        Returns:
-            pd.DataFrame: DataFrame with users and their average VRAM efficiency
-
-        Raises:
-            ValueError: If the filter for expected_value_alloc_vram_efficiency is invalid.
-        """
-        if self.users_with_efficiency_metrics is None:
-            self.calculate_user_efficiency_metrics()
-            print(
-                "Users DataFrame with efficiency metrics was not available. "
-                "Calculated it using the DataFrame of jobs with efficiency metrics."
-            )
-
-        mask = pd.Series(
-            [True] * len(self.users_with_efficiency_metrics), index=self.users_with_efficiency_metrics.index
-        )
-
-        if alloc_vram_efficiency_filter is not None:
-            try:
-                mask &= EfficiencyAnalysis.apply_numeric_filter(
-                    self.users_with_efficiency_metrics["expected_value_alloc_vram_efficiency"],
-                    alloc_vram_efficiency_filter,
-                    {FilterTypeEnum.NUMERIC_SCALAR, FilterTypeEnum.DICTIONARY},
-                    filter_name="expected_value_alloc_vram_efficiency",
-                )
-            except ValueError as e:
-                raise ValueError("Invalid filter for expected_value_alloc_vram_efficiency.") from e
-
-        col = self.users_with_efficiency_metrics["job_count"]
-        mask &= col.ge(min_jobs)
-
-        inefficient_users = self.users_with_efficiency_metrics[mask]
-
-        # Sort by the metric ascending (lower is worse)
-        inefficient_users = inefficient_users.sort_values("expected_value_alloc_vram_efficiency", ascending=True)
-        return inefficient_users
-
-    def find_inefficient_users_by_vram_hours(
-        self, vram_hours_filter: int | float | dict = 200, min_jobs: int = 5
-    ) -> pd.DataFrame:
-        """
-        Identify users with high VRAM-hours across their jobs compared to others.
-
-        Args:
-            vram_hours_filter:
-                - None: no filtering on vram_hours
-                - int | float: select rows where vram_hours == value
-                - dict with 'min'/'max' and required 'inclusive' (bool): select rows in the range
-            min_jobs (int): Minimum number of jobs a user must have to be included in the analysis
-
-        Returns:
-            pd.DataFrame: DataFrame with users and their total VRAM hours
-
-        Raises:
-            ValueError: If the filter is invalid
-        """
-        if self.users_with_efficiency_metrics is None:
-            self.calculate_user_efficiency_metrics()
-            print(
-                "Users DataFrame with efficiency metrics was not available. "
-                "Calculated it using the DataFrame of jobs with efficiency metrics."
-            )
-
-        mask = pd.Series(
-            [True] * len(self.users_with_efficiency_metrics), index=self.users_with_efficiency_metrics.index
-        )
-
-        if vram_hours_filter is not None:
-            try:
-                mask &= EfficiencyAnalysis.apply_numeric_filter(
-                    self.users_with_efficiency_metrics["vram_hours"],
-                    vram_hours_filter,
-                    {FilterTypeEnum.NUMERIC_SCALAR, FilterTypeEnum.DICTIONARY},
-                    filter_name="vram_hours_filter",
-                )
-            except ValueError as e:
-                raise ValueError("Invalid filter for vram_hours.") from e
-
-        col = self.users_with_efficiency_metrics["job_count"]
-        mask &= col.ge(min_jobs)
-
-        inefficient_users = self.users_with_efficiency_metrics[mask]
-
-        # Sort by the metric descending (higher is worse)
-        inefficient_users = inefficient_users.sort_values("vram_hours", ascending=False)
-        return inefficient_users
-
     def calculate_all_efficiency_metrics(
         self,
         filtered_jobs: pd.DataFrame,
@@ -722,57 +623,6 @@ def calculate_pi_account_efficiency_metrics(self) -> pd.DataFrame:
         self.pi_accounts_with_efficiency_metrics = pi_efficiency_metrics
         return self.pi_accounts_with_efficiency_metrics
 
-    def find_inefficient_pis_by_vram_hours(
-        self, vram_hours_filter: int | float | dict = 200, min_jobs: int = 5
-    ) -> pd.DataFrame:
-        """
-        Identify inefficient PI accounts based on VRAM hours.
-
-        Args:
-            vram_hours_filter:
-                - None: no filtering on vram_hours
-                - int | float: select rows where pi_acc_vram_hours == value
-                - dict with 'min'/'max' and required 'inclusive' (bool): select rows in the range
-            min_jobs (int): Minimum number of jobs a PI account must have to be included in the analysis
-
-        Returns:
-            pd.DataFrame: DataFrame with PI accounts and their VRAM hours
-
-        Raises:
-            ValueError: If the filter is invalid
-        """
-        if self.pi_accounts_with_efficiency_metrics is None:
-            self.calculate_pi_account_efficiency_metrics()
-            print(
-                "PI accounts with efficiency metrics DataFrame was not available. "
-                "Calculated it using the DataFrame of users with efficiency metrics."
-            )
-
-        mask = pd.Series(
-            [True] * len(self.pi_accounts_with_efficiency_metrics),
-            index=self.pi_accounts_with_efficiency_metrics.index,
-        )
-
-        if vram_hours_filter is not None:
-            try:
-                mask &= EfficiencyAnalysis.apply_numeric_filter(
-                    self.pi_accounts_with_efficiency_metrics["pi_acc_vram_hours"],
-                    vram_hours_filter,
-                    {FilterTypeEnum.NUMERIC_SCALAR, FilterTypeEnum.DICTIONARY},
-                    filter_name="pi_acc_vram_hours_filter",
-                )
-            except ValueError as e:
-                raise ValueError("Invalid filter for pi_acc_vram_hours.") from e
-
-        col = self.pi_accounts_with_efficiency_metrics["job_count"]
-        mask &= col.ge(min_jobs)
-
-        inefficient_pi_accounts = self.pi_accounts_with_efficiency_metrics[mask]
-
-        # Sort by the metric descending (higher is worse)
-        inefficient_pi_accounts = inefficient_pi_accounts.sort_values("pi_acc_vram_hours", ascending=False)
-        return inefficient_pi_accounts
-
     def sort_and_filter_records_with_metrics(
         self,
         metrics_df_name_enum: MetricsDFNameEnumT,

src/visualization/efficiency_metrics.py

@@ -88,6 +88,7 @@ def _human_readable_value(val: object) -> str:
         # numpy / pandas NA
         try:
             import pandas as _pd  # local import to avoid circular issues
+
             try:
                 _tmp_val = val  # help type checkers
                 isna_func = getattr(_pd, "isna", None)