add documentationchanges

Author: Espiobest

.gitignore

@@ -43,6 +43,9 @@ data/
 /docs/build
 /site
 
-# Quarto Reports
+# Quarto Report files
 .quarto 
-*.html
+*.html
+*.pdf
+*.yml
+*.pkl

docs/analysis/frequency_analysis.md

@@ -2,4 +2,5 @@
 title: Frequency Analysis
 ---
 
-::: src.analysis.frequency_analysis
+<!-- TODO (Ayush): Update when frequency analysis is merged-->
+<!-- ::: src.analysis.frequency_analysis -->

docs/demo.md

@@ -6,7 +6,7 @@ This page showcases the DS4CG Unity Job Analytics project in action with interac
 
 Explore our comprehensive Jupyter notebooks that demonstrate the full capabilities:
 
-### 📊 [Frequency Analysis Demo](../notebooks/Frequency Analysis/)
+### 📊 [Frequency Analysis Demo](notebooks/Frequency Analysis/)
 **Complete end-to-end workflow** showing:
 
 - Database connection and preprocessing
@@ -15,26 +15,26 @@ Explore our comprehensive Jupyter notebooks that demonstrate the full capabiliti
 - Interactive visualizations
 - Best/worst user identification
 
-### 📈 [Basic Visualization](../notebooks/Basic%20Visualization/)
+### 📈 [Basic Visualization](notebooks/Basic%20Visualization/)
 **Column statistics and exploratory analysis** including:
 
 - Data loading and preprocessing
 - Column-level statistical visualizations
 - Distribution analysis
 - Data quality assessment
 
-### 🔍 [Efficiency Analysis](../notebooks/Efficiency%20Analysis/)
+### 🔍 [Efficiency Analysis](notebooks/Efficiency%20Analysis/)
 **Advanced efficiency analysis techniques** covering:
 
 - Job filtering and metrics calculation
 - User and PI group analysis
 - Inefficiency identification
 - Performance comparison workflows
 
-### 🎯 [Clustering Analysis](../notebooks/clustering_analysis/)
+### 🎯 [Clustering Analysis](notebooks/clustering_analysis/)
 **User behavior clustering and pattern analysis**
 
-### 📊 [Frequency Analysis](../notebooks/Frequency%20Analysis/)
+### 📊 [Frequency Analysis](notebooks/Frequency%20Analysis/)
 **Time series frequency analysis and patterns**
 
 ---

mkdocs.yml

@@ -2,9 +2,31 @@ site_name: "Unity Job Analytics"
 
 theme:
   name: "material"
+  features:
+    - navigation.tabs          # Enables horizontal top nav tabs
+    - navigation.tabs.sticky   # Keeps tabs visible when scrolling
+    - navigation.sections      # Groups subsections under tabs
+    - toc.integrate            # Puts table of contents in main pane
+  palette:
+    # Palette toggle for light mode
+    - scheme: default
+      toggle:
+        icon: material/brightness-7
+        name: Switch to dark mode
+
+    # Palette toggle for dark mode
+    - scheme: slate
+      toggle:
+        icon: material/brightness-4
+        name: Switch to light mode
 
 plugins:
   - search
+  - mkdocs-jupyter:
+      include_source: true
+      allow_errors: true 
+      ignore_h1_titles: true
+
   - mkdocstrings:
         handlers:
           python:
@@ -21,20 +43,47 @@ plugins:
               # render Parameters/Returns as a table
               docstring_section_style: list
 
+# Configure markdown extensions
+markdown_extensions:
+  - attr_list
+  - md_in_html
+  - pymdownx.arithmatex:
+      generic: true
+
+extra_javascript:
+  # - javascripts/mathjax.js
+  - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js
+
+# Suppress warnings for external links to notebooks
+validation:
+  omitted_files: warn
+  absolute_links: warn
+  unrecognized_links: warn
+
+docs_dir: docs
+
 nav:
   - Home: 'index.md'
-  - About: 'about.md'
-  - Data:
-    - 'preprocess.md'
-  - Analysis:
-    - 'analysis/efficiency_analysis.md'
-  - Visualization:
-    - 'visualization/visualization.md'
-    - 'visualization/columns.md'
-    - 'visualization/efficiency_metrics.md'
-    - 'visualization/models.md'
-  - MVP Scripts:
-    - 'mvp_scripts/cpu_metrics.md'
-    - 'mvp_scripts/gpu_metrics.md'
-    - 'mvp_scripts/zero_gpu_usage.md'
-
+  - Getting Started: 'getting-started.md'
+  - Data and Efficiency Metrics: 'data-and-metrics.md'
+  - Demo: 'demo.md'
+  - Notebooks:
+    - Efficiency Analysis: 'notebooks/Efficiency Analysis.ipynb'
+    - Visualization: 'notebooks/Basic Visualization.ipynb'
+    # - Frequency Analysis: 'notebooks/Frequency Analysis.ipynb'
+  - Technical Documentation:
+    - Data Processing: 'preprocess.md'
+    - Analysis:
+      - 'analysis/efficiency_analysis.md'
+      - 'analysis/frequency_analysis.md'
+    - Visualization:
+      - 'visualization/visualization.md'
+      - 'visualization/columns.md'
+      - 'visualization/efficiency_metrics.md'
+      - 'visualization/models.md'
+    - MVP Scripts:
+      - 'mvp_scripts/cpu_metrics.md'
+      - 'mvp_scripts/gpu_metrics.md'
+      - 'mvp_scripts/zero_gpu_usage.md'
+  - FAQ: 'faq.md'
+  - Contact: 'contact.md'

mvp_scripts/gpu_metrics.py

@@ -27,7 +27,7 @@
 vram_labels = [0] + vram_cutoffs[2:]
 
 
-def get_requested_vram(constraints):
+def get_requested_vram(constraints) -> int:
     """Get the minimum requested VRAM from job constraints.
 
     Args:
@@ -67,8 +67,6 @@ def __init__(
             metricsfile (str, optional): Path to the DuckDB database file containing job data.
             min_elapsed (int, optional): Minimum elapsed time (in seconds) for jobs to be included.
         """
-        if local:
-            metricsfile = "../data/slurm_data_small.db"
         self.con = duckdb.connect(metricsfile)
         # TODO - handle array jobs properly
         df = self.con.query(

mvp_scripts/zero_gpu_usage_list.py

@@ -23,7 +23,7 @@
 HOURS = "{hours} unused GPU hours. The most recent jobs are the following:"
 
 
-def get_job_type_breakdown(interactive, jobs):
+def get_job_type_breakdown(interactive, jobs) -> str:
     """Generate a summary string describing the breakdown of interactive and batch jobs.
 
     Args:
@@ -45,7 +45,7 @@ def get_job_type_breakdown(interactive, jobs):
     )
 
 
-def pi_report(account, days_back=60):
+def pi_report(account, days_back=60) -> None:
     """Create an efficiency report for a given PI group, summarizing GPU usage and waste.
 
     Args:
@@ -74,7 +74,7 @@ def pi_report(account, days_back=60):
 
 def main(
     dbfile="./modules/admin-resources/reporting/slurm_data.db", userlist="./users.csv", sendEmail=False, days_back=60
-):
+) -> None:
     """Print out a list of users who habitually waste GPU hours, and optionally email them a report.
 
     Args:

src/analysis/efficiency_analysis.py

@@ -53,7 +53,6 @@ def load_preprocessed_jobs_dataframe_from_duckdb(
         raise RuntimeError(f"Failed to load jobs DataFrame: {e}") from e
 
 
-
 class EfficiencyAnalysis:
     """
     Class to encapsulate the efficiency analysis of jobs based on various metrics.

src/preprocess/preprocess.py

@@ -821,4 +821,4 @@ def preprocess_data(
     processing_error_logs.clear()
     error_indices.clear()
 
-    return data
+    return data