Add Report Generation and Configuration Improvements for GuideLLM (#24)

markurtz · web-flow · commit aee89e88d6e1 · 2024-08-02T17:58:40.000-04:00
## Summary

Introduces a report file generation feature for GuideLLM and fixes the
configuration functionality. Once the frontend is live, we will add the
report generation functionality to the CLI and add integration and end
to end tests

## Details

- **Report Generation Feature**:
    - Added logic to generate and inject reports from Pydantic models.
- Created utility functions for loading HTML templates, injecting data,
and saving reports.
- Implemented support for both local file paths and URLs as sources for
HTML templates.
- Included handling for different environments (local, dev, staging,
prod) with environment-specific report URLs.
- **Configuration Improvements**:
- Refactored configuration structure into a more modular and
maintainable format.
- Updated configuration to use Pydantic's `BaseSettings` and
`model_validator` for environment-based settings.
- Improved logging configuration with customizable options for console
and file logging levels.
    - Enhanced OpenAI settings for more robust API connectivity.
- **Testing**:
- Added comprehensive unit and integration tests for the new report
generation feature.
- Updated existing tests to align with the new configuration structure.
- Ensured all tests pass successfully and that code coverage is
maintained.

## Test Plan

- **Automation Testing**:
- Added and verified unit tests for the report generation and
configuration functionalities pass without issues.
- **Manual Testing**:
- Manually tested the report generation process using various Pydantic
models and HTML templates.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -29,6 +29,7 @@ repos:
 
         # dev dependencies
         pytest,
+        pydantic_settings,
 
         # types
         types-click,
diff --git a/pyproject.toml b/pyproject.toml
@@ -48,6 +48,7 @@ dev = [
     "pytest~=8.2.2",
     "pytest-cov~=5.0.0",
     "pytest-mock~=3.14.0",
+    "requests-mock~=1.12.1",
 
     # code quality
     "mypy~=1.10.1",
diff --git a/src/config/__init__.py b/src/config/__init__.py
diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py
@@ -1,13 +1,13 @@
 from typing import Any, Dict, Generator, List, Optional
 
 import openai
-from config import settings
 from loguru import logger
 from openai import OpenAI, Stream
 from openai.types import Completion
 from transformers import AutoTokenizer
 
 from guidellm.backend import Backend, BackendEngine, GenerativeResponse
+from guidellm.config import settings
 from guidellm.core import TextGenerationRequest
 
 __all__ = ["OpenAIBackend"]
diff --git a/src/guidellm/config/__init__.py b/src/guidellm/config/__init__.py
@@ -0,0 +1,3 @@
+from .base import settings
+
+__all__ = ["settings"]
diff --git a/src/guidellm/config/base.py b/src/guidellm/config/base.py
@@ -0,0 +1,106 @@
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+__all__ = [
+    "settings",
+    "Settings",
+    "Environment",
+    "LoggingSettings",
+    "OpenAISettings",
+    "ReportGenerationSettings",
+]
+
+
+class Environment(str, Enum):
+    """
+    Enum for the supported environments
+    """
+
+    LOCAL = "local"
+    DEV = "dev"
+    STAGING = "staging"
+    PROD = "prod"
+
+
+ENV_REPORT_MAPPING = {
+    Environment.PROD: "https://guidellm.neuralmagic.com/local-report/index.html",
+    Environment.STAGING: "https://staging.guidellm.neuralmagic.com/local-report/index.html",
+    Environment.DEV: "https://dev.guidellm.neuralmagic.com/local-report/index.html",
+    Environment.LOCAL: "tests/dummy/report.html",
+}
+
+
+class LoggingSettings(BaseModel):
+    """
+    Logging settings for the application
+    """
+
+    disabled: bool = False
+    clear_loggers: bool = True
+    console_log_level: str = "INFO"
+    log_file: Optional[str] = None
+    log_file_level: Optional[str] = None
+
+
+class OpenAISettings(BaseModel):
+    """
+    OpenAI settings for the application to connect to the API
+    for OpenAI server based pathways
+    """
+
+    # OpenAI API key.
+    api_key: str = "invalid"
+
+    # OpenAI-compatible server URL
+    # NOTE: The default value is default address of llama.cpp web server
+    base_url: str = "http://localhost:8080"
+
+    max_gen_tokens: int = 4096
+
+
+class ReportGenerationSettings(BaseModel):
+    source: str = ""
+
+
+class Settings(BaseSettings):
+    """
+    All the settings are powered by pydantic_settings and could be
+    populated from the .env file.
+
+    The format to populate the settings is next
+
+    ```sh
+    export GUIDELLM__LOGGING__DISABLED=true
+    export GUIDELLM__OPENAI__API_KEY=******
+    ```
+
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="GUIDELLM__",
+        env_nested_delimiter="__",
+        extra="ignore",
+        validate_default=True,
+        env_file=".env",
+    )
+
+    env: Environment = Environment.PROD
+    request_timeout: int = 30
+
+    logging: LoggingSettings = LoggingSettings()
+    openai: OpenAISettings = OpenAISettings()
+    report_generation: ReportGenerationSettings = ReportGenerationSettings()
+
+    @model_validator(mode="after")
+    @classmethod
+    def set_default_source(cls, values):
+        if not values.report_generation.source:
+            values.report_generation.source = ENV_REPORT_MAPPING.get(values.env)
+
+        return values
+
+
+settings = Settings()
diff --git a/src/guidellm/logger.py b/src/guidellm/logger.py
@@ -36,9 +36,11 @@
 
 import sys
 
-from config import LoggingSettings, settings
 from loguru import logger
 
+from guidellm.config import settings
+from guidellm.config.base import LoggingSettings
+
 __all__ = ["configure_logger", "logger"]
 
 
diff --git a/src/guidellm/request/emulated.py b/src/guidellm/request/emulated.py
@@ -10,6 +10,7 @@
 from loguru import logger
 from transformers import PreTrainedTokenizer
 
+from guidellm.config import settings
 from guidellm.core.request import TextGenerationRequest
 from guidellm.request.base import RequestGenerator
 
@@ -117,7 +118,7 @@ def _load_config(self, config: Union[str, Dict]) -> EmulatedConfig:
     def _load_emulated_data(self) -> List[str]:
         url = "https://www.gutenberg.org/files/1342/1342-0.txt"
         logger.info(f"Downloading text corpus from {url}")
-        response = requests.get(url, timeout=30)
+        response = requests.get(url, timeout=settings.request_timeout)
         response.raise_for_status()
 
         content = response.text
diff --git a/src/guidellm/utils/__init__.py b/src/guidellm/utils/__init__.py
@@ -1,11 +1,17 @@
 from .constants import (
     PREFERRED_DATA_COLUMNS,
     PREFERRED_DATA_SPLITS,
-    STANDARD_SLEEP_INTERVAL,
+    REPORT_HTML_MATCH,
+    REPORT_HTML_PLACEHOLDER,
 )
+from .injector import create_report, inject_data, load_html_file
 
 __all__ = [
     "PREFERRED_DATA_COLUMNS",
     "PREFERRED_DATA_SPLITS",
-    "STANDARD_SLEEP_INTERVAL",
+    "REPORT_HTML_MATCH",
+    "REPORT_HTML_PLACEHOLDER",
+    "create_report",
+    "inject_data",
+    "load_html_file",
 ]
diff --git a/src/guidellm/utils/constants.py b/src/guidellm/utils/constants.py
@@ -1,4 +1,9 @@
-__all__ = ["PREFERRED_DATA_COLUMNS", "PREFERRED_DATA_SPLITS", "STANDARD_SLEEP_INTERVAL"]
+__all__ = [
+    "PREFERRED_DATA_COLUMNS",
+    "PREFERRED_DATA_SPLITS",
+    "REPORT_HTML_MATCH",
+    "REPORT_HTML_PLACEHOLDER",
+]
 
 
 PREFERRED_DATA_COLUMNS = [
@@ -16,4 +21,6 @@
 
 PREFERRED_DATA_SPLITS = ["test", "validation", "train"]
 
-STANDARD_SLEEP_INTERVAL = 0.1
+REPORT_HTML_MATCH = "window.report_data = {};"
+
+REPORT_HTML_PLACEHOLDER = "{}"
diff --git a/src/guidellm/utils/injector.py b/src/guidellm/utils/injector.py
@@ -0,0 +1,94 @@
+from pathlib import Path
+from typing import Union
+
+import requests
+from pydantic import BaseModel
+
+from guidellm.config import settings
+from guidellm.utils.constants import (
+    REPORT_HTML_MATCH,
+    REPORT_HTML_PLACEHOLDER,
+)
+
+__all__ = ["create_report", "inject_data", "load_html_file"]
+
+
+def create_report(model: BaseModel, output_path: Union[str, Path]) -> Path:
+    """
+    Creates a report from the model and saves it to the output path.
+
+    :param model: the model to serialize and inject
+    :type model: BaseModel
+    :param output_path: the path, either a file or a directory,
+        to save the report to. If a directory, the report will be saved
+        as "report.html" inside of the directory.
+    :type output_path: str
+    :return: the path to the saved report
+    :rtype: str
+    """
+    if not isinstance(output_path, Path):
+        output_path = Path(output_path)
+
+    html_content = load_html_file(settings.report_generation.source)
+    report_content = inject_data(
+        model, html_content, REPORT_HTML_MATCH, REPORT_HTML_PLACEHOLDER
+    )
+
+    if not output_path.suffix:
+        # assume directory, save as report.html
+        output_path = output_path / "report.html"
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(report_content)
+
+    return output_path
+
+
+def inject_data(
+    model: BaseModel,
+    html: str,
+    match: str,
+    placeholder: str,
+) -> str:
+    """
+    Injects the data from the model into the HTML while replacing the placeholder.
+
+    :param model: the model to serialize and inject
+    :type model: BaseModel
+    :param html: the html to inject the data into
+    :type html: str
+    :param match: the string to match in the html to find the placeholder
+    :type match: str
+    :param placeholder: the placeholder to replace with the model data
+        inside of the placeholder
+    :type placeholder: str
+    :return: the html with the model data injected
+    :rtype: str
+    """
+    model_str = model.json()
+    inject_str = match.replace(placeholder, model_str)
+
+    return html.replace(match, inject_str)
+
+
+def load_html_file(path_or_url: str) -> str:
+    """
+    Load an HTML file from a path or URL
+
+    :param path_or_url: the path or URL to load the HTML file from
+    :type path_or_url: str
+    :return: the HTML content
+    :rtype: str
+    """
+    if path_or_url.startswith("http"):
+        response = requests.get(path_or_url, timeout=settings.request_timeout)
+        response.raise_for_status()
+
+        return response.text
+
+    path = Path(path_or_url)
+
+    if not path.exists():
+        raise FileNotFoundError(f"File not found: {path_or_url}")
+
+    return path.read_text()
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,8 +1,8 @@
 from typing import Callable, Optional
 
 import pytest
-from config import settings
 from guidellm.backend import Backend, BackendEngine, OpenAIBackend
+from guidellm.config import settings
 from loguru import logger
 
 
diff --git a/tests/integration/backend/test_openai_backend_submit.py b/tests/integration/backend/test_openai_backend_submit.py
@@ -3,8 +3,8 @@
 
 import pytest
 import requests
-from config import settings
 from guidellm.backend import OpenAIBackend
+from guidellm.config import settings
 from guidellm.core import TextGenerationRequest, TextGenerationResult
 from openai.pagination import SyncPage
 from openai.types import Model
@@ -32,7 +32,7 @@ def _openai_server_healthcheck():
 
 
 @pytest.mark.skip("OpenAI compatible service is not deployed yet")
-@pytest.mark.integration()
+@pytest.mark.sanity()
 def test_openai_submit_request(
     mocker,
     openai_backend_factory: Callable[..., OpenAIBackend],
diff --git a/tests/unit/config/__init__.py b/tests/unit/config/__init__.py
diff --git a/tests/unit/config/test_base.py b/tests/unit/config/test_base.py
diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py
diff --git a/tests/unit/utils/__init__.py b/tests/unit/utils/__init__.py
diff --git a/tests/unit/utils/test_injector.py b/tests/unit/utils/test_injector.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+from .base import settings`
	`2`	`+`
	`3`	`+__all__ = ["settings"]`