[GuideLLM Refactor] Replace librosa, pydub, and soundfile with torchcodec (#411)

## TODO

- [ ] ~~More flexible version locking in multimodal extras group~~
- Goal with this was to add locking for different torchcodec/torch
versions but honestly its not worth the hassle
- [x] Check for multi-modal libs being installed
- [ ] More testing on `encode_audio`

## Summary

<!--
Include a short paragraph of the changes introduced in this PR.
If this PR requires additional context or rationale, explain why
the changes are necessary.
-->
Replaces audio processing libraries with `torchcodec` which eliminates
19 dependencies and brings us inline with what HuggingFace `datasets` is
doing.

## Details

<!--
Provide a detailed list of all changes introduced in this pull request.
-->
- 

## Test Plan

<!--
List the steps needed to test this PR.
-->
- Run against audio server with

```bash
guidellm benchmark run \
    --target http://localhost:8000 \
    --profile "synchronous" \
    --max-requests 20 \
    --request-type "audio_transcriptions" \
    --data "openslr/librispeech_asr" \
    --data-args '{"name": "clean", "split": "test"}'
```

---

- [x] "I certify that all code in this PR is my own, except as noted
below."

## Use of AI

- [x] Includes AI-assisted code completion
- [ ] Includes code generated by an AI application
- [ ] Includes AI-generated tests (NOTE: AI written tests should have a
docstring that includes `## WRITTEN BY AI ##`)

Author: sjmonson

pylock.toml

@@ -62,26 +62,37 @@ dependencies = [
     "httpx[http2]<1.0.0",
     "loguru",
     "msgpack",
-    "numpy<2.0.0",
-    "pillow",
+    "numpy>=2.0.0",
     "protobuf",
     "pydantic>=2.11.7",
     "pydantic-settings>=2.0.0",
-    "pydub",
     "pyyaml>=6.0.0",
     "rich",
     "sanic",
     "transformers",
     "uvloop>=0.18",
-    "librosa>=0.11.0",
     "torch",
 ]
 
 [project.optional-dependencies]
+# Meta Extras
+all = ["guidellm[perf,openai,multimodal]"]
+recommended = ["guidellm[perf,openai]"]
+# Feature Extras
 perf = ["orjson", "msgpack", "msgspec", "uvloop"]
 openai = ["tiktoken>=0.11.0", "blobfile>=3.1.0"]
-recommended = ["guidellm[perf,openai]"]
+multimodal = [
+    "datasets[audio,vision]>=4.1.0",
+    "pillow",
+    # Torchcodec needs specific torch version
+    "torch==2.9.*",
+    "torchcodec==0.8",
+]
+# Dev Tooling
 dev = [
+    # Install all optional dependencies
+    "guidellm[all]",
+
     # build
     "build>=1.0.0",
     "setuptools>=61.0",

pyproject.toml

@@ -1,13 +1,14 @@
 from __future__ import annotations
 
+from abc import ABCMeta
 from typing import Any
 
 from guidellm.data.preprocessors.preprocessor import (
     DatasetPreprocessor,
     PreprocessorRegistry,
 )
 from guidellm.data.schemas import GenerativeDatasetColumnType
-from guidellm.data.utils import encode_audio, encode_image, encode_video, text_stats
+from guidellm.data.utils import text_stats
 from guidellm.schemas import GenerationRequest, GenerationRequestArguments, UsageMetrics
 
 __all__ = [
@@ -18,8 +19,28 @@
 ]
 
 
+class RequestFormatter(DatasetPreprocessor, metaclass=ABCMeta):
+    @staticmethod
+    def encode_audio(*args, **kwargs):
+        from guidellm.extras.multimodal import encode_audio
+
+        return encode_audio(*args, **kwargs)
+
+    @staticmethod
+    def encode_image(*args, **kwargs):
+        from guidellm.extras.multimodal import encode_image
+
+        return encode_image(*args, **kwargs)
+
+    @staticmethod
+    def encode_video(*args, **kwargs):
+        from guidellm.extras.multimodal import encode_video
+
+        return encode_video(*args, **kwargs)
+
+
 @PreprocessorRegistry.register("text_completions")
-class GenerativeTextCompletionsRequestFormatter(DatasetPreprocessor):
+class GenerativeTextCompletionsRequestFormatter(RequestFormatter):
     def __init__(
         self,
         model: str,
@@ -92,7 +113,7 @@ def __call__(
 
 
 @PreprocessorRegistry.register("chat_completions")
-class GenerativeChatCompletionsRequestFormatter(DatasetPreprocessor):
+class GenerativeChatCompletionsRequestFormatter(RequestFormatter):
     def __init__(
         self,
         model: str,
@@ -120,7 +141,7 @@ def __init__(
             encode_kwargs.get("audio", {}) if encode_kwargs else {}
         )
 
-    def __call__(
+    def __call__(  # noqa: C901, PLR0912, PLR0915
         self, columns: dict[GenerativeDatasetColumnType, list[Any]]
     ) -> GenerationRequest:
         arguments = GenerationRequestArguments(body={})
@@ -200,7 +221,7 @@ def __call__(
             if not image:
                 continue
 
-            image_dict = encode_image(image, **self.encode_image_kwargs)
+            image_dict = self.encode_image(image, **self.encode_image_kwargs)
             if (image_pixels := image_dict.get("image_pixels")) is not None:
                 input_metrics.image_pixels = (
                     input_metrics.image_pixels or 0
@@ -223,7 +244,7 @@ def __call__(
             if not video:
                 continue
 
-            video_dict = encode_video(video, **self.encode_video_kwargs)
+            video_dict = self.encode_video(video, **self.encode_video_kwargs)
             if (video_frames := video_dict.get("video_frames")) is not None:
                 input_metrics.video_frames = (
                     input_metrics.video_frames or 0
@@ -250,7 +271,9 @@ def __call__(
             if not audio:
                 continue
 
-            audio_dict = encode_audio(audio, b64encode=True, **self.encode_audio_kwargs)
+            audio_dict = self.encode_audio(
+                audio, b64encode=True, **self.encode_audio_kwargs
+            )
             if (audio_samples := audio_dict.get("audio_samples")) is not None:
                 input_metrics.audio_samples = (
                     input_metrics.audio_samples or 0
@@ -288,7 +311,7 @@ def __call__(
 
 
 @PreprocessorRegistry.register("audio_transcriptions")
-class GenerativeAudioTranscriptionRequestFormatter(DatasetPreprocessor):
+class GenerativeAudioTranscriptionRequestFormatter(RequestFormatter):
     def __init__(
         self,
         model: str,
@@ -345,7 +368,7 @@ def __call__(  # noqa: C901
                 f"one audio column, but got {len(audio_columns)}."
             )
 
-        audio_dict = encode_audio(
+        audio_dict = self.encode_audio(
             audio_columns[0], b64encode=False, **self.encode_audio_kwargs
         )
         input_metrics.audio_samples = audio_dict.get("audio_samples")

src/guidellm/data/preprocessors/formatters.py

@@ -1,22 +1,10 @@
 from .dataset import DEFAULT_SPLITS, resolve_dataset_split
 from .functions import (
-    encode_audio,
-    encode_image,
-    encode_video,
-    get_file_format,
-    is_url,
-    resize_image,
     text_stats,
 )
 
 __all__ = [
     "DEFAULT_SPLITS",
-    "encode_audio",
-    "encode_image",
-    "encode_video",
-    "get_file_format",
-    "is_url",
-    "resize_image",
     "resolve_dataset_split",
     "text_stats",
 ]