[4975376]Add support for HF model perplexity calculation

Signed-off-by: unknown <[email protected]>

Author: ynankani

examples/windows/accuracy_benchmark/perplexity_metrics/README.md

@@ -25,6 +25,7 @@ This script is originally based on [perplexity_metrics.py](https://github.com/mi
 - Install dependencies:
 
   **For CUDA 12.x (recommended for CUDA 12.1-12.9):**
+
   ```bash
   pip install -r requirements.txt
   ```
@@ -38,11 +39,13 @@ This script is originally based on [perplexity_metrics.py](https://github.com/mi
 ## Supported Models
 
 ### ONNX Runtime GenAI Models
+
 - Any ONNX Runtime GenAI model exported with a compatible `genai_config.json` and tokenizer.
 - Supported architectures include: Gemma, Llama, Mistral, Phi (language + vision), Qwen.
 - Supported execution providers: CPU, DirectML, CUDA, NvTensorRtRtx.
 
 ### HuggingFace Models
+
 - Any HuggingFace causal language model (e.g., `meta-llama/Llama-2-7b-hf`, `gpt2`, `mistralai/Mistral-7B-v0.1`).
 - Models are automatically downloaded from the HuggingFace Hub if not cached locally.
 - Supports custom data types (float16, bfloat16, float32) for efficient inference.
@@ -52,23 +55,27 @@ This script is originally based on [perplexity_metrics.py](https://github.com/mi
 ### Evaluate ONNX Models
 
 #### Single Model
+
 ```bash
 python run_perplexity.py --models /path/to/model
 ```
 
 #### Multiple Models
+
 ```bash
 python run_perplexity.py --models /path/to/model1 /path/to/model2
 ```
 
 #### Custom Input Sequence Length(s)
+
 You can specify the input sequence length(s) to evaluate using the `--i` argument:
 
 ```bash
 python run_perplexity.py --models /path/to/model --i 1024,2048,4096,8192,12288
 ```
 
 #### Custom Prefill Chunk Size
+
 You can specify the prefill chunk size to evaluate using the `--chunk_size` argument:
 
 ```bash
@@ -78,21 +85,25 @@ python run_perplexity.py --models /path/to/model --i 1024,2048,4096,8192,12288 -
 ### Evaluate HuggingFace Models
 
 #### Basic HuggingFace Model Evaluation
+
 ```bash
 python run_perplexity.py --hf_model meta-llama/Llama-2-7b-hf --i 1024
 ```
 
 #### With Custom Data Type (Recommended for Performance)
+
 ```bash
 python run_perplexity.py --hf_model meta-llama/Llama-2-7b-hf --hf_dtype float16 --i 1024
 ```
 
 #### With Multiple Input Lengths
+
 ```bash
 python run_perplexity.py --hf_model meta-llama/Llama-2-7b-hf --hf_dtype float16 --i 1024,2048,4096
 ```
 
 #### On CPU (if no GPU available)
+
 ```bash
 python run_perplexity.py --hf_model gpt2 --hf_device cpu --i 1024
 ```
@@ -189,6 +200,7 @@ Set `DEBUG = True` in `perplexity_metrics.py` for detailed logs.
 ## Common Use Cases
 
 ### Compare ONNX vs. HuggingFace Model
+
 Verify that your ONNX exported model has similar perplexity to the original HuggingFace model:
 
 ```bash
@@ -201,11 +213,13 @@ python run_perplexity.py \
 ```
 
 ### Evaluate Small Models (for quick testing)
+
 ```bash
 python run_perplexity.py --hf_model gpt2 --hf_dtype float16 --i 1024
 ```
 
 ### Benchmark Multiple Quantization Variants
+
 ```bash
 python run_perplexity.py \
   --models /path/to/fp16_model /path/to/int8_model /path/to/int4_model \

examples/windows/accuracy_benchmark/perplexity_metrics/perplexity_metrics.py

@@ -114,9 +114,9 @@ def calculate_perplexity_hf(
     print(f"[INFO] Full input length: {seq_len}")
     print(f"[INFO] max_length: {max_length}, stride: {stride}")
 
-    max_eval_length = seq_len  
+    max_eval_length = seq_len
 
-    # Initialize accumulators for log probabilities (same as ONNX version)
+    # Initialize accumulators for log probabilities
     total_log_probs = 0.0
     total_token_count = 0
     prev_end_loc = 0
@@ -127,14 +127,15 @@ def calculate_perplexity_hf(
         trg_len = end_loc - prev_end_loc
 
         if DEBUG:
-            print(f"\n[LOOP] chunk_idx={chunk_idx} [begin={begin_loc} end={end_loc}] trg_len={trg_len}")
+            print(
+                f"\n[LOOP] chunk_idx={chunk_idx} [begin={begin_loc} end={end_loc}] trg_len={trg_len}"
+            )
 
         # Extract the current chunk of input tokens (keep on CPU until needed)
         input_ids_chunk = input_ids[:, begin_loc:end_loc].to(device)
         target_ids = input_ids_chunk.clone()
 
         # Mask context tokens: only predict for last trg_len tokens in chunk
-        # This matches the ONNX version logic
         mask = np.ones(target_ids.shape, dtype=bool)
         mask[:, :-trg_len] = False
         target_ids_masked = target_ids.clone()
@@ -155,7 +156,7 @@ def calculate_perplexity_hf(
             if DEBUG:
                 print(f"[LOGITS] Shape: {logits.shape}, dtype: {logits.dtype}")
 
-        # Compute log probabilities over vocabulary for each position (same as ONNX)
+        # Compute log probabilities over vocabulary for each position
         log_probs = torch.nn.functional.log_softmax(logits, dim=2).cpu().numpy()
         chunk_seq_len = log_probs.shape[1]
 
@@ -197,12 +198,22 @@ def calculate_perplexity_hf(
         total_token_count += int(valid_log_probs.size)
 
         if DEBUG:
-            print(f"[LOOP] This chunk: valid tokens={valid_log_probs.size}, sum={np.sum(valid_log_probs)}")
+            print(
+                f"[LOOP] This chunk: valid tokens={valid_log_probs.size}, sum={np.sum(valid_log_probs)}"
+            )
             print(f"[TALLY] total_log_probs: {total_log_probs}")
             print(f"[TALLY] total_token_count: {total_token_count}")
 
         # Clear GPU cache to prevent OOM
-        del outputs, logits, log_probs, pred_log_probs, input_ids_chunk, target_ids, target_ids_masked
+        del (
+            outputs,
+            logits,
+            log_probs,
+            pred_log_probs,
+            input_ids_chunk,
+            target_ids,
+            target_ids_masked,
+        )
         if device == "cuda":
             torch.cuda.empty_cache()
 

examples/windows/accuracy_benchmark/perplexity_metrics/requirements.txt

@@ -1,5 +1,6 @@
 # PyTorch with CUDA 12.x support (compatible with CUDA 12.1-12.9)
 --extra-index-url https://download.pytorch.org/whl/cu129
+accelerate
 
 coloredlogs
 datasets
@@ -15,9 +16,8 @@ pytest
 sentencepiece
 sympy
 torch>=2.0.0
-torchvision
 torchaudio
+torchvision
 transformers
-accelerate
 
 

examples/windows/accuracy_benchmark/perplexity_metrics/run_perplexity.py

@@ -116,7 +116,7 @@ def run_perplexity_on_models(
                         "Error": "None",
                     }
                 )
-            except Exception as e:
+            except Exception as e:  # noqa: PERF203
                 print(f"  Error for input length {input_len}: {e!s}")
                 results.append(
                     {
@@ -134,6 +134,7 @@ def run_perplexity_on_models(
         # Unload HuggingFace model from GPU memory before ONNX evaluation
         print("[CLEANUP] Unloading HuggingFace model from GPU memory...")
         import gc
+
         import torch
 
         if torch.cuda.is_available():