Add number of input tokens metric (#559)

* calculate number of input tokens

* Calculate input tokens for triton

* skip empty response

* Remove unused imports

Author: nv-hwoo

src/c++/perf_analyzer/genai-perf/genai_perf/llm_metrics.py

@@ -31,11 +31,15 @@
 from itertools import pairwise
 
 import numpy as np
+from genai_perf.llm_inputs.llm_inputs import OutputFormat
 from genai_perf.tokenizer import AutoTokenizer
 from genai_perf.utils import load_json, remove_sse_prefix
 from rich.console import Console
 from rich.table import Table
 
+_OPENAI_CHAT_COMPLETIONS = OutputFormat.OPENAI_CHAT_COMPLETIONS
+_OPENAI_COMPLETIONS = OutputFormat.OPENAI_COMPLETIONS
+
 
 class Metrics:
     """A base class for all the metrics class that contains common metrics."""
@@ -48,6 +52,7 @@ class Metrics:
         "output_token_throughput_per_request",
         "request_throughput",
         "num_output_token",
+        "num_input_token",
     ]
 
     time_fields = [
@@ -108,13 +113,15 @@ def __init__(
         output_token_throughputs: list[float] = [],
         output_token_throughputs_per_request: list[int] = [],
         num_output_tokens: list[int] = [],
+        num_input_tokens: list[int] = [],
     ) -> None:
         super().__init__(request_throughputs, request_latencies)
         self.time_to_first_tokens = time_to_first_tokens
         self.inter_token_latencies = inter_token_latencies
         self.output_token_throughputs = output_token_throughputs
         self.output_token_throughputs_per_request = output_token_throughputs_per_request
         self.num_output_tokens = num_output_tokens
+        self.num_input_tokens = num_input_tokens
 
         # add base name mapping
         self._base_names["time_to_first_tokens"] = "time_to_first_token"
@@ -124,6 +131,7 @@ def __init__(
             "output_token_throughputs_per_request"
         ] = "output_token_throughput_per_request"
         self._base_names["num_output_tokens"] = "num_output_token"
+        self._base_names["num_input_tokens"] = "num_input_token"
 
 
 class Statistics:
@@ -424,10 +432,15 @@ class LLMProfileDataParser(ProfileDataParser):
     """
 
     def __init__(
-        self, filename: str, service_kind: str, tokenizer: AutoTokenizer
+        self,
+        filename: str,
+        service_kind: str,
+        output_format: OutputFormat,
+        tokenizer: AutoTokenizer,
     ) -> None:
         self._tokenizer = tokenizer
         self._service_kind = service_kind
+        self._output_format = output_format
         super().__init__(filename)
 
     def _parse_requests(self, requests: dict) -> LLMMetrics:
@@ -437,14 +450,21 @@ def _parse_requests(self, requests: dict) -> LLMMetrics:
         time_to_first_tokens = []
         inter_token_latencies = []
         output_token_throughputs_per_request = []
+        num_input_tokens = []
         num_generated_tokens = []
         for request in requests:
             req_timestamp = request["timestamp"]
+            req_inputs = request["request_inputs"]
             res_timestamps = request["response_timestamps"]
             res_outputs = request["response_outputs"]
 
             self._preprocess_response(res_timestamps, res_outputs)
 
+            # Skip requests with empty response. This happens sometimes when the
+            # model returns a single response with empty string.
+            if not res_timestamps:
+                continue
+
             # track entire benchmark duration
             min_req_timestamp = min(min_req_timestamp, req_timestamp)
             max_res_timestamp = max(max_res_timestamp, res_timestamps[-1])
@@ -457,6 +477,10 @@ def _parse_requests(self, requests: dict) -> LLMMetrics:
             # time to first token
             time_to_first_tokens.append(res_timestamps[0] - req_timestamp)
 
+            # number of input tokens
+            input_tokens = self._tokenize_request_inputs(req_inputs)
+            num_input_tokens.append(len(input_tokens))
+
             # output token throughput per request
             output_tokens = self._tokenize_response_outputs(res_outputs)
             num_output_tokens = list(map(len, output_tokens))
@@ -490,6 +514,7 @@ def _parse_requests(self, requests: dict) -> LLMMetrics:
             output_token_throughputs,
             output_token_throughputs_per_request,
             num_generated_tokens,
+            num_input_tokens,
         )
 
     def _preprocess_response(
@@ -513,6 +538,32 @@ def _preprocess_response(
                 res_timestamps.pop()
                 res_outputs.pop()
 
+    def _tokenize_request_inputs(self, req_inputs: dict) -> list[list[int]]:
+        """Deserialize the request input and return tokenized inputs."""
+        if self._service_kind == "triton":
+            return self._tokenize_triton_request_input(req_inputs)
+        elif self._service_kind == "openai":
+            return self._tokenize_openai_request_input(req_inputs)
+        else:
+            raise ValueError(f"Unknown service kind: '{self._service_kind}'.")
+
+    def _tokenize_triton_request_input(self, req_inputs: dict) -> list[list[int]]:
+        """Tokenize the Triton request input texts."""
+        return self._tokenizer(req_inputs["text_input"])["input_ids"]
+
+    def _tokenize_openai_request_input(self, req_inputs: dict) -> list[list[int]]:
+        """Tokenize the OpenAI request input texts."""
+        payload = json.loads(req_inputs["payload"])
+        if self._output_format == _OPENAI_CHAT_COMPLETIONS:
+            input_text = payload["messages"][0]["content"]
+        elif self._output_format == _OPENAI_COMPLETIONS:
+            input_text = payload["prompt"][0]
+        else:
+            raise ValueError(
+                "Failed to parse OpenAI request input in profile export file."
+            )
+        return self._tokenizer(input_text)["input_ids"]
+
     def _tokenize_response_outputs(self, res_outputs: dict) -> list[list[int]]:
         """Deserialize the response output and return tokenized outputs."""
         if self._service_kind == "triton":

src/c++/perf_analyzer/genai-perf/genai_perf/main.py

@@ -71,9 +71,14 @@ def generate_inputs(args: ArgumentParser, tokenizer: AutoTokenizer) -> None:
 
 
 def calculate_metrics(
-    file: str, service_kind: str, tokenizer: AutoTokenizer
+    args: ArgumentParser, tokenizer: AutoTokenizer
 ) -> LLMProfileDataParser:
-    return LLMProfileDataParser(file, service_kind, tokenizer)
+    return LLMProfileDataParser(
+        filename=args.profile_export_file,
+        service_kind=args.service_kind,
+        output_format=args.output_format,
+        tokenizer=tokenizer,
+    )
 
 
 def report_output(metrics: LLMProfileDataParser, args):
@@ -99,9 +104,7 @@ def run():
         tokenizer = get_tokenizer(args.tokenizer)
         generate_inputs(args, tokenizer)
         args.func(args, extra_args)
-        metrics = calculate_metrics(
-            args.profile_export_file, args.service_kind, tokenizer
-        )
+        metrics = calculate_metrics(args, tokenizer)
         report_output(metrics, args)
     except Exception as e:
         raise GenAIPerfException(e)