update

Signed-off-by: Chenfei Zhang <chenfeiz@nvidia.com>

Author: chenfeiz0326

jenkins/runPerfSanityTriage.groovy

@@ -110,4 +110,3 @@ pipeline {
         } // stage Run Perf Sanity Script
     } // stages
 } // pipeline
-

jenkins/scripts/open_search_db.py

@@ -257,7 +257,9 @@ def queryFromOpenSearchDB(json_data, project) -> dict:
             OpenSearchDB.logger.info(
                 f"project {project} is not in read access project list: {json.dumps(READ_ACCESS_PROJECT_NAME)}"
             )
-            print(f"project {project} is not in read access project list: {json.dumps(READ_ACCESS_PROJECT_NAME)}")
+            print(
+                f"project {project} is not in read access project list: {json.dumps(READ_ACCESS_PROJECT_NAME)}"
+            )
             return None
         if not isinstance(json_data, str):
             json_data_dump = json.dumps(json_data)
@@ -279,12 +281,16 @@ def queryFromOpenSearchDB(json_data, project) -> dict:
             OpenSearchDB.logger.info(
                 f"OpenSearchDB query failed, will retry, error:{res.status_code} {res.text}"
             )
-            print(f"OpenSearchDB query failed, will retry, error:{res.status_code} {res.text}")
+            print(
+                f"OpenSearchDB query failed, will retry, error:{res.status_code} {res.text}"
+            )
             retry_time -= 1
         OpenSearchDB.logger.info(
             f"Fail to queryFromOpenSearchDB after {retry_time} retry: {url}, json: {json_data_dump}, error: {res.text}"
         )
-        print(f"Fail to queryFromOpenSearchDB after {retry_time} retry: {url}, json: {json_data_dump}, error: {res.text}")
+        print(
+            f"Fail to queryFromOpenSearchDB after {retry_time} retry: {url}, json: {json_data_dump}, error: {res.text}"
+        )
         return None
 
     @staticmethod

jenkins/scripts/perf/perf_sanity_triage.py

@@ -14,45 +14,31 @@
 QUERY_LOOKBACK_DAYS = 7
 MAX_QUERY_SIZE = 3000
 
+
 def query_regression_data(project_name):
-    """
-    Query regression data from OpenSearch database.
-    Returns data where b_is_regression is True.
-    """
+    """Query regression data from OpenSearch database."""
     last_days = QUERY_LOOKBACK_DAYS
 
     must_clauses = [
-        {
-            "term": {
-                "b_is_valid": True
-            }
-        },
-        {
-            "term": {
-                "b_is_post_merge": True
-            }
-        },
-        {
-            "term": {
-                "b_is_regression": True
-            }
-        },
+        {"term": {"b_is_valid": True}},
+        {"term": {"b_is_post_merge": True}},
+        {"term": {"b_is_regression": True}},
         {
             "range": {
                 "ts_created": {
-                    "gte":
-                    int(time.time() - 24 * 3600 * last_days) // (24 * 3600) *
-                    24 * 3600 * 1000,
+                    "gte": int(time.time() - 24 * 3600 * last_days)
+                    // (24 * 3600)
+                    * 24
+                    * 3600
+                    * 1000,
                 }
             }
         },
     ]
 
     json_data = {
         "query": {
-            "bool": {
-                "must": must_clauses
-            },
+            "bool": {"must": must_clauses},
         },
         "size": MAX_QUERY_SIZE,
     }
@@ -62,38 +48,30 @@ def query_regression_data(project_name):
     try:
         res = OpenSearchDB.queryFromOpenSearchDB(json_data, project_name)
         if res is None:
-            print(
-                f"Failed to query from {project_name}, returned no response")
+            print(f"Failed to query from {project_name}, returned no response")
             return None
         else:
             payload = res.json().get("hits", {}).get("hits", [])
             if len(payload) == 0:
-                print(
-                    f"No regression data found in {project_name}, returned empty list"
-                )
+                print(f"No regression data found in {project_name}, returned empty list")
                 return []
             for hit in payload:
                 data_dict = hit.get("_source", {})
                 data_dict["_id"] = hit.get("_id", "")
                 if data_dict["_id"] == "":
-                    print(
-                        f"Failed to query from {project_name}, returned data with no _id"
-                    )
+                    print(f"Failed to query from {project_name}, returned data with no _id")
                     return None
                 data_list.append(data_dict)
-            print(
-                f"Successfully queried from {project_name}, queried {len(data_list)} entries"
-            )
+            print(f"Successfully queried from {project_name}, queried {len(data_list)} entries")
             return data_list
     except Exception as e:
         print(f"Failed to query from {project_name}, returned error: {e}")
         return None
 
 
 def get_regression_data_by_job_id(data_list, query_job_number):
-    """
-    Categorize regression data by s_job_id.
-    Returns a dict with job_id as key and list of regression data as value.
+    """Returns a dict with job_id as key and list of regression data as value.
+
     Only returns the latest query_job_number jobs.
     """
     if data_list is None or len(data_list) == 0:
@@ -111,14 +89,10 @@ def get_regression_data_by_job_id(data_list, query_job_number):
 
     # Sort job_ids by the latest ts_created in each group (descending)
     def get_latest_timestamp(job_id):
-        timestamps = [
-            d.get("ts_created", 0) for d in job_data_dict[job_id]
-        ]
+        timestamps = [d.get("ts_created", 0) for d in job_data_dict[job_id]]
         return max(timestamps) if timestamps else 0
 
-    sorted_job_ids = sorted(job_data_dict.keys(),
-                            key=get_latest_timestamp,
-                            reverse=True)
+    sorted_job_ids = sorted(job_data_dict.keys(), key=get_latest_timestamp, reverse=True)
 
     # Only keep the latest query_job_number jobs
     latest_job_ids = sorted_job_ids[:query_job_number]
@@ -131,8 +105,8 @@ def get_latest_timestamp(job_id):
 
 
 def send_regression_message(regression_dict, channel_id, bot_token):
-    """
-    Organize regression data into message format and send to Slack channel(s).
+    """Organize regression data into message format and send to Slack channel(s).
+
     channel_id can be a single ID or multiple IDs separated by commas.
     """
     if not regression_dict:
@@ -146,8 +120,7 @@ def send_regression_message(regression_dict, channel_id, bot_token):
         job_header = f"*LLM/main/L0_PostMerge/{job_id}:*\n"
         msg_parts.append(job_header)
 
-        sorted_data_list = sorted(data_list,
-                                  key=lambda x: x.get("s_test_case_name", ""))
+        sorted_data_list = sorted(data_list, key=lambda x: x.get("s_test_case_name", ""))
         for idx, data in enumerate(sorted_data_list, start=1):
             test_case_name = data.get("s_test_case_name", "N/A")
             regression_info = data.get("s_regression_info", "N/A")
@@ -169,9 +142,7 @@ def send_regression_message(regression_dict, channel_id, bot_token):
 
 
 def send_message(msg, channel_id, bot_token):
-    """
-    Send message to Slack channel using slack_sdk.
-    """
+    """Send message to Slack channel using slack_sdk."""
     try:
         client = WebClient(token=bot_token)
 
@@ -196,28 +167,19 @@ def send_message(msg, channel_id, bot_token):
 
 
 def main():
-    parser = argparse.ArgumentParser(
-        description="Perf Sanity Triage Script")
-    parser.add_argument("--project_name",
-                        type=str,
-                        required=True,
-                        help="OpenSearch project name")
-    parser.add_argument("--operation",
-                        type=str,
-                        required=True,
-                        help="Operation to perform")
-    parser.add_argument("--channel_id",
-                        type=str,
-                        default="",
-                        help="Slack channel ID(s), comma-separated for multiple channels")
-    parser.add_argument("--bot_token",
-                        type=str,
-                        default="",
-                        help="Slack bot token")
-    parser.add_argument("--query_job_number",
-                        type=int,
-                        default=1,
-                        help="Number of latest jobs to query")
+    parser = argparse.ArgumentParser(description="Perf Sanity Triage Script")
+    parser.add_argument("--project_name", type=str, required=True, help="OpenSearch project name")
+    parser.add_argument("--operation", type=str, required=True, help="Operation to perform")
+    parser.add_argument(
+        "--channel_id",
+        type=str,
+        default="",
+        help="Slack channel ID(s), comma-separated for multiple channels",
+    )
+    parser.add_argument("--bot_token", type=str, default="", help="Slack bot token")
+    parser.add_argument(
+        "--query_job_number", type=int, default=1, help="Number of latest jobs to query"
+    )
 
     args = parser.parse_args()
 
@@ -233,11 +195,9 @@ def main():
             print("Failed to query regression data")
             return
 
-        regression_dict = get_regression_data_by_job_id(data_list,
-                                                        args.query_job_number)
+        regression_dict = get_regression_data_by_job_id(data_list, args.query_job_number)
 
-        send_regression_message(regression_dict, args.channel_id,
-                                args.bot_token)
+        send_regression_message(regression_dict, args.channel_id, args.bot_token)
     else:
         print(f"Unknown operation: {args.operation}")
 

tests/integration/defs/perf/open_search_db_utils.py

@@ -643,65 +643,19 @@ def _get_metric_keys():
     return metric_keys
 
 
-def _print_perf_data(data):
-    """Print performance metrics and config for a single data entry."""
-    print_info("=== Metrics ===")
-    for metric in MAXIMIZE_METRICS + MINIMIZE_METRICS:
-        if metric in data:
-            value = data.get(metric, "N/A")
-            print_info(f'"{metric}": {value}')
-
-    metric_keys = _get_metric_keys()
-    print_info("\n=== Config ===")
-    config_keys = sorted([key for key in data.keys() if key not in metric_keys])
-    for key in config_keys:
-        value = data[key]
-        print_info(f'"{key}": {value}')
-
-
 def _print_regression_data(data, print_func=None):
     """
-    Print regression info, metrics with baselines/thresholds, and config.
+    Print regression info and config.
     """
     if print_func is None:
         print_func = print_info
 
     if "s_regression_info" in data:
         print_func("=== Regression Info ===")
-        print_func(f"{data['s_regression_info']}")
+        for item in data["s_regression_info"].split(","):
+            print_func(item.strip())
 
     metric_keys = _get_metric_keys()
-    is_post_merge = data.get("b_is_post_merge", False)
-
-    print_func("=== Metrics ===")
-    for metric in MAXIMIZE_METRICS + MINIMIZE_METRICS:
-        metric_suffix = metric[2:]  # Strip "d_" prefix
-        baseline_key = f"d_baseline_{metric_suffix}"
-        if is_post_merge:
-            threshold_key = f"d_threshold_post_merge_{metric_suffix}"
-        else:
-            threshold_key = f"d_threshold_pre_merge_{metric_suffix}"
-        # Only print if at least one of the keys exists
-        if metric in data or baseline_key in data or threshold_key in data:
-            value = data.get(metric, "N/A")
-            baseline = data.get(baseline_key, "N/A")
-            threshold = data.get(threshold_key, "N/A")
-            # Calculate percentage difference between value and baseline
-            # Positive percentage means better perf, negative means regression
-            if (isinstance(value, (int, float))
-                    and isinstance(baseline, (int, float)) and baseline != 0):
-                if metric in MAXIMIZE_METRICS:
-                    # Larger is better: value > baseline is positive (better)
-                    percentage = (value - baseline) / baseline * 100
-                else:
-                    # Smaller is better: value < baseline is positive (better)
-                    percentage = (baseline - value) / baseline * 100
-                percentage_str = f"{percentage:+.2f}%"
-            else:
-                percentage_str = "N/A"
-            print_func(
-                f'"{metric}": {value}, "{baseline_key}": {baseline}, '
-                f'"{threshold_key}": {threshold}, "diff": {percentage_str}')
 
     print_func("\n=== Config ===")
     config_keys = sorted([key for key in data.keys() if key not in metric_keys])
@@ -721,7 +675,6 @@ def check_perf_regression(new_data_dict):
         data for data in new_data_dict.values()
         if data.get("b_is_regression", False)
     ]
-
     # Split regression data into post-merge and pre-merge
     post_merge_regressions = [
         data for data in regressive_data_list
@@ -744,15 +697,16 @@ def check_perf_regression(new_data_dict):
 
     # Print post-merge regression data with print_error
     if len(post_merge_regressions) > 0:
+        print_error(
+            f"Found {len(post_merge_regressions)} post-merge regression data")
         for i, data in enumerate(post_merge_regressions):
             print_error(f"\n{'=' * 60}")
             print_error(f"Post-merge Regression Data #{i + 1}")
             print_error("=" * 60)
             _print_regression_data(data, print_func=print_error)
-        print_error(
-            f"Found {len(post_merge_regressions)} post-merge regression data")
-        raise RuntimeError(
-            f"Found {len(post_merge_regressions)} post-merge regression data")
+        # Regression will not fail the test.
+        # raise RuntimeError(
+        #     f"Found {len(post_merge_regressions)} post-merge regression data")
 
     # Print summary if no regressions
     if len(regressive_data_list) == 0:

tests/integration/defs/perf/test_perf_sanity.py

@@ -1176,9 +1176,7 @@ def get_perf_result(self, outputs: Dict[int, List[str]]):
         """Parse performance results from outputs."""
 
         def parse_metrics_from_output(output: str) -> Dict[str, float] | None:
-            """Parse all metrics from a single output string.
-
-            """
+            """Parse all metrics from a single output string."""
             metrics = {}
             lines = output.split("\n")
             for metric_type, regex in PERF_METRIC_LOG_QUERIES.items():
@@ -1389,16 +1387,19 @@ def check_test_failure(self):
         failed_servers = []
         for server_idx, client_configs in self.server_client_configs.items():
             server_perf_results = self._perf_results.get(server_idx, [])
-            if (
-                len(server_perf_results) != len(client_configs)
-                or any(metrics is None for metrics in server_perf_results)
+            if len(server_perf_results) != len(client_configs) or any(
+                metrics is None for metrics in server_perf_results
             ):
                 failed_servers.append(server_idx)
 
         if failed_servers:
             for server_idx in failed_servers:
-                print_error(f"Server {server_idx} failed: perf results count mismatch or incomplete metrics")
-            raise Exception(f"Test failed: servers {failed_servers} did not produce expected results")
+                print_error(
+                    f"Server {server_idx} failed: perf results count mismatch or incomplete metrics"
+                )
+            raise Exception(
+                f"Test failed: servers {failed_servers} did not produce expected results"
+            )
 
         print_info("All servers passed")
 

tests/integration/test_lists/waives.txt

@@ -497,7 +497,6 @@ disaggregated/test_disaggregated.py::test_disaggregated_benchmark_on_diff_backen
 disaggregated/test_disaggregated.py::test_disaggregated_benchmark_on_diff_backends[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/5769890)
 disaggregated/test_disaggregated.py::test_disaggregated_benchmark_on_diff_backends[llama-v3-8b-hf] SKIP (https://nvbugs/5769890,https://nvbugs/5748683)
 accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_pp4_mtp] SKIP (https://nvbugs/5779536)
-perf/test_perf_sanity.py::test_e2e[disagg_upload-deepseek-r1-fp4_1k1k_ctx1_gen1_dep8_bs768_eplb0_mtp0_ccb-UCX] SKIP (https://nvbugs/5778381)
 unittest/_torch/attention/test_flashinfer_star_attn.py::TestStarAttention::test_flashinfer_star_attention[num_layers:2-num_heads:32-num_kv_heads:8-head_dim:64-anchor_size:64-block_size:64-dtype:torch.float16] SKIP (https://nvbugs/5781389)
 unittest/_torch/ray_orchestrator/multi_gpu/test_ops.py::test_reducescatter_pg_op[var_len:True-seqlen:16-hidden:128] SKIP (https://nvbugs/5781383)
 cpp/test_e2e.py::test_model[-mamba-86] SKIP (https://nvbugs/5781665)