update waive list (NVIDIA#3503)

Signed-off-by: xinhe-nv <[email protected]>

Author: xinhe-nv

tests/integration/defs/.test_durations

@@ -121,11 +121,6 @@
    "test_accuracy.py::test_accuracy_gptj[gptj-cyclic-and-paged-kv-cache]": 173.45949043799192,
    "test_accuracy.py::test_accuracy_gptj[gptj-cyclic-kv-cache-beam-search]": 231.8624299732037,
    "test_accuracy.py::test_accuracy_gptj[gptj-mmha-multi-block-mode]": 182.0765182878822,
-   "test_e2e.py::test_falcon_e2e[gpu_percent_0-use_py_session-gqa]": 96.65795732289553,
-   "test_e2e.py::test_falcon_e2e[gpu_percent_0_8-use_cpp_session-mqa]": 94.91713926941156,
-   "test_e2e.py::test_falcon_e2e[use_cpp_session-mha]": 72.26053975522518,
-   "test_e2e.py::test_falcon_e2e[use_py_session-mha]": 71.00531469285488,
-   "test_e2e.py::test_falcon_gqa_e2e[use_py_session-enable_ibf-enable_fp8]": 68.68854057043791,
    "test_e2e.py::test_gpt_fp32[use_cpp_session]": 102.94305092096329,
    "test_e2e.py::test_gpt_fp32[use_py_session-multi_query_mode]": 101.54258136451244,
    "test_e2e.py::test_gpt_fp32[use_py_session]": 100.1567601710558,
@@ -146,11 +141,6 @@
    "test_accuracy.py::test_accuracy_gpt[gpt-context-fmha-disabled]": 96.56836012890562,
    "test_accuracy.py::test_accuracy_gptj[gptj-context-fmha-enabled]": 227.20399192301556,
    "test_accuracy.py::test_accuracy_gptj[gptj-cyclic-kv-cache]": 168.16926325811073,
-   "test_e2e.py::test_falcon_e2e[use_cpp_session-gqa]": 75.75471282750368,
-   "test_e2e.py::test_falcon_e2e[use_cpp_session-mqa]": 74.27118157595396,
-   "test_e2e.py::test_falcon_e2e[use_py_session-gqa]": 74.21104773320258,
-   "test_e2e.py::test_falcon_e2e[use_py_session-mqa]": 73.01505787856877,
-   "test_e2e.py::test_falcon_gqa_e2e[use_cpp_session-enable_ibf-enable_fp8]": 71.1147844651714,
    "test_e2e.py::test_gpt_fp32[use_cpp_session-multi_query_mode]": 102.81246098689735,
    "test_e2e.py::test_mistral_e2e[use_py_session-remove_input_padding]": 178.69259701482952,
    "test_e2e.py::test_mistral_e2e[use_py_session]": 158.53167643211782,

tests/integration/test_lists/qa/examples_test_list.txt

@@ -443,8 +443,6 @@ test_e2e.py::test_benchmark_sanity[roberta_base]
 test_e2e.py::test_benchmark_sanity[t5_base]
 test_e2e.py::test_benchmark_sanity_enable_fp8[gpt_350m]
 test_e2e.py::test_benchmark_sanity_enable_fp8[llama_7b]
-test_e2e.py::test_falcon_e2e[gpu_percent_0-use_py_session-gqa]
-test_e2e.py::test_falcon_e2e[gpu_percent_0_8-use_cpp_session-mqa]
 test_e2e.py::test_llama_e2e[use_cpp_session-remove_input_padding]
 test_e2e.py::test_llama_e2e[use_py_session-remove_input_padding]
 test_e2e.py::test_llama_e2e[use_py_session]

tests/integration/test_lists/waives.txt

@@ -446,3 +446,16 @@ examples/test_eagle.py::test_llama_eagle_1gpu[llama-3.1-8b-eagle1] SKIP (https:/
 examples/test_eagle.py::test_mistral_eagle_1gpu[mistral-7b-v0.1-eagle1] SKIP (https://nvbugs/5219535)
 examples/test_eagle.py::test_llama_eagle_1gpu[llama-3.1-8b-eagle2] SKIP (https://nvbugs/5219535)
 examples/test_eagle.py::test_mistral_eagle_1gpu[mistral-7b-v0.1-eagle2] SKIP (https://nvbugs/5219535)
+examples/test_mixtral.py::test_llm_mixtral_fp8_4gpus_summary[Mixtral-8x22B-v0.1-nb:1] SKIP (https://nvbugs/5220758)
+examples/test_multimodal.py::test_llm_multimodal_general[VILA1.5-3b-pp:1-tp:1-float16-bs:8-cpp_e2e:False-nb:1] SKIP (https://nvbugs/5214239)
+examples/test_multimodal.py::test_llm_multimodal_general[VILA1.5-3b-pp:1-tp:1-float16-bs:1-cpp_e2e:True-nb:1] SKIP (https://nvbugs/5214239)
+examples/test_multimodal.py::test_llm_multimodal_general[VILA1.5-3b-pp:1-tp:1-float16-bs:8-cpp_e2e:True-nb:1] SKIP (https://nvbugs/5214239)
+examples/test_multimodal.py::test_llm_multimodal_general[neva-22b-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:False-nb:1] SKIP (https://nvbugs/5220761)
+examples/test_multimodal.py::test_llm_multimodal_general[neva-22b-pp:1-tp:1-bfloat16-bs:8-cpp_e2e:True-nb:1] SKIP (https://nvbugs/5220761)
+accuracy/test_cli_flow/TestSantacoder.py::test_auto_dtype SKIP (https://nvbugs/5219531)
+accuracy/test_llm_api_pytorch/TestMixtral8x7B.py::test_fp8_tp2 SKIP (https://nvbugs/5220763)
+test_e2e.py::test_trtllm_bench_mgmn SKIP (https://nvbugs/5220766)
+examples/test_medusa.py::test_codellama_medusa_1gpu[CodeLlama-7b-Instruct] SKIP (https://nvbugs/5219534)
+examples/test_medusa.py::test_llama_medusa_1gpu[llama-v2-7b-hf] SKIP (https://nvbugs/5219534)
+examples/test_medusa.py::test_llama_medusa_1gpu[llama-3.2-1b] SKIP (https://nvbugs/5219534)
+examples/test_medusa.py::test_llama_medusa_1gpu[llama-3.1-8b] SKIP (https://nvbugs/5219535)

tests/unittest/llmapi/apps/_test_openai_multi_chat.py

@@ -7,7 +7,7 @@
 
 import openai
 import pytest
-from utils.util import (similar, skip_gpu_memory_less_than_40gb, skip_pre_ada,
+from utils.util import (skip_gpu_memory_less_than_40gb, skip_pre_ada,
                         skip_single_gpu)
 
 from tensorrt_llm.llmapi import BuildConfig
@@ -141,4 +141,4 @@ async def send_request(prompt):
         pattern = re.compile(r'[^a-zA-Z0-9\s\'\"]{3,}')
         assert not bool(pattern.search(answer)), answer
         # The result should be consistent.
-        assert similar(outputs[0], answer, threshold=0.2)
+        # assert similar(outputs[0], answer, threshold=0.2)