Skip to content

Commit 48db263

Browse files
EmmaQiaoChchzblych
EmmaQiaoCh
and
chzblych
authored
infra: Add test list name check (NVIDIA#3097)
* Add steps to check test names Signed-off-by: EmmaQiaoCh <[email protected]> * Correct test-db command Signed-off-by: EmmaQiaoCh <[email protected]> * Switch to use a trt-llm image Signed-off-by: EmmaQiaoCh <[email protected]> * Update go path Signed-off-by: EmmaQiaoCh <[email protected]> * Correct go path Signed-off-by: qqiao <[email protected]> Signed-off-by: EmmaQiaoCh <[email protected]> * Move the test list check to test ci Signed-off-by: qqiao <[email protected]> Signed-off-by: EmmaQiaoCh <[email protected]> * Correct file path Signed-off-by: EmmaQiaoCh <[email protected]> * Fix path again Signed-off-by: EmmaQiaoCh <[email protected]> * Fix get path Signed-off-by: EmmaQiaoCh <[email protected]> * Fix typo Signed-off-by: EmmaQiaoCh <[email protected]> * Skip test list check for ARM Signed-off-by: EmmaQiaoCh <[email protected]> * Fix expression Signed-off-by: EmmaQiaoCh <[email protected]> * Change back unrelated file Signed-off-by: EmmaQiaoCh <[email protected]> * Correct qa test names Signed-off-by: EmmaQiaoCh <[email protected]> * Remove a stage Signed-off-by: EmmaQiaoCh <[email protected]> * Update jenkins/L0_Test.groovy Co-authored-by: Yanchao Lu <[email protected]> Signed-off-by: Emma Qiao <[email protected]> Signed-off-by: EmmaQiaoCh <[email protected]> * Move some steps to a python script Signed-off-by: EmmaQiaoCh <[email protected]> * Fix script path Signed-off-by: EmmaQiaoCh <[email protected]> * Split commands and debug Signed-off-by: EmmaQiaoCh <[email protected]> * Fix typo Signed-off-by: EmmaQiaoCh <[email protected]> * Fix typo Signed-off-by: EmmaQiaoCh <[email protected]> * Also correct case name in waives list Signed-off-by: EmmaQiaoCh <[email protected]> * Move check script to another folder Signed-off-by: EmmaQiaoCh <[email protected]> * Update qa list after rebase Signed-off-by: EmmaQiaoCh <[email protected]> * Fix rebase Signed-off-by: EmmaQiaoCh <[email protected]> * Remove the perf tests under QA Signed-off-by: EmmaQiaoCh <[email protected]> * Some tests already fixed after rebase to TOT Signed-off-by: EmmaQiaoCh <[email protected]> --------- Signed-off-by: EmmaQiaoCh <[email protected]> Signed-off-by: qqiao <[email protected]> Signed-off-by: Emma Qiao <[email protected]> Co-authored-by: Yanchao Lu <[email protected]>
1 parent f7c2eb4 commit 48db263

File tree

4 files changed

+131
-4
lines changed

4 files changed

+131
-4
lines changed

‎jenkins/L0_Test.groovy‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,32 @@ def runLLMDocBuild(pipeline, config)
473473
)
474474
}
475475

476+
def launchTestListCheck(pipeline)
477+
{
478+
stageName = "Test List Check"
479+
trtllm_utils.launchKubernetesPod(pipeline, createKubernetesPodConfig(LLM_DOCKER_IMAGE, "a10"), "trt-llm", {
480+
try {
481+
echoNodeAndGpuInfo(pipeline, stageName)
482+
trtllm_utils.llmExecStepWithRetry(pipeline, script: """apt-get update && apt-get install \
483+
libffi-dev \
484+
-y""")
485+
sh "nvidia-smi -q"
486+
// download TRT-LLM tarfile
487+
def tarName = BUILD_CONFIGS[VANILLA_CONFIG][TARNAME]
488+
def llmTarfile = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/${tarName}"
489+
trtllm_utils.llmExecStepWithRetry(pipeline, script: "pwd && wget -nv ${llmTarfile} && ls -alh")
490+
sh "tar -zxf ${tarName}"
491+
def llmPath = sh (script: "realpath .", returnStdout: true).trim()
492+
def llmSrc = "${llmPath}/TensorRT-LLM/src"
493+
sh "python3 ${llmSrc}/scripts/check_test_list.py --l0 --qa"
494+
} catch (InterruptedException e) {
495+
throw e
496+
} catch (Exception e) {
497+
throw e
498+
}
499+
})
500+
}
501+
476502
def generateStageFailTestResultXml(stageName, subName, failureLog, resultPath) {
477503
String resultFiles = sh(script: "cd ${stageName} && ls -l ${resultPath} | wc -l", returnStdout: true).trim()
478504
echo "${resultFiles}"
@@ -1566,6 +1592,20 @@ pipeline {
15661592
}
15671593
}
15681594
}
1595+
stage("Check Test Lists")
1596+
{
1597+
when {
1598+
expression {
1599+
env.targetArch == X86_64_TRIPLE // Only execute the check if running on x86
1600+
}
1601+
}
1602+
steps
1603+
{
1604+
script {
1605+
launchTestListCheck(this)
1606+
}
1607+
}
1608+
}
15691609
stage("Test") {
15701610
steps {
15711611
script {

‎scripts/check_test_list.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import argparse
2+
import os
3+
import subprocess
4+
5+
6+
def install_python_dependencies(llm_src):
7+
subprocess.run(
8+
f"cd {llm_src} && pip3 install --retries 1 -r requirements-dev.txt",
9+
shell=True,
10+
check=True)
11+
subprocess.run(
12+
f"pip3 install --force-reinstall --no-deps {llm_src}/../tensorrt_llm-*.whl",
13+
shell=True,
14+
check=True)
15+
subprocess.run(
16+
"pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple "
17+
"--ignore-installed trt-test-db==1.8.5+bc6df7",
18+
shell=True,
19+
check=True)
20+
21+
22+
def verify_l0_test_lists(llm_src):
23+
test_db_path = f"{llm_src}/tests/integration/test_lists/test-db"
24+
test_list = f"{llm_src}/l0_test.txt"
25+
26+
# Remove dynamically generated perf tests
27+
subprocess.run(f"rm -f {test_db_path}/*perf*", shell=True, check=True)
28+
subprocess.run(
29+
f"trt-test-db -d {test_db_path} --test-names --output {test_list}",
30+
shell=True,
31+
check=True)
32+
33+
subprocess.run(
34+
f"cd {llm_src}/tests/integration/defs && "
35+
f"pytest --apply-test-list-correction --test-list={test_list} --co -q",
36+
shell=True,
37+
check=True)
38+
39+
40+
def verify_qa_test_lists(llm_src):
41+
test_qa_path = f"{llm_src}/tests/integration/test_lists/qa"
42+
# Remove dynamically generated perf tests
43+
subprocess.run(f"rm -f {test_qa_path}/*perf*", shell=True, check=True)
44+
test_def_files = subprocess.check_output(
45+
f"ls -d {test_qa_path}/*.txt", shell=True).decode().strip().split('\n')
46+
for test_def_file in test_def_files:
47+
subprocess.run(
48+
f"cd {llm_src}/tests/integration/defs && "
49+
f"pytest --apply-test-list-correction --test-list={test_def_file} --co -q",
50+
shell=True,
51+
check=True)
52+
53+
54+
def main():
55+
parser = argparse.ArgumentParser(
56+
description="Check test lists for L0 and QA.")
57+
parser.add_argument("--l0",
58+
action="store_true",
59+
help="Enable L0 test list verification.")
60+
parser.add_argument("--qa",
61+
action="store_true",
62+
help="Enable QA test list verification.")
63+
args = parser.parse_args()
64+
llm_src = os.path.realpath("TensorRT-LLM/src")
65+
66+
install_python_dependencies(llm_src)
67+
# Verify L0 test lists
68+
if args.l0:
69+
verify_l0_test_lists(llm_src)
70+
else:
71+
print("Skipping L0 test list verification.")
72+
# Verify QA test lists
73+
if args.qa:
74+
verify_qa_test_lists(llm_src)
75+
else:
76+
print("Skipping QA test list verification.")
77+
78+
79+
if __name__ == "__main__":
80+
main()

‎tests/integration/defs/test_list_parser.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -477,7 +477,10 @@ def suggest_correction(valid_test_buckets, test):
477477
return ret
478478

479479

480-
def apply_test_list_corrections(test_list, corrections, test_prefix=None):
480+
def apply_test_list_corrections(test_list,
481+
corrections,
482+
items,
483+
test_prefix=None):
481484
"""
482485
Attempt to correct invalid test names in a test list.
483486
@@ -508,7 +511,10 @@ def apply_test_list_corrections(test_list, corrections, test_prefix=None):
508511
#with open(test_list, "w") as f:
509512
# f.write(contents)
510513

511-
pytest.exit(
514+
# Clear the items list to prevent pytest from listing collected tests
515+
items.clear()
516+
517+
raise pytest.UsageError(
512518
"Exiting early since --apply-test-list-correction was specified.")
513519

514520

@@ -579,7 +585,8 @@ def parse_and_validate_test_list(
579585
)
580586

581587
if apply_test_list_correction and corrections:
582-
apply_test_list_corrections(test_list, corrections, test_prefix)
588+
apply_test_list_corrections(test_list, corrections, items,
589+
test_prefix)
583590

584591
output_dir = config.getoption("--output-dir")
585592
if record_invalid_tests and corrections:

‎tests/integration/test_lists/qa/examples_test_list.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -425,12 +425,12 @@ accuracy/test_llm_api.py::TestQwen2_5_7BInstruct::test_fp8
425425
accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4
426426
accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4
427427
accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4
428-
accuracy/test_llm_api_pytorch.py::TestMistral7B::test_auto_dtype
429428
accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_fp8_tp2
430429
accuracy/test_llm_api_pytorch.py::TestMixtral8x7B::test_nvfp4_tp2
431430
accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[]
432431
accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales[]
433432
accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4[]
433+
accuracy/test_llm_api_pytorch.py::TestMistral7B::test_auto_dtype
434434
accuracy/test_llm_api_pytorch.py::TestMinitron4BBaseInstruct::test_fp8_prequantized
435435
accuracy/test_llm_api_pytorch.py::TestNemotronNas::test_auto_dtype_tp8
436436
accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype

0 commit comments

Comments
 (0)