From 73c67b1c1bdb7ea584f7c1ca1ff72ffba00d7e4b Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 21 Jul 2025 19:07:57 +0530 Subject: [PATCH 01/11] Test case --- qa/L0_backend_python/test.sh | 48 ++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index b2e6b8f034..4658b98bdf 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -407,6 +407,54 @@ and shared memory pages after starting triton equals to $current_num_pages \n*** exit 1 fi + +# Test model with non-existent model file +# The model.py file is intentionally not created to trigger the failure. +rm -fr ./models +mkdir -p models/non_existent_model/1 +cp ../python_models/identity_fp32/config.pbtxt ./models/non_existent_model/config.pbtxt +(cd models/non_existent_model && \ + sed -i "s/^name:.*/name: \"non_existent_model\"/" config.pbtxt) + +SERVER_LOG="./non_existent_model_server.log" +CLIENT_LOG="./non_existent_model_client.log" +ERROR_MESSAGE_1="Failed to preinitialize Python stub: Python model file not found in './models/non_existent_model/1/model.py'" +ERROR_MESSAGE_2="failed to load 'non_existent_model'" + +prev_num_pages=`get_shm_pages` +run_server +if [ "$SERVER_PID" != "0" ]; then + echo -e "*** FAILED: unexpected success starting $SERVER" >> $CLIENT_LOG + RET=1 + kill_server +else + if grep -q "$ERROR_MESSAGE_1" $SERVER_LOG; then + echo -e "Found \"$ERROR_MESSAGE_1\"" >> $CLIENT_LOG + else + echo -e "Not found \"$ERROR_MESSAGE_1\" in $SERVER_LOG" >> $CLIENT_LOG + cat $SERVER_LOG >> $CLIENT_LOG + RET=1 + fi + + if grep -q "$ERROR_MESSAGE_2" $SERVER_LOG; then + echo -e "Found \"$ERROR_MESSAGE_2\"" >> $CLIENT_LOG + else + echo -e "Not found \"$ERROR_MESSAGE_2\" in $SERVER_LOG" >> $CLIENT_LOG + cat $SERVER_LOG >> $CLIENT_LOG + RET=1 + fi +fi + +current_num_pages=`get_shm_pages` +if [ $current_num_pages -ne $prev_num_pages ]; then + cat $SERVER_LOG + ls /dev/shm + echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly. +Shared memory pages before starting triton equals to $prev_num_pages +and shared memory pages after starting triton equals to $current_num_pages \n***" + exit 1 +fi + # Disable env test for Jetson since cloud storage repos are not supported # Disable ensemble, io and bls tests for Jetson since GPU Tensors are not supported # Disable variants test for Jetson since already built without GPU Tensor support From 53901de4a49b48b7355a41844b23fbd705a6bafc Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 21 Jul 2025 21:49:57 +0530 Subject: [PATCH 02/11] Update --- qa/L0_backend_python/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index 4658b98bdf..77d9184568 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -418,7 +418,7 @@ cp ../python_models/identity_fp32/config.pbtxt ./models/non_existent_model/confi SERVER_LOG="./non_existent_model_server.log" CLIENT_LOG="./non_existent_model_client.log" -ERROR_MESSAGE_1="Failed to preinitialize Python stub: Python model file not found in './models/non_existent_model/1/model.py'" +ERROR_MESSAGE_1="Failed to preinitialize Python stub: Python model file not found in '`pwd`/models/non_existent_model/1/model.py'" ERROR_MESSAGE_2="failed to load 'non_existent_model'" prev_num_pages=`get_shm_pages` From d10deecce034f5ac7f40e1559e858260934f9dc6 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Wed, 23 Jul 2025 09:19:07 +0530 Subject: [PATCH 03/11] Remove shm related check --- qa/L0_backend_python/test.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index 77d9184568..2f36659a15 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -445,16 +445,6 @@ else fi fi -current_num_pages=`get_shm_pages` -if [ $current_num_pages -ne $prev_num_pages ]; then - cat $SERVER_LOG - ls /dev/shm - echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly. -Shared memory pages before starting triton equals to $prev_num_pages -and shared memory pages after starting triton equals to $current_num_pages \n***" - exit 1 -fi - # Disable env test for Jetson since cloud storage repos are not supported # Disable ensemble, io and bls tests for Jetson since GPU Tensors are not supported # Disable variants test for Jetson since already built without GPU Tensor support From 060826227d7e3e0d0a26e753e6266d0cda462adf Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Sun, 3 Aug 2025 20:51:14 +0530 Subject: [PATCH 04/11] Add shm check --- qa/L0_backend_python/test.sh | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index 2f36659a15..90770fe35c 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -418,6 +418,7 @@ cp ../python_models/identity_fp32/config.pbtxt ./models/non_existent_model/confi SERVER_LOG="./non_existent_model_server.log" CLIENT_LOG="./non_existent_model_client.log" +SERVER_ARGS=$BASE_SERVER_ARGS ERROR_MESSAGE_1="Failed to preinitialize Python stub: Python model file not found in '`pwd`/models/non_existent_model/1/model.py'" ERROR_MESSAGE_2="failed to load 'non_existent_model'" @@ -445,6 +446,52 @@ else fi fi +if [ $current_num_pages -ne $prev_num_pages ]; then + cat $SERVER_LOG + ls /dev/shm + echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly. +Shared memory pages before starting triton equals to $prev_num_pages +and shared memory pages after starting triton equals to $current_num_pages \n***" + exit 1 +fi + +SERVER_LOG="./non_existent_model_server_auto_config_disabled.log" +CLIENT_LOG="./non_existent_model_client_auto_config_disabled.log" +SERVER_ARGS="$BASE_SERVER_ARGS --disable-auto-complete-config" + +prev_num_pages=`get_shm_pages` +run_server +if [ "$SERVER_PID" != "0" ]; then + echo -e "*** FAILED: unexpected success starting $SERVER" >> $CLIENT_LOG + RET=1 + kill_server +else + if grep -q "$ERROR_MESSAGE_1" $SERVER_LOG; then + echo -e "Found \"$ERROR_MESSAGE_1\"" >> $CLIENT_LOG + else + echo -e "Not found \"$ERROR_MESSAGE_1\" in $SERVER_LOG" >> $CLIENT_LOG + cat $SERVER_LOG >> $CLIENT_LOG + RET=1 + fi + + if grep -q "$ERROR_MESSAGE_2" $SERVER_LOG; then + echo -e "Found \"$ERROR_MESSAGE_2\"" >> $CLIENT_LOG + else + echo -e "Not found \"$ERROR_MESSAGE_2\" in $SERVER_LOG" >> $CLIENT_LOG + cat $SERVER_LOG >> $CLIENT_LOG + RET=1 + fi +fi + +if [ $current_num_pages -ne $prev_num_pages ]; then + cat $SERVER_LOG + ls /dev/shm + echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly. +Shared memory pages before starting triton equals to $prev_num_pages +and shared memory pages after starting triton equals to $current_num_pages \n***" + exit 1 +fi + # Disable env test for Jetson since cloud storage repos are not supported # Disable ensemble, io and bls tests for Jetson since GPU Tensors are not supported # Disable variants test for Jetson since already built without GPU Tensor support From 0e9b131708efc6f813f8f13f54340871f52cd7a7 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 4 Aug 2025 11:37:16 +0530 Subject: [PATCH 05/11] Update venv command to fix CI --- qa/common/util.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/common/util.sh b/qa/common/util.sh index 08d10d7ed0..7c832cba79 100755 --- a/qa/common/util.sh +++ b/qa/common/util.sh @@ -522,7 +522,7 @@ function setup_virtualenv() { if [[ -v MSYSTEM ]]; then pip3 install pytest else - virtualenv --system-site-packages venv + python3 -m venv --system-site-packages venv source venv/bin/activate pip install pytest fi From 180d27e04dafe5865ffa01ae127e67d412de4e5f Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 4 Aug 2025 15:57:10 +0530 Subject: [PATCH 06/11] Update --- qa/L0_backend_python/test.sh | 10 ++++++---- qa/common/util.sh | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index 90770fe35c..72bc6f013a 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -508,10 +508,14 @@ if [ "$TEST_JETSON" == "0" ]; then for TEST in ${SUBTESTS}; do # Run each subtest in a separate virtual environment to avoid conflicts # between dependencies. - setup_virtualenv set +e - (cd ${TEST} && bash -ex test.sh) + ( + setup_virtualenv + cd ${TEST} && bash -ex test.sh + deactivate_virtualenv + ) + EXIT_CODE=$? if [ $EXIT_CODE -ne 0 ]; then echo "Subtest ${TEST} FAILED" @@ -524,8 +528,6 @@ if [ "$TEST_JETSON" == "0" ]; then fi fi set -e - - deactivate_virtualenv done # [DLIS-5969]: Incorporate env test for windows diff --git a/qa/common/util.sh b/qa/common/util.sh index 7c832cba79..08d10d7ed0 100755 --- a/qa/common/util.sh +++ b/qa/common/util.sh @@ -522,7 +522,7 @@ function setup_virtualenv() { if [[ -v MSYSTEM ]]; then pip3 install pytest else - python3 -m venv --system-site-packages venv + virtualenv --system-site-packages venv source venv/bin/activate pip install pytest fi From ff52d7e4be9677e5fa862f5a2af1a355f1d6005d Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 4 Aug 2025 20:08:28 +0530 Subject: [PATCH 07/11] Update --- qa/L0_backend_python/test.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index 72bc6f013a..a0bf3d0179 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -555,18 +555,19 @@ fi for TEST in ${SUBTESTS}; do # Run each subtest in a separate virtual environment to avoid conflicts # between dependencies. - setup_virtualenv set +e - (cd ${TEST} && bash -ex test.sh) + ( + setup_virtualenv + cd ${TEST} && bash -ex test.sh + deactivate_virtualenv + ) if [ $? -ne 0 ]; then echo "Subtest ${TEST} FAILED" RET=1 fi set -e - - deactivate_virtualenv done if [ $RET -eq 0 ]; then From e548869c0571d9d5f3154a6a509512edc5b31d4a Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 7 Aug 2025 11:56:45 +0530 Subject: [PATCH 08/11] Update qa/L0_backend_python/test.sh Co-authored-by: Yingge He <157551214+yinggeh@users.noreply.github.com> --- qa/L0_backend_python/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index a0bf3d0179..824b336dee 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -486,7 +486,7 @@ fi if [ $current_num_pages -ne $prev_num_pages ]; then cat $SERVER_LOG ls /dev/shm - echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly. + echo -e "\n***\n*** Test Failed. Shared memory pages were not cleaned properly. Shared memory pages before starting triton equals to $prev_num_pages and shared memory pages after starting triton equals to $current_num_pages \n***" exit 1 From d13cb76910d4f7b079edde66afa15a3e2bdfd627 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 7 Aug 2025 11:57:23 +0530 Subject: [PATCH 09/11] Update qa/L0_backend_python/test.sh Co-authored-by: Yingge He <157551214+yinggeh@users.noreply.github.com> --- qa/L0_backend_python/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index 824b336dee..6abb708379 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -449,7 +449,7 @@ fi if [ $current_num_pages -ne $prev_num_pages ]; then cat $SERVER_LOG ls /dev/shm - echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly. + echo -e "\n***\n*** Test Failed. Shared memory pages were not cleaned properly. Shared memory pages before starting triton equals to $prev_num_pages and shared memory pages after starting triton equals to $current_num_pages \n***" exit 1 From 12875494965859efd98100f558d7b57fe459b6d3 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 7 Aug 2025 12:17:24 +0530 Subject: [PATCH 10/11] Update test --- qa/L0_backend_python/test.sh | 118 +++++++++++++---------------------- 1 file changed, 44 insertions(+), 74 deletions(-) diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index 6abb708379..e3b986850a 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -416,81 +416,55 @@ cp ../python_models/identity_fp32/config.pbtxt ./models/non_existent_model/confi (cd models/non_existent_model && \ sed -i "s/^name:.*/name: \"non_existent_model\"/" config.pbtxt) -SERVER_LOG="./non_existent_model_server.log" -CLIENT_LOG="./non_existent_model_client.log" -SERVER_ARGS=$BASE_SERVER_ARGS ERROR_MESSAGE_1="Failed to preinitialize Python stub: Python model file not found in '`pwd`/models/non_existent_model/1/model.py'" ERROR_MESSAGE_2="failed to load 'non_existent_model'" -prev_num_pages=`get_shm_pages` -run_server -if [ "$SERVER_PID" != "0" ]; then - echo -e "*** FAILED: unexpected success starting $SERVER" >> $CLIENT_LOG - RET=1 - kill_server -else - if grep -q "$ERROR_MESSAGE_1" $SERVER_LOG; then - echo -e "Found \"$ERROR_MESSAGE_1\"" >> $CLIENT_LOG +for test_mode in "default" "auto_config_disabled"; do + if [ "$test_mode" == "default" ]; then + SERVER_LOG_SUFFIX="" + SERVER_ARGS=$BASE_SERVER_ARGS else - echo -e "Not found \"$ERROR_MESSAGE_1\" in $SERVER_LOG" >> $CLIENT_LOG - cat $SERVER_LOG >> $CLIENT_LOG - RET=1 + SERVER_LOG_SUFFIX="_auto_config_disabled" + SERVER_ARGS="$BASE_SERVER_ARGS --disable-auto-complete-config" fi - if grep -q "$ERROR_MESSAGE_2" $SERVER_LOG; then - echo -e "Found \"$ERROR_MESSAGE_2\"" >> $CLIENT_LOG - else - echo -e "Not found \"$ERROR_MESSAGE_2\" in $SERVER_LOG" >> $CLIENT_LOG - cat $SERVER_LOG >> $CLIENT_LOG - RET=1 - fi -fi + SERVER_LOG="./non_existent_model_server${SERVER_LOG_SUFFIX}.log" + CLIENT_LOG="./non_existent_model_client${SERVER_LOG_SUFFIX}.log" -if [ $current_num_pages -ne $prev_num_pages ]; then - cat $SERVER_LOG - ls /dev/shm - echo -e "\n***\n*** Test Failed. Shared memory pages were not cleaned properly. -Shared memory pages before starting triton equals to $prev_num_pages -and shared memory pages after starting triton equals to $current_num_pages \n***" - exit 1 -fi - -SERVER_LOG="./non_existent_model_server_auto_config_disabled.log" -CLIENT_LOG="./non_existent_model_client_auto_config_disabled.log" -SERVER_ARGS="$BASE_SERVER_ARGS --disable-auto-complete-config" - -prev_num_pages=`get_shm_pages` -run_server -if [ "$SERVER_PID" != "0" ]; then - echo -e "*** FAILED: unexpected success starting $SERVER" >> $CLIENT_LOG - RET=1 - kill_server -else - if grep -q "$ERROR_MESSAGE_1" $SERVER_LOG; then - echo -e "Found \"$ERROR_MESSAGE_1\"" >> $CLIENT_LOG - else - echo -e "Not found \"$ERROR_MESSAGE_1\" in $SERVER_LOG" >> $CLIENT_LOG - cat $SERVER_LOG >> $CLIENT_LOG + prev_num_pages=`get_shm_pages` + run_server + if [ "$SERVER_PID" != "0" ]; then + echo -e "*** FAILED: unexpected success starting $SERVER" >> $CLIENT_LOG RET=1 - fi - - if grep -q "$ERROR_MESSAGE_2" $SERVER_LOG; then - echo -e "Found \"$ERROR_MESSAGE_2\"" >> $CLIENT_LOG + kill_server else - echo -e "Not found \"$ERROR_MESSAGE_2\" in $SERVER_LOG" >> $CLIENT_LOG - cat $SERVER_LOG >> $CLIENT_LOG - RET=1 + if grep -q "$ERROR_MESSAGE_1" $SERVER_LOG; then + echo -e "Found \"$ERROR_MESSAGE_1\"" >> $CLIENT_LOG + else + echo -e "Not found \"$ERROR_MESSAGE_1\" in $SERVER_LOG" >> $CLIENT_LOG + cat $SERVER_LOG >> $CLIENT_LOG + RET=1 + fi + + if grep -q "$ERROR_MESSAGE_2" $SERVER_LOG; then + echo -e "Found \"$ERROR_MESSAGE_2\"" >> $CLIENT_LOG + else + echo -e "Not found \"$ERROR_MESSAGE_2\" in $SERVER_LOG" >> $CLIENT_LOG + cat $SERVER_LOG >> $CLIENT_LOG + RET=1 + fi fi -fi -if [ $current_num_pages -ne $prev_num_pages ]; then - cat $SERVER_LOG - ls /dev/shm - echo -e "\n***\n*** Test Failed. Shared memory pages were not cleaned properly. + current_num_pages=`get_shm_pages` + if [ $current_num_pages -ne $prev_num_pages ]; then + cat $SERVER_LOG + ls /dev/shm + echo -e "\n***\n*** Test Failed. Shared memory pages were not cleaned properly. Shared memory pages before starting triton equals to $prev_num_pages and shared memory pages after starting triton equals to $current_num_pages \n***" - exit 1 -fi + exit 1 + fi +done # Disable env test for Jetson since cloud storage repos are not supported # Disable ensemble, io and bls tests for Jetson since GPU Tensors are not supported @@ -508,14 +482,10 @@ if [ "$TEST_JETSON" == "0" ]; then for TEST in ${SUBTESTS}; do # Run each subtest in a separate virtual environment to avoid conflicts # between dependencies. + setup_virtualenv set +e - ( - setup_virtualenv - cd ${TEST} && bash -ex test.sh - deactivate_virtualenv - ) - + (cd ${TEST} && bash -ex test.sh) EXIT_CODE=$? if [ $EXIT_CODE -ne 0 ]; then echo "Subtest ${TEST} FAILED" @@ -528,6 +498,8 @@ if [ "$TEST_JETSON" == "0" ]; then fi fi set -e + + deactivate_virtualenv done # [DLIS-5969]: Incorporate env test for windows @@ -555,19 +527,17 @@ fi for TEST in ${SUBTESTS}; do # Run each subtest in a separate virtual environment to avoid conflicts # between dependencies. + setup_virtualenv set +e - ( - setup_virtualenv - cd ${TEST} && bash -ex test.sh - deactivate_virtualenv - ) - + (cd ${TEST} && bash -ex test.sh) if [ $? -ne 0 ]; then echo "Subtest ${TEST} FAILED" RET=1 fi set -e + + deactivate_virtualenv done if [ $RET -eq 0 ]; then From d18227f134a757d7eea85afedb06965aaa4b8eb2 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 7 Aug 2025 12:18:29 +0530 Subject: [PATCH 11/11] Update --- qa/L0_backend_python/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index e3b986850a..58b92590e1 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -531,6 +531,7 @@ for TEST in ${SUBTESTS}; do set +e (cd ${TEST} && bash -ex test.sh) + if [ $? -ne 0 ]; then echo "Subtest ${TEST} FAILED" RET=1