ci: Improve error handling for Python backend model initialization failures (#8303)

Author: pskiran1

qa/L0_backend_python/test.sh

@@ -407,6 +407,65 @@ and shared memory pages after starting triton equals to $current_num_pages \n***
     exit 1
 fi
 
+
+# Test model with non-existent model file
+# The model.py file is intentionally not created to trigger the failure.
+rm -fr ./models
+mkdir -p models/non_existent_model/1
+cp ../python_models/identity_fp32/config.pbtxt ./models/non_existent_model/config.pbtxt
+(cd models/non_existent_model && \
+          sed -i "s/^name:.*/name: \"non_existent_model\"/" config.pbtxt)
+
+ERROR_MESSAGE_1="Failed to preinitialize Python stub: Python model file not found in '`pwd`/models/non_existent_model/1/model.py'"
+ERROR_MESSAGE_2="failed to load 'non_existent_model'"
+
+for test_mode in "default" "auto_config_disabled"; do
+    if [ "$test_mode" == "default" ]; then
+        SERVER_LOG_SUFFIX=""
+        SERVER_ARGS=$BASE_SERVER_ARGS
+    else
+        SERVER_LOG_SUFFIX="_auto_config_disabled"
+        SERVER_ARGS="$BASE_SERVER_ARGS --disable-auto-complete-config"
+    fi
+
+    SERVER_LOG="./non_existent_model_server${SERVER_LOG_SUFFIX}.log"
+    CLIENT_LOG="./non_existent_model_client${SERVER_LOG_SUFFIX}.log"
+
+    prev_num_pages=`get_shm_pages`
+    run_server
+    if [ "$SERVER_PID" != "0" ]; then
+        echo -e "*** FAILED: unexpected success starting $SERVER" >> $CLIENT_LOG
+        RET=1
+        kill_server
+    else
+        if grep -q "$ERROR_MESSAGE_1" $SERVER_LOG; then
+            echo -e "Found \"$ERROR_MESSAGE_1\"" >> $CLIENT_LOG
+        else
+            echo -e "Not found \"$ERROR_MESSAGE_1\" in $SERVER_LOG" >> $CLIENT_LOG
+            cat $SERVER_LOG >> $CLIENT_LOG
+            RET=1
+        fi
+
+        if grep -q "$ERROR_MESSAGE_2" $SERVER_LOG; then
+            echo -e "Found \"$ERROR_MESSAGE_2\"" >> $CLIENT_LOG
+        else
+            echo -e "Not found \"$ERROR_MESSAGE_2\" in $SERVER_LOG" >> $CLIENT_LOG
+            cat $SERVER_LOG >> $CLIENT_LOG
+            RET=1
+        fi
+    fi
+
+    current_num_pages=`get_shm_pages`
+    if [ $current_num_pages -ne $prev_num_pages ]; then
+        cat $SERVER_LOG
+        ls /dev/shm
+        echo -e "\n***\n*** Test Failed. Shared memory pages were not cleaned properly.
+Shared memory pages before starting triton equals to $prev_num_pages
+and shared memory pages after starting triton equals to $current_num_pages \n***"
+        exit 1
+    fi
+done
+
 # Disable env test for Jetson since cloud storage repos are not supported
 # Disable ensemble, io and bls tests for Jetson since GPU Tensors are not supported
 # Disable variants test for Jetson since already built without GPU Tensor support