test: Improve L0_logging stability (#7486)

Author: rmccorm4

qa/L0_logging/log_format_test.py

@@ -405,6 +405,9 @@ def _launch_server(self, escaped=None):
         if not os.path.exists(self._server_options["log-file"]):
             raise Exception("Log not found")
 
+        # Give server a little time to have the endpoints up and ready
+        time.sleep(10)
+
     def _validate_log_record(self, record, format_regex, escaped):
         match = format_regex.search(record)
         assert match, "Invalid log line"
@@ -483,18 +486,29 @@ def test_injection(self, log_format, format_regex, injected_record):
             # TODO Refactor server launch, shutdown into reusable class
             wait_time = 10
 
-            while wait_time and not triton_client.is_server_ready():
+            while wait_time:
+                try:
+                    if triton_client.is_server_ready():
+                        break
+                # Gracefully handle connection error if server endpoint isn't up yet
+                except Exception as e:
+                    print(
+                        f"Client failed to connect, retries remaining: {wait_time}. Error: {e}"
+                    )
+
                 time.sleep(1)
                 wait_time -= 1
+                print(f"Server not ready yet, retries remaining: {wait_time}")
 
             while wait_time and not triton_client.is_model_ready("simple"):
                 time.sleep(1)
                 wait_time -= 1
 
-            if not triton_client.is_server_ready() or not triton_client.is_model_ready(
-                "simple"
-            ):
-                raise Exception("Model or Server not Ready")
+            if not triton_client.is_server_ready():
+                raise Exception("Server not Ready")
+
+            if not triton_client.is_model_ready("simple"):
+                raise Exception("Model not Ready")
 
         except Exception as e:
             self._shutdown_server()

qa/L0_logging/test.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -60,6 +60,11 @@ source ../common/util.sh
 rm -f *.log
 rm -fr $MODELSDIR && mkdir -p $MODELSDIR
 
+if [ ! -d ${DATADIR} ]; then
+  echo -e "\n***\n*** ${DATADIR} does not exist!\n***"
+  exit 1
+fi
+
 # set up simple repository MODELBASE
 rm -fr $MODELSDIR && mkdir -p $MODELSDIR && \
     cp -r $DATADIR/$MODELBASE $MODELSDIR/simple && \