Misc fixes (#799)

* Fix tests so that artifacts show up on failure

* Fix bug in extract_model_name_from_variant_name

* Discard auto-generated preferred_batch_size

* prep for ensemble fix. Boyscout some types

* fix type checking

* PR feedback

Author: tgerdesnv

model_analyzer/config/generate/base_model_config_generator.py

@@ -309,7 +309,13 @@ def extract_model_name_from_variant_name(variant_name: str) -> str:
         Removes '_config_#/default' from the variant name and returns
         the model name, eg. model_name_config_10 -> model_name
         """
-        return variant_name[: variant_name.find("_config_")]
+        model_name = variant_name
+        config_index = variant_name.find("_config_")
+
+        if config_index != -1:
+            model_name = variant_name[:config_index]
+
+        return model_name
 
     @staticmethod
     def create_original_config_from_variant(variant_config: ModelConfig) -> ModelConfig:

model_analyzer/config/run/run_config.py

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import List
+
 from model_analyzer.config.run.model_run_config import ModelRunConfig
 
 
@@ -33,15 +35,15 @@ def __init__(self, triton_env):
         """
 
         self._triton_env = triton_env
-        self._model_run_configs = []
+        self._model_run_configs: List[ModelRunConfig] = []
 
     def add_model_run_config(self, model_run_config):
         """
         Add a ModelRunConfig to this RunConfig
         """
         self._model_run_configs.append(model_run_config)
 
-    def model_run_configs(self):
+    def model_run_configs(self) -> List[ModelRunConfig]:
         """
         Returns the list of ModelRunConfigs to run concurrently
         """

model_analyzer/record/metrics_manager.py

@@ -426,10 +426,11 @@ def _create_remote_mode_model_variant(
             # Ignore if the dir already exists
             pass
 
-    def _load_model_variants(self, run_config):
+    def _load_model_variants(self, run_config: RunConfig) -> bool:
         """
         Loads all model variants in the client
         """
+        # TODO TMA-1487: Make BLS and ensemble both load all composing model variants first
         for mrc in run_config.model_run_configs():
             if not self._load_model_variant(variant_config=mrc.model_config_variant()):
                 return False
@@ -444,7 +445,7 @@ def _load_model_variants(self, run_config):
 
         return True
 
-    def _load_model_variant(self, variant_config):
+    def _load_model_variant(self, variant_config: ModelConfigVariant) -> bool:
         """
         Conditionally loads a model variant in the client
         """
@@ -458,7 +459,7 @@ def _load_model_variant(self, variant_config):
             retval = self._do_load_model_variant(variant_config)
         return retval
 
-    def _do_load_model_variant(self, variant_config):
+    def _do_load_model_variant(self, variant_config: ModelConfigVariant) -> bool:
         """
         Loads a model variant in the client
         """

model_analyzer/triton/model/model_config.py

@@ -92,6 +92,16 @@ def create_model_config_dict(config, client, gpus, model_repository, model_name)
                 config = ModelConfig._get_default_config_from_server(
                     config, client, gpus, model_name
                 )
+
+                # An auto-completed triton model config will set preferred_batch_size
+                # to a default value. We do not want to keep and honor that
+                # value when we are searching, so we discard it here
+                if (
+                    "dynamic_batching" in config
+                    and "preferred_batch_size" in config["dynamic_batching"]
+                ):
+                    del config["dynamic_batching"]["preferred_batch_size"]
+
             else:
                 ModelConfig._check_default_config_exceptions(config, model_path)
 

qa/L0_multi_model_profile/test.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 source ../common/util.sh
-create_logs_dir
+create_logs_dir "L0_multi_model_profile"
 
 # Set test parameters
 MODEL_ANALYZER="`which model-analyzer`"

qa/L0_perf_analyzer/test.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 source ../common/util.sh
-create_logs_dir
+create_logs_dir "L0_perf_analyzer"
 
 # Set test parameters
 MODEL_ANALYZER="`which model-analyzer`"

tests/test_model_config_generator.py

@@ -807,6 +807,20 @@ def test_early_exit_on_manual(self):
                                                       early_exit_enable=True)
         # yapf: enable
 
+    def test_extract_model_name_from_variant_name(self):
+        input_output_pairs = {}
+        input_output_pairs[
+            "onnx_int32_int32_int32_config_default"
+        ] = "onnx_int32_int32_int32"
+        input_output_pairs["onnx_int32_int32_int32_config_2"] = "onnx_int32_int32_int32"
+        input_output_pairs["onnx_int32_int32_int32"] = "onnx_int32_int32_int32"
+
+        for variant_name, expected_model_name in input_output_pairs.items():
+            model_name = BaseModelConfigGenerator.extract_model_name_from_variant_name(
+                variant_name
+            )
+            self.assertEqual(model_name, expected_model_name)
+
     def _run_and_test_model_config_generator(
         self,
         yaml_str,