Check if other models on server (#756)

* Warn if non-profiled model is loaded on remote server

* HTTP and GRPC have different API calls

Author: nv-braf

model_analyzer/analyzer.py

@@ -116,6 +116,9 @@ def profile(
         self._create_metrics_manager(client, gpus)
         self._create_model_manager(client, gpus)
 
+        if self._config.triton_launch_mode == "remote":
+            self._warn_if_other_models_loaded_on_remote_server(client)
+
         if self._config.model_repository:
             self._get_server_only_metrics(client, gpus)
             self._profile_models()
@@ -395,3 +398,15 @@ def _create_report_config(self, args: list) -> ConfigCommandReport:
         cli.add_subcommand(cmd="report", help="", config=config)
         cli.parse(args)
         return config
+
+    def _warn_if_other_models_loaded_on_remote_server(self, client):
+        repository_index = client.get_model_repository_index()
+        profile_model_names = [pm.model_name() for pm in self._config.profile_models]
+
+        for model in repository_index:
+            if model["name"] not in profile_model_names:
+                model_name = model["name"]
+                logger.warning(
+                    f"A model not being profiled ({model_name}) is loaded on the remote Tritonserver. "
+                    "This could impact the profile results."
+                )

model_analyzer/triton/client/grpc_client.py

@@ -78,3 +78,9 @@ def get_model_config(self, model_name, num_retries):
         self.wait_for_model_ready(model_name, num_retries)
         model_config_dict = self._client.get_model_config(model_name, as_json=True)
         return model_config_dict["config"]
+
+    def get_model_repository_index(self):
+        """
+        Returns the JSON dict holding the model repository index.
+        """
+        return self._client.get_model_repository_index(as_json=True)["models"]

model_analyzer/triton/client/http_client.py

@@ -93,3 +93,9 @@ def __init__(self, server_url, ssl_options={}):
             ssl_context_factory=ssl_context_factory,
             insecure=insecure,
         )
+
+    def get_model_repository_index(self):
+        """
+        Returns the JSON dict holding the model repository index.
+        """
+        return self._client.get_model_repository_index()