Fix: Check whether a tool was successfully started (#2500)

This fix tracks the status of 'tool.up' command(s) as well as the
output generated by Docker. If the output contains 'warning...platform'
text, we might be dealing with a container started on an incompatible
platform, in which case we wait a bit (because it will crash very soon),
check its status, and report the outcome.

The fix also adds better error reporting on tool start/stop failures
and partial-success reporting.

Closes #2456

Author: ipspace

netsim/cli/external_commands.py

@@ -251,15 +251,30 @@ def get_local_addr() -> str:
 #
 # Execute external tool commands
 #
-def execute_tool_commands(cmds: list, topology: Box) -> None:
+def execute_tool_commands(cmds: list, topology: Box) -> typing.Optional[str]:
   topology.sys.docker_net = ""
   topology.sys.ipaddr = get_local_addr()
   if docker_is_used(topology):
     topology.sys.docker_net = f"--network={topology.addressing.mgmt.get('_network',None) or 'netlab_mgmt'}"
 
+  output = ''
   for cmd in cmds:
     cmd = strings.eval_format(cmd,topology)
-    run_command(cmd = [ 'bash', '-c', cmd ],check_result=True)
+    status = run_command(
+              cmd = [ 'bash', '-c', cmd + " 2>&1"],   # Redirect STDERR to STDOUT to collect it
+              check_result=True,                      # I want to get the status
+              ignore_errors=True,                     # ... returned but not reported
+              return_stdout=True)                     # ... and we need STDOUT content to look for the warnings
+    if not isinstance(status,str):
+      log.error(
+        f'Failed to execute {cmd}',module='tools',
+        category=log.ErrorAbort,
+        skip_header=True)
+      return None
+    else:
+      output += status
+
+  return output
 
 #
 # Get the "how to connect to the tool" message

netsim/cli/up.py

@@ -8,7 +8,9 @@
 import typing
 import argparse
 import os
+import re
 import sys
+import time
 
 from box import Box
 from pathlib import Path
@@ -259,6 +261,24 @@ def reload_saved_config(args: argparse.Namespace, topology: Box) -> None:
   log.status_success()
   print("Saved configurations reloaded")
 
+"""
+Check the state of the external tool container
+"""
+def check_tool_status(tool: str, status: typing.Optional[str], topology: Box) -> bool:
+  if status is None:
+    return False
+
+  if not re.search('(?i)warning.*platform.*match',status):
+    return True
+
+  log.info(f'Platform mismatch between {tool} container image and your hardware, checking tool status')
+  time.sleep(0.5)
+  c_stat = external_commands.run_command(
+              f'docker inspect {topology.name}_{tool}',
+              ignore_errors=True,return_stdout=True,check_result=True)
+
+  return bool(c_stat)
+
 """
 Deploy external tools
 """
@@ -273,21 +293,31 @@ def start_external_tools(args: argparse.Namespace, topology: Box) -> None:
 
   lab_status_change(topology,f'starting external tools')
   log.section_header('Starting','external tools')
+  t_count = 0
+  t_success = 0
   for tool in topology.tools.keys():
     cmds = external_commands.get_tool_command(tool,'up',topology)
     if cmds is None:
       continue
-    external_commands.execute_tool_commands(cmds,topology)
+
+    t_count += 1
+    status = external_commands.execute_tool_commands(cmds,topology)
+    if not is_dry_run() and not check_tool_status(tool,status,topology):
+      log.error(f'Failed to start {tool}',module='tools',category=Warning,skip_header=True)
+      continue
     msg = external_commands.get_tool_message(tool,topology)
     if not is_dry_run():
-      print(f"... {tool} tool started")
+      t_success += 1
+      log.status_success()
+      print(f"{tool} tool started")
 
     if msg:
       print(("DRY_RUN: " if is_dry_run() else "") + msg + "\n")
 
-  lab_status_change(topology,f'external tools started')
-  log.status_success()
-  print("External tools started")
+  lab_status_change(topology,f'{t_success}/{t_count} external tools started')
+  if not is_dry_run():
+    log.partial_success(t_success,t_count)
+    print(f"{t_success}/{t_count} external tools started")
 
 """
 Main "lab start" process