Reread validation tests from modified lab topology file

docs/netlab/validate.md

@@ -6,12 +6,12 @@
 ## Usage
 
 ```text
-$ netlab validate -h
-usage: netlab inspect [-h] [-v] [-q] [--list] [--node NODES] [--skip-wait] [-e]
-                      [--dump {result} [{result} ...]] [-i INSTANCE]
-                      [tests ...]
+usage: netlab validate [-h] [-v] [-q] [--list] [--node NODES] [--skip-wait] [-e]
+                       [--source TEST_SOURCE] [--dump {result} [{result} ...]]
+                       [-i INSTANCE]
+                       [tests ...]
 
-Inspect data structures in transformed lab topology
+Run lab validation tests specified in the lab topology
 
 positional arguments:
   tests                 Validation test(s) to execute (default: all)
@@ -24,10 +24,11 @@ options:
   --node NODES          Execute validation tests only on selected node(s)
   --skip-wait           Skip the waiting period
   -e, --error-only      Display only validation errors (on stderr)
+  --source TEST_SOURCE  Read tests from the specified YAML file
   --dump {result} [{result} ...]
-                        Dump additional information during validation process
-  -i INSTANCE, --instance INSTANCE
-                        Specify lab instance to validate
+                        Dump additional information during the validation process
+  -i, --instance INSTANCE
+                        Specify the lab instance to validate
 ```
 
 The **netlab validate** command returns the overall test results in its exit code:
@@ -48,3 +49,10 @@ The **netlab validate** command returns the overall test results in its exit cod
 ```{tip}
 * Use **‌netlab validate --error-only** to shorten the printout and display only the validation errors.
 ```
+
+## Developing Validation Tests
+
+Validation test development is usually an interactive process that requires several changes to the **validate** lab topology attribute before you get them just right. Restarting the lab every time you change the validation tests just to have them transformed and stored in the snapshot file is tedious; these changes to **netlab validate** (introduced in release 25.12) streamline the process:
+
+* The **netlab validate** command compares the timestamp of the lab topology file with the timestamp of the snapshot file. When necessary, it rereads the validation tests from the changed lab topology file.
+* You can develop the validation tests in a separate YAML file and run them with the **netlab validate --source** CLI option. After the validation tests are complete, copy them into the lab topology.

netsim/cli/__init__.py

@@ -76,7 +76,7 @@ def parser_lab_location(
       *i_flags,
       dest='instance',
       action='store',
-      help=argparse.SUPPRESS if hide else f'Specify lab instance to {action}')
+      help=argparse.SUPPRESS if hide else f'Specify the lab instance to {action}')
   if snapshot:
     parser.add_argument(
       '--snapshot',
@@ -240,7 +240,10 @@ def change_lab_instance(instance: typing.Union[int,str], quiet: bool = False) ->
 #
 # Snapshot loading code -- loads the specified snapshot file and checks its modification date
 #
-def load_snapshot(args: typing.Union[argparse.Namespace,Box],ghosts: bool = True) -> Box:
+def load_snapshot(
+      args: typing.Union[argparse.Namespace,Box],
+      ghosts: bool = True,
+      warn_modified: bool = True) -> Box:
   if 'instance' in args and args.instance:
     change_lab_instance(args.instance,args.quiet if 'quiet' in args else False)
 
@@ -265,14 +268,16 @@ def load_snapshot(args: typing.Union[argparse.Namespace,Box],ghosts: bool = True
       log.fatal(f"Cannot read the topology snapshot file {args.snapshot}")
     topology = yaml_topology
 
+  topology._input.snapshot = snapshot
+
   if '_netlab_version' not in topology:
     log.fatal(f"{args.snapshot} is either not a netlab snapshot file or was created with an older netlab version")
 
   if not ghosts:
     topology = augment.nodes.ghost_buster(topology)
 
   global_vars.init(topology)
-  check_modified_source(snapshot,topology)
+  check_modified_source(snapshot,topology,warn_modified)
   return topology
 
 """
@@ -316,9 +321,13 @@ def load_data_source(args: argparse.Namespace, ghosts: bool = True) -> Box:
     f'Could not get the data from {args.snapshot} or lab topology {args.topology}',
     more_hints='Start the lab or specify an alternate topology file with the --topology flag')
 
-def check_modified_source(snapshot: str, topology: typing.Optional[Box] = None) -> None:
+def check_modified_source(
+      snapshot: str,
+      topology: typing.Optional[Box] = None,
+      warning: bool = True) -> typing.Optional[str]:
+
   if topology is None:
-    return
+    return None
 
   snap_time = os.path.getmtime(snapshot)
 
@@ -329,12 +338,18 @@ def check_modified_source(snapshot: str, topology: typing.Optional[Box] = None)
     if in_time <= snap_time:
       continue
 
-    log.warning(
-      text=f'Lab topology source file {infile} has been modified',
-      module='cli',
-      flag='snapshot.modified',
-      more_data=f'after the snapshot {snapshot} has been created',
-      hint='recreate')
+    if warning:
+      log.warning(
+        text=f'Lab topology source file {infile} has been modified',
+        module='cli',
+        flag='snapshot.modified',
+        more_data=f'after the snapshot {snapshot} has been created',
+        hint='recreate')
+
+    topology._input.modified = infile
+    return infile
+
+  return None
 
 # get_message: get action-specific message from topology file
 #

netsim/cli/validate/__init__.py

@@ -14,7 +14,7 @@
 from ...utils import log, strings
 from ...utils import status as _status
 from .. import load_snapshot
-from . import parse, report, tests
+from . import parse, report, source, tests
 
 # I'm cheating. Declaring a global variable is easier than passing 'args' argument around
 #
@@ -42,8 +42,17 @@ def list_tests(topology: Box) -> None:
 def run(cli_args: typing.List[str]) -> None:
   global TEST_COUNT,ERROR_ONLY
   args = parse.validate_parse(cli_args)
+  if args.error_only:
+    args.quiet = True
   log.set_logging_flags(args)
-  topology = load_snapshot(args)
+  topology = load_snapshot(args,warn_modified=False)
+  if args.test_source:
+    source.update_validation_tests(topology,args.test_source)
+  elif topology.get('_input.modified'):
+    log.warning(
+      text=f'Topology source file(s) have changed since the lab has started',
+      module='-')
+    source.update_validation_tests(topology,topology.input[0])
 
   if 'validate' not in topology:
     if args.skip_missing:

netsim/cli/validate/parse.py

@@ -37,6 +37,10 @@ def validate_parse(args: typing.List[str]) -> argparse.Namespace:
     '-e','--error-only',
     dest='error_only', action='store_true',
     help='Display only validation errors (on stderr)')
+  parser.add_argument(
+    '--source',
+    dest='test_source',action='store',
+    help='Read tests from the specified YAML file')
   parser.add_argument(
     '--skip-missing',
     dest='skip_missing', action='store_true',
@@ -47,7 +51,7 @@ def validate_parse(args: typing.List[str]) -> argparse.Namespace:
     choices=['result'],
     nargs='+',
     default=[],
-    help='Dump additional information during validation process')
+    help='Dump additional information during the validation process')
   parser.add_argument(
     dest='tests', action='store',
     nargs='*',

netsim/cli/validate/source.py

@@ -0,0 +1,54 @@
+#
+# Common validation utility functions
+#
+
+
+from box import Box
+
+from ...augment import validate as _validate
+from ...data.types import must_be_id
+from ...data.validate import validate_attributes
+from ...utils import log, read
+from .. import error_and_exit
+
+
+def validate_test_attributes(topology: Box) -> None:
+  for t_name,t_data in topology.validate.items():
+    must_be_id(
+      parent=None,
+      key=t_name,
+      path=f'NOATTR:test name {t_name}',
+      module='validate')
+    validate_attributes(
+      data=t_data,                                    # Validate test description
+      topology=topology,
+      data_path=f'validate.{t_name}',                 # Topology path to test entry
+      data_name=f'validation test',
+      attr_list=['_v_entry'],                         # We're checking validation entries
+      module='validate')                              # Function is called from 'validate' command
+
+  log.exit_on_error()
+
+def update_validation_tests(topology: Box, src: str) -> None:
+  log.info(f'Reading validation tests from {src}')
+  add_topo = read.read_yaml(filename=src)         # Read tests or whole topology from input file
+  if not isinstance(add_topo,Box):                # We have to redo most of the sanity checks done by 'netlab create'
+    error_and_exit('The input file is not a dictionary')
+
+  if 'validate' in add_topo:                      # If we have a 'validate' element in the input dictionary
+    v_tests = add_topo.validate                   # We're assuming we read a whole lab topology
+    if not isinstance(v_tests,Box):               # ... and validate element must be a dictionary
+      error_and_exit('The "validate" element in the input file is not a dictionary')
+  else:
+    v_tests = add_topo                            # Maybe we just read the tests source file?
+
+  if 'nodes' in v_tests or 'links' in v_tests:    # Anyway, final bit of a sanity check...
+    error_and_exit(                               # ... maybe we got a topology file with no 'validate' element?
+      'The source file contains "nodes" or "links" but no "validate" element',
+      more_hints="It looks like your topology file has no validation tests")
+
+  topology.validate = v_tests                     # Hope we got it right; replace validation tests
+  validate_test_attributes(topology)
+  _validate.process_validation(topology)          # Do checks and data transformation on validation tests
+
+  return