From 524be5baa59d803a5bff5c06defed20c91b83727 Mon Sep 17 00:00:00 2001 From: Ivan Pepelnjak Date: Thu, 4 Dec 2025 18:28:32 +0100 Subject: [PATCH 1/2] Reread validation tests from modified lab topology file The commit gives 'netlab validate' the ability to reread the validation tests if the topology file has changed, or read them from the specified source file. Also changed: * More control over the warnings generated by the 'read topology from the snapshot' process * Store the name of the snapshot and modified topology file in the '_input' element --- docs/netlab/validate.md | 24 ++++++++++----- netsim/cli/__init__.py | 37 +++++++++++++++------- netsim/cli/validate/__init__.py | 13 ++++++-- netsim/cli/validate/parse.py | 6 +++- netsim/cli/validate/source.py | 54 +++++++++++++++++++++++++++++++++ 5 files changed, 112 insertions(+), 22 deletions(-) create mode 100644 netsim/cli/validate/source.py diff --git a/docs/netlab/validate.md b/docs/netlab/validate.md index 1553746b33..233771b272 100644 --- a/docs/netlab/validate.md +++ b/docs/netlab/validate.md @@ -6,12 +6,12 @@ ## Usage ```text -$ netlab validate -h -usage: netlab inspect [-h] [-v] [-q] [--list] [--node NODES] [--skip-wait] [-e] - [--dump {result} [{result} ...]] [-i INSTANCE] - [tests ...] +usage: netlab validate [-h] [-v] [-q] [--list] [--node NODES] [--skip-wait] [-e] + [--source TEST_SOURCE] [--dump {result} [{result} ...]] + [-i INSTANCE] + [tests ...] -Inspect data structures in transformed lab topology +Run lab validation tests specified in the lab topology positional arguments: tests Validation test(s) to execute (default: all) @@ -24,10 +24,11 @@ options: --node NODES Execute validation tests only on selected node(s) --skip-wait Skip the waiting period -e, --error-only Display only validation errors (on stderr) + --source TEST_SOURCE Read tests from the specified YAML file --dump {result} [{result} ...] - Dump additional information during validation process - -i INSTANCE, --instance INSTANCE - Specify lab instance to validate + Dump additional information during the validation process + -i, --instance INSTANCE + Specify the lab instance to validate ``` The **netlab validate** command returns the overall test results in its exit code: @@ -48,3 +49,10 @@ The **netlab validate** command returns the overall test results in its exit cod ```{tip} * Use **‌netlab validate --error-only** to shorten the printout and display only the validation errors. ``` + +## Developing Validation Tests + +Validation test development is usually an interactive process that requires several changes to the **validate** lab topology attribute before you get them just right. Restarting the lab every time you change the validation tests just to have them transformed and stored in the snapshot file is tedious; these changes to **netlab validate** (introduced in release 25.12) streamline the process: + +* The **netlab validate** command compares the timestamp of the lab topology file with the timestamp of the snapshot file. When necessary, it rereads the validation tests from the changed lab topology file. +* You can develop the validation tests in a separate YAML file and run them with the **netlab validate --source** CLI option. After the validation tests are complete, copy them into the lab topology. diff --git a/netsim/cli/__init__.py b/netsim/cli/__init__.py index de5ec91ec6..493f108ee1 100755 --- a/netsim/cli/__init__.py +++ b/netsim/cli/__init__.py @@ -76,7 +76,7 @@ def parser_lab_location( *i_flags, dest='instance', action='store', - help=argparse.SUPPRESS if hide else f'Specify lab instance to {action}') + help=argparse.SUPPRESS if hide else f'Specify the lab instance to {action}') if snapshot: parser.add_argument( '--snapshot', @@ -240,7 +240,10 @@ def change_lab_instance(instance: typing.Union[int,str], quiet: bool = False) -> # # Snapshot loading code -- loads the specified snapshot file and checks its modification date # -def load_snapshot(args: typing.Union[argparse.Namespace,Box],ghosts: bool = True) -> Box: +def load_snapshot( + args: typing.Union[argparse.Namespace,Box], + ghosts: bool = True, + warn_modified: bool = True) -> Box: if 'instance' in args and args.instance: change_lab_instance(args.instance,args.quiet if 'quiet' in args else False) @@ -265,6 +268,8 @@ def load_snapshot(args: typing.Union[argparse.Namespace,Box],ghosts: bool = True log.fatal(f"Cannot read the topology snapshot file {args.snapshot}") topology = yaml_topology + topology._input.snapshot = snapshot + if '_netlab_version' not in topology: log.fatal(f"{args.snapshot} is either not a netlab snapshot file or was created with an older netlab version") @@ -272,7 +277,7 @@ def load_snapshot(args: typing.Union[argparse.Namespace,Box],ghosts: bool = True topology = augment.nodes.ghost_buster(topology) global_vars.init(topology) - check_modified_source(snapshot,topology) + check_modified_source(snapshot,topology,warn_modified) return topology """ @@ -316,9 +321,13 @@ def load_data_source(args: argparse.Namespace, ghosts: bool = True) -> Box: f'Could not get the data from {args.snapshot} or lab topology {args.topology}', more_hints='Start the lab or specify an alternate topology file with the --topology flag') -def check_modified_source(snapshot: str, topology: typing.Optional[Box] = None) -> None: +def check_modified_source( + snapshot: str, + topology: typing.Optional[Box] = None, + warning: bool = True) -> typing.Optional[str]: + if topology is None: - return + return None snap_time = os.path.getmtime(snapshot) @@ -329,12 +338,18 @@ def check_modified_source(snapshot: str, topology: typing.Optional[Box] = None) if in_time <= snap_time: continue - log.warning( - text=f'Lab topology source file {infile} has been modified', - module='cli', - flag='snapshot.modified', - more_data=f'after the snapshot {snapshot} has been created', - hint='recreate') + if warning: + log.warning( + text=f'Lab topology source file {infile} has been modified', + module='cli', + flag='snapshot.modified', + more_data=f'after the snapshot {snapshot} has been created', + hint='recreate') + + topology._input.modified = infile + return infile + + return None # get_message: get action-specific message from topology file # diff --git a/netsim/cli/validate/__init__.py b/netsim/cli/validate/__init__.py index 60e6659283..bf28b12235 100644 --- a/netsim/cli/validate/__init__.py +++ b/netsim/cli/validate/__init__.py @@ -14,7 +14,7 @@ from ...utils import log, strings from ...utils import status as _status from .. import load_snapshot -from . import parse, report, tests +from . import parse, report, source, tests # I'm cheating. Declaring a global variable is easier than passing 'args' argument around # @@ -42,8 +42,17 @@ def list_tests(topology: Box) -> None: def run(cli_args: typing.List[str]) -> None: global TEST_COUNT,ERROR_ONLY args = parse.validate_parse(cli_args) + if args.error_only: + args.quiet = True log.set_logging_flags(args) - topology = load_snapshot(args) + topology = load_snapshot(args,warn_modified=False) + if args.test_source: + source.update_validation_tests(topology,args.test_source) + elif topology.get('_input.modified'): + log.warning( + text=f'Topology source file(s) have changed since the lab has started', + module='-') + source.update_validation_tests(topology,topology.input[0]) if 'validate' not in topology: if args.skip_missing: diff --git a/netsim/cli/validate/parse.py b/netsim/cli/validate/parse.py index 1207fdef60..48ae3e3396 100644 --- a/netsim/cli/validate/parse.py +++ b/netsim/cli/validate/parse.py @@ -37,6 +37,10 @@ def validate_parse(args: typing.List[str]) -> argparse.Namespace: '-e','--error-only', dest='error_only', action='store_true', help='Display only validation errors (on stderr)') + parser.add_argument( + '--source', + dest='test_source',action='store', + help='Read tests from the specified YAML file') parser.add_argument( '--skip-missing', dest='skip_missing', action='store_true', @@ -47,7 +51,7 @@ def validate_parse(args: typing.List[str]) -> argparse.Namespace: choices=['result'], nargs='+', default=[], - help='Dump additional information during validation process') + help='Dump additional information during the validation process') parser.add_argument( dest='tests', action='store', nargs='*', diff --git a/netsim/cli/validate/source.py b/netsim/cli/validate/source.py new file mode 100644 index 0000000000..f68920c764 --- /dev/null +++ b/netsim/cli/validate/source.py @@ -0,0 +1,54 @@ +# +# Common validation utility functions +# + + +from box import Box + +from ...augment import validate as _validate +from ...data.types import must_be_id +from ...data.validate import validate_attributes +from ...utils import log, read +from .. import error_and_exit + + +def validate_test_attributes(topology: Box) -> None: + for t_name,t_data in topology.validate.items(): + must_be_id( + parent=None, + key=t_name, + path=f'NOATTR:test name {t_name}', + module='validate') + validate_attributes( + data=t_data, # Validate test description + topology=topology, + data_path=f'validate.{t_name}', # Topology path to test entry + data_name=f'validation test', + attr_list=['_v_entry'], # We're checking validation entries + module='validate') # Function is called from 'validate' command + + log.exit_on_error() + +def update_validation_tests(topology: Box, src: str) -> None: + log.info(f'Reading validation tests from {src}') + add_topo = read.read_yaml(filename=src) # Read tests or whole topology from input file + if not isinstance(add_topo,Box): # We have to redo most of the sanity checks done by 'netlab create' + error_and_exit('The input file is not a dictionary') + + if 'validate' in add_topo: # If we have a 'validate' element in the input dictionary + v_tests = add_topo.validate # We're assuming we read a whole lab topology + if not isinstance(v_tests,Box): # ... and validate element must be a dictionary + error_and_exit('The "validate" element in the input file is not a dictionary') + else: + v_tests = add_topo # Maybe we just read the tests source file? + + if 'nodes' in v_tests or 'links' in v_tests: # Anyway, final bit of a sanity check... + error_and_exit( # ... maybe we got a topology file with no 'validate' element? + 'The source file contains "nodes" or "links" but no "validate" element', + more_hints="It looks like your topology file has no validation tests") + + topology.validate = v_tests # Hope we got it right; replace validation tests + validate_test_attributes(topology) + _validate.process_validation(topology) # Do checks and data transformation on validation tests + + return From 4759a485fb9aea19b35741bf7c9f00f5c9a439cb Mon Sep 17 00:00:00 2001 From: Ivan Pepelnjak Date: Fri, 5 Dec 2025 12:35:07 +0100 Subject: [PATCH 2/2] Further input file validation checks suggested by Copilot --- netsim/cli/validate/source.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/netsim/cli/validate/source.py b/netsim/cli/validate/source.py index f68920c764..5f0fc67c16 100644 --- a/netsim/cli/validate/source.py +++ b/netsim/cli/validate/source.py @@ -3,6 +3,8 @@ # +import os + from box import Box from ...augment import validate as _validate @@ -30,10 +32,14 @@ def validate_test_attributes(topology: Box) -> None: log.exit_on_error() def update_validation_tests(topology: Box, src: str) -> None: + if not os.path.exists(src): + error_and_exit(f'{src} does not exist') log.info(f'Reading validation tests from {src}') add_topo = read.read_yaml(filename=src) # Read tests or whole topology from input file + if add_topo is None: + error_and_exit(f'The input file ({src}) is not a YAML file') if not isinstance(add_topo,Box): # We have to redo most of the sanity checks done by 'netlab create' - error_and_exit('The input file is not a dictionary') + error_and_exit(f'The input file ({src}) is not a dictionary') if 'validate' in add_topo: # If we have a 'validate' element in the input dictionary v_tests = add_topo.validate # We're assuming we read a whole lab topology