Create daemon config files in 'netlab initial' Python code (ipspace#2867)

The daemon config files and other container configuration templates
are no longer rendered in an Ansible playbook, ensuring the created
configurations match the actual daemon configurations mapped into
containers.

Also:
* Add '--clean' parameter to cleanup the output directory
* Handle the output directory cleanup/creation in Python code
* Add tags to 'create-config' playbook to correctly handle the
  '-m', '-i' and '-c' parameters
* Define '-l' parameter as a valid 'netlab initial' parameter
  (previously it was just passed to Ansible) and use it to limit
  the nodes for which the container config templates are rendered.

Author: ipspace

docs/netlab/initial.md

@@ -34,8 +34,8 @@ Jinja2 templates are used together with **_device_\_config** Ansible modules to
 
 ```text
 $ netlab initial -h
-usage: netlab initial [-h] [--log] [-v] [-q] [-i] [-m [MODULE]] [-c] [--ready] [--fast]
-                      [-o [OUTPUT]] [--instance INSTANCE]
+usage: netlab initial [-h] [--log] [-v] [-q] [-i] [-m [MODULE]] [-l LIMIT] [-c]
+                      [--ready] [--fast] [-o [OUTPUT]] [--clean] [--instance INSTANCE]
 
 Initial device configurations
 
@@ -45,17 +45,20 @@ options:
   -v, --verbose         Verbose logging (add multiple flags for increased verbosity)
   -q, --quiet           Report only major errors
   -i, --initial         Deploy just the initial configuration
-  -m [MODULE], --module [MODULE]
+  -m, --module [MODULE]
                         Deploy module-specific configuration (optionally including a
                         list of modules separated by commas)
+  -l, --limit LIMIT     Limit the operation to a subset of nodes
   -c, --custom          Deploy custom configuration templates (specified in "config"
                         group or node attribute)
   --ready               Wait for devices to become ready
   --fast                Use "free" strategy in Ansible playbook for faster configuration
                         deployment
-  -o [OUTPUT], --output [OUTPUT]
+  -o, --output [OUTPUT]
                         Create a directory with initial configurations instead of
                         deploying them (default output directory: config)
+  --clean               Clean up the output directory before creating the initial
+                        configuration
   --instance INSTANCE   Specify lab instance to configure
 
 All other arguments are passed directly to ansible-playbook
@@ -119,10 +122,10 @@ You'll find more details in _[](custom-config)_ and _[](dev-find-custom)_ docume
 
 The **netlab initial** command deploys all initial device configurations when started without additional parameters. To control the deployment of initial configurations:
 
-* use the `-i` flag to deploy initial device configurations. 
-* use the `-m` flag to deploy module-specific configurations. 
-* use the `-m` flag followed by a module name (example: `-m ospf -m bgp`) to deploy device configuration for specific modules. You can use the `-m` flag multiple times.
-* use the `-c` flag to deploy custom configuration templates. 
+* Use the `-i` flag to deploy initial device configurations.
+* Use the `-m` flag to deploy all module-specific configurations.
+* Use the `-m` flag followed by a module name (example: `-m ospf`) or a comma-separated list of module names (example: `-m ospf,bgp`) to deploy device configuration for specific module(s).
+* Use the `-c` flag to deploy custom configuration templates.
 
 All unrecognized parameters are passed to the internal `initial-config.ansible` Ansible playbook. You can use **ansible-playbook** CLI parameters to modify the configuration deployment, for example:
 

netsim/ansible/create-config.ansible

@@ -7,19 +7,11 @@
     config_dir: "{{ lookup('env','PWD') }}/config"
     node_provider: "{{ provider|default(netlab_provider) }}"
   tasks:
-  - block:
-    - name: Set variables that cannot be set with vars
-      set_fact:
-        netlab_device_type: "{{ netlab_device_type|default(ansible_network_os) }}"
-        netlab_interfaces: "{{ ([ loopback ] if loopback is defined else []) + interfaces|default([]) }}"
-
-    - name: Create config directory in {{ config_dir }}
-      file:
-        path: "{{ config_dir }}"
-        state: directory
-      run_once: true
-
-    delegate_to: localhost
+  - name: Set variables that cannot be set with vars
+    set_fact:
+      netlab_device_type: "{{ netlab_device_type|default(ansible_network_os) }}"
+      netlab_interfaces: "{{ ([ loopback ] if loopback is defined else []) + interfaces|default([]) }}"
+    tags: [ always ]
 
   - name: Create initial device configuration
     include_tasks: "tasks/create-config.yml"
@@ -28,6 +20,8 @@
         vars:
           config_item: initial
           paths: "{{ paths_templates.dirs }}"
+        tags: [ always ]
+    tags: [ initial ]
 
 - name: Create module-specific configurations
   hosts: modules
@@ -50,9 +44,11 @@
         loop_var: config_item
       args:
         apply:
+          tags: [ always ]
           vars:
             paths: "{{ paths_templates.dirs }}"
 
+    tags: [ module ]
     delegate_to: localhost
 
 - name: Create custom deployment templates
@@ -68,27 +64,7 @@
       loop_var: custom_config
     args:
       apply:
+        tags: [ always ]
         vars:
           paths: "{{ paths_custom.dirs }}"
-
-- name: Create daemon configuration files
-  hosts: daemons
-  serial: 1
-  vars:
-    node_provider: "{{ provider|default(netlab_provider) }}"
-    extra_config: >
-      {{ _daemon_config|default({})
-            |list
-            |difference(modules|default([]))
-            |difference(config|default([])) }}
-  tasks:
-  - name: Create daemon configurations
-    include_tasks: "tasks/create-config.yml"
-    loop: "{{ extra_config }}"
-    when: "'@' not in config_item"
-    loop_control:
-      loop_var: config_item
-    args:
-      apply:
-        vars:
-          paths: "{{ paths_templates.dirs }}"
+    tags: [ custom ]

netsim/cli/initial.py

@@ -3,106 +3,34 @@
 #
 # Deploys initial device configurations
 #
-import argparse
-import os
 import typing
 
 from .. import devices
 from ..utils import log
 from ..utils import status as _status
 from . import (
   ansible,
-  common_parse_args,
   external_commands,
   get_message,
+  initial_actions,
   lab_status_change,
   load_snapshot,
-  parser_lab_location,
 )
 
 
-#
-# CLI parser for 'netlab initial' command
-#
-def initial_config_parse(args: typing.List[str]) -> typing.Tuple[argparse.Namespace, typing.List[str]]:
-  parser = argparse.ArgumentParser(
-    parents=[ common_parse_args() ],
-    prog="netlab initial",
-    description='Initial device configurations',
-    epilog='All other arguments are passed directly to ansible-playbook')
-
-  parser.add_argument(
-    '-i','--initial',
-    dest='initial', action='store_true',
-    help='Deploy just the initial configuration')
-  parser.add_argument(
-    '-m','--module',
-    dest='module', action='store',nargs='?',const='*',
-    help='Deploy module-specific configuration (optionally including a list of modules separated by commas)')
-  parser.add_argument(
-    '-c','--custom',
-    dest='custom', action='store_true',
-    help='Deploy custom configuration templates (specified in "config" group or node attribute)')
-  parser.add_argument(
-    '--ready',
-    dest='ready', action='store_true',
-    help='Wait for devices to become ready')
-  parser.add_argument(
-    '--fast',
-    dest='fast', action='store_true',
-    help='Use "free" strategy in Ansible playbook for faster configuration deployment')
-  parser.add_argument(
-    '-o','--output',
-    dest='output', action='store',nargs='?',const='config',
-    help='Create a directory with initial configurations instead of deploying them (default output directory: config)')
-  parser.add_argument(
-    '--no-message',
-    dest='no_message', action='store_true',
-    help=argparse.SUPPRESS)
-  parser_lab_location(parser,instance=True,i_used=True,action='configure')
-
-  return parser.parse_known_args(args)
-
 def run_initial(cli_args: typing.List[str]) -> None:
-  (args,rest) = initial_config_parse(cli_args)
-  if args.output:
-    rest = ['-e',f'config_dir="{os.path.abspath(args.output)}"' ] + rest
-
+  (args,rest) = initial_actions.initial_config_parse(cli_args)
   topology = load_snapshot(args)
 
-  deploy_parts = []
-  if args.verbose:
-    rest = ['-' + 'v' * args.verbose] + rest
-
-  if args.initial:
-    rest = ['-t','initial'] + rest
-    deploy_parts.append("initial configuration")
-
-  if args.quiet:
-    os.environ["ANSIBLE_STDOUT_CALLBACK"] = "selective"
-
-  if args.module:
-    if args.module != "*":
-      deploy_parts.append("module(s): " + args.module)
-      rest = ['-e','modlist='+args.module] + rest
-    else:
-      deploy_parts.append("modules")
-    rest = ['-t','module'] + rest
-  
-  if args.custom:
-    deploy_parts.append("custom")
-    rest = ['-t','custom'] + rest
-
-  if args.fast or os.environ.get('NETLAB_FAST_CONFIG',None):
-    rest = ['-e','netlab_strategy=free'] + rest
+  rest = rest + initial_actions.ansible_args(args)
+  deploy_parts = initial_actions.get_deploy_parts(args)
 
   if args.logging or args.verbose:
     print("Ansible playbook args: %s" % rest)
 
   ansible.check_version()
   if args.output:
-    ansible.playbook('create-config.ansible',rest)
-    print("\nInitial configurations have been created in the %s directory" % args.output)
+    initial_actions.configs.run(topology,args,rest)
     return
   elif args.ready:
     ansible.playbook('device-ready.ansible',rest)

netsim/cli/initial_actions/__init__.py

@@ -0,0 +1,107 @@
+"""
+Common functions for the "netlab initial" actions
+"""
+
+import argparse
+import os
+import typing
+
+from .. import common_parse_args, parser_lab_location
+from . import configs
+
+
+#
+# CLI parser for 'netlab initial' command
+#
+def initial_config_parse(args: typing.List[str]) -> typing.Tuple[argparse.Namespace, typing.List[str]]:
+  parser = argparse.ArgumentParser(
+    parents=[ common_parse_args() ],
+    prog="netlab initial",
+    description='Initial device configurations',
+    epilog='All other arguments are passed directly to ansible-playbook')
+
+  parser.add_argument(
+    '-i','--initial',
+    dest='initial', action='store_true',
+    help='Deploy just the initial configuration')
+  parser.add_argument(
+    '-m','--module',
+    dest='module', action='store',nargs='?',const='*',
+    help='Deploy module-specific configuration (optionally including a list of modules separated by commas)')
+  parser.add_argument(
+    '-l','--limit',
+    dest='limit', action='store',
+    help='Limit the operation to a subset of nodes')
+  parser.add_argument(
+    '-c','--custom',
+    dest='custom', action='store_true',
+    help='Deploy custom configuration templates (specified in "config" group or node attribute)')
+  parser.add_argument(
+    '--ready',
+    dest='ready', action='store_true',
+    help='Wait for devices to become ready')
+  parser.add_argument(
+    '--fast',
+    dest='fast', action='store_true',
+    help='Use "free" strategy in Ansible playbook for faster configuration deployment')
+  parser.add_argument(
+    '-o','--output',
+    dest='output', action='store',nargs='?',const='config',
+    help='Create a directory with initial configurations instead of deploying them (default output directory: config)')
+  parser.add_argument(
+    '--clean',
+    dest='clean', action='store_true',
+    help='Clean up the output directory before creating the initial configuration')
+  parser.add_argument(
+    '--no-message',
+    dest='no_message', action='store_true',
+    help=argparse.SUPPRESS)
+  parser_lab_location(parser,instance=True,i_used=True,action='configure')
+
+  return parser.parse_known_args(args)
+
+"""
+Build Ansible arguments based on 'netlab initial' parameters
+"""
+def ansible_args(args: argparse.Namespace) -> list:
+  rest: typing.List[str] = []
+  if args.verbose:
+    rest = ['-' + 'v' * args.verbose] + rest
+
+  if args.limit:
+    rest = ['--limit',args.limit] + rest
+
+  if args.initial:
+    rest = ['-t','initial'] + rest
+
+  if args.quiet:
+    os.environ["ANSIBLE_STDOUT_CALLBACK"] = "selective"
+
+  if args.module:
+    if args.module != "*":
+      rest = ['-e','modlist='+args.module] + rest
+    rest = ['-t','module'] + rest
+
+  if args.custom:
+    rest = ['-t','custom'] + rest
+
+  if args.fast or os.environ.get('NETLAB_FAST_CONFIG',None):
+    rest = ['-e','netlab_strategy=free'] + rest
+
+  return rest
+
+def get_deploy_parts(args: argparse.Namespace) -> list:
+  deploy_parts = []
+  if args.initial:
+    deploy_parts.append("initial configuration")
+
+  if args.module:
+    if args.module != "*":
+      deploy_parts.append("module(s): " + args.module)
+    else:
+      deploy_parts.append("modules")
+
+  if args.custom:
+    deploy_parts.append("custom")
+
+  return deploy_parts