CLI: add options to specify alternate config files (#336)

thiell · thiell · commit 5fa13e3da9f2 · 2018-04-19T10:54:03.000-04:00
Added the following command line options: --conf to specify alternative clush.conf (clush only) --groupsconf to specify alternative groups.conf (all CLIs) For example, this can be useful to programmatically create/update clustershell config files with an alternate group source setup. Closes #336 Change-Id: I3aff60057274198b700b44b4d5bbdc916a5b5d0b
diff --git a/doc/man/man1/clubak.1 b/doc/man/man1/clubak.1
@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH CLUBAK 1 "2017-10-23" "1.8" "ClusterShell User Manual"
+.TH CLUBAK 1 "2018-04-16" "1.8" "ClusterShell User Manual"
 .SH NAME
 clubak \- format output from clush/pdsh-like output and more
 .
@@ -71,6 +71,9 @@ fold nodeset using node groups
 .BI \-s \ GROUPSOURCE\fP,\fB \ \-\-groupsource\fB= GROUPSOURCE
 optional \fBgroups.conf\fP(5) group source to use
 .TP
+.BI \-\-groupsconf\fB= FILE
+use alternate config file for groups.conf(5)
+.TP
 .B \-G\fP,\fB  \-\-groupbase
 do not display group source prefix (always \fI@groupname\fP)
 .TP
diff --git a/doc/man/man1/cluset.1 b/doc/man/man1/cluset.1
@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH CLUSET 1 "2017-10-23" "1.8" "ClusterShell User Manual"
+.TH CLUSET 1 "2018-04-16" "1.8" "ClusterShell User Manual"
 .SH NAME
 cluset \- compute advanced cluster node set operations
 .
@@ -61,6 +61,9 @@ show this help message and exit
 .TP
 .BI \-s \ GROUPSOURCE\fP,\fB \ \-\-groupsource\fB= GROUPSOURCE
 optional \fBgroups.conf\fP(5) group source to use
+.TP
+.BI \-\-groupsconf\fB= FILE
+use alternate config file for groups.conf(5)
 .UNINDENT
 .INDENT 0.0
 .TP
diff --git a/doc/man/man1/clush.1 b/doc/man/man1/clush.1
@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH CLUSH 1 "2017-10-23" "1.8" "ClusterShell User Manual"
+.TH CLUSH 1 "2018-04-16" "1.8" "ClusterShell User Manual"
 .SH NAME
 clush \- execute shell commands on a cluster
 .
@@ -162,6 +162,12 @@ optional \fBgroups.conf\fP(5) group source to use
 .B \-n\fP,\fB  \-\-nostdin
 do not watch for possible input from stdin; this should be used when \fBclush\fP is run in the background (or in scripts).
 .TP
+.BI \-\-groupsconf\fB= FILE
+use alternate config file for groups.conf(5)
+.TP
+.BI \-\-conf\fB= FILE
+use alternate config file for clush.conf(5)
+.TP
 .BI \-O \ <KEY=VALUE>\fP,\fB \ \-\-option\fB= <KEY=VALUE>
 override any key=value \fBclush.conf\fP(5) options (repeat as needed)
 .UNINDENT
diff --git a/doc/man/man1/nodeset.1 b/doc/man/man1/nodeset.1
@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH NODESET 1 "2017-10-23" "1.8" "ClusterShell User Manual"
+.TH NODESET 1 "2018-04-16" "1.8" "ClusterShell User Manual"
 .SH NAME
 nodeset \- compute advanced nodeset operations
 .
@@ -61,6 +61,9 @@ show this help message and exit
 .TP
 .BI \-s \ GROUPSOURCE\fP,\fB \ \-\-groupsource\fB= GROUPSOURCE
 optional \fBgroups.conf\fP(5) group source to use
+.TP
+.BI \-\-groupsconf\fB= FILE
+use alternate config file for groups.conf(5)
 .UNINDENT
 .INDENT 0.0
 .TP
diff --git a/doc/txt/clubak.txt b/doc/txt/clubak.txt
@@ -7,7 +7,7 @@ format output from clush/pdsh-like output and more
 --------------------------------------------------
 
 :Author: Stephane Thiell <sthiell@stanford.edu>
-:Date:   2017-10-23
+:Date:   2018-04-16
 :Copyright: GNU Lesser General Public License version 2.1 or later (LGPLv2.1+)
 :Version: 1.8
 :Manual section: 1
@@ -49,6 +49,8 @@ OPTIONS
 -r, --regroup  fold nodeset using node groups
 -s GROUPSOURCE, --groupsource=GROUPSOURCE
                optional ``groups.conf``\(5) group source to use
+--groupsconf=FILE
+               use alternate config file for groups.conf(5)
 -G, --groupbase
                do not display group source prefix (always `@groupname`)
 -S SEPARATOR, --separator=SEPARATOR
diff --git a/doc/txt/cluset.txt b/doc/txt/cluset.txt
@@ -7,7 +7,7 @@ compute advanced cluster node set operations
 --------------------------------------------
 
 :Author: Stephane Thiell <sthiell@stanford.edu>
-:Date:   2017-10-23
+:Date:   2018-04-16
 :Copyright: GNU Lesser General Public License version 2.1 or later (LGPLv2.1+)
 :Version: 1.8
 :Manual section: 1
@@ -41,6 +41,7 @@ OPTIONS
   -h, --help            show this help message and exit
   -s GROUPSOURCE, --groupsource=GROUPSOURCE
                         optional ``groups.conf``\(5) group source to use
+  --groupsconf=FILE     use alternate config file for groups.conf(5)
 
   Commands:
     -c, --count         show number of nodes in nodeset(s)
diff --git a/doc/txt/clush.txt b/doc/txt/clush.txt
@@ -7,7 +7,7 @@ execute shell commands on a cluster
 -----------------------------------
 
 :Author: Stephane Thiell <sthiell@stanford.edu>
-:Date:   2017-10-23
+:Date:   2018-04-16
 :Copyright: GNU Lesser General Public License version 2.1 or later (LGPLv2.1+)
 :Version: 1.8
 :Manual section: 1
@@ -131,6 +131,8 @@ OPTIONS
 -s GROUPSOURCE, --groupsource=GROUPSOURCE
                       optional ``groups.conf``\(5) group source to use
 -n, --nostdin         do not watch for possible input from stdin; this should be used when ``clush`` is run in the background (or in scripts).
+--groupsconf=FILE     use alternate config file for groups.conf(5)
+--conf=FILE           use alternate config file for clush.conf(5)
 -O <KEY=VALUE>, --option=<KEY=VALUE>
                       override any key=value ``clush.conf``\(5) options (repeat as needed)
 
diff --git a/doc/txt/nodeset.txt b/doc/txt/nodeset.txt
@@ -7,7 +7,7 @@ compute advanced nodeset operations
 -----------------------------------
 
 :Author: Stephane Thiell <sthiell@stanford.edu>
-:Date:   2017-10-23
+:Date:   2018-04-16
 :Copyright: GNU Lesser General Public License version 2.1 or later (LGPLv2.1+)
 :Version: 1.8
 :Manual section: 1
@@ -41,6 +41,7 @@ OPTIONS
   -h, --help            show this help message and exit
   -s GROUPSOURCE, --groupsource=GROUPSOURCE
                         optional ``groups.conf``\(5) group source to use
+  --groupsconf=FILE     use alternate config file for groups.conf(5)
 
   Commands:
     -c, --count         show number of nodes in nodeset(s)
diff --git a/lib/ClusterShell/CLI/Clubak.py b/lib/ClusterShell/CLI/Clubak.py
@@ -1,6 +1,6 @@
 #
 # Copyright (C) 2010-2012 CEA/DAM
-# Copyright (C) 2017 Stephane Thiell <sthiell@stanford.edu>
+# Copyright (C) 2017-2018 Stephane Thiell <sthiell@stanford.edu>
 #
 # This file is part of ClusterShell.
 #
@@ -31,9 +31,10 @@
 
 from ClusterShell.MsgTree import MsgTree, MODE_DEFER, MODE_TRACE
 from ClusterShell.NodeSet import NodeSet, NodeSetParseError, std_group_resolver
+from ClusterShell.NodeSet import set_std_group_resolver_config
 
 from ClusterShell.CLI.Display import Display, THREE_CHOICES
-from ClusterShell.CLI.Display import sys_stdin, sys_stdout, sys_stderr
+from ClusterShell.CLI.Display import sys_stdin, sys_stdout
 from ClusterShell.CLI.Error import GENERIC_ERRORS, handle_generic_error
 from ClusterShell.CLI.OptionParser import OptionParser
 from ClusterShell.CLI.Utils import nodeset_cmpkey
@@ -94,12 +95,15 @@ def clubak():
 
     # Argument management
     parser = OptionParser("%prog [options]")
+    parser.install_groupsconf_option()
     parser.install_display_options(verbose_options=True,
                                    separator_option=True,
                                    dshbak_compat=True,
                                    msgtree_mode=True)
     options = parser.parse_args()[0]
 
+    set_std_group_resolver_config(options.groupsconf)
+
     if options.interpret_keys == THREE_CHOICES[-1]: # auto?
         enable_nodeset_key = None # AUTO
     else:
diff --git a/lib/ClusterShell/CLI/Clush.py b/lib/ClusterShell/CLI/Clush.py
@@ -1,6 +1,6 @@
 #
 # Copyright (C) 2007-2016 CEA/DAM
-# Copyright (C) 2015-2017 Stephane Thiell <sthiell@stanford.edu>
+# Copyright (C) 2015-2018 Stephane Thiell <sthiell@stanford.edu>
 #
 # This file is part of ClusterShell.
 #
@@ -59,8 +59,8 @@
 
 from ClusterShell.Event import EventHandler
 from ClusterShell.MsgTree import MsgTree
-from ClusterShell.NodeSet import RESOLVER_NOGROUP, std_group_resolver
-from ClusterShell.NodeSet import NodeSet, NodeSetParseError
+from ClusterShell.NodeSet import RESOLVER_NOGROUP, set_std_group_resolver_config
+from ClusterShell.NodeSet import NodeSet, NodeSetParseError, std_group_resolver
 from ClusterShell.Task import Task, task_self
 
 
@@ -793,18 +793,21 @@ def main():
     parser.add_option("-n", "--nostdin", action="store_true", dest="nostdin",
                       help="don't watch for possible input from stdin")
 
-    parser.install_config_options('clush.conf(5)')
+    parser.install_groupsconf_option()
+    parser.install_clush_config_options()
     parser.install_nodes_options()
     parser.install_display_options(verbose_options=True)
     parser.install_filecopy_options()
     parser.install_connector_options()
 
     (options, args) = parser.parse_args()
 
+    set_std_group_resolver_config(options.groupsconf)
+
     #
     # Load config file and apply overrides
     #
-    config = ClushConfig(options)
+    config = ClushConfig(options, options.conf)
 
     # Initialize logging
     if config.verbosity >= VERB_DEBUG:
diff --git a/lib/ClusterShell/CLI/Nodeset.py b/lib/ClusterShell/CLI/Nodeset.py
@@ -1,6 +1,6 @@
 #
 # Copyright (C) 2008-2016 CEA/DAM
-# Copyright (C) 2015-2017 Stephane Thiell <sthiell@stanford.edu>
+# Copyright (C) 2015-2018 Stephane Thiell <sthiell@stanford.edu>
 #
 # This file is part of ClusterShell.
 #
@@ -36,8 +36,8 @@
 from ClusterShell.CLI.Error import GENERIC_ERRORS, handle_generic_error
 from ClusterShell.CLI.OptionParser import OptionParser
 
-from ClusterShell.NodeSet import NodeSet, RangeSet
-from ClusterShell.NodeSet import grouplist, std_group_resolver
+from ClusterShell.NodeSet import NodeSet, RangeSet, std_group_resolver
+from ClusterShell.NodeSet import grouplist, set_std_group_resolver_config
 from ClusterShell.NodeUtils import GroupSourceNoUpcall
 
 
@@ -154,11 +154,13 @@ def nodeset():
     usage = "%prog [COMMAND] [OPTIONS] [ns1 [-ixX] ns2|...]"
 
     parser = OptionParser(usage)
+    parser.install_groupsconf_option()
     parser.install_nodeset_commands()
     parser.install_nodeset_operations()
     parser.install_nodeset_options()
     (options, args) = parser.parse_args()
 
+    set_std_group_resolver_config(options.groupsconf)
     group_resolver = std_group_resolver()
 
     if options.debug:
diff --git a/lib/ClusterShell/CLI/OptionParser.py b/lib/ClusterShell/CLI/OptionParser.py
@@ -83,11 +83,20 @@ def __init__(self, usage, **kwargs):
                         type="safestring", dest="groupsource",
                         help="optional groups.conf(5) group source to use")
 
-    def install_config_options(self, filename=''):
-        """Install config options override"""
+    def install_clush_config_options(self):
+        """Install config options for clush"""
+        # config file override (--conf)
+        self.add_option("--conf", action="store", metavar='FILE',
+                        help="use alternate config file for clush.conf(5)")
+        # config file option overrides (-O)
         self.add_option("-O", "--option", action="append", metavar="KEY=VALUE",
                         dest="option", default=[],
-                        help="override any key=value %s options" % filename)
+                        help="override any key=value clush.conf(5) options")
+
+    def install_groupsconf_option(self):
+        """Install an alternate groups.conf file option"""
+        self.add_option("--groupsconf", action="store", metavar='FILE',
+                        help="use alternate config file for groups.conf(5)")
 
     def install_nodes_options(self):
         """Install nodes selection options"""
diff --git a/lib/ClusterShell/CLI/Utils.py b/lib/ClusterShell/CLI/Utils.py
@@ -1,5 +1,6 @@
 #
 # Copyright (C) 2010-2015 CEA/DAM
+# Copyright (C) 2018 Stephane Thiell <sthiell@stanford.edu>
 #
 # This file is part of ClusterShell.
 #
@@ -21,8 +22,6 @@
 CLI utility functions
 """
 
-import sys
-
 
 (KIBI, MEBI, GIBI, TEBI) = (1024.0, 1024.0 ** 2, 1024.0 ** 3, 1024.0 ** 4)
 
diff --git a/lib/ClusterShell/NodeSet.py b/lib/ClusterShell/NodeSet.py
@@ -1572,3 +1572,18 @@ def set_std_group_resolver(new_resolver):
     global RESOLVER_STD_GROUP
     RESOLVER_STD_GROUP = new_resolver or _DEF_RESOLVER_STD_GROUP
 
+def set_std_group_resolver_config(groupsconf, illegal_chars=None):
+    """
+    Helper to create and set std group resolver from a config file path.
+
+    By default, the GroupResolverConfig object is created using
+    illegal_chars=NodeSet.ILLEGAL_GROUP_CHARS.
+
+    This method does nothing if groupsconf is not defined.
+    """
+    if groupsconf:
+        if illegal_chars is None:
+            illegal_chars = ILLEGAL_GROUP_CHARS
+        group_resolver = NodeUtils.GroupResolverConfig(groupsconf,
+                                                       illegal_chars)
+        set_std_group_resolver(group_resolver)
diff --git a/tests/CLIClubakTest.py b/tests/CLIClubakTest.py
@@ -10,6 +10,9 @@
 from TLib import *
 from ClusterShell.CLI.Clubak import main
 
+from ClusterShell.NodeSet import set_std_group_resolver, \
+                                 set_std_group_resolver_config
+
 
 def _outfmt(*args):
     outfmt = "---------------\n%s\n---------------\n bar\n"
@@ -166,3 +169,42 @@ def test_009_diff(self):
                        b"line_mode=False gather=True tree_depth=1\n")
         self._clubak_t(["--diff", "-L"], b"foo1: bar\nfoo2: bar", b'', 2,
                        b"error: option mismatch (diff not supported in line_mode)\n")
+
+
+class CLIClubakTestGroupsConf(CLIClubakTest):
+    """Unit test class for testing --groupsconf option"""
+
+    def setUp(self):
+        self.gconff = make_temp_file(dedent("""
+            [Main]
+            default: global_default
+
+            [global_default]
+            map: echo foo[1-2]
+            all: echo @foo
+            list: echo foo
+            """).encode())
+        set_std_group_resolver_config(self.gconff.name)
+
+        # passed to --groupsconf
+        self.custf = make_temp_file(dedent("""
+            [Main]
+            default: custom
+
+            [custom]
+            map: echo foo[1-2]
+            all: echo @bar
+            list: echo bar
+            """).encode())
+
+    def tearDown(self):
+        set_std_group_resolver(None)
+        self.gconff = None
+        self.custf = None
+
+    def test_groupsconf_option(self):
+        """test nodeset with --groupsconf"""
+        # use -r (--regroup) to test group resolution
+        self._clubak_t(["-r"], b"foo1: bar\nfoo2: bar", _outfmt("@foo (2)"))
+        self._clubak_t(["--groupsconf", self.custf.name, "-r"],
+                       b"foo1: bar\nfoo2: bar", _outfmt("@bar (2)"))
diff --git a/tests/CLIClushTest.py b/tests/CLIClushTest.py
diff --git a/tests/CLIConfigTest.py b/tests/CLIConfigTest.py
diff --git a/tests/CLINodesetTest.py b/tests/CLINodesetTest.py
