NodeSet: do not require 'all' special group for file-based source

Fix 'all' fallback mechanism for file-based source. With this change,
the all special group for file-based source is only used if defined,
otherwise we just include all nodes from the source.
Used by NodeSet.fromall() and @* group syntax.

Change-Id: I3b3de11566421de94ff302698f3d2a8a0be18eb5

Author: thiell

conf/groups.d/cluster.yaml.example

@@ -17,29 +17,32 @@
 #
 # Break and adapt to fit your own needs. Use nodeset CLI to test config.
 
-# Group source cluster:
-# define groups @cluster:adm, @cluster:io, etc.
-cluster:
+# Group source roles:
+# define groups @roles:adm, @roles:io, etc.
+roles:
     adm: 'example0'
     io: '@racks:rack2,example2'
     compute: '@racks:rack[3-4]'
     gpu: '@racks:rack4'
-    all: '@adm,@io,@compute'
+    # the 'all' special group is only needed if we don't want all nodes from
+    # this group source included, here we don't want example0 for clush -a
+    all: '@io,@compute'
 
 # Group source racks:
-# define groups @racks:rack[1-4] (ranges work for groups too!) and @racks:all
+# define groups @racks:rack[1-4], @racks:old and @racks:new
 racks:
     rack1: 'example[0,2]'
     rack2: 'example[4-5]'
     rack3: 'example[32-159]'
     rack4: 'example[156-159]'
-    # other groups in same source may be referenced without the "source:" prefix
-    all: '@rack[1-4]'
+    # groups from same source may be referenced without the "source:" prefix
+    # and yes, ranges work for groups too!
+    old: '@rack[1,3]'
+    new: '@rack[2,4]'
 
 # Group source cpu:
 # define groups @cpu:ivy, @cpu:hsw and @cpu:all
 cpu:
     ivy: 'example[32-63]'
     # groups from other sources must be prefixed with "source:"
-    hsw: '@cluster:compute!@ivy'
-    all: '@ivy,@hsw'
+    hsw: '@roles:compute!@ivy'

lib/ClusterShell/NodeSet.py

@@ -890,24 +890,24 @@ def all_nodes(self, namespace=None):
         """Get all nodes from group resolver as a list of strings."""
         # namespace is the optional group source
         assert self.group_resolver is not None
-        all = []
+        alln = []
         try:
             # Ask resolver to provide all nodes.
-            all = self.group_resolver.all_nodes(namespace)
+            alln = self.group_resolver.all_nodes(namespace)
         except NodeUtils.GroupSourceNoUpcall:
             try:
                 # As the resolver is not able to provide all nodes directly,
                 # failback to list + map(s) method:
                 for grp in self.grouplist(namespace):
-                    all += self.group_resolver.group_nodes(grp, namespace)
+                    alln += self.group_resolver.group_nodes(grp, namespace)
             except NodeUtils.GroupSourceNoUpcall:
                 # We are not able to find "all" nodes, definitely.
-                raise NodeSetExternalError("Not enough working external " \
-                    "calls (all, or map + list) defined to get all nodes")
+                msg = "Not enough working methods (all or map + list) to " \
+                      "get all nodes"
+                raise NodeSetExternalError(msg)
         except NodeUtils.GroupSourceQueryFailed, exc:
-            raise NodeSetExternalError("Unable to get all nodes due to the " \
-                "following external failure:\n\t%s" % exc)
-        return all
+            raise NodeSetExternalError("Failed to get all nodes: %s" % exc)
+        return alln
 
     def _next_op(self, pat):
         """Opcode parsing subroutine."""
@@ -1316,8 +1316,12 @@ def _find_groups(self, node, namespace, allgroups):
                     yield grp
         else:
             # find node groups using resolver
-            for group in self._resolver.node_groups(node, namespace):
-                yield group
+            try:
+                for group in self._resolver.node_groups(node, namespace):
+                    yield group
+            except NodeUtils.GroupSourceQueryFailed, exc:
+                msg = "Group source query failed: %s" % exc
+                raise NodeSetExternalError(msg)
 
     def _groups2(self, groupsource=None, autostep=None):
         """Find node groups this nodeset belongs to. [private]"""

lib/ClusterShell/NodeUtils.py

@@ -59,7 +59,7 @@ def __init__(self, message, group_source):
         self.group_source = group_source
 
 class GroupSourceNoUpcall(GroupSourceError):
-    """Raised when upcall is not available"""
+    """Raised when upcall or method is not available"""
 
 class GroupSourceQueryFailed(GroupSourceError):
     """Raised when a query failed (eg. no group found)"""
@@ -111,14 +111,14 @@ def resolv_list(self):
     def resolv_all(self):
         """Return the content of all groups as defined by this GroupSource"""
         if self.allgroups is None:
-            raise GroupSourceQueryFailed("All groups info not available", self)
+            raise GroupSourceNoUpcall("All groups info not available", self)
         return self.allgroups
 
     def resolv_reverse(self, node):
         """
         Return the group name matching the provided node.
         """
-        raise GroupSourceQueryFailed("Not implemented", self)
+        raise GroupSourceNoUpcall("Not implemented", self)
 
 
 class FileGroupSource(GroupSource):

tests/NodeSetGroupTest.py

@@ -191,7 +191,7 @@ def testAllNoResolver(self):
         """test NodeSet.fromall() with no resolver"""
         self.assertRaises(NodeSetExternalError, NodeSet.fromall,
                           resolver=RESOLVER_NOGROUP)
-            
+
     def testGroupsNoResolver(self):
         """test NodeSet.groups() with no resolver"""
         nodeset = NodeSet("foo", resolver=RESOLVER_NOGROUP)
@@ -214,7 +214,7 @@ def testGroupResolverAddSourceError(self):
 
     def testGroupResolverMinimal(self):
         """test NodeSet with minimal GroupResolver"""
-        
+
         test_groups1 = makeTestG1()
 
         source = UpcallGroupSource("minimal",
@@ -230,7 +230,6 @@ def testGroupResolverMinimal(self):
 
         self.assertRaises(NodeSetExternalError, NodeSet.fromall, resolver=res)
 
-    
     def testConfigEmpty(self):
         """test groups with an empty configuration file"""
         f = make_temp_file("")
@@ -408,7 +407,7 @@ def testConfigQueryFailed(self):
 
 [local]
 map: false
-#all:
+all: false
 list: echo foo
 #reverse:
         """)
@@ -417,6 +416,27 @@ def testConfigQueryFailed(self):
         self.assertEqual(str(nodeset), "example[1-100]")
         self.assertRaises(NodeSetExternalError, nodeset.regroup)
 
+        # all_nodes()
+        self.assertRaises(NodeSetExternalError, NodeSet.fromall, resolver=res)
+
+    def testConfigQueryFailedReverse(self):
+        """test groups with config and failed query (reverse)"""
+        f = make_temp_file("""
+# A comment
+
+[Main]
+default: local
+
+[local]
+map: echo example1
+list: echo foo
+reverse: false
+        """)
+        res = GroupResolverConfig(f.name)
+        nodeset = NodeSet("@foo", resolver=res)
+        self.assertEqual(str(nodeset), "example1")
+        self.assertRaises(NodeSetExternalError, nodeset.regroup)
+
     def testConfigRegroupWrongNamespace(self):
         """test groups by calling regroup(wrong_namespace)"""
         f = make_temp_file("""
@@ -1236,8 +1256,8 @@ def test_base_class0(self):
         self.assertEqual(gs.resolv_map('gr1'), '')
         self.assertEqual(gs.resolv_map('gr2'), '')
         self.assertEqual(gs.resolv_list(), [])
-        self.assertRaises(GroupSourceQueryFailed, gs.resolv_all)
-        self.assertRaises(GroupSourceQueryFailed, gs.resolv_reverse, 'n4')
+        self.assertRaises(GroupSourceNoUpcall, gs.resolv_all)
+        self.assertRaises(GroupSourceNoUpcall, gs.resolv_reverse, 'n4')
 
     def test_base_class1(self):
         """test base GroupSource class (map and list)"""
@@ -1246,8 +1266,8 @@ def test_base_class1(self):
         self.assertEqual(gs.resolv_map('gr1'), ['n1', 'n4', 'n3', 'n2'])
         self.assertEqual(gs.resolv_map('gr2'), ['n9', 'n4'])
         self.assertEqual(sorted(gs.resolv_list()), ['gr1', 'gr2'])
-        self.assertRaises(GroupSourceQueryFailed, gs.resolv_all)
-        self.assertRaises(GroupSourceQueryFailed, gs.resolv_reverse, 'n4')
+        self.assertRaises(GroupSourceNoUpcall, gs.resolv_all)
+        self.assertRaises(GroupSourceNoUpcall, gs.resolv_reverse, 'n4')
 
     def test_base_class2(self):
         """test base GroupSource class (all)"""
@@ -1405,6 +1425,12 @@ def test_yaml_basic(self):
 
         # No 'all' defined: all_nodes() should raise an error
         self.assertRaises(GroupSourceError, res.all_nodes)
+        # but then NodeSet falls back to the union of all groups
+        nodeset = NodeSet.fromall(resolver=res)
+        self.assertEqual(str(nodeset), "example[1-100]")
+        # regroup doesn't use @all in that case
+        self.assertEqual(nodeset.regroup(), "@bar,@foo")
+
         # No 'reverse' defined: node_groups() should raise an error
         self.assertRaises(GroupSourceError, res.node_groups, "example1")
 
@@ -1420,6 +1446,27 @@ def test_yaml_basic(self):
         nodeset = NodeSet("example[102-200]", resolver=res)
         self.assertEqual(nodeset.regroup(), "example[102-200]")
 
+    def test_yaml_fromall(self):
+        """test groups special all group"""
+        dname = make_temp_dir()
+        f = make_temp_file("""
+[Main]
+default: yaml
+autodir: %s
+        """ % dname)
+        yamlfile = make_temp_file("""
+yaml:
+    foo: example[1-4,91-100],example90
+    bar: example[5-89]
+    all: example[90-100]
+        """, suffix=".yaml", dir=dname)
+
+        res = GroupResolverConfig(f.name)
+        nodeset = NodeSet.fromall(resolver=res)
+        self.assertEqual(str(nodeset), "example[90-100]")
+        # regroup uses @all if it is defined
+        self.assertEqual(nodeset.regroup(), "@all")
+
     def test_yaml_invalid_groups_not_dict(self):
         """test groups with an invalid YAML config file (1)"""
         dname = make_temp_dir()