NodeUtils: change cache_delay to cache_time

conf: improve slurm group config example (again): add cache_time and use
%R format instead of %P to filter '*' from partition name also in
reverse upcall

Change-Id: I7c5546361efafcaca2865ec45f119aa533d8b510

Author: thiell

conf/groups.conf.d/slurm.conf.example

@@ -9,7 +9,7 @@
 map: sinfo -h -o "%N" -p $GROUP
 all: sinfo -h -o "%N"
 list: sinfo -h -o "%R"
-reverse: sinfo -h -N -o "%P" -n $NODE
+reverse: sinfo -h -N -o "%R" -n $NODE
 
 # SLURM state bindings
 #
@@ -18,5 +18,4 @@ map: if [ "$GROUP" == "broken" ]; then sinfo -h -o "%N" -t down,drained; else si
 all: sinfo -h -o "%N"
 list: sinfo -h -o "%T"
 reverse: sinfo -h -N -o "%T" -n $NODE
-
-# TODO: add slurm jobs bindings here
+cache_time: 300

doc/extras/vim/syntax/groupsconf.vim

@@ -21,7 +21,7 @@ syn match groupsGroupsDirKey "^\(groupsdir\|confdir\|autodir\)\(:\|=\)"
 " Sources
 syn match groupsVars "\(\$GROUP\|\$NODE\|$SOURCE\)" contained
 syn match groupsKeys "^\w\+\(:\|=\)"me=e-1 contained
-syn match groupsKeyValue "^\(map\|all\|list\|reverse\|cache_delay\)\+\(:\|=\).*$" contains=groupsKeys,groupsVars
+syn match groupsKeyValue "^\(map\|all\|list\|reverse\|cache_time\)\+\(:\|=\).*$" contains=groupsKeys,groupsVars
 
 syn match  groupsComment    "#.*$"
 syn match  groupsComment    ";.*$"

doc/man/man5/groups.conf.5

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH GROUPS.CONF 5 "2015-10-12" "1.7" "ClusterShell User Manual"
+.TH GROUPS.CONF 5 "2015-11-06" "1.7" "ClusterShell User Manual"
 .SH NAME
 groups.conf \- Configuration file for ClusterShell node groups
 .
@@ -131,7 +131,7 @@ the \fIlist\fP and \fImap\fP external calls. Also, if the number of nodes to
 reverse is greater than the number of available groups, the \fIreverse\fP
 external command is avoided automatically.
 .TP
-.B cache_delay
+.B cache_time
 Number of seconds each upcall result is kept in cache, in memory only.
 Default is 3600 seconds. This is useful only for daemons using nodegroups.
 .UNINDENT
@@ -157,7 +157,7 @@ group.  The desired behaviour is up to the system administrator.
 .SH RESOURCE USAGE
 .sp
 All external command results are cached in memory to avoid multiple calls. Each
-result is kept for a limited amount of time. See cache_delay option to tune
+result is kept for a limited amount of time. See cache_time option to tune
 this behaviour.
 .SH EXAMPLES
 .sp

doc/txt/groups.conf.txt

@@ -7,7 +7,7 @@ Configuration file for ClusterShell node groups
 -----------------------------------------------
 
 :Author: Stephane Thiell, <[email protected]>
-:Date:   2015-10-12
+:Date:   2015-11-06
 :Copyright: CeCILL-C V1
 :Version: 1.7
 :Manual section: 5
@@ -102,7 +102,7 @@ reverse
   the *list* and *map* external calls. Also, if the number of nodes to
   reverse is greater than the number of available groups, the *reverse*
   external command is avoided automatically.
-cache_delay
+cache_time
   Number of seconds each upcall result is kept in cache, in memory only.
   Default is 3600 seconds. This is useful only for daemons using nodegroups.
 
@@ -127,7 +127,7 @@ RESOURCE USAGE
 ==============
 
 All external command results are cached in memory to avoid multiple calls. Each
-result is kept for a limited amount of time. See cache_delay option to tune
+result is kept for a limited amount of time. See cache_time option to tune
 this behaviour.
 
 

lib/ClusterShell/NodeUtils.py

@@ -1,6 +1,6 @@
-# Copyright CEA/DAM/DIF (2010, 2012, 2013, 2014)
+# Copyright CEA/DAM/DIF (2010-2015)
 #  Contributors:
-#   Stephane THIELL <[email protected]>
+#   Stephane THIELL <[email protected]>
 #   Aurelien DEGREMONT <[email protected]>
 #
 # This file is part of the ClusterShell library.
@@ -77,7 +77,7 @@ class GroupResolverConfigError(GroupResolverError):
     """Raised when a configuration error is encountered"""
 
 
-_DEFAULT_CACHE_DELAY = 3600
+_DEFAULT_CACHE_TIME = 3600
 
 
 class GroupSource(object):
@@ -153,12 +153,12 @@ class UpcallGroupSource(GroupSource):
     GroupSource class managing external calls for nodegroup support.
 
     Upcall results are cached for a customizable amount of time. This is
-    controlled by `cache_delay` attribute. Default is 3600 seconds.
+    controlled by `cache_time` attribute. Default is 3600 seconds.
     """
 
     def __init__(self, name, map_upcall, all_upcall=None,
                  list_upcall=None, reverse_upcall=None, cfgdir=None,
-                 cache_delay=None):
+                 cache_time=None):
         GroupSource.__init__(self, name)
         self.verbosity = 0
         self.cfgdir = cfgdir
@@ -175,10 +175,10 @@ def __init__(self, name, map_upcall, all_upcall=None,
             self.has_reverse = True
 
         # Cache upcall data
-        if cache_delay is None:
-            self.cache_delay = _DEFAULT_CACHE_DELAY
+        if cache_time is None:
+            self.cache_time = _DEFAULT_CACHE_TIME
         else:
-            self.cache_delay = cache_delay
+            self.cache_time = cache_time
         self._cache = {}
         self.clear_cache()
 
@@ -219,7 +219,7 @@ def _upcall_cache(self, upcall, cache, key, **args):
         corresponding `upcall'.
 
         If `key' is missing, it is added to provided `cache'. Each entry in a
-        cache is kept only for a limited time equal to self.cache_delay .
+        cache is kept only for a limited time equal to self.cache_time .
         """
         if not self.upcalls.get(upcall):
             raise GroupSourceNoUpcall(upcall, self)
@@ -231,7 +231,7 @@ def _upcall_cache(self, upcall, cache, key, **args):
 
         # Fetch the data if unknown of just purged
         if key not in cache:
-            cache_expiry = time.time() + self.cache_delay
+            cache_expiry = time.time() + self.cache_time
             # $CFGDIR and $SOURCE always replaced
             args['CFGDIR'] = self.cfgdir
             args['SOURCE'] = self.name
@@ -277,20 +277,20 @@ class YAMLGroupLoader(object):
 
     - create GroupSource objects
     - gather groups dict content on load
-    - reload the file once cache_delay has expired
+    - reload the file once cache_time has expired
     """
 
-    def __init__(self, filename, cache_delay=None):
+    def __init__(self, filename, cache_time=None):
         """
         Initialize YAMLGroupLoader and load file.
 
         :param filename: YAML file path
-        :param cache_delay: cache delay
+        :param cache_time: cache time (seconds)
         """
-        if cache_delay is None:
-            self.cache_delay = _DEFAULT_CACHE_DELAY
+        if cache_time is None:
+            self.cache_time = _DEFAULT_CACHE_TIME
         else:
-            self.cache_delay = cache_delay
+            self.cache_time = cache_time
         self.cache_expiry = 0
         self.filename = filename
         self.sources = {}
@@ -335,7 +335,7 @@ def _load(self):
             # else: cannot add new source on reload - just ignore it
 
         # groups are loaded, set cache expiry
-        self.cache_expiry = time.time() + self.cache_delay
+        self.cache_expiry = time.time() + self.cache_time
 
     def __iter__(self):
         """Iterate over GroupSource objects."""
@@ -347,10 +347,10 @@ def groups(self, sourcename):
         Groups dict accessor for sourcename.
 
         This method is called by associated FileGroupSource objects and simply
-        returns dict content, after reloading file if cache_delay has expired.
+        returns dict content, after reloading file if cache_time has expired.
         """
         if self.cache_expiry < time.time():
-            # reload whole file if cache delay expired
+            # reload whole file if cache time expired
             self._load()
 
         return self._groups[sourcename]
@@ -607,21 +607,21 @@ def _sources_from_cfg(self, cfg, cfgdir):
                     if srcname != self.SECTION_MAIN:
                         # only map is a mandatory upcall
                         map_upcall = cfg.get(section, 'map', True)
-                        all_upcall = list_upcall = reverse_upcall = delay = None
+                        all_upcall = list_upcall = reverse_upcall = ctime = None
                         if cfg.has_option(section, 'all'):
                             all_upcall = cfg.get(section, 'all', True)
                         if cfg.has_option(section, 'list'):
                             list_upcall = cfg.get(section, 'list', True)
                         if cfg.has_option(section, 'reverse'):
                             reverse_upcall = cfg.get(section, 'reverse', True)
-                        if cfg.has_option(section, 'cache_delay'):
-                            delay = float(cfg.get(section, 'cache_delay', True))
+                        if cfg.has_option(section, 'cache_time'):
+                            ctime = float(cfg.get(section, 'cache_time', True))
                         # add new group source
                         self.add_source(UpcallGroupSource(srcname, map_upcall,
                                                           all_upcall,
                                                           list_upcall,
                                                           reverse_upcall,
-                                                          cfgdir, delay))
+                                                          cfgdir, ctime))
         except (NoSectionError, NoOptionError, ValueError), exc:
             raise GroupResolverConfigError(str(exc))
 

tests/NodeSetGroupTest.py

@@ -1201,29 +1201,29 @@ def test_expired_cache(self):
         """test UpcallGroupSource cache entries expired according to config"""
         # create custom resolver with default source
         source = StaticGroupSource('cache', {'map': {'a': 'foo1', 'b': 'foo2'} })
-        source.cache_delay = 0.2
+        source.cache_time = 0.2
         res = GroupResolver(source)
 
         # Populate map cache
         self.assertEqual("foo1", str(NodeSet("@a", resolver=res)))
         self.assertEqual("foo2", str(NodeSet("@b", resolver=res)))
         self.assertEqual(len(source._cache['map']), 2)
 
-        # Be sure 0.2 cache delay is expired (especially for old Python version)
+        # Be sure 0.2 cache time is expired (especially for old Python version)
         time.sleep(0.25)
 
         source._data['map']['a'] = 'something_else'
         self.assertEqual('something_else', str(NodeSet("@a", resolver=res)))
 
-    def test_config_cache_delay(self):
-        """test group config cache_delay options"""
+    def test_config_cache_time(self):
+        """test group config cache_time options"""
         f = make_temp_file("""
 [local]
-cache_delay: 0.2
+cache_time: 0.2
 map: echo foo1
         """)
         res = GroupResolverConfig(f.name)
-        self.assertEqual(res._sources['local'].cache_delay, 0.2)
+        self.assertEqual(res._sources['local'].cache_time, 0.2)
         self.assertEqual("foo1", str(NodeSet("@local:foo", resolver=res)))
 
 
@@ -1303,21 +1303,21 @@ def test_multi_sources(self):
                          { 'cherry': 'client-4-2' })
 
     def test_reload(self):
-        """test YAMLGroupLoader cache_delay"""
+        """test YAMLGroupLoader cache_time"""
         f = make_temp_file("""
 vendors:
     apricot: "node[1-10]"
     avocado: 'node[11-20]'
     banana: node[21-30]
 customers:
     cherry: client-4-2""")
-        loader = YAMLGroupLoader(f.name, cache_delay=1)
+        loader = YAMLGroupLoader(f.name, cache_time=1)
         self.assertEqual(loader.groups("vendors"),
                          { 'apricot': 'node[1-10]',
                            'avocado': 'node[11-20]',
                            'banana': 'node[21-30]' })
 
-        # modify YAML file and check that it is reloaded after cache_delay
+        # modify YAML file and check that it is reloaded after cache_time
         f.write("\n    nut: node42\n")
         # oh and BTW for ultimate code coverage, test if we add a new source
         # on-the-fly, this is not supported but should be ignored
@@ -1350,7 +1350,7 @@ def test_iter(self):
 src3:
     src3grp1: node31
     src3grp2: node32""")
-        loader = YAMLGroupLoader(f.name, cache_delay = 0.1)
+        loader = YAMLGroupLoader(f.name, cache_time = 0.1)
         # iterate sources with cache expired
         for source in loader:
             time.sleep(0.5) # force reload