parent b6a7c6e

author Andy Georges <[email protected]> 1674744933 +0100
committer Andy Georges <[email protected]> 1677589584 +0100
gpgsig -----BEGIN PGP SIGNATURE-----

 iQIzBAABCAAdFiEEsRL1Gh5rbMUbS7S8e6TNo671utAFAmP9/FAACgkQe6TNo671
 utCySA//ZFeuoh5ZG3son8CXs0jZvfQBnxlnP0M3e4HbGi/KBINO95TB8TLjIbQ/
 ypvzEiHDeUIUsJn69ef645DGvNkop9H+1IepTh73WRNw/z13KBPjV/X7XoMUk4iZ
 7scmeHHbA6FH6KP4hgI5XCslGQKn6obucaCZgDE8e1bZldD8NuJxgQXapJvK0gQ/
 xJwarkZQ+4OTKxxRtCyWac33Ftx2z6mCbtdERae5KRuq+8oUOI1SG1q9kp+MGITa
 6L5GEED/7fsurfLEo0GRD9dHxlHd7RMO6GziCaIjcDeipFDP2bFM7x5uM0B9Nw2g
 dxO3gz7WG8YZF9vKtYuN1Sas1yGTLYBB5TjcI6lXhyCtQy2NVAeeAjbjCqr9dbzV
 n03oA9PdTojGq5dhXPwrkR17fXyoxiND7JI9ibdxL/yaGCZPJN0NgCHyZl5CnhOC
 o7qR3nhVZoJ9HalVzZnbjJcYjECWBYKYuRZoicclf56gvp6QvC/bDESQBBpe8SbF
 qIqc7SLQ+gddE8ajFRwiBeWU3UX9t/XPWpEvixvI4dXaZIpFydQGLsYQ3/9OehPv
 z0IzWp648H5hXfmYk3zpe25QfjGDj23OzwqHlNkr7KdReXck/2HhhDHL2Fw/h4jB
 CpYOlt05wLZlPnLt35sKomirx6yIOFU5rfkUdTU/0Ris90ZQWtI=
 =pjru
 -----END PGP SIGNATURE-----

test: fixes and refactoring to work with DiskCache

Author: itkovian

lib/vsc/utils/cache.py

@@ -29,12 +29,10 @@
 @author: Andy Georges (Ghent University)
 """
 import diskcache as dc
-import logging
-import shutil
 import time
 
 
-class FileCache(object):
+class FileCache(dc.Cache):
     """File cache with a timestamp safety.
 
     Wrapper around diskcache to retain the old API until all usage can be replaced.
@@ -67,17 +65,12 @@ def __init__(self, filename, retain_old=True, raise_unpickable=False):
         """
         del raise_unpickable
 
-        self.retain_old = retain_old  # this is no longer used
+        super().__init__(filename)
 
-        self.filename = filename
-        try:
-            self.cache = dc.Cache(filename)
-        except:
-            shutil.rmtree(filename)
-            self.cache = dc.Cache(filename)
+        self.retain_old = retain_old  # this is no longer used
 
         if not retain_old:
-            self.cache.clear()
+            self.clear()
 
     def update(self, key, data, threshold=None):
         """Update the given data if the existing data is older than the given threshold.
@@ -90,11 +83,13 @@ def update(self, key, data, threshold=None):
         @param data: whatever needs to be stored
         @param threshold: time in seconds
         """
-        old, old_timestamp = self.cache.get(key, default=(None, None))
         now = time.time()
+        stored = self.set(key=key, value=(now, data), expire=threshold)
 
-        if not old or now - old_timestamp > threshold:
-            self.cache[key] = (data, now)
+        if stored:
+            return (now, data)
+        else:
+            return (None, None)
 
     def load(self, key):
         """Load the stored data for the given key along with the timestamp it was stored.
@@ -103,7 +98,7 @@ def load(self, key):
 
         @returns: (timestamp, data) if there is data for the given key, None otherwise.
         """
-        return self.cache.get(key, default=None)
+        return self.get(key, default=(None, None))
 
     @DeprecationWarning
     def retain(self):
@@ -115,8 +110,3 @@ def discard(self):
         """Discard non-updated data on close."""
         self.retain_old = False
 
-    def close(self):
-        """Close the cache."""
-        self.cache.close()
-
-        logging.info('closing the file cache at %s', self.filename)

lib/vsc/utils/nagios.py

@@ -260,24 +260,26 @@ def __init__(self, header, filename, threshold, nagios_username="nagios", world_
         self.nagios_username = nagios_username
 
     def report_and_exit(self):
-        """Unzips the cache file and reads the JSON data back in, prints the data and exits accordingly.
+        """Reads the cache, prints the data and exits accordingly.
 
         If the cache data is too old (now - cache timestamp > self.threshold), a critical exit is produced.
         """
         try:
-            nagios_cache = FileCache(self.filename, True)
+            nagios_cache = FileCache(self.filename)
         except (IOError, OSError):
             logging.critical("Error opening file %s for reading", self.filename)
-            unknown_exit("%s nagios gzipped JSON file unavailable (%s)" % (self.header, self.filename))
+            unknown_exit("%s nagios cache unavailable (%s)" % (self.header, self.filename))
 
-        (timestamp, ((nagios_exit_code, nagios_exit_string), nagios_message)) = nagios_cache.load('nagios')
+        (_, nagios_exit_info) = nagios_cache.load('nagios')
+
+        if nagios_exit_info is None:
+            unknown_exit("%s nagios exit info expired" % self.header)
+
+        ((nagios_exit_code, nagios_exit_string), nagios_message) = nagios_exit_info
+
+        print("%s %s" % (nagios_exit_string, nagios_message))
+        sys.exit(nagios_exit_code)
 
-        if self.threshold <= 0 or time.time() - timestamp < self.threshold:
-            logging.info("Nagios check cache file %s contents delivered: %s", self.filename, nagios_message)
-            print("%s %s" % (nagios_exit_string, nagios_message))
-            sys.exit(nagios_exit_code)
-        else:
-            unknown_exit("%s gzipped JSON file too old (timestamp = %s)" % (self.header, time.ctime(timestamp)))
 
     def cache(self, nagios_exit, nagios_message):
         """Store the result in the cache file with a timestamp.
@@ -290,29 +292,37 @@ def cache(self, nagios_exit, nagios_message):
         """
         try:
             nagios_cache = FileCache(self.filename)
-            nagios_cache.update('nagios', (nagios_exit, nagios_message), 0)  # always update
+            nagios_cache.update('nagios', (nagios_exit, nagios_message), threshold=self.threshold)
             nagios_cache.close()
             logging.info("Wrote nagios check cache file %s at about %s", self.filename, time.ctime(time.time()))
         except (IOError, OSError):
             # raising an error is ok, since we usually do this as the very last thing in the script
-            logging.error("Cannot save to the nagios gzipped JSON file (%s)" % self.filename)
+            logging.error("Cannot save to the nagios cache (%s)", self.filename)
             raise Exception()
 
         try:
             p = pwd.getpwnam(self.nagios_username)
             if self.world_readable:
-                os.chmod(self.filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)
+                os.chmod(
+                    self.filename,
+                    stat.S_IRUSR | stat.S_IWUSR |
+                    stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH |
+                    stat.S_IXUSR | stat.S_IXGRP
+                )
             else:
-                os.chmod(self.filename, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP)
+                os.chmod(
+                    self.filename,
+                    stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXUSR | stat.S_IXGRP
+                )
 
             # only change owner/group when run as root
             if os.geteuid() == 0:
                 os.chown(self.filename, p.pw_uid, p.pw_gid)
             else:
                 logging.warning("Not running as root: Cannot chown the nagios check file %s to %s",
                               self.filename, self.nagios_username)
-        except (OSError, FileNotFoundErrorExc):
-            logging.error("Cannot chown the nagios check file %s to the nagios user" % (self.filename))
+        except (OSError, FileNotFoundError):
+            logging.error("Cannot chown the nagios check file %s to the nagios user", self.filename)
             raise Exception()
 
         return True
@@ -439,7 +449,7 @@ def __init__(self, **kwargs):
         self._final = None
         self._final_state = None
 
-        self._threshold = 0
+        self._threshold = None
         self._report_and_exit = False
 
         self._world_readable = False

lib/vsc/utils/script_tools.py

@@ -37,7 +37,6 @@
 from copy import deepcopy
 
 import logging
-from vsc.utils import fancylogger
 from vsc.utils.availability import proceed_on_ha_service
 from vsc.utils.generaloption import SimpleOption
 from vsc.utils.lock import lock_or_bork, release_or_bork, LOCKFILE_DIR, LOCKFILE_FILENAME_TEMPLATE
@@ -138,8 +137,6 @@ def __init__(self, options, run_prologue=True, excepthook=None, **kwargs):
         else:
             sys.excepthook = excepthook
 
-        self.log = fancylogger.getLogger()
-
     def prologue(self):
         """Checks the options given for settings and takes appropriate action.
 
@@ -160,7 +157,7 @@ def prologue(self):
 
         # check for HA host
         if self.options.ha and not proceed_on_ha_service(self.options.ha):
-            self.log.warning("Not running on the target host %s in the HA setup. Stopping.", self.options.ha)
+            logging.warning("Not running on the target host %s in the HA setup. Stopping.", self.options.ha)
             self.nagios_reporter.ok("Not running on the HA master.")
             sys.exit(NAGIOS_EXIT_OK)
 
@@ -169,7 +166,7 @@ def prologue(self):
                                                    threshold=self.options.nagios_check_interval_threshold * 2)
             lock_or_bork(self.lockfile, self.nagios_reporter)
 
-        self.log.info("%s has started", _script_name(sys.argv[0]))
+        logging.info("%s has started", _script_name(sys.argv[0]))
 
     def _epilogue(self):
         if not self.options.disable_locking and not self.options.dry_run:
@@ -184,7 +181,7 @@ def epilogue(self, nagios_message, nagios_thresholds=None):
 
         nagios_thresholds['message'] = nagios_message
         self.nagios_reporter._eval_and_exit(**nagios_thresholds)
-        self.log.info("%s has finished", _script_name(sys.argv[0]))  # may not be reached
+        logging.info("%s has finished", _script_name(sys.argv[0]))  # may not be reached
 
     def ok(self, nagios_message):
         """Run at the end of a script and force an OK exit."""
@@ -221,8 +218,8 @@ def critical_exception_handler(self, tp, value, traceback):
 
         This function is meant to be used as sys.excepthook
         """
-        self.log.exception("unhandled exception detected: %s - %s", tp, value)
-        self.log.debug("traceback %s", traceback)
+        logging.exception("unhandled exception detected: %s - %s", tp, value)
+        logging.debug("traceback %s", traceback)
         message = "Script failure: %s - %s" % (tp, value)
         self.critical(message)
 

test/cache.py

@@ -38,6 +38,7 @@
 import sys
 import random
 
+from pathlib import PurePath
 from vsc.install.testing import TestCase
 from vsc.utils.cache import FileCache
 
@@ -61,9 +62,7 @@ def test_contents(self):
         data, threshold = get_rand_data()
 
         # create a tempfilename
-        (handle, filename) = tempfile.mkstemp(dir='/tmp')
-        os.unlink(filename)
-        os.close(handle)
+        filename = PurePath("/tmp") / next(tempfile._get_candidate_names())
         cache = FileCache(filename)
         for (key, value) in data.items():
             cache.update(key, value, threshold)
@@ -72,45 +71,31 @@ def test_contents(self):
         for key in data.keys():
             info = cache.load(key)
             self.assertFalse(info is None)
-            (ts, value) = info
+            ts, value = info
             self.assertTrue(value == data[key])
             self.assertTrue(ts <= now)
 
     def test_save_and_load(self):
         """Check if the loaded data is the same as the saved data."""
         # test with random data
         data, threshold = get_rand_data()
-        tempdir = tempfile.mkdtemp()
-        # create a tempfilename
-        (handle, filename) = tempfile.mkstemp(dir=tempdir)
-        os.close(handle)
-        shutil.rmtree(tempdir)
+
+        filename = PurePath("/tmp") / next(tempfile._get_candidate_names())
         cache = FileCache(filename)
+
         for (key, value) in data.items():
             cache.update(key, value, threshold)
         cache.close()
 
         now = time.time()
         new_cache = FileCache(filename)
         for key in data.keys():
-            info = cache.load(key)
+            info = new_cache.load(key)
             self.assertTrue(info is not None)
             (ts, value) = info
             self.assertTrue(value == data[key])
             self.assertTrue(ts <= now)
         new_cache.close()
 
-        shutil.rmtree(tempdir)
-
-    def test_corrupt_cache(self):
-        """Test to see if we can handle a corrupt cache file"""
-        tempdir = tempfile.mkdtemp()
-
-        # create a bollocks cache file
-        with open(os.path.join(tempdir, "cache.db", "w")) as f:
-            f.write("blabla")
-
-        # this should clear the cache and create a new one
-        FileCache(tempdir)
+        shutil.rmtree(filename)
 
-        shutil.rmtree(tempdir)