Merge pull request #147 from matyasselmeci/wip/sw3360-stashcache

StashCache tests (SOFTWARE-3360)

Author: matyasselmeci

files/test_sequence

@@ -9,6 +9,7 @@ test_110_condor_cron
 test_130_gridftp
 test_140_lcmaps
 test_150_xrootd
+test_155_stashcache
 test_160_rsv
 test_170_pbs
 test_180_cvmfs
@@ -32,6 +33,7 @@ test_420_gridftp
 test_430_uberftp
 test_440_glexec
 test_450_xrootd
+test_460_stashcache
 test_470_rsv
 test_490_jobs
 test_510_edgmkgridmap
@@ -50,6 +52,7 @@ test_790_condorce
 test_800_gratia
 test_820_cvmfs
 test_830_rsv
+test_835_stashcache
 test_840_xrootd
 test_850_lcmaps
 test_860_gridftp

osg-test

@@ -229,7 +229,7 @@ if __name__ == '__main__':
 
     # Check for osg-ca-generator, which may have been forgotten in a source install
     try:
-        __import__('cagen')
+        import cagen
     except ImportError:
         sys.exit("Cannot find 'cagen' library. Please install osg-ca-generator")
 
@@ -254,5 +254,6 @@ if __name__ == '__main__':
         signal.signal(signal.SIGALRM, signal.SIG_DFL)
     else:
         print('No tests to run.')
+        EXIT_CODE = 1
     wrap_up()
     sys.exit(EXIT_CODE)

osgtest/library/core.py

@@ -416,13 +416,17 @@ def version_compare(evr1, evr2):
     as a 3-element tuple or list.
 
     """
-    if isinstance(evr1, basestring):
+    if is_string(evr1):
         epoch1, version1, release1 = stringToVersion(evr1)
+    elif isinstance(evr1, bytes):
+        epoch1, version1, release1 = stringToVersion(evr1.decode())
     else:
         epoch1, version1, release1 = evr1
 
-    if isinstance(evr2, basestring):
+    if is_string(evr2):
         epoch2, version2, release2 = stringToVersion(evr2)
+    elif isinstance(evr2, bytes):
+        epoch2, version2, release2 = stringToVersion(evr2.decode())
     else:
         epoch2, version2, release2 = evr2
 
@@ -755,3 +759,12 @@ def run_fn_if_el_release_ok(*args, **kwargs):
         return run_fn_if_el_release_ok
     return el_release_decorator
 
+
+try:
+    unicode
+except NameError:  # python 3
+    unicode = str
+
+
+def is_string(var):
+    return isinstance(var, (str, unicode))

osgtest/library/files.py

@@ -237,3 +237,12 @@ def filesBackedup(path, owner):
         return True
     else:
         return False
+
+
+def safe_makedirs(directory, mode=0o777):
+    """Create a directory and all its parent directories, unless it already
+    exists.
+
+    """
+    if not os.path.isdir(directory):
+        os.makedirs(directory, mode)

osgtest/library/osgunittest.py

@@ -12,6 +12,19 @@
 import unittest
 import time
 
+
+# Copied from unittest.util, Python 3.6
+_MAX_LENGTH = 80
+def safe_repr(obj, short=False):
+    try:
+        result = repr(obj)
+    except Exception:
+        result = object.__repr__(obj)
+    if not short or len(result) < _MAX_LENGTH:
+        return result
+    return result[:_MAX_LENGTH] + ' [truncated]...'
+
+
 # Define the classes we need to handle the two new types of test results: ok
 # skip, and bad skip.
 
@@ -111,6 +124,14 @@ def failIfSubsetOf(self, a, b, message=None):
         if set(a).issubset(set(b)):
             raise AssertionError(message)
 
+    def assertEqualVerbose(self, actual, expected, message=None):
+        aftermessage = "actual %s != expected %s" % (safe_repr(actual), safe_repr(expected))
+        if message:
+            fullmessage = "%s (%s)" % (message, aftermessage)
+        else:
+            fullmessage = aftermessage
+        self.assertEqual(actual, expected, fullmessage)
+
     # This is mostly a copy of the method from unittest in python 2.4.
     # There is some code here to test if the 'result' object accepts 'skips',
     # since the original TestResult object does not. If it does not, an

osgtest/tests/test_150_xrootd.py

@@ -5,6 +5,9 @@
 import osgtest.library.service as service
 import osgtest.library.osgunittest as osgunittest
 
+
+XROOTD_PORT = 1096  # chosen so it doesn't conflict w/ the stashcache instances
+
 XROOTD_CFG_TEXT = """\
 cms.space min 2g 5g
 xrootd.seclib /usr/lib64/libXrdSec-4.so
@@ -17,6 +20,7 @@
     %s
 acc.authdb /etc/xrootd/auth_file
 ofs.authorize
+xrd.port %d
 """
 
 AUTHFILE_TEXT = """\
@@ -32,6 +36,7 @@ def test_01_start_xrootd(self):
         core.config['certs.xrootdcert'] = '/etc/grid-security/xrd/xrdcert.pem'
         core.config['certs.xrootdkey'] = '/etc/grid-security/xrd/xrdkey.pem'
         core.config['xrootd.config'] = '/etc/xrootd/xrootd-clustered.cfg'
+        core.config['xrootd.port'] = XROOTD_PORT
         core.config['xrootd.gsi'] = "ON"
         core.state['xrootd.started-server'] = False
         core.state['xrootd.backups-exist'] = False
@@ -58,7 +63,9 @@ def test_01_start_xrootd(self):
                             owner="xrootd",
                             chown=(user.pw_uid, user.pw_gid))
 
-            files.append(core.config['xrootd.config'], XROOTD_CFG_TEXT % sec_protocol, owner='xrootd', backup=True)
+            files.append(core.config['xrootd.config'],
+                         XROOTD_CFG_TEXT % (sec_protocol, core.config['xrootd.port']),
+                         owner='xrootd', backup=True)
             authfile = '/etc/xrootd/auth_file'
             files.write(authfile, AUTHFILE_TEXT, owner="xrootd", chown=(user.pw_uid, user.pw_gid))
 

osgtest/tests/test_155_stashcache.py

@@ -0,0 +1,157 @@
+import os
+import pwd
+
+from osgtest.library import core
+from osgtest.library import files
+from osgtest.library.osgunittest import OSGTestCase
+from osgtest.library import service
+
+
+CACHE_DIR = "/tmp/sccache"
+CACHE_XROOT_PORT = 1094  # can't change this - stashcp doesn't allow you to specify port
+CACHE_HTTP_PORT = 8001
+ORIGIN_XROOT_PORT = 1095
+ORIGIN_DIR = "/tmp/scorigin"
+CACHE_AUTHFILE_PATH = "/etc/xrootd/Authfile-cache"
+CACHE_CONFIG_PATH = "/etc/xrootd/xrootd-stashcache-cache-server.cfg"
+ORIGIN_CONFIG_PATH = "/etc/xrootd/xrootd-stashcache-origin-server.cfg"
+CACHES_JSON_PATH = "/etc/stashcache/caches.json"
+
+
+# TODO Set up authenticated stashcache as well
+CACHE_CONFIG_TEXT = """\
+all.export  /
+set cachedir = {CACHE_DIR}
+xrd.allow host *
+sec.protocol  host
+all.adminpath /var/spool/xrootd
+
+xrootd.trace emsg login stall redirect
+ofs.trace all
+xrd.trace all
+cms.trace all
+
+ofs.osslib  libXrdPss.so
+# normally this is the redirector but we don't have one in this environment
+pss.origin localhost:{ORIGIN_XROOT_PORT}
+pss.cachelib libXrdFileCache.so
+pss.setopt DebugLevel 1
+
+oss.localroot $(cachedir)
+
+pfc.blocksize 512k
+pfc.ram       1024m
+# ^ xrootd won't start without a gig
+pfc.prefetch  10
+pfc.diskusage 0.90 0.95
+
+ofs.authorize 1
+acc.audit deny grant
+
+acc.authdb {CACHE_AUTHFILE_PATH}
+sec.protbind * none
+xrd.protocol http:{CACHE_HTTP_PORT} libXrdHttp.so
+
+xrd.port {CACHE_XROOT_PORT}
+
+http.listingdeny yes
+http.staticpreload http://static/robots.txt /etc/xrootd/stashcache-robots.txt
+
+# Tune the client timeouts to more aggressively timeout.
+pss.setopt ParallelEvtLoop 10
+pss.setopt RequestTimeout 25
+#pss.setopt TimeoutResolution 1
+pss.setopt ConnectTimeout 25
+pss.setopt ConnectionRetry 2
+#pss.setopt StreamTimeout 35
+
+all.sitename osgtest
+
+xrootd.diglib * /etc/xrootd/digauth.cf
+""".format(**globals())
+
+
+CACHE_AUTHFILE_TEXT = """\
+u * / rl
+"""
+
+
+ORIGIN_CONFIG_TEXT = """\
+xrd.allow host *
+sec.protocol  host
+sec.protbind  * none
+all.adminpath /var/spool/xrootd
+all.pidpath /var/run/xrootd
+
+# The directory on local disk containing the files to share, e.g. "/stash".
+oss.localroot {ORIGIN_DIR}
+all.export /
+
+xrd.port {ORIGIN_XROOT_PORT}
+all.role server
+
+xrootd.trace emsg login stall redirect
+ofs.trace all
+xrd.trace all
+cms.trace all
+""".format(**globals())
+
+
+CACHES_JSON_TEXT = """\
+[
+{"name":"root://localhost", "status":1, "longitude":-89.4012, "latitude":43.0731}
+]
+"""
+
+
+_NAMESPACE = "stashcache"
+
+
+def _getcfg(key):
+    return core.config["%s.%s" % (_NAMESPACE, key)]
+
+
+def _setcfg(key, val):
+    core.config["%s.%s" % (_NAMESPACE, key)] = val
+
+
+class TestStartStashCache(OSGTestCase):
+    @core.elrelease(7,8)
+    def setUp(self):
+        core.skip_ok_unless_installed("stashcache-origin-server", "stashcache-cache-server", "stashcache-client")
+
+    def test_01_configure(self):
+        for key, val in [
+            ("cache_authfile_path", CACHE_AUTHFILE_PATH),
+            ("cache_config_path", CACHE_CONFIG_PATH),
+            ("origin_config_path", ORIGIN_CONFIG_PATH),
+            ("caches_json_path", CACHES_JSON_PATH),
+            ("cache_http_port", CACHE_HTTP_PORT),
+            ("origin_dir", ORIGIN_DIR),
+            ("cache_dir", CACHE_DIR),
+            ("origin_xroot_port", ORIGIN_XROOT_PORT),
+            ("cache_xroot_port", CACHE_XROOT_PORT)
+        ]:
+            _setcfg(key, val)
+
+        xrootd_user = pwd.getpwnam("xrootd")
+        for d in [_getcfg("origin_dir"), _getcfg("cache_dir"),
+                  os.path.dirname(_getcfg("caches_json_path"))]:
+            files.safe_makedirs(d)
+            os.chown(d, xrootd_user.pw_uid, xrootd_user.pw_gid)
+
+        for key, text in [
+            ("cache_config_path", CACHE_CONFIG_TEXT),
+            ("cache_authfile_path", CACHE_AUTHFILE_TEXT),
+            ("origin_config_path", ORIGIN_CONFIG_TEXT),
+            ("caches_json_path", CACHES_JSON_TEXT)
+        ]:
+            files.write(_getcfg(key), text, owner=_NAMESPACE, chmod=0o644)
+
+    def test_02_start_origin(self):
+        if not service.is_running("xrootd@stashcache-origin-server"):
+            service.check_start("xrootd@stashcache-origin-server")
+
+    def test_03_start_cache(self):
+        if not service.is_running("xrootd@stashcache-cache-server"):
+            service.check_start("xrootd@stashcache-cache-server")

osgtest/tests/test_450_xrootd.py

@@ -30,7 +30,7 @@ def test_01_xrdcp_local_to_server(self):
         else:
             temp_dir = tempfile.mkdtemp()
         os.chmod(temp_dir, 0o777)
-        xrootd_url = 'root://%s/%s/copied_file.txt' % (hostname, temp_dir)
+        xrootd_url = 'root://%s:%d/%s/copied_file.txt' % (hostname, core.config['xrootd.port'], temp_dir)
         command = ('xrdcp', '--debug', '3', TestXrootd.__data_path, xrootd_url)
 
         status, stdout, stderr = core.system(command, user=True)
@@ -57,7 +57,7 @@ def test_02_xrdcp_server_to_local(self):
         f = open(temp_source_dir + "/copied_file.txt", "w")
         f.write("This is some test data for an xrootd test.")
         f.close()
-        xrootd_url = 'root://%s/%s/copied_file.txt' % (hostname, temp_source_dir)
+        xrootd_url = 'root://%s:%d/%s/copied_file.txt' % (hostname, core.config['xrootd.port'], temp_source_dir)
         local_path = temp_target_dir + '/copied_file.txt'
         command = ('xrdcp', '--debug', '3', xrootd_url, local_path)