More tests, including stashcp and xroot copy from cache

Author: matyasselmeci

osgtest/tests/test_155_stashcache.py

@@ -8,9 +8,9 @@
 
 
 CACHE_DIR = "/tmp/sccache"
-CACHE_PORT = 1094  # can't change this - stashcp doesn't allow you to specify port
+CACHE_XROOT_PORT = 1094  # can't change this - stashcp doesn't allow you to specify port
 CACHE_HTTP_PORT = 8001
-ORIGIN_PORT = 1095
+ORIGIN_XROOT_PORT = 1095
 ORIGIN_DIR = "/tmp/scorigin"
 CACHE_AUTHFILE_PATH = "/etc/xrootd/Authfile-cache"
 CACHE_CONFIG_PATH = "/etc/xrootd/xrootd-stashcache-cache-server.cfg"
@@ -33,7 +33,7 @@
 
 ofs.osslib  libXrdPss.so
 # normally this is the redirector but we don't have one in this environment
-pss.origin localhost:{ORIGIN_PORT}
+pss.origin localhost:{ORIGIN_XROOT_PORT}
 pss.cachelib libXrdFileCache.so
 pss.setopt DebugLevel 1
 
@@ -52,7 +52,7 @@
 sec.protbind * none
 xrd.protocol http:{CACHE_HTTP_PORT} libXrdHttp.so
 
-xrd.port {CACHE_PORT}
+xrd.port {CACHE_XROOT_PORT}
 
 http.listingdeny yes
 http.staticpreload http://static/robots.txt /etc/xrootd/stashcache-robots.txt
@@ -87,7 +87,7 @@
 oss.localroot {ORIGIN_DIR}
 all.export /
 
-xrd.port {ORIGIN_PORT}
+xrd.port {ORIGIN_XROOT_PORT}
 all.role server
 
 xrootd.trace emsg login stall redirect
@@ -129,7 +129,8 @@ def test_01_configure(self):
             ("cache_http_port", CACHE_HTTP_PORT),
             ("origin_dir", ORIGIN_DIR),
             ("cache_dir", CACHE_DIR),
-            ("origin_port", ORIGIN_PORT)
+            ("origin_xroot_port", ORIGIN_XROOT_PORT),
+            ("cache_xroot_port", CACHE_XROOT_PORT)
         ]:
             _setcfg(key, val)
 

osgtest/tests/test_460_stashcache.py

@@ -1,5 +1,7 @@
 import os
 import pwd
+import random
+import tempfile
 
 from osgtest.library import core
 from osgtest.library import files
@@ -21,35 +23,66 @@ def _setcfg(key, val):
 
 
 class TestStashCache(OSGTestCase):
-    _text = "this is a test"
+    # testfiles with random contents
+    testfiles = [
+        ("testfile%d" % x, str(random.random()))
+        for x in range(4)
+    ]
+
+    def assertCached(self, name, contents):
+        fpath = os.path.join(_getcfg("cache_dir"), name)
+        self.assertTrue(os.path.exists(fpath),
+                        name + " not cached")
+        self.assertEqual(contents, files.read(fpath, as_single_string=True),
+                         "cached file %s contents do not match expected" % name)
 
     @core.elrelease(7,8)
     def setUp(self):
         core.skip_ok_unless_installed("stashcache-origin-server", "stashcache-cache-server", "stashcache-client")
         self.skip_bad_unless(service.is_running("xrootd@stashcache-origin-server"))
         self.skip_bad_unless(service.is_running("xrootd@stashcache-cache-server"))
 
-    def test_01_create_file(self):
+    def test_01_create_files(self):
         xrootd_user = pwd.getpwnam("xrootd")
-        files.write(os.path.join(_getcfg("origin_dir"), "testfile"),
-                    self._text, backup=False, chmod=0o644,
-                    chown=(xrootd_user.pw_uid, xrootd_user.pw_gid))
+        for name, contents in self.testfiles:
+            files.write(os.path.join(_getcfg("origin_dir"), name),
+                        contents, backup=False, chmod=0o644,
+                        chown=(xrootd_user.pw_uid, xrootd_user.pw_gid))
 
-    def test_02_fetch_from_origin(self):
+    def test_02_xroot_fetch_from_origin(self):
+        name, contents = self.testfiles[0]
         result, _, _ = \
             core.check_system(["xrdcp", "-d1", "-N", "-f",
-                               "root://localhost:%d//testfile" % _getcfg("origin_port"),
+                               "root://localhost:%d//%s" % (_getcfg("origin_xroot_port"), name),
                                "-"], "Checking xroot copy from origin")
-        self.assertEqual(result, self._text, "downloaded file does not match expected")
+        self.assertEqual(result, contents, "downloaded file does not match expected")
 
     def test_03_http_fetch_from_cache(self):
+        name, contents = self.testfiles[1]
         try:
             f = urlopen(
-                "http://localhost:%d/testfile" % _getcfg("cache_http_port")
+                "http://localhost:%d/%s" % (_getcfg("cache_http_port"), name)
             )
             result = f.read()
         except IOError as e:
             self.fail("Unable to download from cache via http: %s" % e)
-        self.assertEqual(result, self._text, "downloaded file does not match expected")
-        self.assertTrue(os.path.exists(os.path.join(_getcfg("cache_dir"), "testfile")),
-                        "testfile not cached")
+        self.assertEqual(result, contents, "downloaded file does not match expected")
+        self.assertCached(name, contents)
+
+    def test_04_xroot_fetch_from_cache(self):
+        name, contents = self.testfiles[2]
+        result, _, _ = \
+            core.check_system(["xrdcp", "-d1", "-N", "-f",
+                               "root://localhost:%d//%s" % (_getcfg("cache_xroot_port"), name),
+                               "-"], "Checking xroot copy from cache")
+        self.assertEqual(result, contents, "downloaded file does not match expected")
+        self.assertCached(name, contents)
+
+    def test_05_stashcp(self):
+        name, contents = self.testfiles[3]
+        with tempfile.NamedTemporaryFile(mode="r+b") as tf:
+            core.check_system(["stashcp", "-d", "/"+name, tf.name],
+                              "Checking stashcp")
+            result = tf.read()
+        self.assertEqual(result, contents, "stashcp'ed file does match expected")
+        self.assertCached(name, contents)