Merge pull request #185 from efajardo/SOFTWARE-3936

Adding HTTP(S) tests to the cache (SOFTWARE-3936)

Author: matyasselmeci

osgtest/tests/test_155_stashcache.py

@@ -37,6 +37,7 @@
 PARAMS_CFG_CONTENTS = "\n".join("setenv {0} = {1}\nset {0} = {1}".format(k, v)
                                 for k, v in PARAMS.items()) + "\n"
 
+
 PRE_CFG_PATH = "/etc/xrootd/config.d/11-pre.cfg"
 PRE_CFG_CONTENTS = """
 set DisableOsgMonitoring = 1
@@ -52,6 +53,7 @@
 
     pss.origin localhost:$(OriginAuthXrootPort)
     xrd.protocol http:$(CacheHTTPSPort) libXrdHttp.so
+    setenv XrdSecGSISRVNAMES=*
 else if named stash-cache
     xrd.port $(CacheXrootPort)
     set rootdir = $(CacheRootdir)
@@ -77,10 +79,19 @@
 """
 
 CACHE_AUTHFILE_PATH = PARAMS["StashCacheAuthfile"]
-CACHE_AUTHFILE_CONTENTS = "u * / rl\n"
+# The hash of the vdttest user DN
+# "/DC=org/DC=opensciencegrid/C=US/O=OSG Software/OU=People/CN=vdttest"
+# is b64f6609.0
+user_DN_hash = "b64f6609.0"
+CACHE_AUTHFILE_CONTENTS = "u %s /osgtest/PROTECTED rl\n" % user_DN_hash
+
 
 CACHE_PUBLIC_AUTHFILE_PATH = PARAMS["StashCachePublicAuthfile"]
-CACHE_PUBLIC_AUTHFILE_CONTENTS = "u * / rl\n"
+CACHE_PUBLIC_AUTHFILE_CONTENTS = """
+u * /osgtest/PROTECTED -rl \
+/ rl
+"""
+
 
 ORIGIN_AUTHFILE_PATH = PARAMS["StashOriginAuthfile"]
 ORIGIN_AUTHFILE_CONTENTS = "u * /osgtest/PROTECTED rl\n"

osgtest/tests/test_460_stashcache.py

@@ -27,7 +27,10 @@ class TestStashCache(OSGTestCase):
         for x in range(4)
     ]
 
-    def assertCached(self, name, contents):
+    def assertCached(self, name, contents, auth=False):
+        OriginExport = getcfg("OriginExport")
+        if auth:
+            OriginExport = getcfg("OriginAuthExport")
         fpath = os.path.join(getcfg("CacheRootdir"), getcfg("OriginExport").lstrip("/"), name)
         self.assertTrue(os.path.exists(fpath),
                         name + " not cached")
@@ -47,16 +50,20 @@ def setUp(self):
                                       by_dependency=True)
         if core.rpm_is_installed("xcache"):
             self.skip_ok_if(core.PackageVersion("xcache") < "1.0.2", "needs xcache 1.0.2+")
-        self.skip_bad_unless_running("xrootd@stash-origin", "xrootd@stash-cache")
+        self.skip_bad_unless_running("xrootd@stash-origin", "xrootd@stash-cache", "xrootd@stash-origin-auth",
+                                     "xrootd@stash-cache-auth")
 
     def test_01_create_files(self):
         xrootd_user = pwd.getpwnam("xrootd")
         for name, contents in self.testfiles:
             files.write(os.path.join(getcfg("OriginRootdir"), getcfg("OriginExport").lstrip("/"), name),
                         contents, backup=False, chmod=0o644,
                         chown=(xrootd_user.pw_uid, xrootd_user.pw_gid))
+            files.write(os.path.join(getcfg("OriginRootdir"), getcfg("OriginAuthExport").lstrip("/"), name),
+                        contents, backup=False, chmod=0o644,
+                        chown=(xrootd_user.pw_uid, xrootd_user.pw_gid))
 
-    def test_02_xroot_fetch_from_origin(self):
+    def test_02_xrootd_fetch_from_origin(self):
         name, contents = self.testfiles[0]
         path = os.path.join(getcfg("OriginExport"), name)
         result, _, _ = \
@@ -100,3 +107,53 @@ def test_05_stashcp(self):
             result = tf.read()
         self.assertEqualVerbose(result, contents, "stashcp'ed file mismatch")
         self.assertCached(name, contents)
+
+    def test_06_xrootd_fetch_from_origin_auth(self):
+        core.skip_ok_unless_installed('globus-proxy-utils', by_dependency=True)
+        self.skip_bad_unless(core.state['proxy.valid'], 'requires a proxy cert')
+        name, contents = self.testfiles[0]
+        path = os.path.join(getcfg("OriginAuthExport"), name)
+        dest_file = '/tmp/testfileFromOriginAuth'
+        os.environ["XrdSecGSISRVNAMES"] = "*"
+        result, _, _ = core.check_system(["xrdcp", "-d1", '-f', 
+                                          "root://localhost:%d/%s" % (getcfg("OriginAuthXrootPort"), path),
+                                          dest_file],
+                                         "Checking xrootd copy from authenticated origin", user=True)
+        origin_file = os.path.join(getcfg("OriginRootdir"), getcfg("OriginAuthExport").lstrip("/"), name)
+        checksum_match = files.checksum_files_match(origin_file, dest_file)
+        self.assert_(checksum_match, 'Origin and directly downloaded file have the same contents')
+
+    def test_07_xrootd_fetch_from_auth_cache(self):
+        core.skip_ok_unless_installed('globus-proxy-utils', by_dependency=True)
+        self.skip_bad_unless(core.state['proxy.valid'], 'requires a proxy cert')
+        name, contents = self.testfiles[2]
+        path = os.path.join(getcfg("OriginAuthExport"), name)
+        os.environ["XrdSecGSISRVNAMES"] = "*"
+        dest_file = '/tmp/testfileXrootdFromAuthCache'
+        result, _, _ = \
+            core.check_system(["xrdcp", "-d1","-f",
+                               "root://%s:%d/%s" % (core.get_hostname(),getcfg("CacheHTTPSPort"), path),
+                               dest_file], "Checking xrootd copy from Authenticated cache", user=True)
+        origin_file = os.path.join(getcfg("OriginRootdir"), getcfg("OriginAuthExport").lstrip("/"), name)
+        checksum_match = files.checksum_files_match(origin_file, dest_file)
+        self.assert_(checksum_match, 'Origin and file downloaded via cache have the same contents')
+
+    def test_08_https_fetch_from_auth_cache(self):
+        core.skip_ok_unless_installed('globus-proxy-utils', 'gfal2-plugin-http', 'gfal2-util', 
+                                      'gfal2-plugin-file', by_dependency=True)
+        self.skip_bad_unless(core.state['proxy.valid'], 'requires a proxy cert')
+        name, contents = self.testfiles[3]
+        path = os.path.join(getcfg("OriginAuthExport"), name)
+        dest_file = '/tmp/testfileHTTPsFromAuthCache'
+        uid = pwd.getpwnam(core.options.username)[2]
+        usercert = '/tmp/x509up_u%d' % uid
+        userkey = '/tmp/x509up_u%d' % uid
+        result, _, _ = \
+            core.check_system(["gfal-copy", "-vf",
+                               "--cert", usercert, "--key", userkey,
+                               "https://%s:%d%s" % (core.get_hostname(),getcfg("CacheHTTPSPort"), path),
+                               "file://%s"%dest_file],
+                              "Checking xrootd copy from Authenticated cache", user=True)
+        origin_file = os.path.join(getcfg("OriginRootdir"), getcfg("OriginAuthExport").lstrip("/"), name)
+        checksum_match = files.checksum_files_match(origin_file, dest_file)
+        self.assert_(checksum_match, 'Origin and file downloaded via cache have the same contents')