Merge pull request #155 from efajardo/sw3362

Xrootd Third Party Transfers SOFTWARE-3362

Author: brianhlin

files/test_sequence

@@ -10,6 +10,7 @@ test_130_gridftp
 test_140_lcmaps
 test_150_xrootd
 test_155_stashcache
+test_158_xrootd_tpc
 test_160_rsv
 test_170_pbs
 test_180_cvmfs
@@ -34,6 +35,7 @@ test_430_uberftp
 test_440_glexec
 test_450_xrootd
 test_460_stashcache
+test_465_xrootd_tpc
 test_470_rsv
 test_490_jobs
 test_510_edgmkgridmap
@@ -53,6 +55,7 @@ test_800_gratia
 test_820_cvmfs
 test_830_rsv
 test_835_stashcache
+test_838_xrootd_tpc
 test_840_xrootd
 test_850_lcmaps
 test_860_gridftp

osgtest/library/files.py

@@ -25,7 +25,7 @@
 import re
 import shutil
 import tempfile
-
+import hashlib
 import osgtest.library.core as core
 
 
@@ -246,3 +246,21 @@ def safe_makedirs(directory, mode=0o777):
     """
     if not os.path.isdir(directory):
         os.makedirs(directory, mode)
+
+def checksum_file(path):
+    """Return the md5 checksum of a file """
+    if os.path.exists(path):
+        return hashlib.md5(open(path,'rb').read()).hexdigest()
+    else:
+        return False
+
+def checksum_files_match(file1, file2):
+    """Returns true if the checksum of the files matches
+    
+    """
+    checksum1 = checksum_file(file1)
+    checksum2 = checksum_file(file2)
+    if checksum1 and checksum2:
+        return checksum1==checksum2
+    else:
+        return False

osgtest/library/service.py

@@ -2,6 +2,7 @@
 import time
 
 import osgtest.library.core as core
+from osgtest.library import files
 
 STATUS_RUNNING = 0 # LSB: program is running or service is OK
 STATUS_STOPPED = 3 # LSB: program is not running according to LSB init standards
@@ -35,13 +36,13 @@ def start(service_name):
     core.check_system(command, 'Start ' + service_name + ' service')
     core.state[service_name + '.started-service'] = True
 
-def check_start(service_name, timeout=10):
+def check_start(service_name, timeout=10, log_to_check = None):
     """
     Start a service, 'service_name' via init script or systemd and ensure that
     it starts running within a 'timeout' second window (default=10s)
     """
     start(service_name)
-    assert is_running(service_name, timeout=10), "%s is not running" % service_name
+    assert is_running(service_name, timeout=10, log_to_check = log_to_check), "%s is not running" % service_name
 
 def stop(service_name):
     """
@@ -89,7 +90,7 @@ def status(service_name):
     status_rc, _, _ = core.system(command)
     return status_rc
 
-def check_status(service_name, expected_status, timeout=10):
+def check_status(service_name, expected_status, timeout=10, log_to_check = None):
     """
     Return True if the exit code of the 'service_name' status check is
     expected_status before 'timeout' seconds. Otherwise, False.
@@ -101,15 +102,20 @@ def check_status(service_name, expected_status, timeout=10):
         time.sleep(1)
         timer += 1
 
+    if status_rc != expected_status and log_to_check:
+        log_file_contents = files.read(log_to_check)
+        core.log_message("Last lines of log: %s" % log_to_check)
+        for line in log_file_contents[-9:]:
+            core.log_message(line)
     return status_rc == expected_status
 
-def is_running(service_name, timeout=1):
+def is_running(service_name, timeout=1, log_to_check = None):
     """
     Return True if 'service_name' is determined to be running via init script or
     systemd, according to LSB init standards, before 'timeout'
     seconds. Otherwise, False.
     """
-    return check_status(service_name, STATUS_RUNNING, timeout)
+    return check_status(service_name, STATUS_RUNNING, timeout, log_to_check)
 
 def is_stopped(service_name, timeout=1):
     """

osgtest/tests/test_150_xrootd.py

@@ -24,7 +24,7 @@
 """
 
 AUTHFILE_TEXT = """\
-u * /tmp a
+u * /tmp a /usr/share/ r
 u = /tmp/@=/ a
 u xrootd /tmp a
 """

osgtest/tests/test_158_xrootd_tpc.py

@@ -0,0 +1,109 @@
+import os
+import pwd
+import osgtest.library.core as core
+import osgtest.library.files as files
+import osgtest.library.service as service
+import osgtest.library.osgunittest as osgunittest
+
+
+HTTP_PORT1 = 9001  # chosen so it doesn't conflict w/ the stashcache instances
+HTTP_PORT2 = 9002
+
+XROOTD_CFG_TEXT = """\
+cms.space min 2g 5g
+xrootd.seclib /usr/lib64/libXrdSec.so
+http.secxtractor /usr/lib64/libXrdLcmaps.so
+
+sec.protocol /usr/lib64 gsi -d 2 -certdir:/etc/grid-security/certificates \
+    -cert:/etc/grid-security/xrd/xrdcert.pem \
+    -key:/etc/grid-security/xrd/xrdkey.pem \
+    -crl:1 \
+    -ca:0 \
+    --gmapopt:10 \
+    --gmapto:0 \
+    %s
+
+acc.authdb /etc/xrootd/auth_file
+ofs.authorize
+all.export    /
+
+if exec xrootd
+  http.cadir /etc/grid-security/certificates
+  http.cert /etc/grid-security/xrd/xrdcert.pem
+  http.key /etc/grid-security/xrd/xrdkey.pem
+  http.listingdeny yes
+  http.desthttps yes
+  http.trace all debug
+  # Enable third-party-copy
+  http.exthandler xrdtpc libXrdHttpTPC.so
+  # Pass the bearer token to the Xrootd authorization framework.
+  http.header2cgi Authorization authz
+
+  # Enable Macaroons
+  ofs.authlib libXrdMacaroons.so
+  xrd.port %d
+  xrd.protocol http:%d /usr/lib64/libXrdHttp-4.so
+fi
+http.exthandler xrdmacaroons libXrdMacaroons.so
+all.sitename VDTTESTSITE
+
+"""
+
+class TestStartXrootdTPC(osgunittest.OSGTestCase):
+    @core.elrelease(7,8)
+    def test_01_configure_xrootd(self):
+        core.config['xrootd.tpc.config-1'] = '/etc/xrootd/xrootd-third-party-copy-1.cfg'
+        core.config['xrootd.tpc.config-2'] = '/etc/xrootd/xrootd-third-party-copy-2.cfg'
+        core.config['xrootd.tpc.http-port1'] = HTTP_PORT1
+        core.config['xrootd.tpc.http-port2'] = HTTP_PORT2
+        core.state['xrootd.started-http-server-1'] = False
+        core.state['xrootd.started-http-server-2'] = False
+        core.state['xrootd.tpc.backups-exist'] = False
+
+        self.skip_ok_unless(core.options.adduser, 'user not created')
+        core.skip_ok_unless_installed('globus-proxy-utils', 'xrootd', 'xrootd-scitokens', by_dependency=True)
+
+        user = pwd.getpwnam("xrootd")
+
+        lcmaps_packages = ('lcmaps', 'lcmaps-db-templates', 'xrootd-lcmaps', 'vo-client', 'vo-client-lcmaps-voms')
+        if all([core.rpm_is_installed(x) for x in lcmaps_packages]):
+            core.log_message("Using xrootd-lcmaps authentication")
+            sec_protocol = '-authzfun:libXrdLcmaps.so -authzfunparms:--loglevel,5'
+            sec_protocol += ',--policy,authorize_only'
+        else:
+            core.log_message("Using XRootD mapfile authentication")
+            sec_protocol = '-gridmap:/etc/grid-security/xrd/xrdmapfile'
+
+        files.write(core.config['xrootd.tpc.config-1'],
+                     XROOTD_CFG_TEXT % (sec_protocol, core.config['xrootd.tpc.http-port1'], core.config['xrootd.tpc.http-port1']),
+                     owner='xrootd', backup=True, chown=(user.pw_uid, user.pw_gid))
+        files.write(core.config['xrootd.tpc.config-2'],
+                     XROOTD_CFG_TEXT % (sec_protocol, core.config['xrootd.tpc.http-port2'], core.config['xrootd.tpc.http-port2']),
+                     owner='xrootd', backup=True, chown=(user.pw_uid, user.pw_gid))
+        core.state['xrootd.tpc.backups-exist'] = True
+ 
+    def test_02_create_secrets(self):
+        core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', by_dependency=True)
+        core.config['xrootd.tpc.macaroon-secret-1'] = '/etc/xrootd/macaroon-secret-1'
+        core.config['xrootd.tpc.macaroon-secret-2'] = '/etc/xrootd/macaroon-secret-2'
+        core.check_system(["openssl", "rand", "-base64", "-out",
+                               core.config['xrootd.tpc.macaroon-secret-1'], "64"], "Creating symmetric key")
+        core.check_system(["openssl", "rand", "-base64", "-out",
+                               core.config['xrootd.tpc.macaroon-secret-2'], "64"], "Creating symmetric key")
+        files.append(core.config['xrootd.tpc.config-1'], 
+                         "macaroons.secretkey %s"%(core.config['xrootd.tpc.macaroon-secret-1']),
+                         owner='xrootd', backup=False)
+        files.append(core.config['xrootd.tpc.config-2'],
+                         "macaroons.secretkey %s"%(core.config['xrootd.tpc.macaroon-secret-2']),
+                      owner='xrootd', backup=False)
+
+
+    def test_03_start_xrootd(self):
+        core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', by_dependency=True)
+        core.config['xrootd_tpc_service_1'] = "xrootd@third-party-copy-1"
+        core.config['xrootd_tpc_service_2'] = "xrootd@third-party-copy-2"
+        service.check_start(core.config['xrootd_tpc_service_1'], log_to_check = '/var/log/xrootd/third-party-copy-1/xrootd.log')
+        service.check_start(core.config['xrootd_tpc_service_2'], log_to_check = '/var/log/xrootd/third-party-copy-2/xrootd.log')
+        core.state['xrootd.started-http-server-1'] = True
+        core.state['xrootd.started-http-server-2'] = True
+

osgtest/tests/test_465_xrootd_tpc.py

@@ -0,0 +1,50 @@
+import os
+import json
+import pwd
+
+import osgtest.library.core as core
+import osgtest.library.files as files
+import osgtest.library.osgunittest as osgunittest
+
+
+class TestXrootdTPC(osgunittest.OSGTestCase):
+
+    def test_01_create_macaroons(self):
+        core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', 'x509-scitokens-issuer-client', by_dependency=True)
+        self.skip_bad_unless(core.state['proxy.created'], 'Proxy creation failed')
+        
+        uid = pwd.getpwnam(core.options.username)[2]
+        usercert = '/tmp/x509up_u%d' % uid
+        userkey = '/tmp/x509up_u%d' % uid
+        
+        core.config['xrootd.tpc.url-1'] = "https://" + core.get_hostname() + ":9001" + "/usr/share/osg-test/test_gridftp_data.txt".strip()
+        command = ('macaroon-init', core.config['xrootd.tpc.url-1'], '20', 'DOWNLOAD')
+
+        status, stdout, stderr = core.system(command, user=True)
+        fail = core.diagnose('Obtain Macaroon one',
+                             command, status, stdout, stderr)
+        core.config['xrootd.tpc.macaroon-1'] = stdout.strip()
+
+        core.config['xrootd.tpc.url-2'] = "https://" + core.get_hostname() + ":9002" + "/tmp/test_gridftp_data_tpc.txt".strip()
+        command = ('macaroon-init', core.config['xrootd.tpc.url-2'], '20', 'UPLOAD')
+        status, stdout, stderr = core.system(command, user=True)
+        fail = core.diagnose('Obtain Macaroon number two',
+                             command, status, stdout, stderr)
+        core.config['xrootd.tpc.macaroon-2'] = stdout.strip()
+        
+    def test_02_initate_tpc(self):
+        core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', 'x509-scitokens-issuer-client', by_dependency=True)
+        headers = {}
+        command = ('curl', '-A', 'Test', "-vk", "-X", "COPY",
+                   '-H', "Authorization: Bearer %s" % core.config['xrootd.tpc.macaroon-1'],
+                   '-H', "Source: %s" % core.config['xrootd.tpc.url-1'], 
+                   '-H', 'Overwrite: T', 
+                   '-H', 'Copy-Header:  Authorization: Bearer %s'% core.config['xrootd.tpc.macaroon-2'],
+                   core.config['xrootd.tpc.url-2'])
+        status, stdout, stderr = core.system(command, user=True)
+        fail = core.diagnose('Initiate third party copy',
+                             command, status, stdout, stderr)
+        file_copied = os.path.exists("/tmp/test_gridftp_data_tpc.txt")
+        self.assert_(file_copied, 'Copied file missing')
+        chechskum_match = files.checksum_files_match("/tmp/test_gridftp_data_tpc.txt", "/usr/share/osg-test/test_gridftp_data.txt")
+        self.assert_(chechskum_match, 'Files have same contents')

osgtest/tests/test_838_xrootd_tpc.py

@@ -0,0 +1,21 @@
+import osgtest.library.core as core
+import osgtest.library.files as files
+import osgtest.library.service as service
+import osgtest.library.osgunittest as osgunittest
+
+class TestStopXrootdTPC(osgunittest.OSGTestCase):
+    @core.elrelease(7,8)
+    def test_01_stop_xrootd(self):
+        if core.state['xrootd.tpc.backups-exist']:
+            files.restore(core.config['xrootd.tpc.config-1'], "xrootd")
+            files.restore(core.config['xrootd.tpc.config-2'], "xrootd")
+
+        core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', by_dependency=True)
+        self.skip_ok_if(not core.state['xrootd.started-http-server-1'] and 
+                        not core.state['xrootd.started-http-server-2'], 
+                        'did not start any of the http servers')
+        service.check_stop(core.config['xrootd_tpc_service_1'])
+        service.check_stop(core.config['xrootd_tpc_service_2'])
+
+    def test_02_clean_test_files(self):
+        files.remove("/tmp/test_gridftp_data_tpc.txt", force=True)