initiate TPC

Author: efajardo

.travis.yml

@@ -1,11 +1,11 @@
 sudo: required
 env:
   matrix:
-#   - OS_TYPE=centos OS_VERSION=6 PACKAGES=osg-ce-condor,rsv,vo-client-lcmaps-voms,llrun
-#   - OS_TYPE=centos OS_VERSION=6 PACKAGES=osg-gridftp,rsv,lcmaps,lcmaps-db-templates,vo-client-lcmaps-voms,rsv,llrun
-#   - OS_TYPE=centos OS_VERSION=7 PACKAGES=osg-ce-condor,rsv,vo-client-lcmaps-voms,llrun,singularity-runtime,osg-oasis
-#   - OS_TYPE=centos OS_VERSION=7 PACKAGES=osg-gridftp,rsv,lcmaps,lcmaps-db-templates,vo-client-lcmaps-voms,rsv,llrun,singularity-runtime,osg-oasis
-   - OS_TYPE=centos OS_VERSION=7 PACKAGES=xrootd,xrootd-client,xrootd-scitokens,globus-proxy-utils,lcmaps,lcmaps-db-templates,xrootd-lcmaps,vo-client,vo-client-lcmaps-voms
+   - OS_TYPE=centos OS_VERSION=6 PACKAGES=osg-ce-condor,rsv,vo-client-lcmaps-voms,llrun
+   - OS_TYPE=centos OS_VERSION=6 PACKAGES=osg-gridftp,rsv,lcmaps,lcmaps-db-templates,vo-client-lcmaps-voms,rsv,llrun
+   - OS_TYPE=centos OS_VERSION=7 PACKAGES=osg-ce-condor,rsv,vo-client-lcmaps-voms,llrun,singularity-runtime,osg-oasis
+   - OS_TYPE=centos OS_VERSION=7 PACKAGES=osg-gridftp,rsv,lcmaps,lcmaps-db-templates,vo-client-lcmaps-voms,rsv,llrun,singularity-runtime,osg-oasis
+   - OS_TYPE=centos OS_VERSION=7 PACKAGES=xrootd,xrootd-client,xrootd-multiuser,globus-proxy-utils
 
 services:
   - docker

files/test_sequence

@@ -35,6 +35,7 @@ test_430_uberftp
 test_440_glexec
 test_450_xrootd
 test_460_stashcache
+test_465_xrootd_tpc
 test_470_rsv
 test_490_jobs
 test_510_edgmkgridmap

osgtest/library/files.py

@@ -25,7 +25,7 @@
 import re
 import shutil
 import tempfile
-
+import hashlib
 import osgtest.library.core as core
 
 
@@ -176,7 +176,7 @@ def append(path, contents, force=False, owner=None, backup=True):
         return
 
     new_contents = old_contents + [contents]
-    write(path, new_contents, owner, backup=False)
+    write(path, new_contents, backup=False)
 
 
 def restore(path, owner):
@@ -246,3 +246,21 @@ def safe_makedirs(directory, mode=0o777):
     """
     if not os.path.isdir(directory):
         os.makedirs(directory, mode)
+
+def checksum_file(path):
+    """Return the md5 checksum of a file """
+    if os.path.exists(path):
+        return hashlib.md5(open(path,'rb').read()).hexdigest()
+    else:
+        return False
+
+def checksum_files_match(file1, file2):
+    """Returns true if the checksum of the files matches
+    
+    """
+    checksum1 = checksum_file(file1)
+    checksum2 = checksum_file(file2)
+    if checksum1 and checksum2:
+        return checksum1==checksum2
+    else:
+        return False

osgtest/library/service.py

@@ -36,13 +36,13 @@ def start(service_name):
     core.check_system(command, 'Start ' + service_name + ' service')
     core.state[service_name + '.started-service'] = True
 
-def check_start(service_name, timeout=10, logToCheck = None):
+def check_start(service_name, timeout=10, log_to_check = None):
     """
     Start a service, 'service_name' via init script or systemd and ensure that
     it starts running within a 'timeout' second window (default=10s)
     """
     start(service_name)
-    assert is_running(service_name, timeout=10, logToCheck = logToCheck), "%s is not running" % service_name
+    assert is_running(service_name, timeout=10, log_to_check = log_to_check), "%s is not running" % service_name
 
 def stop(service_name):
     """
@@ -90,7 +90,7 @@ def status(service_name):
     status_rc, _, _ = core.system(command)
     return status_rc
 
-def check_status(service_name, expected_status, timeout=10, logToCheck = None):
+def check_status(service_name, expected_status, timeout=10, log_to_check = None):
     """
     Return True if the exit code of the 'service_name' status check is
     expected_status before 'timeout' seconds. Otherwise, False.
@@ -102,20 +102,20 @@ def check_status(service_name, expected_status, timeout=10, logToCheck = None):
         time.sleep(1)
         timer += 1
 
-    if status_rc != expected_status and logToCheck!= None:
-        LogFileContents = files.read(logToCheck)
-        core.log_message("Last lines of log: %s" % logToCheck)
-        for line in LogFileContents[-9:]:
+    if status_rc != expected_status and log_to_check:
+        log_file_contents = files.read(log_to_check)
+        core.log_message("Last lines of log: %s" % log_to_check)
+        for line in log_file_contents[-9:]:
             core.log_message(line)
     return status_rc == expected_status
 
-def is_running(service_name, timeout=1, logToCheck = None):
+def is_running(service_name, timeout=1, log_to_check = None):
     """
     Return True if 'service_name' is determined to be running via init script or
     systemd, according to LSB init standards, before 'timeout'
     seconds. Otherwise, False.
     """
-    return check_status(service_name, STATUS_RUNNING, timeout, logToCheck)
+    return check_status(service_name, STATUS_RUNNING, timeout, log_to_check)
 
 def is_stopped(service_name, timeout=1):
     """

osgtest/tests/test_150_xrootd.py

@@ -24,7 +24,7 @@
 """
 
 AUTHFILE_TEXT = """\
-u * /tmp a
+u * /tmp a /usr/share/ r
 u = /tmp/@=/ a
 u xrootd /tmp a
 """

osgtest/tests/test_158_xrootd_tpc.py

@@ -11,17 +11,21 @@
 
 XROOTD_CFG_TEXT = """\
 cms.space min 2g 5g
-xrootd.seclib /usr/lib64/libXrdSec-4.so
-sec.protocol /usr/lib64 gsi -certdir:/etc/grid-security/certificates \
+xrootd.seclib /usr/lib64/libXrdSec.so
+http.secxtractor /usr/lib64/libXrdLcmaps.so
+
+sec.protocol /usr/lib64 gsi -d 2 -certdir:/etc/grid-security/certificates \
     -cert:/etc/grid-security/xrd/xrdcert.pem \
     -key:/etc/grid-security/xrd/xrdkey.pem \
-    -crl:3 \
+    -crl:1 \
+    -ca:0 \
     --gmapopt:10 \
     --gmapto:0 \
     %s
 
 acc.authdb /etc/xrootd/auth_file
 ofs.authorize
+all.export    /
 
 if exec xrootd
   http.cadir /etc/grid-security/certificates
@@ -36,7 +40,7 @@
   http.header2cgi Authorization authz
 
   # Enable Macaroons
-  ofs.authlib libXrdMacaroons.so libXrdAccSciTokens.so
+  ofs.authlib libXrdMacaroons.so
   xrd.port %d
   xrd.protocol http:%d /usr/lib64/libXrdHttp-4.so
 fi
@@ -57,16 +61,14 @@ def test_01_configure_xrootd(self):
         core.state['xrootd.tpc.backups-exist'] = False
 
         self.skip_ok_unless(core.options.adduser, 'user not created')
-        core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', by_dependency=True)
+        core.skip_ok_unless_installed('globus-proxy-utils', 'xrootd', 'xrootd-scitokens', by_dependency=True)
 
         user = pwd.getpwnam("xrootd")
-        core.skip_ok_unless_installed('globus-proxy-utils')
 
         lcmaps_packages = ('lcmaps', 'lcmaps-db-templates', 'xrootd-lcmaps', 'vo-client', 'vo-client-lcmaps-voms')
         if all([core.rpm_is_installed(x) for x in lcmaps_packages]):
             core.log_message("Using xrootd-lcmaps authentication")
             sec_protocol = '-authzfun:libXrdLcmaps.so -authzfunparms:--loglevel,5'
-            #XROOTD_CFG_TEXT += "http.secxtractor /usr/lib64/libXrdLcmaps.so/n"
             sec_protocol += ',--policy,authorize_only'
         else:
             core.log_message("Using XRootD mapfile authentication")
@@ -76,7 +78,7 @@ def test_01_configure_xrootd(self):
                      XROOTD_CFG_TEXT % (sec_protocol, core.config['xrootd.tpc.http-port1'], core.config['xrootd.tpc.http-port1']),
                      owner='xrootd', backup=True, chown=(user.pw_uid, user.pw_gid))
         files.write(core.config['xrootd.tpc.config-2'],
-                     XROOTD_CFG_TEXT % (sec_protocol, core.config['xrootd.tpc.http-port2'], core.config['xrootd.tpc.http-port1']),
+                     XROOTD_CFG_TEXT % (sec_protocol, core.config['xrootd.tpc.http-port2'], core.config['xrootd.tpc.http-port2']),
                      owner='xrootd', backup=True, chown=(user.pw_uid, user.pw_gid))
         core.state['xrootd.tpc.backups-exist'] = True
 
@@ -85,9 +87,9 @@ def test_02_create_secrets(self):
         core.config['xrootd.tpc.macaroon-secret-1'] = '/etc/xrootd/macaroon-secret-1'
         core.config['xrootd.tpc.macaroon-secret-2'] = '/etc/xrootd/macaroon-secret-2'
         core.check_system(["openssl", "rand", "-base64", "-out",
-                               core.config['xrootd.tpc.macaroon-secret-1'], "64"], "Creating simmetric key")
+                               core.config['xrootd.tpc.macaroon-secret-1'], "64"], "Creating symmetric key")
         core.check_system(["openssl", "rand", "-base64", "-out",
-                               core.config['xrootd.tpc.macaroon-secret-2'], "64"], "Creating simmetric key")
+                               core.config['xrootd.tpc.macaroon-secret-2'], "64"], "Creating symmetric key")
         files.append(core.config['xrootd.tpc.config-1'], 
                          "macaroons.secretkey %s"%(core.config['xrootd.tpc.macaroon-secret-1']),
                          owner='xrootd', backup=False)
@@ -100,8 +102,8 @@ def test_03_start_xrootd(self):
         core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', by_dependency=True)
         core.config['xrootd_tpc_service_1'] = "xrootd@third-party-copy-1"
         core.config['xrootd_tpc_service_2'] = "xrootd@third-party-copy-2"
-        service.check_start(core.config['xrootd_tpc_service_1'], logToCheck = '/var/log/xrootd/third-party-copy-1/xrootd.log')
-        service.check_start(core.config['xrootd_tpc_service_2'], logToCheck = '/var/log/xrootd/third-party-copy-2/xrootd.log')
+        service.check_start(core.config['xrootd_tpc_service_1'], log_to_check = '/var/log/xrootd/third-party-copy-1/xrootd.log')
+        service.check_start(core.config['xrootd_tpc_service_2'], log_to_check = '/var/log/xrootd/third-party-copy-2/xrootd.log')
         core.state['xrootd.started-http-server-1'] = True
         core.state['xrootd.started-http-server-2'] = True
 

osgtest/tests/test_465_xrootd-tpc.py

@@ -0,0 +1,54 @@
+import os
+import json
+import pwd
+import requests
+
+import osgtest.library.core as core
+import osgtest.library.files as files
+import osgtest.library.osgunittest as osgunittest
+
+
+class TestXrootdTPC(osgunittest.OSGTestCase):
+
+    def test_01_create_macaroons(self):
+        core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', 'x509-scitokens-issuer-client', by_dependency=True)
+        self.skip_bad_unless(core.state['proxy.created'], 'Proxy creation failed')
+        
+        uid = pwd.getpwnam(core.options.username)[2]
+        usercert = '/tmp/x509up_u%d' % uid
+        userkey = '/tmp/x509up_u%d' % uid
+        
+        core.config['xrootd.tpc.url-1'] = "https://" + core.get_hostname() + ":9001" + "/usr/share/osg-test/test_gridftp_data.txt"
+        command = ('macaroon-init', core.config['xrootd.tpc.url-1'], '20', 'DOWNLOAD')
+
+        status, stdout, stderr = core.system(command, user=True)
+        fail = core.diagnose('Obtain Macaroon one',
+                             command, status, stdout, stderr)
+        core.config['xrootd.tpc.macaroon-1'] = stdout.strip('\n')
+
+        core.config['xrootd.tpc.url-2'] = "https://" + core.get_hostname() + ":9002" + "/tmp/test_gridftp_data_tpc.txt"
+        command = ('macaroon-init', core.config['xrootd.tpc.url-2'], '20', 'UPLOAD')
+        status, stdout, stderr = core.system(command, user=True)
+        fail = core.diagnose('Obtain Macaroon number two',
+                             command, status, stdout, stderr)
+        core.config['xrootd.tpc.macaroon-2'] = stdout.strip('\n')
+        
+
+    def test_02_initate_tpc(self):
+        core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', by_dependency=True)
+        session = requests.Session()
+        session.verify = False
+        headers = {}
+        headers['Overwrite'] = 'T'
+        headers['Authorization'] = 'Bearer %s' % core.config['xrootd.tpc.macaroon-1']
+        headers['Source'] = core.config['xrootd.tpc.url-1']
+        headers['Copy-Header'] = 'Authorization: Bearer %s' % core.config['xrootd.tpc.macaroon-2']
+        resp = session.request('COPY',
+                               core.config['xrootd.tpc.url-2'], headers=headers,
+                               allow_redirects=True)
+        file_copied = os.path.exists("/tmp/test_gridftp_data_tpc.txt")
+        self.assert_(file_copied, 'Copied file missing')
+        chechskum_match = files.checksum_files_match("/tmp/test_gridftp_data_tpc.txt", "/usr/share/osg-test/test_gridftp_data.txt")
+        self.assert_(chechskum_match, 'Files have same contents')
+
+        

osgtest/tests/test_465_xrootd_tpc.py

@@ -57,4 +57,4 @@ export _condor_CONDOR_CE_TRACE_ATTEMPTS=60
 # Ok, do actual testing
 INSTALL_STR="--install ${PACKAGES//,/ --install }"
 echo "------------ OSG Test --------------"
-osg-test --extra-repo osg-testing -vad --hostcert --no-cleanup ${INSTALL_STR}
+osg-test -vad --hostcert --no-cleanup ${INSTALL_STR}