Adding the TPC tests to the test sequences

Author: efajardo

.travis.yml

@@ -5,7 +5,7 @@ env:
 #   - OS_TYPE=centos OS_VERSION=6 PACKAGES=osg-gridftp,rsv,lcmaps,lcmaps-db-templates,vo-client-lcmaps-voms,rsv,llrun
 #   - OS_TYPE=centos OS_VERSION=7 PACKAGES=osg-ce-condor,rsv,vo-client-lcmaps-voms,llrun,singularity-runtime,osg-oasis
 #   - OS_TYPE=centos OS_VERSION=7 PACKAGES=osg-gridftp,rsv,lcmaps,lcmaps-db-templates,vo-client-lcmaps-voms,rsv,llrun,singularity-runtime,osg-oasis
-   - OS_TYPE=centos OS_VERSION=7 PACKAGES=xrootd,xrootd-client,xrootd-multiuser,xrootd-scitokens,globus-proxy-utils
+   - OS_TYPE=centos OS_VERSION=7 PACKAGES=xrootd,xrootd-client,xrootd-multiuser,xrootd-scitokens,globus-proxy-utils,lcmaps,lcmaps-db-templates,xrootd-lcmaps,vo-client,vo-client-lcmaps-voms
 
 services:
   - docker

files/test_sequence

@@ -10,6 +10,7 @@ test_130_gridftp
 test_140_lcmaps
 test_150_xrootd
 test_155_stashcache
+test_158_xrootd_tpc
 test_160_rsv
 test_170_pbs
 test_180_cvmfs
@@ -53,6 +54,7 @@ test_800_gratia
 test_820_cvmfs
 test_830_rsv
 test_835_stashcache
+test_838_xrootd_tpc
 test_840_xrootd
 test_850_lcmaps
 test_860_gridftp

osgtest/library/files.py

@@ -176,7 +176,7 @@ def append(path, contents, force=False, owner=None, backup=True):
         return
 
     new_contents = old_contents + [contents]
-    write(path, new_contents, backup=False)
+    write(path, new_contents, owner, backup=False)
 
 
 def restore(path, owner):

osgtest/library/service.py

@@ -2,6 +2,7 @@
 import time
 
 import osgtest.library.core as core
+from osgtest.library import files
 
 STATUS_RUNNING = 0 # LSB: program is running or service is OK
 STATUS_STOPPED = 3 # LSB: program is not running according to LSB init standards
@@ -35,13 +36,13 @@ def start(service_name):
     core.check_system(command, 'Start ' + service_name + ' service')
     core.state[service_name + '.started-service'] = True
 
-def check_start(service_name, timeout=10):
+def check_start(service_name, timeout=10, logToCheck = None):
     """
     Start a service, 'service_name' via init script or systemd and ensure that
     it starts running within a 'timeout' second window (default=10s)
     """
     start(service_name)
-    assert is_running(service_name, timeout=10), "%s is not running" % service_name
+    assert is_running(service_name, timeout=10, logToCheck = logToCheck), "%s is not running" % service_name
 
 def stop(service_name):
     """
@@ -89,7 +90,7 @@ def status(service_name):
     status_rc, _, _ = core.system(command)
     return status_rc
 
-def check_status(service_name, expected_status, timeout=10):
+def check_status(service_name, expected_status, timeout=10, logToCheck = None):
     """
     Return True if the exit code of the 'service_name' status check is
     expected_status before 'timeout' seconds. Otherwise, False.
@@ -101,15 +102,20 @@ def check_status(service_name, expected_status, timeout=10):
         time.sleep(1)
         timer += 1
 
+    if status_rc != expected_status and logToCheck!= None:
+        LogFileContents = files.read(logToCheck)
+        core.log_message("Last lines of log: %s" % logToCheck)
+        for line in LogFileContents[-9:]:
+            core.log_message(line)
     return status_rc == expected_status
 
-def is_running(service_name, timeout=1):
+def is_running(service_name, timeout=1, logToCheck = None):
     """
     Return True if 'service_name' is determined to be running via init script or
     systemd, according to LSB init standards, before 'timeout'
     seconds. Otherwise, False.
     """
-    return check_status(service_name, STATUS_RUNNING, timeout)
+    return check_status(service_name, STATUS_RUNNING, timeout, logToCheck)
 
 def is_stopped(service_name, timeout=1):
     """

osgtest/tests/test_158_xrootd_tpc.py

@@ -27,44 +27,38 @@
   http.cadir /etc/grid-security/certificates
   http.cert /etc/grid-security/xrd/xrdcert.pem
   http.key /etc/grid-security/xrd/xrdkey.pem
-  http.secxtractor /usr/lib64/libXrdLcmaps.so
   http.listingdeny yes
   http.desthttps yes
-
+  http.trace all debug
   # Enable third-party-copy
   http.exthandler xrdtpc libXrdHttpTPC.so
   # Pass the bearer token to the Xrootd authorization framework.
   http.header2cgi Authorization authz
 
   # Enable Macaroons
   ofs.authlib libXrdMacaroons.so libXrdAccSciTokens.so
-
-fi
-
-if named xrd-TPC-1
+  xrd.port %d
   xrd.protocol http:%d /usr/lib64/libXrdHttp-4.so
 fi
+http.exthandler xrdmacaroons libXrdMacaroons.so
+macaroons.secretkey /etc/xrootd/macaroon-secret
+all.sitename VDTTESTSITE
 
-if named xrd-TPC-2
-  xrd.protocol http:%d /usr/lib64/libXrdHttp-4.so
-fi
-end
 """
 
 class TestStartXrootdTPC(osgunittest.OSGTestCase):
     @core.elrelease(7,8)
-    def test_01_start_xrootd(self):
-        core.config['certs.xrootdcert'] = '/etc/grid-security/xrd/xrdcert.pem'
-        core.config['certs.xrootdkey'] = '/etc/grid-security/xrd/xrdkey.pem'
-        core.config['xrootd.tpc.config'] = '/etc/xrootd/xrootd-third-party-copy.cfg'
+    def test_01_configure_xrootd(self):
+        core.config['xrootd.tpc.config-1'] = '/etc/xrootd/xrootd-third-party-copy-1.cfg'
+        core.config['xrootd.tpc.config-2'] = '/etc/xrootd/xrootd-third-party-copy-2.cfg'
         core.config['xrootd.tpc.http-port1'] = HTTP_PORT1
         core.config['xrootd.tpc.http-port2'] = HTTP_PORT2
         core.state['xrootd.started-http-server-1'] = False
         core.state['xrootd.started-http-server-2'] = False
         core.state['xrootd.tpc.backups-exist'] = False
 
         self.skip_ok_unless(core.options.adduser, 'user not created')
-        core.skip_ok_unless_installed('xrootd', by_dependency=True)
+        core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', by_dependency=True)
 
         user = pwd.getpwnam("xrootd")
         core.skip_ok_unless_installed('globus-proxy-utils')
@@ -73,23 +67,33 @@ def test_01_start_xrootd(self):
         if all([core.rpm_is_installed(x) for x in lcmaps_packages]):
             core.log_message("Using xrootd-lcmaps authentication")
             sec_protocol = '-authzfun:libXrdLcmaps.so -authzfunparms:--loglevel,5'
-            if core.PackageVersion('xrootd-lcmaps') >= '1.4.0':
-                sec_protocol += ',--policy,authorize_only'
+            #XROOTD_CFG_TEXT += "http.secxtractor /usr/lib64/libXrdLcmaps.so/n"
+            sec_protocol += ',--policy,authorize_only'
         else:
             core.log_message("Using XRootD mapfile authentication")
             sec_protocol = '-gridmap:/etc/grid-security/xrd/xrdmapfile'
 
-        files.append(core.config['xrootd.tpc.config'],
-                     XROOTD_CFG_TEXT % (sec_protocol, core.config['xrootd.port']),
-                     owner='xrootd', backup=True)
+        files.write(core.config['xrootd.tpc.config-1'],
+                     XROOTD_CFG_TEXT % (sec_protocol, core.config['xrootd.tpc.http-port1'], core.config['xrootd.tpc.http-port1']),
+                     owner='xrootd', backup=True, chown=(user.pw_uid, user.pw_gid))
+        files.write(core.config['xrootd.tpc.config-2'],
+                     XROOTD_CFG_TEXT % (sec_protocol, core.config['xrootd.tpc.http-port2'], core.config['xrootd.tpc.http-port1']),
+                     owner='xrootd', backup=True, chown=(user.pw_uid, user.pw_gid))
         core.state['xrootd.tpc.backups-exist'] = True
+ 
+    def test_02_create_secret_key(self):
+        core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', by_dependency=True)
+        core.config['xrootd.tpc.macaroon-secret'] = '/etc/xrootd/macaroon-secret'
+        core.check_system(["openssl", "rand", "-base64", "-out",
+                               core.config['xrootd.tpc.macaroon-secret'], "64"], "Creating simmetric key")
 
-    def test_02_start_xrootd(self):
+
+    def test_03_start_xrootd(self):
         core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', by_dependency=True)
-        core.config['xrootd_tpc_service_1'] = "xrd-TPC-1@third-party-copy"
-        core.config['xrootd_tpc_service_2'] = "xrd-TPC-2@third-party-copy"
-        service.check_start(core.config['xrootd_tpc_service_1'])
-        service.check_start(core.config['xrootd_tpc_service_2'])
-        
-        core.state['xrootd_tpc_service_1'] = True
-        core.state['xrootd_tpc_service_2'] = True
+        core.config['xrootd_tpc_service_1'] = "xrootd@third-party-copy-1"
+        core.config['xrootd_tpc_service_2'] = "xrootd@third-party-copy-2"
+        service.check_start(core.config['xrootd_tpc_service_1'], logToCheck = '/var/log/xrootd/third-party-copy-1/xrootd.log')
+        service.check_start(core.config['xrootd_tpc_service_2'], logToCheck = '/var/log/xrootd/third-party-copy-2/xrootd.log')
+        core.state['xrootd.started-http-server-1'] = True
+        core.state['xrootd.started-http-server-2'] = True
+

osgtest/tests/test_848_xrootd_tpc.py renamed to osgtest/tests/test_838_xrootd_tpc.py

@@ -7,10 +7,13 @@ class TestStopXrootdTPC(osgunittest.OSGTestCase):
     @core.elrelease(7,8)
     def test_01_stop_xrootd(self):
         if core.state['xrootd.tpc.backups-exist']:
-            files.restore(core.config['xrootd.tpc.config'], "xrootd")
+            files.restore(core.config['xrootd.tpc.config-1'], "xrootd")
+            files.restore(core.config['xrootd.tpc.config-2'], "xrootd")
 
-        core.skip_ok_unless_installed('xrootd', by_dependency=True)
-        self.skip_ok_if(core.state['xrootd_tpc_service'], 'did not start server')
+        core.skip_ok_unless_installed('xrootd', 'xrootd-scitokens', by_dependency=True)
+        self.skip_ok_if(not core.state['xrootd.started-http-server-1'] and 
+                        not core.state['xrootd.started-http-server-2'], 
+                        'did not start any of the http servers')
         service.check_stop(core.config['xrootd_tpc_service_1'])
         service.check_stop(core.config['xrootd_tpc_service_2'])
 

travis-ci/test_inside_docker.sh

@@ -57,4 +57,4 @@ export _condor_CONDOR_CE_TRACE_ATTEMPTS=60
 # Ok, do actual testing
 INSTALL_STR="--install ${PACKAGES//,/ --install }"
 echo "------------ OSG Test --------------"
-osg-test -vad --hostcert --no-cleanup ${INSTALL_STR}
+osg-test --extra-repo osg-testing -vad --hostcert --no-cleanup ${INSTALL_STR}