Merge pull request #213 from brianhlin/SOFTWARE-4221.add-scitokens-tests

Add scitokens tests (SOFTWARE-4221)

Author: brianhlin

files/test_sequence

@@ -19,6 +19,7 @@ test_280_gsiopenssh
 test_290_slurm
 test_380_cvmfs
 test_400_proxy
+test_401_scitokens
 test_403_myproxy
 test_407_voms
 test_410_jobs

osgtest/tests/test_190_condorce.py

@@ -1,14 +1,13 @@
 import cagen
-import re
 
 import osgtest.library.core as core
 import osgtest.library.files as files
 import osgtest.library.osgunittest as osgunittest
 import osgtest.library.condor as condor
 import osgtest.library.service as service
 
+
 class TestStartCondorCE(osgunittest.OSGTestCase):
-    # Tests 01-02 are needed to reconfigure condor to work with HTCondor-CE
     def test_01_configure_condor(self):
         core.skip_ok_unless_installed('condor', 'htcondor-ce', 'htcondor-ce-client')
 
@@ -34,31 +33,41 @@ def test_02_reconfigure_condor(self):
 
         command = ('condor_reconfig', '-debug')
         core.check_system(command, 'Reconfigure Condor')
-        self.assert_(service.is_running('condor', timeout=10), 'Condor not running after reconfig')
+        self.assertTrue(service.is_running('condor', timeout=10), 'Condor not running after reconfig')
+
+    def test_02_scitoken_mapping(self):
+        core.state['condor-ce.wrote-mapfile'] = False
+        core.skip_ok_unless_installed('condor', 'htcondor-ce')
+        self.skip_ok_if(core.PackageVersion('condor') <= '8.9.4',
+                        'HTCondor version does not support SciToken submission')
+
+        condorce_version = core.PackageVersion('htcondor-ce')
+        scitoken_mapping = f'SCITOKENS "https://demo.scitokens.org" {core.options.username}\n'
+
+        # Write the mapfile to the admin mapfile directory
+        # https://github.com/htcondor/htcondor-ce/pull/425
+        if condorce_version >= '5.1.0':
+            core.config['condor-ce.mapfile'] = '/etc/condor-ce/mapfiles.d/01-osg-test.conf'
+        else:
+            core.config['condor-ce.mapfile'] = '/etc/condor-ce/condor_mapfile'
+            mapfile_contents = files.read(core.config['condor-ce.mapfile'], as_single_string=True)
+            scitoken_mapping += mapfile_contents
+
+        files.write(core.config['condor-ce.mapfile'],
+                    scitoken_mapping,
+                    owner='condor-ce',
+                    chmod=0o644)
+        core.state['condor-ce.wrote-mapfile'] = True
 
     def test_03_configure_ce(self):
         core.skip_ok_unless_installed('condor', 'htcondor-ce', 'htcondor-ce-client')
 
         # Set up Condor, PBS, and Slurm routes
         # Leave the GRIDMAP knob in tact to verify that it works with the LCMAPS VOMS plugin
         core.config['condor-ce.condor-ce-cfg'] = '/etc/condor-ce/config.d/99-osgtest.condor-ce.conf'
-        # Add host DN to condor_mapfile
-        if core.options.hostcert:
-            core.config['condor-ce.condorce_mapfile'] = '/etc/condor-ce/condor_mapfile.osg-test'
-            hostcert_dn, _ = cagen.certificate_info(core.config['certs.hostcert'])
-            mapfile_contents = files.read('/etc/condor-ce/condor_mapfile')
-            mapfile_contents.insert(0, re.sub(r'([/=\.])', r'\\\1', "GSI \"^%s$\" " % hostcert_dn) + \
-                                              "%[email protected]\n" % core.get_hostname())
-            files.write(core.config['condor-ce.condorce_mapfile'],
-                        mapfile_contents,
-                        owner='condor-ce',
-                        chmod=0o644)
-        else:
-            core.config['condor-ce.condorce_mapfile'] = '/etc/condor-ce/condor_mapfile'
 
         condor_contents = """GRIDMAP = /etc/grid-security/grid-mapfile
-CERTIFICATE_MAPFILE = %s
-ALL_DEBUG=D_FULLDEBUG
+ALL_DEBUG=D_CAT D_ALWAYS:2
 JOB_ROUTER_DEFAULTS = $(JOB_ROUTER_DEFAULTS) [set_default_maxMemory = 128;]
 JOB_ROUTER_ENTRIES = \\
    [ \\
@@ -82,7 +91,16 @@ def test_03_configure_ce(self):
 JOB_ROUTER_SCHEDD2_SPOOL=/var/lib/condor/spool
 JOB_ROUTER_SCHEDD2_NAME=$(FULL_HOSTNAME)
 JOB_ROUTER_SCHEDD2_POOL=$(FULL_HOSTNAME):9618
-""" % core.config['condor-ce.condorce_mapfile']
+
+AUTH_SSL_SERVER_CERTFILE = /etc/grid-security/hostcert.pem
+AUTH_SSL_SERVER_KEYFILE = /etc/grid-security/hostkey.pem
+AUTH_SSL_SERVER_CADIR = /etc/grid-security/certificates
+AUTH_SSL_SERVER_CAFILE =
+AUTH_SSL_CLIENT_CERTFILE = /etc/grid-security/hostcert.pem
+AUTH_SSL_CLIENT_KEYFILE = /etc/grid-security/hostkey.pem
+AUTH_SSL_CLIENT_CADIR = /etc/grid-security/certificates
+AUTH_SSL_CLIENT_CAFILE =
+"""
 
         if core.rpm_is_installed('htcondor-ce-view'):
             condor_contents += "\nDAEMON_LIST = $(DAEMON_LIST), CEVIEW, GANGLIAD, SCHEDD"

osgtest/tests/test_401_scitokens.py

@@ -0,0 +1,61 @@
+import json
+import os
+import pwd
+import time
+
+from urllib import error, request
+
+import osgtest.library.core as core
+import osgtest.library.files as files
+import osgtest.library.osgunittest as osgunittest
+
+# Headers so that heroku doesn't block us
+HEADERS = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko)' +
+                         'Chrome/35.0.1916.47 Safari/537.36',
+           'Content-Type': 'application/json'}
+
+
+def request_demo_scitoken(scope, audience='ANY'):
+    """Request a token with 'scope' from the demo SciTokens issuer
+    """
+
+    payload_dict = {'aud': audience,
+                    'ver': 'scitokens:2.0',
+                    'scope': scope,
+                    'exp': int(time.time() + 3600),
+                    'sub': 'osg-test'}
+    payload = json.dumps({'payload': json.dumps(payload_dict),
+                          'algorithm': 'ES256'}).encode()
+
+    req = request.Request('https://demo.scitokens.org/issue',
+                          data=payload,
+                          headers=HEADERS)
+
+    return request.urlopen(req).read()
+
+
+class TestTokens(osgunittest.OSGTestCase):
+
+    def test_01_request_condor_write_scitoken(self):
+        core.state['token.condor_write_created'] = False
+        core.config['token.condor_write'] = '/tmp/condor_write.scitoken'
+
+        core.skip_ok_unless_installed('htcondor-ce', 'condor')
+        self.skip_ok_if(core.PackageVersion('condor') <= '8.9.4',
+                        'HTCondor version does not support SciToken submission')
+        self.skip_ok_if(os.path.exists(core.config['token.condor_write']),
+                        'SciToken with HTCondor WRITE already exists')
+
+        hostname = core.get_hostname()
+        try:
+            token = request_demo_scitoken('condor:/READ condor:/WRITE', audience=f'{hostname}:9619')
+        except error.URLError as exc:
+            self.fail(f"Failed to request token from demo.scitokens.org:\n{exc}")
+
+        ids = (0, 0)
+        if core.state['general.user_added']:
+            user = pwd.getpwnam(core.options.username)
+            ids = (user.pw_uid, user.pw_gid)
+
+        files.write(core.config['token.condor_write'], core.to_str(token), backup=False, chown=ids)
+        core.state['token.condor_write_created'] = True

osgtest/tests/test_410_jobs.py

@@ -58,7 +58,10 @@ def test_02_condor_ce_run_condor(self):
         self.skip_bad_unless(service.is_running('condor-ce'), 'ce not running')
         self.skip_bad_unless(service.is_running('condor'), 'condor not running')
         self.skip_bad_unless(core.state['jobs.env-set'], 'job environment not set')
-        self.skip_bad_unless(core.state['proxy.valid'], 'requires a proxy cert')
+        self.skip_bad_unless(core.state['proxy.valid'] or core.state['token.condor_write_created'],
+                             'requires a scitoken or a proxy')
 
-        command = ('condor_ce_run', '-r', '%s:9619' % core.get_hostname(), '/bin/env')
+        command = ['condor_ce_run', '-r', '%s:9619' % core.get_hostname(), '/bin/env']
+        if core.state['token.condor_write_created']:
+            command.insert(0, f"_condor_SCITOKENS_FILE={core.config['token.condor_write']}")
         self.run_job_in_tmp_dir(command, 'condor_ce_run a Condor job')

osgtest/tests/test_550_condorce.py

@@ -8,7 +8,6 @@
 except ImportError:
     from urllib.request import urlopen
 
-import osgtest.library.condor as condor
 import osgtest.library.core as core
 import osgtest.library.files as files
 import osgtest.library.service as service
@@ -18,20 +17,40 @@
 class TestCondorCE(osgunittest.OSGTestCase):
 
     def setUp(self):
-        # Enforce GSI auth for testing
-        os.environ['_condor_SEC_CLIENT_AUTHENTICATION_METHODS'] = 'GSI'
+        # Enforce SciToken or GSI auth for testing
+        os.environ['_condor_SEC_CLIENT_AUTHENTICATION_METHODS'] = 'SCITOKENS, GSI'
+        core.skip_ok_unless_installed('condor', 'htcondor-ce')
+        self.skip_bad_unless(service.is_running('condor-ce'), 'ce not running')
+
+        self.command = []
+        if core.state['token.condor_write_created']:
+            self.command = [f"_condor_SCITOKENS_FILE={core.config['token.condor_write']}"]
 
     def tearDown(self):
         os.environ.pop('_condor_SEC_CLIENT_AUTHENTICATION_METHODS')
 
+    def check_write_creds(self):
+        """Check for credentials necessary for HTCondor-CE WRITE
+        """
+        self.skip_bad_unless(core.state['proxy.valid'] or core.state['token.condor_write_created'],
+                             'requires a scitoken or a proxy')
+
+    def check_schedd_ready(self):
+        """Check if the HTCondor-CE schedd is up as expected
+        """
+        self.skip_bad_unless(core.state['condor-ce.schedd-ready'], 'CE schedd not ready to accept jobs')
+
     def run_blahp_trace(self, lrms):
         """Run condor_ce_trace() against a non-HTCondor backend and verify the cache"""
         lrms_cache_prefix = {'pbs': 'qstat', 'slurm': 'slurm'}
 
         cwd = os.getcwd()
         os.chdir('/tmp')
-        command = ('condor_ce_trace', '-a osgTestBatchSystem = %s' % lrms.lower(), '--debug', core.get_hostname())
-        trace_out, _, _ = core.check_system(command, 'ce trace against %s' % lrms.lower(), user=True)
+        self.command += ['condor_ce_trace',
+                         '-a osgTestBatchSystem = %s' % lrms.lower(),
+                         '--debug',
+                         core.get_hostname()]
+        trace_out, _, _ = core.check_system(self.command, 'ce trace against %s' % lrms.lower(), user=True)
 
         try:
             backend_jobid = re.search(r'%s_JOBID=(\d+)' % lrms.upper(), trace_out).group(1)
@@ -64,69 +83,55 @@ def run_blahp_trace(self, lrms):
         # S'2'
         # p6
         # s."
-        self.assert_(re.search(r'BatchJobId[=\s"\'p1S]+%s' % backend_jobid, cache),
-                     'Job %s not found in %s blahp cache:\n%s' % (backend_jobid, lrms.upper(), cache))
+        self.assertTrue(re.search(r'BatchJobId[=\s"\'p1S]+%s' % backend_jobid, cache),
+                        'Job %s not found in %s blahp cache:\n%s' % (backend_jobid, lrms.upper(), cache))
 
         os.chdir(cwd)
 
-    def general_requirements(self):
-        core.skip_ok_unless_installed('condor', 'htcondor-ce', 'htcondor-ce-client')
-        self.skip_bad_unless(service.is_running('condor-ce'), 'ce not running')
-
     def test_01_status(self):
-        self.general_requirements()
-
         command = ('condor_ce_status', '-any')
         core.check_system(command, 'ce status', user=True)
 
     def test_02_queue(self):
-        self.general_requirements()
-
         command = ('condor_ce_q', '-verbose')
         core.check_system(command, 'ce queue', user=True)
 
     def test_03_ping(self):
-        self.general_requirements()
-        self.skip_bad_unless(core.state['proxy.valid'], 'requires a proxy cert')
-
-        command = ('condor_ce_ping', 'WRITE', '-verbose')
-        stdout, _, _ = core.check_system(command, 'ping using GSI and gridmap', user=True)
-        self.assert_(re.search(r'Authorized:\s*TRUE', stdout), 'could not authorize with GSI')
+        self.check_write_creds()
+        self.command += ['condor_ce_ping', 'WRITE', '-verbose']
+        stdout, _, _ = core.check_system(self.command, 'ping using SCITOKENS or GSI', user=True)
+        self.assertTrue(re.search(r'Authorized:\s*TRUE', stdout), 'could not authorize with SCITOKENS or GSI')
 
     def test_04_trace(self):
-        self.general_requirements()
-        self.skip_bad_unless(core.state['condor-ce.schedd-ready'], 'CE schedd not ready to accept jobs')
-        self.skip_bad_unless(core.state['proxy.valid'], 'requires a proxy cert')
+        self.check_schedd_ready()
+        self.check_write_creds()
 
         cwd = os.getcwd()
         os.chdir('/tmp')
 
-        command = ('condor_ce_trace', '--debug', core.get_hostname())
-        core.check_system(command, 'ce trace', user=True)
+        self.command += ['condor_ce_trace', '--debug', core.get_hostname()]
+        core.check_system(self.command, 'ce trace', user=True)
 
         os.chdir(cwd)
 
     def test_05_pbs_trace(self):
-        self.general_requirements()
-        self.skip_bad_unless(core.state['condor-ce.schedd-ready'], 'CE schedd not ready to accept jobs')
         core.skip_ok_unless_installed('torque-mom', 'torque-server', 'torque-scheduler', 'torque-client', 'munge',
                                       by_dependency=True)
         self.skip_ok_unless(service.is_running('pbs_server'), 'pbs service not running')
-        self.skip_bad_unless(core.state['proxy.valid'], 'requires a proxy cert')
+        self.check_schedd_ready()
+        self.check_write_creds()
         self.run_blahp_trace('pbs')
 
     def test_06_slurm_trace(self):
-        self.general_requirements()
         core.skip_ok_unless_installed(core.SLURM_PACKAGES)
-        self.skip_bad_unless(service.is_running('munge'), 'slurm requires munge')
         self.skip_bad_unless(core.state['condor-ce.schedd-ready'], 'CE schedd not ready to accept jobs')
         self.skip_ok_unless(service.is_running(core.config['slurm.service-name']), 'slurm service not running')
-        self.skip_bad_unless(core.state['proxy.valid'], 'requires a proxy cert')
+        self.check_schedd_ready()
+        self.check_write_creds()
         self.run_blahp_trace('slurm')
 
     def test_07_ceview(self):
         core.config['condor-ce.view-listening'] = False
-        self.general_requirements()
         core.skip_ok_unless_installed('htcondor-ce-view')
         view_url = 'http://%s:%s' % (core.get_hostname(), int(core.config['condor-ce.view-port']))
         try:
@@ -141,6 +146,9 @@ def test_07_ceview(self):
                 debug_contents += 'Failed to read %s\n' % debug_file
             core.log_message(debug_contents)
             self.fail('Could not reach HTCondor-CE View at %s: %s' % (view_url, err))
-        self.assert_(re.search(r'HTCondor-CE Overview', src), 'Failed to find expected CE View contents')
+        self.assertTrue(re.search(r'HTCondor-CE Overview', src), 'Failed to find expected CE View contents')
         core.config['condor-ce.view-listening'] = True
 
+    def test_08_config_val(self):
+        command = ('condor_ce_config_val', '-dump')
+        core.check_system(command, 'condor_ce_config_val as non-root', user=True)

osgtest/tests/test_790_condorce.py

@@ -14,5 +14,5 @@ def test_02_restore_config(self):
 
         files.restore(core.config['condor-ce.condor-cfg'], 'condor-ce')
         files.restore(core.config['condor-ce.condor-ce-cfg'], 'condor-ce')
-        if core.options.hostcert:
-            files.restore(core.config['condor-ce.condorce_mapfile'], 'condor-ce')
+        if core.state['condor-ce.wrote-mapfile']:
+            files.restore(core.config['condor-ce.mapfile'], 'condor-ce')