Dump condor_ce_trace held job reasons

Author: brianhlin

osgtest/tests/test_550_condorce.py

@@ -40,17 +40,33 @@ def check_schedd_ready(self):
         """
         self.skip_bad_unless(core.state['condor-ce.schedd-ready'], 'CE schedd not ready to accept jobs')
 
+    def run_trace(self, *args):
+        """Run condor_ce_trace along with any additional *args. If trace completes with a held job, also return output
+        from 'condor_ce_q -held'.
+        """
+
+        cwd = os.getcwd()
+        os.chdir('/tmp')
+        self.command += ['condor_ce_trace', '--debug'] + list(args) + [core.get_hostname()]
+        trace_rc, stdout, stderr = core.system(self.command, user=True)
+        os.chdir(cwd)
+
+        if trace_rc:
+            msg = 'condor_ce_trace failed'
+            if stdout.find(', was held'):
+                msg = 'condor_ce_trace job held'
+                _, hold_out, hold_err = core.system(('condor_ce_q', '-held'))
+                stdout += hold_out
+                stderr += hold_err
+            self.fail(core.diagnose(msg, self.command, trace_rc, stdout, stderr))
+
+        return stdout, stderr
+
     def run_blahp_trace(self, lrms):
         """Run condor_ce_trace() against a non-HTCondor backend and verify the cache"""
         lrms_cache_prefix = {'pbs': 'qstat', 'slurm': 'slurm'}
 
-        cwd = os.getcwd()
-        os.chdir('/tmp')
-        self.command += ['condor_ce_trace',
-                         '-a osgTestBatchSystem = %s' % lrms.lower(),
-                         '--debug',
-                         core.get_hostname()]
-        trace_out, _, _ = core.check_system(self.command, 'ce trace against %s' % lrms.lower(), user=True)
+        trace_out, _ = self.run_trace(f'-a osgTestBatchSystem = {lrms.lower()}')
 
         try:
             backend_jobid = re.search(r'%s_JOBID=(\d+)' % lrms.upper(), trace_out).group(1)
@@ -86,8 +102,6 @@ def run_blahp_trace(self, lrms):
         self.assertTrue(re.search(r'BatchJobId[=\s"\'p1S]+%s' % backend_jobid, cache),
                         'Job %s not found in %s blahp cache:\n%s' % (backend_jobid, lrms.upper(), cache))
 
-        os.chdir(cwd)
-
     def test_01_status(self):
         command = ('condor_ce_status', '-any')
         core.check_system(command, 'ce status', user=True)
@@ -105,14 +119,8 @@ def test_03_ping(self):
     def test_04_trace(self):
         self.check_schedd_ready()
         self.check_write_creds()
+        self.run_trace()
 
-        cwd = os.getcwd()
-        os.chdir('/tmp')
-
-        self.command += ['condor_ce_trace', '--debug', core.get_hostname()]
-        core.check_system(self.command, 'ce trace', user=True)
-
-        os.chdir(cwd)
 
     def test_05_pbs_trace(self):
         core.skip_ok_unless_installed('torque-mom', 'torque-server', 'torque-scheduler', 'torque-client', 'munge',