From 3f0f8720d67b067cb91a993cc8529e68643dc267 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 29 Jan 2026 15:09:03 -0700 Subject: [PATCH 01/10] wait_for_tests: Refactor and restore commit/time upload --- CIME/wait_for_tests.py | 364 ++++++++++++++++++++--------------------- 1 file changed, 175 insertions(+), 189 deletions(-) diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index 928b010ba30..735815c131b 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -1,6 +1,5 @@ # pylint: disable=import-error -import queue -import os, time, threading, socket, signal, shutil, glob +import queue, os, time, threading, socket, signal, shutil, glob, tempfile # pylint: disable=import-error import logging @@ -92,24 +91,16 @@ def create_cdash_xml_boiler( utc_time, current_time, hostname, - git_commit, ): ############################################################################### site_elem = xmlet.Element("Site") - if "JENKINS_START_TIME" in os.environ: - time_info_str = "Total testing time: {:d} seconds".format( - int(current_time) - int(os.environ["JENKINS_START_TIME"]) - ) - else: - time_info_str = "" - site_elem.attrib["BuildName"] = cdash_build_name site_elem.attrib["BuildStamp"] = "{}-{}".format(utc_time, cdash_build_group) site_elem.attrib["Name"] = hostname site_elem.attrib["OSName"] = "Linux" site_elem.attrib["Hostname"] = hostname - site_elem.attrib["OSVersion"] = "Commit: {}{}".format(git_commit, time_info_str) + site_elem.attrib["OSVersion"] = "Unknown" phase_elem = xmlet.SubElement(site_elem, phase) @@ -130,7 +121,6 @@ def create_cdash_config_xml( current_time, hostname, data_rel_path, - git_commit, ): ############################################################################### site_elem, config_elem = create_cdash_xml_boiler( @@ -140,7 +130,6 @@ def create_cdash_config_xml( utc_time, current_time, hostname, - git_commit, ) xmlet.SubElement(config_elem, "ConfigureCommand").text = "namelists" @@ -165,7 +154,7 @@ def create_cdash_config_xml( xmlet.SubElement(config_elem, "ElapsedMinutes").text = "0" # Skip for now etree = xmlet.ElementTree(site_elem) - etree.write(os.path.join(data_rel_path, "Configure.xml")) + etree.write(data_rel_path / "Configure.xml") ############################################################################### @@ -177,7 +166,6 @@ def create_cdash_build_xml( current_time, hostname, data_rel_path, - git_commit, ): ############################################################################### site_elem, build_elem = create_cdash_xml_boiler( @@ -187,7 +175,6 @@ def create_cdash_build_xml( utc_time, current_time, hostname, - git_commit, ) xmlet.SubElement(build_elem, "ConfigureCommand").text = "case.build" @@ -214,7 +201,7 @@ def create_cdash_build_xml( xmlet.SubElement(build_elem, "ElapsedMinutes").text = "0" # Skip for now etree = xmlet.ElementTree(site_elem) - etree.write(os.path.join(data_rel_path, "Build.xml")) + etree.write(data_rel_path / "Build.xml") ############################################################################### @@ -226,7 +213,6 @@ def create_cdash_test_xml( current_time, hostname, data_rel_path, - git_commit, ): ############################################################################### site_elem, testing_elem = create_cdash_xml_boiler( @@ -236,7 +222,6 @@ def create_cdash_test_xml( utc_time, current_time, hostname, - git_commit, ) test_list_elem = xmlet.SubElement(testing_elem, "TestList") @@ -298,28 +283,14 @@ def create_cdash_test_xml( xmlet.SubElement(testing_elem, "ElapsedMinutes").text = "0" # Skip for now etree = xmlet.ElementTree(site_elem) - - etree.write(os.path.join(data_rel_path, "Test.xml")) + etree.write(data_rel_path / "Test.xml") ############################################################################### def create_cdash_xml_fakes( - results, cdash_build_name, cdash_build_group, utc_time, current_time, hostname + results, cdash_build_name, cdash_build_group, utc_time, current_time, hostname, data_rel_path ): ############################################################################### - # We assume all cases were created from the same code repo - first_result_case = os.path.dirname(list(results.items())[0][1][0]) - try: - srcroot = run_cmd_no_fail( - "./xmlquery --value SRCROOT", from_dir=first_result_case - ) - except CIMEError: - # Use repo containing this script as last resort - srcroot = os.path.join(CIME.utils.get_cime_root(), "..") - - git_commit = CIME.utils.get_current_commit(repo=srcroot) - - data_rel_path = os.path.join("Testing", utc_time) create_cdash_config_xml( results, @@ -329,7 +300,6 @@ def create_cdash_xml_fakes( current_time, hostname, data_rel_path, - git_commit, ) create_cdash_build_xml( @@ -340,7 +310,6 @@ def create_cdash_xml_fakes( current_time, hostname, data_rel_path, - git_commit, ) create_cdash_test_xml( @@ -351,88 +320,72 @@ def create_cdash_xml_fakes( current_time, hostname, data_rel_path, - git_commit, ) - ############################################################################### def create_cdash_upload_xml( - results, cdash_build_name, cdash_build_group, utc_time, hostname, force_log_upload + results, cdash_build_name, cdash_build_group, utc_time, hostname, force_log_upload, tmp_path, data_rel_path ): ############################################################################### - data_rel_path = os.path.join("Testing", utc_time) - - try: - log_dir = "{}_logs".format(cdash_build_name) - - need_to_upload = False + log_dirname = f"{cdash_build_name}_logs" + log_path = tmp_path / log_dirname + + need_to_upload = False + + for test_name, test_data in results.items(): + test_path, test_status, _ = test_data + + if test_status != TEST_PASS_STATUS or force_log_upload: + test_case_dir = os.path.dirname(test_path) + + case_dirs = [test_case_dir] + case_base = os.path.basename(test_case_dir) + test_case2_dir = os.path.join(test_case_dir, "case2", case_base) + if os.path.exists(test_case2_dir): + case_dirs.append(test_case2_dir) + + for case_dir in case_dirs: + for param in ["EXEROOT", "RUNDIR", "CASEDIR"]: + if param == "CASEDIR": + log_src_dir = case_dir + else: + # it's possible that tests that failed very badly/early, and fake cases for testing + # will not be able to support xmlquery + try: + log_src_dir = run_cmd_no_fail( + "./xmlquery {} --value".format(param), + from_dir=case_dir, + ) + except: + continue + + log_dst_dir = log_path / "{}{}_{}_logs".format( + test_name, + "" if case_dir == test_case_dir else ".case2", + param, + ) + log_dst_dir.mkdir(parents=True) + for log_file in glob.glob(os.path.join(log_src_dir, "*log*")): + if os.path.isdir(log_file): + shutil.copytree(log_file, log_dst_dir / os.path.basename(log_file)) + else: + safe_copy(log_file, str(log_dst_dir)) + for log_file in glob.glob(os.path.join(log_src_dir, "*.cprnc.out*")): + safe_copy(log_file, str(log_dst_dir)) - for test_name, test_data in results.items(): - test_path, test_status, _ = test_data + need_to_upload = True - if test_status != TEST_PASS_STATUS or force_log_upload: - test_case_dir = os.path.dirname(test_path) + if need_to_upload: - case_dirs = [test_case_dir] - case_base = os.path.basename(test_case_dir) - test_case2_dir = os.path.join(test_case_dir, "case2", case_base) - if os.path.exists(test_case2_dir): - case_dirs.append(test_case2_dir) + tarball = "{}.tar.gz".format(log_dirname) - for case_dir in case_dirs: - for param in ["EXEROOT", "RUNDIR", "CASEDIR"]: - if param == "CASEDIR": - log_src_dir = case_dir - else: - # it's possible that tests that failed very badly/early, and fake cases for testing - # will not be able to support xmlquery - try: - log_src_dir = run_cmd_no_fail( - "./xmlquery {} --value".format(param), - from_dir=case_dir, - ) - except: - continue - - log_dst_dir = os.path.join( - log_dir, - "{}{}_{}_logs".format( - test_name, - "" if case_dir == test_case_dir else ".case2", - param, - ), - ) - os.makedirs(log_dst_dir) - for log_file in glob.glob(os.path.join(log_src_dir, "*log*")): - if os.path.isdir(log_file): - shutil.copytree( - log_file, - os.path.join( - log_dst_dir, os.path.basename(log_file) - ), - ) - else: - safe_copy(log_file, log_dst_dir) - for log_file in glob.glob( - os.path.join(log_src_dir, "*.cprnc.out*") - ): - safe_copy(log_file, log_dst_dir) - - need_to_upload = True - - if need_to_upload: - - tarball = "{}.tar.gz".format(log_dir) - if os.path.exists(tarball): - os.remove(tarball) - - run_cmd_no_fail( - "tar -cf - {} | gzip -c".format(log_dir), arg_stdout=tarball - ) - base64 = run_cmd_no_fail("base64 {}".format(tarball)) + run_cmd_no_fail( + "tar -cf - {} | gzip -c".format(log_dirname), arg_stdout=tarball, from_dir=str(tmp_path) + ) + base64 = run_cmd_no_fail("base64 {}".format(tarball), from_dir=str(tmp_path)) - xml_text = r""" + xml_text = r""" "?> @@ -444,20 +397,16 @@ def create_cdash_upload_xml( """.format( - cdash_build_name, - utc_time, - cdash_build_group, - hostname, - os.path.abspath(tarball), - base64, - ) - - with open(os.path.join(data_rel_path, "Upload.xml"), "w") as fd: - fd.write(xml_text) + cdash_build_name, + utc_time, + cdash_build_group, + hostname, + str((tmp_path / tarball).absolute()), + base64, +) - finally: - if os.path.isdir(log_dir): - shutil.rmtree(log_dir) + with (data_rel_path / "Upload.xml").open(mode="w") as fd: + fd.write(xml_text) ############################################################################### @@ -482,84 +431,121 @@ def create_cdash_xml( "Could not convert hostname '{}' into an E3SM machine name".format(hostname) ) - for drop_method in ["https", "http"]: - dart_config = """ -SourceDirectory: {0} -BuildDirectory: {0} - -# Site is something like machine.domain, i.e. pragmatic.crd -Site: {1} - -# Build name is osname-revision-compiler, i.e. Linux-2.4.2-2smp-c++ -BuildName: {2} - -# Submission information -IsCDash: TRUE -CDashVersion: -QueryCDashVersion: -DropSite: my.cdash.org -DropLocation: /submit.php?project={3} -DropSiteUser: -DropSitePassword: -DropSiteMode: -DropMethod: {6} -TriggerSite: -ScpCommand: {4} - -# Dashboard start time -NightlyStartTime: {5} UTC - -UseLaunchers: -CurlOptions: CURLOPT_SSL_VERIFYPEER_OFF;CURLOPT_SSL_VERIFYHOST_OFF -""".format( - os.getcwd(), - hostname, - cdash_build_name, - cdash_project, - shutil.which("scp"), - cdash_timestamp, - drop_method, + # We assume all cases were created from the same code repo + first_result_case = os.path.dirname(list(results.items())[0][1][0]) + try: + srcroot = run_cmd_no_fail( + "./xmlquery --value SRCROOT", from_dir=first_result_case ) + except CIMEError: + # Use repo containing this script as last resort + srcroot = os.path.join(CIME.utils.get_cime_root(), "..") - with open("DartConfiguration.tcl", "w") as dart_fd: - dart_fd.write(dart_config) + git_commit = CIME.utils.get_current_commit(repo=srcroot) - utc_time = time.strftime("%Y%m%d-%H%M", utc_time_tuple) - testing_dir = os.path.join("Testing", utc_time) - if os.path.isdir(testing_dir): - shutil.rmtree(testing_dir) + # Get total elapsed time + if "JENKINS_START_TIME" in os.environ: + time_info = int(current_time) - int(os.environ["JENKINS_START_TIME"]) + ) + else: + time_info = "unknown" + + prefixes = [None, first_result_case, os.getcwd()] + for prexix in prefixes: + try: + with tempfile.TemporaryDirectory(prefix=prefix) as tmpdir: + tmp_path = Path(tmpdir) + utc_time = time.strftime("%Y%m%d-%H%M", utc_time_tuple) + dart_path = tmp_path / "DartConfiguration.tcl" + testing_path = tmp_path / "Testing" + testtime_dir = testing_path / utc_time # Most action happens here + tag_file = testing_path / "TAG" + notes_file = tmp_path / "notes.txt" + + testtime_dir.mkdir(parents=True) + + # Make tag file + with tag_file.open(mode="w") as tag_fd: + tag_fd.write(f"{utc_time}\n{cdash_build_group}\n") + + # Make notes file + with notes_file.open(mode="w") as notes_fd: + notes_fd.write(f"Commit {git_commit}\nTotal testing time {time_info} seconds\n") + + create_cdash_xml_fakes( + results, + cdash_build_name, + cdash_build_group, + utc_time, + current_time, + hostname, + testtime_dir + ) - os.makedirs(os.path.join("Testing", utc_time)) + create_cdash_upload_xml( + results, + cdash_build_name, + cdash_build_group, + utc_time, + hostname, + force_log_upload, + tmp_path, + testtime_dir + ) - # Make tag file - with open("Testing/TAG", "w") as tag_fd: - tag_fd.write("{}\n{}\n".format(utc_time, cdash_build_group)) + for drop_method in ["https", "http"]: + dart_config = """ + SourceDirectory: {0} + BuildDirectory: {0} + + # Site is something like machine.domain, i.e. pragmatic.crd + Site: {1} + + # Build name is osname-revision-compiler, i.e. Linux-2.4.2-2smp-c++ + BuildName: {2} + + # Submission information + IsCDash: TRUE + CDashVersion: + QueryCDashVersion: + DropSite: my.cdash.org + DropLocation: /submit.php?project={3} + DropSiteUser: + DropSitePassword: + DropSiteMode: + DropMethod: {6} + TriggerSite: + ScpCommand: {4} + + # Dashboard start time + NightlyStartTime: {5} UTC + + UseLaunchers: + CurlOptions: CURLOPT_SSL_VERIFYPEER_OFF;CURLOPT_SSL_VERIFYHOST_OFF + """.format( + str(tmp_path.absolute()), + hostname, + cdash_build_name, + cdash_project, + shutil.which("scp"), + cdash_timestamp, + drop_method, + ) + with dart_path.open(mode="w") as dart_fd: + dart_fd.write(dart_config) - create_cdash_xml_fakes( - results, - cdash_build_name, - cdash_build_group, - utc_time, - current_time, - hostname, - ) + stat, out, _ = run_cmd("ctest -VV -D NightlySubmit", combine_output=True, from_dir=str(tmp_path)) + if stat != 0: + logging.warning( + "ctest upload drop method {} FAILED:\n{}".format(drop_method, out) + ) + else: + logging.info("Upload SUCCESS:\n{}".format(out)) - create_cdash_upload_xml( - results, - cdash_build_name, - cdash_build_group, - utc_time, - hostname, - force_log_upload, - ) + except Exception as e: + logging.info(f"Prexix '{prefix}' failed with error {e}") - stat, out, _ = run_cmd("ctest -VV -D NightlySubmit", combine_output=True) - if stat != 0: - logging.warning( - "ctest upload drop method {} FAILED:\n{}".format(drop_method, out) - ) else: - logging.info("Upload SUCCESS:\n{}".format(out)) return expect(False, "All cdash upload attempts failed") From 70ddd2df7ff349a53118d1df0101b20613554310 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 29 Jan 2026 15:11:26 -0700 Subject: [PATCH 02/10] Fix --- CIME/wait_for_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index 735815c131b..c49f4e601ad 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -446,7 +446,6 @@ def create_cdash_xml( # Get total elapsed time if "JENKINS_START_TIME" in os.environ: time_info = int(current_time) - int(os.environ["JENKINS_START_TIME"]) - ) else: time_info = "unknown" From 4a4e1d09b36107dfc827166cafef21a4199486fe Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 29 Jan 2026 15:14:18 -0700 Subject: [PATCH 03/10] Fix --- CIME/wait_for_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index c49f4e601ad..e8aa3de80e6 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -1,5 +1,6 @@ # pylint: disable=import-error import queue, os, time, threading, socket, signal, shutil, glob, tempfile +from pathlib import Path # pylint: disable=import-error import logging @@ -450,7 +451,7 @@ def create_cdash_xml( time_info = "unknown" prefixes = [None, first_result_case, os.getcwd()] - for prexix in prefixes: + for prefix in prefixes: try: with tempfile.TemporaryDirectory(prefix=prefix) as tmpdir: tmp_path = Path(tmpdir) From 93bea972593966795ae58dfe5ba8920e95ed05ea Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 29 Jan 2026 15:45:42 -0700 Subject: [PATCH 04/10] Fixes --- CIME/wait_for_tests.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index e8aa3de80e6..1174aed6bca 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -534,20 +534,18 @@ def create_cdash_xml( with dart_path.open(mode="w") as dart_fd: dart_fd.write(dart_config) - stat, out, _ = run_cmd("ctest -VV -D NightlySubmit", combine_output=True, from_dir=str(tmp_path)) + stat, out, _ = run_cmd("ctest -VV -D NightlySubmit -A notes.txt", combine_output=True, from_dir=str(tmp_path)) if stat != 0: logging.warning( "ctest upload drop method {} FAILED:\n{}".format(drop_method, out) ) else: logging.info("Upload SUCCESS:\n{}".format(out)) + return except Exception as e: logging.info(f"Prexix '{prefix}' failed with error {e}") - else: - return - expect(False, "All cdash upload attempts failed") From 5214b170c3eeb266d01b09b083bb5fcbf654d0ef Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 29 Jan 2026 14:46:54 -0800 Subject: [PATCH 05/10] black --- CIME/wait_for_tests.py | 82 ++++++++++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index 1174aed6bca..0986eef2205 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -289,7 +289,13 @@ def create_cdash_test_xml( ############################################################################### def create_cdash_xml_fakes( - results, cdash_build_name, cdash_build_group, utc_time, current_time, hostname, data_rel_path + results, + cdash_build_name, + cdash_build_group, + utc_time, + current_time, + hostname, + data_rel_path, ): ############################################################################### @@ -323,14 +329,22 @@ def create_cdash_xml_fakes( data_rel_path, ) + ############################################################################### def create_cdash_upload_xml( - results, cdash_build_name, cdash_build_group, utc_time, hostname, force_log_upload, tmp_path, data_rel_path + results, + cdash_build_name, + cdash_build_group, + utc_time, + hostname, + force_log_upload, + tmp_path, + data_rel_path, ): ############################################################################### log_dirname = f"{cdash_build_name}_logs" - log_path = tmp_path / log_dirname + log_path = tmp_path / log_dirname need_to_upload = False @@ -369,10 +383,14 @@ def create_cdash_upload_xml( log_dst_dir.mkdir(parents=True) for log_file in glob.glob(os.path.join(log_src_dir, "*log*")): if os.path.isdir(log_file): - shutil.copytree(log_file, log_dst_dir / os.path.basename(log_file)) + shutil.copytree( + log_file, log_dst_dir / os.path.basename(log_file) + ) else: safe_copy(log_file, str(log_dst_dir)) - for log_file in glob.glob(os.path.join(log_src_dir, "*.cprnc.out*")): + for log_file in glob.glob( + os.path.join(log_src_dir, "*.cprnc.out*") + ): safe_copy(log_file, str(log_dst_dir)) need_to_upload = True @@ -382,7 +400,9 @@ def create_cdash_upload_xml( tarball = "{}.tar.gz".format(log_dirname) run_cmd_no_fail( - "tar -cf - {} | gzip -c".format(log_dirname), arg_stdout=tarball, from_dir=str(tmp_path) + "tar -cf - {} | gzip -c".format(log_dirname), + arg_stdout=tarball, + from_dir=str(tmp_path), ) base64 = run_cmd_no_fail("base64 {}".format(tarball), from_dir=str(tmp_path)) @@ -398,13 +418,13 @@ def create_cdash_upload_xml( """.format( - cdash_build_name, - utc_time, - cdash_build_group, - hostname, - str((tmp_path / tarball).absolute()), - base64, -) + cdash_build_name, + utc_time, + cdash_build_group, + hostname, + str((tmp_path / tarball).absolute()), + base64, + ) with (data_rel_path / "Upload.xml").open(mode="w") as fd: fd.write(xml_text) @@ -458,7 +478,7 @@ def create_cdash_xml( utc_time = time.strftime("%Y%m%d-%H%M", utc_time_tuple) dart_path = tmp_path / "DartConfiguration.tcl" testing_path = tmp_path / "Testing" - testtime_dir = testing_path / utc_time # Most action happens here + testtime_dir = testing_path / utc_time # Most action happens here tag_file = testing_path / "TAG" notes_file = tmp_path / "notes.txt" @@ -470,7 +490,9 @@ def create_cdash_xml( # Make notes file with notes_file.open(mode="w") as notes_fd: - notes_fd.write(f"Commit {git_commit}\nTotal testing time {time_info} seconds\n") + notes_fd.write( + f"Commit {git_commit}\nTotal testing time {time_info} seconds\n" + ) create_cdash_xml_fakes( results, @@ -479,7 +501,7 @@ def create_cdash_xml( utc_time, current_time, hostname, - testtime_dir + testtime_dir, ) create_cdash_upload_xml( @@ -490,7 +512,7 @@ def create_cdash_xml( hostname, force_log_upload, tmp_path, - testtime_dir + testtime_dir, ) for drop_method in ["https", "http"]: @@ -523,21 +545,27 @@ def create_cdash_xml( UseLaunchers: CurlOptions: CURLOPT_SSL_VERIFYPEER_OFF;CURLOPT_SSL_VERIFYHOST_OFF """.format( - str(tmp_path.absolute()), - hostname, - cdash_build_name, - cdash_project, - shutil.which("scp"), - cdash_timestamp, - drop_method, - ) + str(tmp_path.absolute()), + hostname, + cdash_build_name, + cdash_project, + shutil.which("scp"), + cdash_timestamp, + drop_method, + ) with dart_path.open(mode="w") as dart_fd: dart_fd.write(dart_config) - stat, out, _ = run_cmd("ctest -VV -D NightlySubmit -A notes.txt", combine_output=True, from_dir=str(tmp_path)) + stat, out, _ = run_cmd( + "ctest -VV -D NightlySubmit -A notes.txt", + combine_output=True, + from_dir=str(tmp_path), + ) if stat != 0: logging.warning( - "ctest upload drop method {} FAILED:\n{}".format(drop_method, out) + "ctest upload drop method {} FAILED:\n{}".format( + drop_method, out + ) ) else: logging.info("Upload SUCCESS:\n{}".format(out)) From e43b9acccb26165bd5124a092fd9c057c26b7648 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 30 Jan 2026 09:15:13 -0700 Subject: [PATCH 06/10] Fixes from copilot --- CIME/wait_for_tests.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index 0986eef2205..6900997fc52 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -7,7 +7,7 @@ import xml.etree.ElementTree as xmlet import CIME.utils -from CIME.utils import expect, Timeout, run_cmd_no_fail, safe_copy, CIMEError +from CIME.utils import expect, Timeout, run_cmd, run_cmd_no_fail, safe_copy, CIMEError from CIME.XML.machines import Machines from CIME.test_status import * from CIME.provenance import save_test_success @@ -372,7 +372,7 @@ def create_cdash_upload_xml( "./xmlquery {} --value".format(param), from_dir=case_dir, ) - except: + except CIMEError: continue log_dst_dir = log_path / "{}{}_{}_logs".format( @@ -470,10 +470,10 @@ def create_cdash_xml( else: time_info = "unknown" - prefixes = [None, first_result_case, os.getcwd()] - for prefix in prefixes: + tmproots = [None, first_result_case, os.getcwd()] + for tmproot in tmproots: try: - with tempfile.TemporaryDirectory(prefix=prefix) as tmpdir: + with tempfile.TemporaryDirectory(dir=tmproot) as tmpdir: tmp_path = Path(tmpdir) utc_time = time.strftime("%Y%m%d-%H%M", utc_time_tuple) dart_path = tmp_path / "DartConfiguration.tcl" @@ -572,7 +572,7 @@ def create_cdash_xml( return except Exception as e: - logging.info(f"Prexix '{prefix}' failed with error {e}") + logging.info(f"Temp dir '{tmpdir}' failed with error {e}") expect(False, "All cdash upload attempts failed") From 7d7b7b742a4002bf9c1e2fafad396fd1b482c59e Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 30 Jan 2026 11:10:44 -0700 Subject: [PATCH 07/10] Add support for picking a tmproot. Add support for testing --- CIME/Tools/wait_for_tests | 12 ++++++++++-- CIME/tests/test_sys_wait_for_tests.py | 4 ++++ CIME/wait_for_tests.py | 20 +++++++++++++++++--- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/CIME/Tools/wait_for_tests b/CIME/Tools/wait_for_tests index c166061c99b..484b393b6f4 100755 --- a/CIME/Tools/wait_for_tests +++ b/CIME/Tools/wait_for_tests @@ -87,7 +87,7 @@ OR ) parser.add_argument( - "--force-log-upload", + "--cdash-force-log-upload", action="store_true", help="Always upload logs to cdash, even if test passed", ) @@ -105,6 +105,11 @@ OR help="The name of the CDash project where results should be uploaded", ) + parser.add_argument( + "--cdash-tmproot", + help="Where to put temporary files needed to do cdash submission. Default=/tmp", + ) + parser.add_argument( "-g", "--cdash-build-group", @@ -132,9 +137,10 @@ OR args.ignore_memleak, args.cdash_build_name, args.cdash_project, + args.cdash_tmproot, args.cdash_build_group, args.timeout, - args.force_log_upload, + args.cdash_force_log_upload, args.no_run, args.update_success, ) @@ -153,6 +159,7 @@ def _main_func(description): ignore_memleak, cdash_build_name, cdash_project, + cdash_tmproot, cdash_build_group, timeout, force_log_upload, @@ -172,6 +179,7 @@ def _main_func(description): ignore_memleak=ignore_memleak, cdash_build_name=cdash_build_name, cdash_project=cdash_project, + cdash_tmproot=cdash_tmproot, cdash_build_group=cdash_build_group, timeout=timeout, force_log_upload=force_log_upload, diff --git a/CIME/tests/test_sys_wait_for_tests.py b/CIME/tests/test_sys_wait_for_tests.py index 0377d65771e..3484c9af7a5 100644 --- a/CIME/tests/test_sys_wait_for_tests.py +++ b/CIME/tests/test_sys_wait_for_tests.py @@ -9,6 +9,7 @@ from CIME import utils from CIME import test_status +from CIME.wait_for_tests import ENV_VAR_KEEP_CDASH from CIME.tests import base from CIME.tests import utils as test_utils @@ -110,6 +111,8 @@ def tearDown(self): for testdir in self._testdirs: shutil.rmtree(testdir) + os.environ.pop(ENV_VAR_KEEP_CDASH, None) + def simple_test(self, testdir, expected_results, extra_args="", build_name=None): # Need these flags to test dashboard if e3sm if self._config.create_test_flag_mode == "e3sm" and build_name is not None: @@ -270,6 +273,7 @@ def test_wait_for_test_cdash_pass(self): def test_wait_for_test_cdash_kill(self): expected_results = ["PEND" if item == 5 else "PASS" for item in range(10)] + os.environ[ENV_VAR_KEEP_CDASH] = "True" build_name = "regression_test_kill_" + self._timestamp run_thread = threading.Thread( target=self.threaded_test, diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index 6900997fc52..91cc60e6206 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -17,6 +17,7 @@ E3SM_MAIN_CDASH = "E3SM" CDASH_DEFAULT_BUILD_GROUP = "ACME_Latest" SLEEP_INTERVAL_SEC = 0.1 +ENV_VAR_KEEP_CDASH = "CIME_TEST_CDASH_WFT" ############################################################################### def signal_handler(*_): @@ -432,7 +433,7 @@ def create_cdash_upload_xml( ############################################################################### def create_cdash_xml( - results, cdash_build_name, cdash_project, cdash_build_group, force_log_upload=False + results, cdash_build_name, cdash_project, cdash_build_group, force_log_upload=False, cdash_tmproot=None ): ############################################################################### @@ -470,7 +471,14 @@ def create_cdash_xml( else: time_info = "unknown" - tmproots = [None, first_result_case, os.getcwd()] + if cdash_tmproot: + tmproots = [cdash_tmproot] + else: + tmproots = [None, first_result_case, os.getcwd()] + + # Try multiple tmproots if necessary. The default /tmp will be tried first + # unless cdash_tmproot was provided. The location of the default can be + # modified via the TMPDIR environment variable. for tmproot in tmproots: try: with tempfile.TemporaryDirectory(dir=tmproot) as tmpdir: @@ -569,6 +577,10 @@ def create_cdash_xml( ) else: logging.info("Upload SUCCESS:\n{}".format(out)) + if ENV_VAR_KEEP_CDASH in os.environ: + logging.info(f"Test mode enabled, copying {str(tmp_path)} to {os.getcwd()}") + safe_copy(str(tmp_path / "Testing"), os.getcwd()) + return except Exception as e: @@ -739,6 +751,7 @@ def wait_for_tests( ignore_memleak=False, cdash_build_name=None, cdash_project=E3SM_MAIN_CDASH, + cdash_tmproot=None, cdash_build_group=CDASH_DEFAULT_BUILD_GROUP, timeout=None, force_log_upload=False, @@ -839,12 +852,13 @@ def wait_for_tests( ) if cdash_build_name: - create_cdash_xml( + tmpdir = create_cdash_xml( test_results, cdash_build_name, cdash_project, cdash_build_group, force_log_upload, + cdash_tmproot ) return all_pass From 303fe74f39c8babb202a8ced15b0485c9f49fa4a Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 30 Jan 2026 10:13:27 -0800 Subject: [PATCH 08/10] black --- CIME/wait_for_tests.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index 91cc60e6206..b971aba6855 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -433,7 +433,12 @@ def create_cdash_upload_xml( ############################################################################### def create_cdash_xml( - results, cdash_build_name, cdash_project, cdash_build_group, force_log_upload=False, cdash_tmproot=None + results, + cdash_build_name, + cdash_project, + cdash_build_group, + force_log_upload=False, + cdash_tmproot=None, ): ############################################################################### @@ -578,7 +583,9 @@ def create_cdash_xml( else: logging.info("Upload SUCCESS:\n{}".format(out)) if ENV_VAR_KEEP_CDASH in os.environ: - logging.info(f"Test mode enabled, copying {str(tmp_path)} to {os.getcwd()}") + logging.info( + f"Test mode enabled, copying {str(tmp_path)} to {os.getcwd()}" + ) safe_copy(str(tmp_path / "Testing"), os.getcwd()) return @@ -858,7 +865,7 @@ def wait_for_tests( cdash_project, cdash_build_group, force_log_upload, - cdash_tmproot + cdash_tmproot, ) return all_pass From 8831bb4f14633c680e7a40ff60618e35df419e48 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 30 Jan 2026 11:51:12 -0700 Subject: [PATCH 09/10] Fix copilot review items --- CIME/wait_for_tests.py | 76 +++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index b971aba6855..f2b58e8cda9 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -530,42 +530,42 @@ def create_cdash_xml( for drop_method in ["https", "http"]: dart_config = """ - SourceDirectory: {0} - BuildDirectory: {0} - - # Site is something like machine.domain, i.e. pragmatic.crd - Site: {1} - - # Build name is osname-revision-compiler, i.e. Linux-2.4.2-2smp-c++ - BuildName: {2} - - # Submission information - IsCDash: TRUE - CDashVersion: - QueryCDashVersion: - DropSite: my.cdash.org - DropLocation: /submit.php?project={3} - DropSiteUser: - DropSitePassword: - DropSiteMode: - DropMethod: {6} - TriggerSite: - ScpCommand: {4} - - # Dashboard start time - NightlyStartTime: {5} UTC - - UseLaunchers: - CurlOptions: CURLOPT_SSL_VERIFYPEER_OFF;CURLOPT_SSL_VERIFYHOST_OFF - """.format( - str(tmp_path.absolute()), - hostname, - cdash_build_name, - cdash_project, - shutil.which("scp"), - cdash_timestamp, - drop_method, - ) +SourceDirectory: {0} +BuildDirectory: {0} + +# Site is something like machine.domain, i.e. pragmatic.crd +Site: {1} + +# Build name is osname-revision-compiler, i.e. Linux-2.4.2-2smp-c++ +BuildName: {2} + +# Submission information +IsCDash: TRUE +CDashVersion: +QueryCDashVersion: +DropSite: my.cdash.org +DropLocation: /submit.php?project={3} +DropSiteUser: +DropSitePassword: +DropSiteMode: +DropMethod: {6} +TriggerSite: +ScpCommand: {4} + +# Dashboard start time +NightlyStartTime: {5} UTC + +UseLaunchers: +CurlOptions: CURLOPT_SSL_VERIFYPEER_OFF;CURLOPT_SSL_VERIFYHOST_OFF +""".format( + str(tmp_path.absolute()), + hostname, + cdash_build_name, + cdash_project, + shutil.which("scp"), + cdash_timestamp, + drop_method, +) with dart_path.open(mode="w") as dart_fd: dart_fd.write(dart_config) @@ -591,7 +591,7 @@ def create_cdash_xml( return except Exception as e: - logging.info(f"Temp dir '{tmpdir}' failed with error {e}") + logging.warning(f"Using temp root '{tmproot}', cdash submission failed with error {e}") expect(False, "All cdash upload attempts failed") @@ -859,7 +859,7 @@ def wait_for_tests( ) if cdash_build_name: - tmpdir = create_cdash_xml( + create_cdash_xml( test_results, cdash_build_name, cdash_project, From b8f71257bec375acd059ff68b07d64e8809ff8b4 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 30 Jan 2026 10:52:29 -0800 Subject: [PATCH 10/10] black --- CIME/wait_for_tests.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index f2b58e8cda9..1f6940138a9 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -558,14 +558,14 @@ def create_cdash_xml( UseLaunchers: CurlOptions: CURLOPT_SSL_VERIFYPEER_OFF;CURLOPT_SSL_VERIFYHOST_OFF """.format( - str(tmp_path.absolute()), - hostname, - cdash_build_name, - cdash_project, - shutil.which("scp"), - cdash_timestamp, - drop_method, -) + str(tmp_path.absolute()), + hostname, + cdash_build_name, + cdash_project, + shutil.which("scp"), + cdash_timestamp, + drop_method, + ) with dart_path.open(mode="w") as dart_fd: dart_fd.write(dart_config) @@ -591,7 +591,9 @@ def create_cdash_xml( return except Exception as e: - logging.warning(f"Using temp root '{tmproot}', cdash submission failed with error {e}") + logging.warning( + f"Using temp root '{tmproot}', cdash submission failed with error {e}" + ) expect(False, "All cdash upload attempts failed")