diff --git a/bin/extract-job-output b/bin/extract-job-output index 0f1e4c4..f774e2e 100755 --- a/bin/extract-job-output +++ b/bin/extract-job-output @@ -61,6 +61,9 @@ for image_path in paths: # print " - creating directory" os.makedirs(local_dir) # print " - extracting" - g.download(full_image_path, os.path.join(output_dir, image_path)) + try: + g.download(full_image_path, os.path.join(output_dir, image_path)) + except Exception as e: + print("Can't download path %s" % full_image_path, e) print "ok" diff --git a/bin/run-job b/bin/run-job index 9467be8..526f452 100755 --- a/bin/run-job +++ b/bin/run-job @@ -44,7 +44,7 @@ run_command_with_retries() osg_series=`grep series $INPUT_DIR/osg-test.conf | sed -e 's/series *= *//'` # Set variables based on OS major version -os_major_version=`sed -e 's/^[^0-9]*//' -e 's/\..*$//' /etc/redhat-release` +os_major_version=`sed -e 's/^[^0-9]*//' -e 's/[^0-9].*$//' /etc/redhat-release` epel_url="https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm" # Starting with OSG 23, we dump release RPMs in the XX-main part of the tree @@ -67,6 +67,11 @@ case $os_major_version in python_lib_dir='/usr/lib/python3.9' python='/usr/bin/python3' ;; + 10 ) + priorities_rpm= + python_lib_dir='/usr/lib/python3.12' + python='/usr/bin/python3' + ;; * ) echo "Could not determine OS major version from '$os_major_version'" exit 2 diff --git a/bin/vmu.py b/bin/vmu.py index 6b14325..8d084de 100755 --- a/bin/vmu.py +++ b/bin/vmu.py @@ -83,7 +83,7 @@ def canonical_os_string(os_release, param_name=False): result = result.replace('CentOS Stream', 'C. Stream') result = result.replace('Rocky Linux', 'Rocky') result = result.replace('AlmaLinux', 'Alma') - result = re.sub(r'(\d)(\.\d+)?(.*)', r'\1 (\3)', result) + result = re.sub(r'(\d{1,2})(\.\d+)?(.*)', r'\1 (\3)', result) else: # Handle OS string from 'platforms' test parameters result = os_release.replace('rhel', 'RHEL') @@ -92,7 +92,7 @@ def canonical_os_string(os_release, param_name=False): result = result.replace('centos', 'CentOS') result = result.replace('rocky', 'Rocky') result = result.replace('alma', 'Alma') - result = re.sub(r'_(\d)\.(.*)', r' \1 (\2)', result) + result = re.sub(r'_(\d{1,2})\.(.*)', r' \1 (\2)', result) return result def canonical_src_string(sources): diff --git a/parameters.d/osg25-el10-aarch64.yaml b/parameters.d/osg25-el10-aarch64.yaml new file mode 100644 index 0000000..0f2e7e1 --- /dev/null +++ b/parameters.d/osg25-el10-aarch64.yaml @@ -0,0 +1,87 @@ +################### +# OSG 24 tests for EL9 +# File format documention: +# https://github.com/opensciencegrid/vm-test-runs#running-osg-test-in-vm-universe +################### + +platforms: + - centos_stream_10.aarch64 + - rocky_10.aarch64 + - alma_10.aarch64 + +sources: + ################### + # Format: + # [:;] ; [> /] + # Example: + # Run osg-test (from 3.2-minefield) with packages from 3.2-release and 3.2-testing that are then upgraded to + # 3.3-testing and 3-3-upcoming-testing: + # 3.2; osg, osg-testing > 3.3/osg-testing, osg-upcoming-testing + ################### + - opensciencegrid:master; 25; osg-minefield + +package_sets: + #### Required #### + # label - used for reporting, should be consistent across param files + # packages - list of packages to install in the test run + #### Optional #### + # selinux - enable SELinux for the package set, otherwise Permissive mode (default: True) + # osg_java - Pre-install OSG java packages (default: False) + # rng - Install entropy generation package (default: False) + ################## + - label: Compute Entrypoint (Condor) + packages: + - osg-ce-condor + - htcondor-ce-view + - label: Compute Entrypoint (Slurm) + packages: + - osg-ce-slurm + - htcondor-ce-view + - slurm + - slurm-slurmd + - slurm-slurmctld + - slurm-perlapi + - slurm-slurmdbd + - mariadb-server + - label: Compute Entrypoint (Torque) + packages: + - osg-ce-pbs + - htcondor-ce-view + - torque + - torque-server + - torque-mom + - torque-client + - torque-scheduler + - mariadb-server + - label: Central Collector + packages: + - htcondor-ce-collector + - htcondor-ce-view + - fetch-crl + - label: Worker Node (privileged) + packages: + - osg-wn-client + - osg-oasis + - apptainer-suid + - label: Worker Node (privileged, tarball deps) + packages: + - osg-update-data + - osg-wn-client + - label: XRootD + packages: + - osg-xrootd-standalone + - xrootd-multiuser + - xrootd-client + - voms-clients-cpp + - label: GlideinwmsFrontend + packages: + - glideinwms-vofrontend + - label: GlideinwmsFactory + packages: + - glideinwms-factory + - label: HTVault-Config + packages: + - htvault-config + - htgettoken + + diff --git a/parameters.d/osg25-el10.yaml b/parameters.d/osg25-el10.yaml new file mode 100644 index 0000000..dc95c6d --- /dev/null +++ b/parameters.d/osg25-el10.yaml @@ -0,0 +1,87 @@ +################### +# OSG 24 tests for EL9 +# File format documention: +# https://github.com/opensciencegrid/vm-test-runs#running-osg-test-in-vm-universe +################### + +platforms: + - centos_stream_10.x86_64 + - rocky_10.x86_64 + - alma_10.x86_64 + +sources: + ################### + # Format: + # [:;] ; [> /] + # Example: + # Run osg-test (from 3.2-minefield) with packages from 3.2-release and 3.2-testing that are then upgraded to + # 3.3-testing and 3-3-upcoming-testing: + # 3.2; osg, osg-testing > 3.3/osg-testing, osg-upcoming-testing + ################### + - opensciencegrid:master; 25; osg-minefield + +package_sets: + #### Required #### + # label - used for reporting, should be consistent across param files + # packages - list of packages to install in the test run + #### Optional #### + # selinux - enable SELinux for the package set, otherwise Permissive mode (default: True) + # osg_java - Pre-install OSG java packages (default: False) + # rng - Install entropy generation package (default: False) + ################## + - label: Compute Entrypoint (Condor) + packages: + - osg-ce-condor + - htcondor-ce-view + - label: Compute Entrypoint (Slurm) + packages: + - osg-ce-slurm + - htcondor-ce-view + - slurm + - slurm-slurmd + - slurm-slurmctld + - slurm-perlapi + - slurm-slurmdbd + - mariadb-server + - label: Compute Entrypoint (Torque) + packages: + - osg-ce-pbs + - htcondor-ce-view + - torque + - torque-server + - torque-mom + - torque-client + - torque-scheduler + - mariadb-server + - label: Central Collector + packages: + - htcondor-ce-collector + - htcondor-ce-view + - fetch-crl + - label: Worker Node (privileged) + packages: + - osg-wn-client + - osg-oasis + - apptainer-suid + - label: Worker Node (privileged, tarball deps) + packages: + - osg-update-data + - osg-wn-client + - label: XRootD + packages: + - osg-xrootd-standalone + - xrootd-multiuser + - xrootd-client + - voms-clients-cpp + - label: GlideinwmsFrontend + packages: + - glideinwms-vofrontend + - label: GlideinwmsFactory + packages: + - glideinwms-factory + - label: HTVault-Config + packages: + - htvault-config + - htgettoken + + diff --git a/parameters.d/osg25-el8.yaml b/parameters.d/osg25-el8.yaml new file mode 100644 index 0000000..068daf9 --- /dev/null +++ b/parameters.d/osg25-el8.yaml @@ -0,0 +1,77 @@ +################### +# OSG 24 tests for EL8 +# File format documention: +# https://github.com/opensciencegrid/vm-test-runs#running-osg-test-in-vm-universe +################### + +platforms: + - rocky_8.x86_64 + - alma_8.x86_64 + +sources: + ################### + # Format: + # [:;] ; [> /] + # Example: + # Run osg-test (from 3.2-minefield) with packages from 3.2-release and 3.2-testing that are then upgraded to + # 3.3-testing and 3-3-upcoming-testing: + # 3.2; osg, osg-testing > 3.3/osg-testing, osg-upcoming-testing + ################### + - opensciencegrid:master; 24; osg, osg-upcoming > 25/osg-minefield + - opensciencegrid:master; 25; osg-minefield + +package_sets: + #### Required #### + # label - used for reporting, should be consistent across param files + # packages - list of packages to install in the test run + #### Optional #### + # selinux - enable SELinux for the package set, otherwise Permissive mode (default: True) + # osg_java - Pre-install OSG java packages (default: False) + # rng - Install entropy generation package (default: False) + ################## + - label: Compute Entrypoint (Condor) + packages: + - osg-ce-condor + - htcondor-ce-view + - label: Compute Entrypoint (Slurm) + packages: + - osg-ce-slurm + - htcondor-ce-view + - slurm + - slurm-slurmd + - slurm-slurmctld + - slurm-perlapi + - slurm-slurmdbd + - mariadb-server + - label: Central Collector + packages: + - htcondor-ce-collector + - htcondor-ce-view + - fetch-crl + - label: Worker Node (privileged) + packages: + - osg-wn-client + - osg-oasis + - apptainer-suid + - label: Worker Node (privileged, tarball deps) + packages: + - osg-update-data + - osg-wn-client + - label: XRootD + packages: + - osg-xrootd-standalone + - xrootd-multiuser + - xrootd-client + - voms-clients-cpp + - label: GlideinwmsFrontend + packages: + - glideinwms-vofrontend + - label: GlideinwmsFactory + packages: + - glideinwms-factory + - label: HTVault-Config + packages: + - htvault-config + - htgettoken + + diff --git a/parameters.d/osg25-el9-aarch64.yaml b/parameters.d/osg25-el9-aarch64.yaml new file mode 100644 index 0000000..f6bfcd5 --- /dev/null +++ b/parameters.d/osg25-el9-aarch64.yaml @@ -0,0 +1,88 @@ +################### +# OSG 24 tests for EL9 +# File format documention: +# https://github.com/opensciencegrid/vm-test-runs#running-osg-test-in-vm-universe +################### + +platforms: + - centos_stream_9.aarch64 + - rocky_9.aarch64 + - alma_9.aarch64 + +sources: + ################### + # Format: + # [:;] ; [> /] + # Example: + # Run osg-test (from 3.2-minefield) with packages from 3.2-release and 3.2-testing that are then upgraded to + # 3.3-testing and 3-3-upcoming-testing: + # 3.2; osg, osg-testing > 3.3/osg-testing, osg-upcoming-testing + ################### + - opensciencegrid:master; 24; osg, osg-upcoming > 25/osg-minefield + - opensciencegrid:master; 25; osg-minefield + +package_sets: + #### Required #### + # label - used for reporting, should be consistent across param files + # packages - list of packages to install in the test run + #### Optional #### + # selinux - enable SELinux for the package set, otherwise Permissive mode (default: True) + # osg_java - Pre-install OSG java packages (default: False) + # rng - Install entropy generation package (default: False) + ################## + - label: Compute Entrypoint (Condor) + packages: + - osg-ce-condor + - htcondor-ce-view + - label: Compute Entrypoint (Slurm) + packages: + - osg-ce-slurm + - htcondor-ce-view + - slurm + - slurm-slurmd + - slurm-slurmctld + - slurm-perlapi + - slurm-slurmdbd + - mariadb-server + - label: Compute Entrypoint (Torque) + packages: + - osg-ce-pbs + - htcondor-ce-view + - torque + - torque-server + - torque-mom + - torque-client + - torque-scheduler + - mariadb-server + - label: Central Collector + packages: + - htcondor-ce-collector + - htcondor-ce-view + - fetch-crl + - label: Worker Node (privileged) + packages: + - osg-wn-client + - osg-oasis + - apptainer-suid + - label: Worker Node (privileged, tarball deps) + packages: + - osg-update-data + - osg-wn-client + - label: XRootD + packages: + - osg-xrootd-standalone + - xrootd-multiuser + - xrootd-client + - voms-clients-cpp + - label: GlideinwmsFrontend + packages: + - glideinwms-vofrontend + - label: GlideinwmsFactory + packages: + - glideinwms-factory + - label: HTVault-Config + packages: + - htvault-config + - htgettoken + + diff --git a/parameters.d/osg25-el9.yaml b/parameters.d/osg25-el9.yaml new file mode 100644 index 0000000..3a6008c --- /dev/null +++ b/parameters.d/osg25-el9.yaml @@ -0,0 +1,88 @@ +################### +# OSG 24 tests for EL9 +# File format documention: +# https://github.com/opensciencegrid/vm-test-runs#running-osg-test-in-vm-universe +################### + +platforms: + - centos_stream_9.x86_64 + - rocky_9.x86_64 + - alma_9.x86_64 + +sources: + ################### + # Format: + # [:;] ; [> /] + # Example: + # Run osg-test (from 3.2-minefield) with packages from 3.2-release and 3.2-testing that are then upgraded to + # 3.3-testing and 3-3-upcoming-testing: + # 3.2; osg, osg-testing > 3.3/osg-testing, osg-upcoming-testing + ################### + - opensciencegrid:master; 24; osg, osg-upcoming > 25/osg-minefield + - opensciencegrid:master; 25; osg-minefield + +package_sets: + #### Required #### + # label - used for reporting, should be consistent across param files + # packages - list of packages to install in the test run + #### Optional #### + # selinux - enable SELinux for the package set, otherwise Permissive mode (default: True) + # osg_java - Pre-install OSG java packages (default: False) + # rng - Install entropy generation package (default: False) + ################## + - label: Compute Entrypoint (Condor) + packages: + - osg-ce-condor + - htcondor-ce-view + - label: Compute Entrypoint (Slurm) + packages: + - osg-ce-slurm + - htcondor-ce-view + - slurm + - slurm-slurmd + - slurm-slurmctld + - slurm-perlapi + - slurm-slurmdbd + - mariadb-server + - label: Compute Entrypoint (Torque) + packages: + - osg-ce-pbs + - htcondor-ce-view + - torque + - torque-server + - torque-mom + - torque-client + - torque-scheduler + - mariadb-server + - label: Central Collector + packages: + - htcondor-ce-collector + - htcondor-ce-view + - fetch-crl + - label: Worker Node (privileged) + packages: + - osg-wn-client + - osg-oasis + - apptainer-suid + - label: Worker Node (privileged, tarball deps) + packages: + - osg-update-data + - osg-wn-client + - label: XRootD + packages: + - osg-xrootd-standalone + - xrootd-multiuser + - xrootd-client + - voms-clients-cpp + - label: GlideinwmsFrontend + packages: + - glideinwms-vofrontend + - label: GlideinwmsFactory + packages: + - glideinwms-factory + - label: HTVault-Config + packages: + - htvault-config + - htgettoken + +