Merge branch 'develop' into bugfix/warn-on-filter-expr-errors

Author: vkarak

.github/workflows/main.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
@@ -48,12 +48,15 @@ jobs:
 
   unittest-macos:
     runs-on: macos-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.9
+    - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         ./bootstrap.sh
@@ -115,12 +118,15 @@ jobs:
 
   wheelvalidation:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
     steps:
       - uses: actions/checkout@v4
-      - name: Setup up Python 3.8
+      - name: Setup up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: ${{ matrix.python-version }}
       - name: Generate Wheel
         run: |
           python -m pip install --upgrade pip setuptools build
@@ -137,7 +143,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}

reframe/__init__.py

@@ -6,7 +6,7 @@
 import os
 import sys
 
-VERSION = '4.7.0-dev.9'
+VERSION = '4.7.0-dev.10'
 INSTALL_PREFIX = os.path.normpath(
     os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 )

reframe/core/config.py

@@ -447,8 +447,10 @@ def validate(self):
         try:
             jsonschema.validate(site_config, self._schema)
         except jsonschema.ValidationError as e:
-            raise ConfigError(f"could not validate configuration files: "
-                              f"'{self._sources}'") from e
+            getlogger().debug(str(e))
+            sources = ', '.join(f'`{f}`' for f in self._sources)
+            raise ConfigError('could not validate configuration files: '
+                              f'{sources}') from e
 
         def _warn_variables(config, opt_path):
             opt_path = '/'.join(opt_path + ['variables'])

reframe/core/exceptions.py

@@ -8,6 +8,7 @@
 #
 
 import inspect
+import jsonschema
 import os
 
 import reframe
@@ -54,7 +55,10 @@ def message(self):
     def __str__(self):
         ret = self._message or ''
         if self.__cause__ is not None:
-            ret += ': ' + str(self.__cause__)
+            if isinstance(self.__cause__, jsonschema.ValidationError):
+                ret += ': ' + self.__cause__.message
+            else:
+                ret += ': ' + str(self.__cause__)
 
         return ret
 

reframe/core/schedulers/__init__.py

@@ -160,7 +160,7 @@ def filter_nodes_by_state(nodelist, state):
     :arg state: The state of the nodes.
         If ``all``, the initial list is returned untouched.
         If ``avail``, only the available nodes will be returned.
-        All other values are interpretes as a state string.
+        All other values are interpreted as a state string.
         State match is exclusive unless the ``*`` is added at the end of the
         state string.
     :returns: the filtered node list
@@ -169,7 +169,7 @@ def filter_nodes_by_state(nodelist, state):
         nodelist = {n for n in nodelist if n.is_avail()}
     elif state != 'all':
         if state.endswith('*'):
-            # non-exclusive stat match
+            # non-exclusive state match
             state = state[:-1]
             nodelist = {
                 n for n in nodelist if n.in_state(state)
@@ -606,7 +606,15 @@ def guess_num_tasks(self):
         available_nodes = filter_nodes_by_state(
             available_nodes, self.sched_flex_alloc_nodes.lower()
         )
+        getlogger().debug(
+            f'[F] Total available in state='
+            f'{self.sched_flex_alloc_nodes.lower()}: {len(available_nodes)}'
+        )
         available_nodes = self.scheduler.filternodes(self, available_nodes)
+        getlogger().debug(
+            f'[F] Total available after scheduler filter: '
+            f'{len(available_nodes)}'
+        )
         return len(available_nodes) * num_tasks_per_node
 
     def submit(self):

reframe/core/schedulers/slurm.py

@@ -310,6 +310,21 @@ def _get_default_partition(self):
 
         return None
 
+    def _get_actual_partition(self, options):
+        try:
+            completed = _run_strict(
+                ' '.join(['srun'] + options + ['--test-only', 'true'])
+            )
+            partition_match = re.search(r'partition (?P<partition>\S+)\s+',
+                                        completed.stderr)
+            if partition_match:
+                return partition_match.group('partition')
+
+        except SpawnedProcessError as e:
+            self.log('could not retrieve actual partition')
+
+        return None
+
     def _merge_files(self, job):
         with osext.change_dir(job.workdir):
             out_glob = glob.glob(job.stdout + '_*')
@@ -345,13 +360,21 @@ def filternodes(self, job, nodes):
         if reservation:
             reservation = reservation.strip()
             nodes &= self._get_reservation_nodes(reservation)
-            self.log(f'[F] Filtering nodes by reservation {reservation}: '
-                     f'available nodes now: {len(nodes)}')
+        else:
+            nodes = {node for node in nodes if not node.in_state('RESERVED')}
+
+        self.log(f'[F] Filtering nodes by reservation={reservation}: '
+                 f'available nodes now: {len(nodes)}')
 
         if partitions:
             partitions = set(partitions.strip().split(','))
         else:
-            default_partition = self._get_default_partition()
+            # Use a default partition if one is configured. Otherwise,
+            # fallback to the partition Slurm chooses for this set of options.
+            default_partition = (
+                self._get_default_partition() or
+                self._get_actual_partition(options)
+            )
             partitions = {default_partition} if default_partition else set()
             self.log(
                 f'[F] No partition specified; using {default_partition!r}'
@@ -676,8 +699,13 @@ def in_statex(self, state):
         return self._states == set(state.upper().split('+'))
 
     def is_avail(self):
-        return any(self.in_statex(s)
-                   for s in ('ALLOCATED', 'COMPLETING', 'IDLE'))
+        available_states = {
+            'ALLOCATED',
+            'COMPLETING',
+            'IDLE',
+            'RESERVED',
+        }
+        return self._states <= available_states
 
     def is_down(self):
         return not self.is_avail()

reframe/frontend/autodetect.py

@@ -106,6 +106,7 @@ def _load_info(filename, schema=None):
         )
         return {}
     except jsonschema.ValidationError as e:
+        getlogger().debug(str(e))
         raise ConfigError(
             f'could not validate meta-config file {filename!r}'
         ) from e

reframe/frontend/printer.py

@@ -124,6 +124,7 @@ def _print_failure_info(rec, runid, total_runs):
             self.info(f"  * Description: {rec['descr']}")
             self.info(f"  * System partition: {rec['system']}")
             self.info(f"  * Environment: {rec['environ']}")
+            self.info(f"  * Test file: {rec['filename']}")
             self.info(f"  * Stage directory: {rec['stagedir']}")
             self.info(f"  * Node list: "
                       f"{nodelist_abbrev(rec['job_nodelist'])}")

reframe/frontend/reporting/__init__.py

@@ -215,12 +215,12 @@ def _restore_session(filename):
         except KeyError:
             found_ver = 'n/a'
 
-        getlogger().verbose(f'JSON validation error: {e}')
+        getlogger().debug(str(e))
         raise ReframeError(
             f'failed to validate report {filename!r}: {e.args[0]} '
             f'(check report data version: required {DATA_VERSION}, '
             f'found: {found_ver})'
-        ) from None
+        ) from e
 
     return _RestoredSessionInfo(report)
 

setup.cfg

@@ -16,6 +16,7 @@ classifiers =
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
     Programming Language :: Python :: 3.12
+    Programming Language :: Python :: 3.13
     License :: OSI Approved :: BSD License
     Operating System :: MacOS
     Operating System :: POSIX :: Linux