Merge pull request #443 from ExaWorks/per_test_custom_attrs

Added a way to specify test filters for custom attributes.

Author: hategan

testing.conf

@@ -126,10 +126,15 @@ project_name =
 #
 # If a site needs specific job attributes to be specified, such as billing
 # accounts, they can be added here. The value is a comma separated list
-# of "executor.name": "value" pairs.
+# of "executor.attr_name": "value" pairs. A regular expression filter enclosed
+# in square brackets can be used after "custom_attributes" to specify that the
+# attributes should only apply to certain tests. The specified custom_attributes
+# directives are processed in the order in which they appear and a lack of a
+# filter is equivalent to a ".*" filter.
 #
 # custom_attributes = "slurm.account": "xyz", \
 #                     "slurm.constraint": "knl"
+# custom_attributes[test_nodefile\[slurm:.*\]] = "slurm.qos": "debug"
 
 custom_attributes =
 

tests/ci_runner.py

@@ -35,7 +35,7 @@ def read_line(f: TextIO) -> Optional[str]:
 
 
 def read_conf(fname: str) -> Dict[str, str]:
-    conf = {}
+    conf: Dict[str, str] = {}
     with open(fname, 'r') as f:
         line = read_line(f)
         while line is not None:
@@ -47,11 +47,28 @@ def read_conf(fname: str) -> Dict[str, str]:
                 kv = line.split('=', 2)
                 if len(kv) != 2:
                     raise ValueError('Invalid line in configuration file: "%s"' % line)
-                conf[kv[0].strip()] = kv[1].strip()
+                add_conf(conf, kv[0].strip(), kv[1].strip())
             line = read_line(f)
     return conf
 
 
+def add_conf(conf: Dict[str, str], key: str, value: str) -> None:
+    if key.startswith('custom_attributes'):
+        if not key.startswith('custom_attributes['):
+            key = key + '[.*]'
+        if not key.endswith(']'):
+            raise ValueError('Invalid custom_attributes entry. Missing closing bracket.')
+        filter = key[len('custom_attributes['):-1]
+        key = 'custom_attributes'
+        value = '{"filter": "%s", "value": {%s}}' % (filter.replace('\\', '\\\\'), value)
+        if key in conf:
+            conf[key] = conf[key] + ', ' + value
+        else:
+            conf[key] = value
+    else:
+        conf[key] = value
+
+
 def run(*args: str, cwd: Optional[str] = None) -> str:
     p = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False,
                        cwd=cwd, text=True)

tests/conftest.py

@@ -154,7 +154,7 @@ def pytest_generate_tests(metafunc):
         for x in _get_executors((metafunc.config)):
             etp = ExecutorTestParams(x, queue_name=metafunc.config.option.queue_name,
                                      project_name=metafunc.config.option.project_name,
-                                     custom_attributes=metafunc.config.option.custom_attributes)
+                                     custom_attributes_raw=metafunc.config.option.custom_attributes)
             etps.append(etp)
 
         metafunc.parametrize('execparams', etps, ids=lambda x: str(x))
@@ -361,7 +361,13 @@ def _parse_custom_attributes(s: Optional[str]) -> Dict[str, object]:
     if s is None:
         return None
     else:
-        return json.loads('{' + s + '}')
+        attrspec = json.loads('[' + s + ']')
+        d = {}
+        for item in attrspec:
+            if item['filter'] not in d:
+                d[item['filter']] = {}
+            d[item['filter']].update(item['value'])
+        return d
 
 
 def _strip_home(path: List[str]) -> List[str]:
@@ -434,8 +440,31 @@ def _now():
     return datetime.datetime.now(tz=datetime.timezone.utc).isoformat(' ')
 
 
+def _process_custom_attributes(item):
+    if not hasattr(item, 'callspec'):
+        return
+    if 'execparams' not in item.callspec.params:
+        return
+    ep: Optional[List[Dict[str, Dict[str, object]]]] = item.callspec.params['execparams']
+    if ep.custom_attributes_raw is None:
+        return
+    if ep is None:
+        return
+    test_name = item.name
+    for filter, attrs in ep.custom_attributes_raw.items():
+        if re.match(filter, test_name):
+            ep.custom_attributes.update(attrs)
+
+
+def _set_attrs(execparams, attrs):
+    for k, v in attrs.items():
+        execparams.customattributes[k] = v
+
+
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
 def pytest_runtest_makereport(item, call):
+    _process_custom_attributes(item)
+
     outcome = yield
     report = outcome.get_result()
 
@@ -588,6 +617,8 @@ def _upload_report(config, data):
     env = config.option.environment
 
     url = config.getoption('server_url')
+    if not url:
+        return
     minimal = config.getoption('minimal_uploads')
     if minimal:
         data = _sanitize(data)

tests/executor_test_params.py

@@ -7,7 +7,8 @@ class ExecutorTestParams:
 
     def __init__(self, spec: str, queue_name: Optional[str] = None,
                  project_name: Optional[str] = None,
-                 custom_attributes: Optional[Dict[str, object]] = None) -> None:
+                 custom_attributes_raw: Optional[Dict[str, Dict[str, object]]] = None) \
+            -> None:
         """
         Construct a new instance.
 
@@ -35,7 +36,8 @@ def __init__(self, spec: str, queue_name: Optional[str] = None,
 
         self.queue_name = queue_name
         self.project_name = project_name
-        self.custom_attributes = custom_attributes
+        self.custom_attributes_raw = custom_attributes_raw
+        self.custom_attributes: Dict[str, object] = {}
 
     def __repr__(self) -> str:
         """Returns a string representation of this object."""