Enable Initializing Second Experiment From Library

.github/workflows/run.yml

@@ -5,43 +5,6 @@ on:
       BENCHPARK_CODECOV_TOKEN:
         required: true
 jobs:
-  check_errors:
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Checkout Benchpark
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - name: Add needed Python libs
-        run: |
-          pip install -r ./requirements.txt
-      - name: Run negative tests via helper
-        shell: bash
-        run: |
-          fail() { echo "TEST FAILED: $*" >&2; exit 1; }
-
-          # one-time setup
-          ./bin/benchpark system init --dest dane llnl-cluster
-
-          # 1) Non-MPIOnlyExperiment Test
-          set +e
-          # babelstream is not an MPIOnlyExperiment
-          stderr_output="$({ ./bin/benchpark experiment init --dest babelstream --system dane babelstream; } 2>&1)"
-          status=$?
-          set -e
-          [[ $status -ne 0 ]] || fail "Expected non-zero exit status, got $status"
-          # Check for the specific BenchparkError message
-          expected="cannot run with MPI only"
-          grep -Fq "$expected" <<< "$stderr_output" && echo "failed as expected: $expected" || fail "Expected error message not found in stderr. Got: $stderr_output"
-
-          # 2) Multiple scaling options
-          set +e
-          # Cannot use both +strong and +weak
-          stderr_output="$({ ./bin/benchpark experiment init --dest kripke --system dane kripke+strong+weak; } 2>&1)"
-          status=$?
-          set -e
-          [[ $status -ne 0 ]] || fail "Expected non-zero exit status, got $status"
-          # Check for the specific BenchparkError message
-          expected="spec cannot specify multiple scaling options"
-          grep -Fq "$expected" <<<"$stderr_output" && echo "failed as expected: $expected" || fail "Expected error message not found in stderr. Got: $stderr_output"
   saxpy:
     runs-on: ubuntu-24.04
     steps:

experiments/babelstream/experiment.py

@@ -6,17 +6,13 @@
 from benchpark.directives import variant, maintainers
 from benchpark.experiment import Experiment
 from benchpark.caliper import Caliper
-from benchpark.cuda import CudaExperiment
-from benchpark.rocm import ROCmExperiment
-from benchpark.openmp import OpenMPExperiment
+from benchpark.models import ModelsType, Models
 
 
 class Babelstream(
     Experiment,
     Caliper,
-    CudaExperiment,
-    ROCmExperiment,
-    OpenMPExperiment,
+    Models(ModelsType.Openmp, ModelsType.Cuda, ModelsType.Rocm),
 ):
     variant(
         "workload",

experiments/kripke/experiment.py

@@ -5,20 +5,14 @@
 
 from benchpark.directives import variant, maintainers
 from benchpark.experiment import Experiment
-from benchpark.mpi import MpiOnlyExperiment
-from benchpark.openmp import OpenMPExperiment
-from benchpark.cuda import CudaExperiment
-from benchpark.rocm import ROCmExperiment
+from benchpark.models import ModelsType, Models
 from benchpark.scaling import ScalingMode, Scaling
 from benchpark.caliper import Caliper
 
 
 class Kripke(
     Experiment,
-    MpiOnlyExperiment,
-    OpenMPExperiment,
-    CudaExperiment,
-    ROCmExperiment,
+    Models(ModelsType.Mpionly, ModelsType.Openmp, ModelsType.Cuda, ModelsType.Rocm),
     Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput),
     Caliper,
 ):

lib/benchpark/experiment.py

@@ -256,30 +256,39 @@ def __init__(self, spec):
 
         self.package_specs = {}
 
+        # Set available programming models for checks
+        models = set()
+        for cls in self.__class__.mro():
+            models.update(getattr(cls, "_available_programming_models", ()))
+        self.programming_models = list(models)
+
         # Explicitly ordered list. "mpi" first
         models = ["mpi"] + ["openmp", "cuda", "rocm"]
+        valid_models = []
         invalid_models = []
         for model in models:
-            # Experiment specifying model in add_package_spec that it doesn't implement
-            if (
-                self.spec.satisfies("+" + model)
-                and model not in self.programming_models
-            ):
-                invalid_models.append(model)
+            if self.spec.satisfies("+" + model):
+                valid_models.append(model)
+                # Experiment specifying model in add_package_spec that it doesn't implement
+                if model not in self.programming_models:
+                    invalid_models.append(model)
+        # MPI is always valid if with another programming model, even if no mpionly
+        if "mpi" in invalid_models and len(valid_models) > 1:
+            invalid_models.remove("mpi")
         # Case where there are no experiments specified in experiment.py
         if len(self.programming_models) == 0:
-            raise NotImplementedError(
+            raise BenchparkError(
                 f"Please specify a programming model in your {self.name}/experiment.py (e.g. MpiOnlyExperiment, OpenMPExperiment, CudaExperiment, ROCmExperiment). See other experiments for examples."
             )
         elif len(invalid_models) > 0:
-            raise NotImplementedError(
+            raise BenchparkError(
                 f'{invalid_models} are not valid programming models for "{self.name}". Choose from {self.programming_models}.'
             )
         # Check if experiment is trying to run in MpiOnly mode without being an MpiOnlyExperiment
         elif "mpi" not in str(self.spec) and not any(
             self.spec.satisfies("+" + model) for model in models[1:]
         ):
-            raise NotImplementedError(
+            raise BenchparkError(
                 f'"{self.name}" cannot run with MPI only without inheriting from MpiOnlyExperiment. Choose from {self.programming_models}'
             )
 
@@ -456,7 +465,10 @@ def compute_applications_section_wrapper(self):
         for cls in self.helpers:
             helper_prefix = cls.get_helper_name_prefix()
             if helper_prefix:
-                expr_helper_list.append(helper_prefix)
+                if isinstance(helper_prefix, list):
+                    expr_helper_list.extend(helper_prefix)
+                else:
+                    expr_helper_list.append(helper_prefix)
         expr_name_suffix = "_".join(expr_helper_list + self.expr_var_names)
 
         self.check_required_variables()

lib/benchpark/models.py

@@ -0,0 +1,85 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+
+from benchpark.error import BenchparkError
+from benchpark.directives import variant, requires
+from benchpark.experiment import ExperimentHelper
+from enum import Enum
+
+
+class ModelsType(Enum):
+    Mpionly = "mpi"
+    Openmp = "openmp"
+    Cuda = "cuda"
+    Rocm = "rocm"
+
+
+def Models(*types):
+    for ty in types:
+        if not isinstance(ty, ModelsType):
+            raise ValueError(f"Invalid programming model: {ty}")
+
+    # Normalize once so we can reuse
+    _available = tuple(t.value for t in types)
+
+    class BaseModel:
+        requires("mpi", when="+mpi")
+        requires("rocm", when="+rocm")
+        requires("cuda", when="+cuda")
+        requires("openmp", when="+openmp")
+
+        variant("mpi", default=True, description="Run with MPI")
+        variant("rocm", default=False, description="Build and run with ROCm")
+        variant("cuda", default=False, description="Build and run with CUDA")
+        variant("openmp", default=False, description="Build and run with OpenMP")
+
+        # Class-level list of supported models for any class that includes this mixin
+        _available_programming_models = _available
+
+        # Handy instance-level property (works on Experiment instances)
+        @property
+        def available_programming_models(self):
+            # If multiple mixins contribute, merge them from MRO at runtime
+            models = set()
+            for cls in type(self).mro():
+                models.update(getattr(cls, "_available_programming_models", ()))
+            return tuple(sorted(models))
+
+        # Quick check helper
+        @staticmethod
+        def supports_model(name: str) -> bool:
+            return name in _available
+
+    # Helper class (unchanged except for optional new method)
+    class Helper(ExperimentHelper):
+        def get_helper_name_prefix(self):
+            models = []
+            for s in [
+                ModelsType.Mpionly.value,
+                ModelsType.Openmp.value,
+                ModelsType.Cuda.value,
+                ModelsType.Rocm.value,
+            ]:
+                if self.spec.satisfies("+" + s):
+                    models.append(s)
+            if len(models) > 0:
+                return models
+            return "no_model"
+
+        # Optional: expose *available* (not selected) models via helper, too
+        def get_available_models(self):
+            models = set()
+            for cls in type(self).__mro__:
+                models.update(getattr(cls, "_available_programming_models", ()))
+            return tuple(sorted(models))
+
+    return type(
+        "ProgrammingModel",
+        (BaseModel,),
+        {
+            "Helper": Helper,
+        },
+    )

lib/benchpark/test/experiment_errors.py

@@ -0,0 +1,28 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+import benchpark.spec
+from benchpark.error import BenchparkError
+
+
+def test_programming_model_checks():
+    # babelstream mpi-only not valid
+    with pytest.raises(BenchparkError, match=r"mpi.*are not valid programming models"):
+        spec = benchpark.spec.ExperimentSpec("babelstream").concretize()
+        experiment = spec.experiment  # noqa: F841
+
+    # stream+openmp not valid
+    with pytest.raises(Exception, match="not a valid variant"):
+        spec = benchpark.spec.ExperimentSpec(
+            "stream+openmp workload=stream"
+        ).concretize()
+        experiment = spec.experiment
+
+    # Multiple scaling options not valid
+    with pytest.raises(BenchparkError, match="cannot specify multiple scaling options"):
+        spec = benchpark.spec.ExperimentSpec("kripke+strong+weak").concretize()
+        experiment = spec.experiment