Enable Initializing Second Experiment From Library

.github/utils/dryruns.py

@@ -23,7 +23,7 @@
     "stream generic-x86",
     # Broken URL's in application.py going to cause dryrun failure
     "genesis",
-    # Not mpionlyexperiment
+    # Not ProgrammingModelType.Mpionly
     "py-scaffold+strong",
     "py-scaffold+weak",
 ]

.github/workflows/run.yml

@@ -5,43 +5,6 @@ on:
       BENCHPARK_CODECOV_TOKEN:
         required: true
 jobs:
-  check_errors:
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Checkout Benchpark
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
-      - name: Add needed Python libs
-        run: |
-          pip install -r ./requirements.txt
-      - name: Run negative tests via helper
-        shell: bash
-        run: |
-          fail() { echo "TEST FAILED: $*" >&2; exit 1; }
-
-          # one-time setup
-          ./bin/benchpark system init --dest dane llnl-cluster
-
-          # 1) Non-MPIOnlyExperiment Test
-          set +e
-          # babelstream is not an MPIOnlyExperiment
-          stderr_output="$({ ./bin/benchpark experiment init dane babelstream; } 2>&1)"
-          status=$?
-          set -e
-          [[ $status -ne 0 ]] || fail "Expected non-zero exit status, got $status"
-          # Check for the specific BenchparkError message
-          expected="cannot run with MPI only"
-          grep -Fq "$expected" <<< "$stderr_output" && echo "failed as expected: $expected" || fail "Expected error message not found in stderr. Got: $stderr_output"
-
-          # 2) Multiple scaling options
-          set +e
-          # Cannot use both +strong and +weak
-          stderr_output="$({ ./bin/benchpark experiment init dane kripke+strong+weak; } 2>&1)"
-          status=$?
-          set -e
-          [[ $status -ne 0 ]] || fail "Expected non-zero exit status, got $status"
-          # Check for the specific BenchparkError message
-          expected="spec cannot specify multiple scaling options"
-          grep -Fq "$expected" <<<"$stderr_output" && echo "failed as expected: $expected" || fail "Expected error message not found in stderr. Got: $stderr_output"
   saxpy:
     runs-on: ubuntu-24.04
     steps:

docs/add-an-experiment.rst

@@ -57,17 +57,17 @@ We create the ``experiment.py`` file under ``benchpark/experiments/hpl/experimen
 The naming of this directory will affect how the experiment is initialized, e.g.,
 ``benchpark experiment init ... hpl``. There are multiple scaling options, modifiers,
 and programming models we can inherit from, but at minimum our experiment should inherit
-from the base ``Experiment`` class and ``MpiOnlyExperiment`` indicating that our
+from the base ``Experiment`` class and ``ProgrammingModel(ProgrammingModelType.Mpionly)`` indicating that our
 experiment can be executed with MPI.
 
 ::
 
     from benchpark.experiment import Experiment
-    from benchpark.mpi import MpiOnlyExperiment
+    from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
 
     class Hpl(
       Experiment,
-      MpiOnlyExperiment,
+      ProgrammingModel(ProgrammingModelType.Mpionly),
     ):
 
 Looking at the `HPL package
@@ -78,21 +78,19 @@ supports the ``OpenMP`` programming model. The HPL package.py defines the ``Cali
 variant because the HPL source code is instrumented with the ``Caliper`` performance
 profiling library (via a `fork <https://github.com/daboehme/HPL-caliper.git>`__ of the
 source code) and the build links to Caliper. Enabling these variants in our Benchpark
-experiment only requires inheritance from the pre-defined ``OpenMPExperiment`` and
+experiment only requires inheritance from the pre-defined ``ProgrammingModelType.Openmp`` and
 ``Caliper`` classes. For more details on the configurability of experiment variants, see
 :ref:`experiment-variants`.
 
 ::
 
     from benchpark.experiment import Experiment
-    from benchpark.mpi import MpiOnlyExperiment
-    from benchpark.openmp import OpenMPExperiment
+    from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
     from benchpark.caliper import Caliper
 
     class Hpl(
       Experiment,
-      MpiOnlyExperiment,
-      OpenMPExperiment,
+      ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp),
       Caliper,
     ):
 
@@ -118,15 +116,13 @@ experiment.
 ::
 
     from benchpark.experiment import Experiment
-    from benchpark.mpi import MpiOnlyExperiment
-    from benchpark.openmp import OpenMPExperiment
+    from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
     from benchpark.caliper import Caliper
     from benchpark.directives import variant, maintainers
 
     class Hpl(
       Experiment,
-      MpiOnlyExperiment,
-      OpenMPExperiment,
+      ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp),
       Caliper,
     ):
 
@@ -190,15 +186,13 @@ by default.
 ::
 
     from benchpark.experiment import Experiment
-    from benchpark.mpi import MpiOnlyExperiment
-    from benchpark.openmp import OpenMPExperiment
+    from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
     from benchpark.caliper import Caliper
     from benchpark.directives import variant, maintainers
 
     class Hpl(
       Experiment,
-      MpiOnlyExperiment,
-      OpenMPExperiment,
+      ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp),
       Caliper,
     ):
 
@@ -285,16 +279,14 @@ runtime parameters during experiment initialization, e.g., ``benchpark experimen
 ::
 
     from benchpark.experiment import Experiment
-    from benchpark.mpi import MpiOnlyExperiment
-    from benchpark.openmp import OpenMPExperiment
+    from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
     from benchpark.scaling import ScalingMode, Scaling
     from benchpark.caliper import Caliper
     from benchpark.directives import variant, maintainers
 
     class Hpl(
       Experiment,
-      MpiOnlyExperiment,
-      OpenMPExperiment,
+      ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp),
       Scaling(ScalingMode.Strong, ScalingMode.Weak),
       Caliper,
     ):
@@ -352,16 +344,14 @@ not list required packages for the benchmark here, since they are already define
 ::
 
     from benchpark.experiment import Experiment
-    from benchpark.mpi import MpiOnlyExperiment
-    from benchpark.openmp import OpenMPExperiment
+    from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
     from benchpark.scaling import ScalingMode, Scaling
     from benchpark.caliper import Caliper
     from benchpark.directives import variant, maintainers
 
     class Hpl(
       Experiment,
-      MpiOnlyExperiment,
-      OpenMPExperiment,
+      ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp),
       Scaling(ScalingMode.Strong, ScalingMode.Weak),
       Caliper,
     ):
@@ -423,10 +413,12 @@ experiment using CUDA (on an NVIDIA GPU), or ``openmp`` for an experiment using
 
     class Amg2023(
       Experiment,
-      MpiOnlyExperiment
-      OpenMPExperiment,
-      CudaExperiment,
-      ROCmExperiment,
+      ProgrammingModel(
+          ProgrammingModelType.Mpionly,
+          ProgrammingModelType.Openmp,
+          ProgrammingModelType.Cuda,
+          ProgrammingModelType.Rocm,
+      ),
       Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput),
       Caliper,
     ):

experiments/ad/experiment.py

@@ -5,10 +5,13 @@
 
 from benchpark.directives import maintainers, variant
 from benchpark.experiment import Experiment
-from benchpark.mpi import MpiOnlyExperiment
+from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
 
 
-class Ad(Experiment, MpiOnlyExperiment):
+class Ad(
+    Experiment,
+    ProgrammingModel(ProgrammingModelType.Mpionly),
+):
     variant(
         "workload",
         default="ad",

experiments/amg2023/experiment.py

@@ -4,21 +4,20 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from benchpark.caliper import Caliper
-from benchpark.cuda import CudaExperiment
 from benchpark.directives import maintainers, variant
 from benchpark.experiment import Experiment
-from benchpark.mpi import MpiOnlyExperiment
-from benchpark.openmp import OpenMPExperiment
-from benchpark.rocm import ROCmExperiment
+from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
 from benchpark.scaling import Scaling, ScalingMode
 
 
 class Amg2023(
     Experiment,
-    MpiOnlyExperiment,
-    OpenMPExperiment,
-    CudaExperiment,
-    ROCmExperiment,
+    ProgrammingModel(
+        ProgrammingModelType.Mpionly,
+        ProgrammingModelType.Openmp,
+        ProgrammingModelType.Cuda,
+        ProgrammingModelType.Rocm,
+    ),
     Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput),
     Caliper,
 ):

experiments/babelstream/experiment.py

@@ -4,19 +4,19 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from benchpark.caliper import Caliper
-from benchpark.cuda import CudaExperiment
 from benchpark.directives import maintainers, variant
 from benchpark.experiment import Experiment
-from benchpark.openmp import OpenMPExperiment
-from benchpark.rocm import ROCmExperiment
+from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
 
 
 class Babelstream(
     Experiment,
     Caliper,
-    CudaExperiment,
-    ROCmExperiment,
-    OpenMPExperiment,
+    ProgrammingModel(
+        ProgrammingModelType.Openmp,
+        ProgrammingModelType.Cuda,
+        ProgrammingModelType.Rocm,
+    ),
 ):
     variant(
         "workload",

experiments/branson/experiment.py

@@ -4,21 +4,20 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from benchpark.caliper import Caliper
-from benchpark.cuda import CudaExperiment
 from benchpark.directives import variant
 from benchpark.experiment import Experiment
-from benchpark.mpi import MpiOnlyExperiment
-from benchpark.openmp import OpenMPExperiment
-from benchpark.rocm import ROCmExperiment
+from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
 from benchpark.scaling import Scaling, ScalingMode
 
 
 class Branson(
     Experiment,
-    MpiOnlyExperiment,
-    OpenMPExperiment,
-    CudaExperiment,
-    ROCmExperiment,
+    ProgrammingModel(
+        ProgrammingModelType.Mpionly,
+        ProgrammingModelType.Openmp,
+        ProgrammingModelType.Cuda,
+        ProgrammingModelType.Rocm,
+    ),
     Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput),
     Caliper,
 ):

experiments/commbench/experiment.py

@@ -3,16 +3,14 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from benchpark.cuda import CudaExperiment
 from benchpark.directives import maintainers, variant
 from benchpark.experiment import Experiment
-from benchpark.rocm import ROCmExperiment
+from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
 
 
 class Commbench(
     Experiment,
-    CudaExperiment,
-    ROCmExperiment,
+    ProgrammingModel(ProgrammingModelType.Cuda, ProgrammingModelType.Rocm),
 ):
     variant("workload", description="workload name", default="basic")
     maintainers("arhag23")

experiments/genesis/experiment.py

@@ -5,11 +5,13 @@
 
 from benchpark.directives import maintainers, variant
 from benchpark.experiment import Experiment
-from benchpark.mpi import MpiOnlyExperiment
-from benchpark.openmp import OpenMPExperiment
+from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
 
 
-class Genesis(Experiment, MpiOnlyExperiment, OpenMPExperiment):
+class Genesis(
+    Experiment,
+    ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp),
+):
 
     variant(
         "workload",

experiments/gpcnet/experiment.py

@@ -5,10 +5,13 @@
 
 from benchpark.directives import maintainers, variant
 from benchpark.experiment import Experiment
-from benchpark.mpi import MpiOnlyExperiment
+from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
 
 
-class Gpcnet(Experiment, MpiOnlyExperiment):
+class Gpcnet(
+    Experiment,
+    ProgrammingModel(ProgrammingModelType.Mpionly),
+):
     variant(
         "workload",
         default="network_test",

experiments/gromacs/experiment.py

@@ -3,20 +3,19 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from benchpark.cuda import CudaExperiment
 from benchpark.directives import maintainers, variant
 from benchpark.experiment import Experiment
-from benchpark.mpi import MpiOnlyExperiment
-from benchpark.openmp import OpenMPExperiment
-from benchpark.rocm import ROCmExperiment
+from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
 
 
 class Gromacs(
     Experiment,
-    MpiOnlyExperiment,
-    OpenMPExperiment,
-    CudaExperiment,
-    ROCmExperiment,
+    ProgrammingModel(
+        ProgrammingModelType.Mpionly,
+        ProgrammingModelType.Openmp,
+        ProgrammingModelType.Cuda,
+        ProgrammingModelType.Rocm,
+    ),
 ):
     variant(
         "workload",

experiments/hpcg/experiment.py

@@ -6,15 +6,13 @@
 from benchpark.caliper import Caliper
 from benchpark.directives import maintainers, variant
 from benchpark.experiment import Experiment
-from benchpark.mpi import MpiOnlyExperiment
-from benchpark.openmp import OpenMPExperiment
+from benchpark.programming_model import ProgrammingModel, ProgrammingModelType
 from benchpark.scaling import Scaling, ScalingMode
 
 
 class Hpcg(
     Experiment,
-    MpiOnlyExperiment,
-    OpenMPExperiment,
+    ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp),
     Scaling(ScalingMode.Strong, ScalingMode.Weak),
     Caliper,
 ):