diff --git a/.github/utils/dryruns.py b/.github/utils/dryruns.py index b0631cd8e..9a383900f 100644 --- a/.github/utils/dryruns.py +++ b/.github/utils/dryruns.py @@ -23,7 +23,7 @@ "stream generic-x86", # Broken URL's in application.py going to cause dryrun failure "genesis", - # Not mpionlyexperiment + # Not ProgrammingModelType.Mpionly "py-scaffold+strong", "py-scaffold+weak", ] diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 83edafdf1..7bb2d6b8d 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -5,43 +5,6 @@ on: BENCHPARK_CODECOV_TOKEN: required: true jobs: - check_errors: - runs-on: ubuntu-24.04 - steps: - - name: Checkout Benchpark - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - - name: Add needed Python libs - run: | - pip install -r ./requirements.txt - - name: Run negative tests via helper - shell: bash - run: | - fail() { echo "TEST FAILED: $*" >&2; exit 1; } - - # one-time setup - ./bin/benchpark system init --dest dane llnl-cluster - - # 1) Non-MPIOnlyExperiment Test - set +e - # babelstream is not an MPIOnlyExperiment - stderr_output="$({ ./bin/benchpark experiment init dane babelstream; } 2>&1)" - status=$? - set -e - [[ $status -ne 0 ]] || fail "Expected non-zero exit status, got $status" - # Check for the specific BenchparkError message - expected="cannot run with MPI only" - grep -Fq "$expected" <<< "$stderr_output" && echo "failed as expected: $expected" || fail "Expected error message not found in stderr. Got: $stderr_output" - - # 2) Multiple scaling options - set +e - # Cannot use both +strong and +weak - stderr_output="$({ ./bin/benchpark experiment init dane kripke+strong+weak; } 2>&1)" - status=$? - set -e - [[ $status -ne 0 ]] || fail "Expected non-zero exit status, got $status" - # Check for the specific BenchparkError message - expected="spec cannot specify multiple scaling options" - grep -Fq "$expected" <<<"$stderr_output" && echo "failed as expected: $expected" || fail "Expected error message not found in stderr. Got: $stderr_output" saxpy: runs-on: ubuntu-24.04 steps: diff --git a/docs/add-an-experiment.rst b/docs/add-an-experiment.rst index 72c0c2b18..47b6c4c49 100644 --- a/docs/add-an-experiment.rst +++ b/docs/add-an-experiment.rst @@ -57,17 +57,18 @@ We create the ``experiment.py`` file under ``benchpark/experiments/hpl/experimen The naming of this directory will affect how the experiment is initialized, e.g., ``benchpark experiment init ... hpl``. There are multiple scaling options, modifiers, and programming models we can inherit from, but at minimum our experiment should inherit -from the base ``Experiment`` class and ``MpiOnlyExperiment`` indicating that our -experiment can be executed with MPI. +from the base ``Experiment`` class and +``ProgrammingModel(ProgrammingModelType.Mpionly)`` indicating that our experiment can be +executed with MPI. :: from benchpark.experiment import Experiment - from benchpark.mpi import MpiOnlyExperiment + from benchpark.programming_model import ProgrammingModel, ProgrammingModelType class Hpl( Experiment, - MpiOnlyExperiment, + ProgrammingModel(ProgrammingModelType.Mpionly), ): Looking at the `HPL package @@ -78,21 +79,19 @@ supports the ``OpenMP`` programming model. The HPL package.py defines the ``Cali variant because the HPL source code is instrumented with the ``Caliper`` performance profiling library (via a `fork `__ of the source code) and the build links to Caliper. Enabling these variants in our Benchpark -experiment only requires inheritance from the pre-defined ``OpenMPExperiment`` and -``Caliper`` classes. For more details on the configurability of experiment variants, see -:ref:`experiment-variants`. +experiment only requires inheritance from the pre-defined +``ProgrammingModelType.Openmp`` and ``Caliper`` classes. For more details on the +configurability of experiment variants, see :ref:`experiment-variants`. :: from benchpark.experiment import Experiment - from benchpark.mpi import MpiOnlyExperiment - from benchpark.openmp import OpenMPExperiment + from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.caliper import Caliper class Hpl( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, + ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp), Caliper, ): @@ -118,15 +117,13 @@ experiment. :: from benchpark.experiment import Experiment - from benchpark.mpi import MpiOnlyExperiment - from benchpark.openmp import OpenMPExperiment + from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.caliper import Caliper from benchpark.directives import variant, maintainers class Hpl( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, + ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp), Caliper, ): @@ -190,15 +187,13 @@ by default. :: from benchpark.experiment import Experiment - from benchpark.mpi import MpiOnlyExperiment - from benchpark.openmp import OpenMPExperiment + from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.caliper import Caliper from benchpark.directives import variant, maintainers class Hpl( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, + ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp), Caliper, ): @@ -285,16 +280,14 @@ runtime parameters during experiment initialization, e.g., ``benchpark experimen :: from benchpark.experiment import Experiment - from benchpark.mpi import MpiOnlyExperiment - from benchpark.openmp import OpenMPExperiment + from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import ScalingMode, Scaling from benchpark.caliper import Caliper from benchpark.directives import variant, maintainers class Hpl( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, + ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp), Scaling(ScalingMode.Strong, ScalingMode.Weak), Caliper, ): @@ -352,16 +345,14 @@ not list required packages for the benchmark here, since they are already define :: from benchpark.experiment import Experiment - from benchpark.mpi import MpiOnlyExperiment - from benchpark.openmp import OpenMPExperiment + from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import ScalingMode, Scaling from benchpark.caliper import Caliper from benchpark.directives import variant, maintainers class Hpl( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, + ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp), Scaling(ScalingMode.Strong, ScalingMode.Weak), Caliper, ): @@ -423,10 +414,12 @@ experiment using CUDA (on an NVIDIA GPU), or ``openmp`` for an experiment using class Amg2023( Experiment, - MpiOnlyExperiment - OpenMPExperiment, - CudaExperiment, - ROCmExperiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Openmp, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput), Caliper, ): diff --git a/experiments/ad/experiment.py b/experiments/ad/experiment.py index a9d09195d..5a3de9c0e 100644 --- a/experiments/ad/experiment.py +++ b/experiments/ad/experiment.py @@ -5,10 +5,13 @@ from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType -class Ad(Experiment, MpiOnlyExperiment): +class Ad( + Experiment, + ProgrammingModel(ProgrammingModelType.Mpionly), +): variant( "workload", default="ad", diff --git a/experiments/amg2023/experiment.py b/experiments/amg2023/experiment.py index 40e33fd14..8f0c5de51 100644 --- a/experiments/amg2023/experiment.py +++ b/experiments/amg2023/experiment.py @@ -4,21 +4,20 @@ # SPDX-License-Identifier: Apache-2.0 from benchpark.caliper import Caliper -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class Amg2023( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, - CudaExperiment, - ROCmExperiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Openmp, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput), Caliper, ): diff --git a/experiments/babelstream/experiment.py b/experiments/babelstream/experiment.py index 1b4a8486a..949676b31 100644 --- a/experiments/babelstream/experiment.py +++ b/experiments/babelstream/experiment.py @@ -4,19 +4,19 @@ # SPDX-License-Identifier: Apache-2.0 from benchpark.caliper import Caliper -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.openmp import OpenMPExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType class Babelstream( Experiment, Caliper, - CudaExperiment, - ROCmExperiment, - OpenMPExperiment, + ProgrammingModel( + ProgrammingModelType.Openmp, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), ): variant( "workload", diff --git a/experiments/branson/experiment.py b/experiments/branson/experiment.py index cde2c6999..88c56c3a9 100644 --- a/experiments/branson/experiment.py +++ b/experiments/branson/experiment.py @@ -4,21 +4,20 @@ # SPDX-License-Identifier: Apache-2.0 from benchpark.caliper import Caliper -from benchpark.cuda import CudaExperiment from benchpark.directives import variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class Branson( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, - CudaExperiment, - ROCmExperiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Openmp, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput), Caliper, ): diff --git a/experiments/commbench/experiment.py b/experiments/commbench/experiment.py index bc3894544..2242eaef6 100644 --- a/experiments/commbench/experiment.py +++ b/experiments/commbench/experiment.py @@ -3,16 +3,14 @@ # # SPDX-License-Identifier: Apache-2.0 -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType class Commbench( Experiment, - CudaExperiment, - ROCmExperiment, + ProgrammingModel(ProgrammingModelType.Cuda, ProgrammingModelType.Rocm), ): variant("workload", description="workload name", default="basic") maintainers("arhag23") diff --git a/experiments/genesis/experiment.py b/experiments/genesis/experiment.py index 565af2085..e47036f21 100644 --- a/experiments/genesis/experiment.py +++ b/experiments/genesis/experiment.py @@ -5,11 +5,13 @@ from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType -class Genesis(Experiment, MpiOnlyExperiment, OpenMPExperiment): +class Genesis( + Experiment, + ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp), +): variant( "workload", diff --git a/experiments/gpcnet/experiment.py b/experiments/gpcnet/experiment.py index 463c8f4d8..ad2aba04e 100644 --- a/experiments/gpcnet/experiment.py +++ b/experiments/gpcnet/experiment.py @@ -5,10 +5,13 @@ from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType -class Gpcnet(Experiment, MpiOnlyExperiment): +class Gpcnet( + Experiment, + ProgrammingModel(ProgrammingModelType.Mpionly), +): variant( "workload", default="network_test", diff --git a/experiments/gromacs/experiment.py b/experiments/gromacs/experiment.py index e8ad020a0..f06b8e783 100644 --- a/experiments/gromacs/experiment.py +++ b/experiments/gromacs/experiment.py @@ -3,20 +3,19 @@ # # SPDX-License-Identifier: Apache-2.0 -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType class Gromacs( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, - CudaExperiment, - ROCmExperiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Openmp, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), ): variant( "workload", diff --git a/experiments/hpcg/experiment.py b/experiments/hpcg/experiment.py index 3f4738dec..a595b1dd4 100644 --- a/experiments/hpcg/experiment.py +++ b/experiments/hpcg/experiment.py @@ -6,15 +6,13 @@ from benchpark.caliper import Caliper from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class Hpcg( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, + ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp), Scaling(ScalingMode.Strong, ScalingMode.Weak), Caliper, ): diff --git a/experiments/hpl/experiment.py b/experiments/hpl/experiment.py index 695ed9fb9..7171483e1 100644 --- a/experiments/hpl/experiment.py +++ b/experiments/hpl/experiment.py @@ -6,15 +6,13 @@ from benchpark.caliper import Caliper from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class Hpl( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, + ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp), Scaling(ScalingMode.Strong, ScalingMode.Weak), Caliper, ): diff --git a/experiments/ior/experiment.py b/experiments/ior/experiment.py index abc336990..0c86f88a4 100644 --- a/experiments/ior/experiment.py +++ b/experiments/ior/experiment.py @@ -5,11 +5,15 @@ from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode -class Ior(Experiment, MpiOnlyExperiment, Scaling(ScalingMode.Strong, ScalingMode.Weak)): +class Ior( + Experiment, + ProgrammingModel(ProgrammingModelType.Mpionly), + Scaling(ScalingMode.Strong, ScalingMode.Weak), +): variant( "workload", default="mpiio-write", diff --git a/experiments/kripke/experiment.py b/experiments/kripke/experiment.py index 12f6cfcdb..ecc48e6f5 100644 --- a/experiments/kripke/experiment.py +++ b/experiments/kripke/experiment.py @@ -4,21 +4,20 @@ # SPDX-License-Identifier: Apache-2.0 from benchpark.caliper import Caliper -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class Kripke( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, - CudaExperiment, - ROCmExperiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Openmp, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput), Caliper, ): diff --git a/experiments/laghos/experiment.py b/experiments/laghos/experiment.py index bae785b59..9190c21ca 100644 --- a/experiments/laghos/experiment.py +++ b/experiments/laghos/experiment.py @@ -4,19 +4,19 @@ # SPDX-License-Identifier: Apache-2.0 from benchpark.caliper import Caliper -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class Laghos( Experiment, - MpiOnlyExperiment, - CudaExperiment, - ROCmExperiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput), Caliper, ): diff --git a/experiments/lammps/experiment.py b/experiments/lammps/experiment.py index efce39198..20e11e326 100644 --- a/experiments/lammps/experiment.py +++ b/experiments/lammps/experiment.py @@ -3,21 +3,20 @@ # # SPDX-License-Identifier: Apache-2.0 -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class Lammps( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, - CudaExperiment, - ROCmExperiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Openmp, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), Scaling(ScalingMode.Strong), ): variant( diff --git a/experiments/md-test/experiment.py b/experiments/md-test/experiment.py index 608a7494b..bb52fba1e 100644 --- a/experiments/md-test/experiment.py +++ b/experiments/md-test/experiment.py @@ -5,13 +5,13 @@ from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class MdTest( Experiment, - MpiOnlyExperiment, + ProgrammingModel(ProgrammingModelType.Mpionly), Scaling(ScalingMode.Strong), ): variant( diff --git a/experiments/osu-micro-benchmarks/experiment.py b/experiments/osu-micro-benchmarks/experiment.py index 7d129effc..17e10becf 100644 --- a/experiments/osu-micro-benchmarks/experiment.py +++ b/experiments/osu-micro-benchmarks/experiment.py @@ -3,18 +3,18 @@ # # SPDX-License-Identifier: Apache-2.0 -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType class OsuMicroBenchmarks( Experiment, - MpiOnlyExperiment, - ROCmExperiment, - CudaExperiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), ): variant( diff --git a/experiments/phloem/experiment.py b/experiments/phloem/experiment.py index 437414057..e6971bb03 100644 --- a/experiments/phloem/experiment.py +++ b/experiments/phloem/experiment.py @@ -5,10 +5,13 @@ from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType -class Phloem(Experiment, MpiOnlyExperiment): +class Phloem( + Experiment, + ProgrammingModel(ProgrammingModelType.Mpionly), +): variant( "workload", default="sqmr", diff --git a/experiments/py-scaffold/experiment.py b/experiments/py-scaffold/experiment.py index d07c3f3b1..d933dcbac 100644 --- a/experiments/py-scaffold/experiment.py +++ b/experiments/py-scaffold/experiment.py @@ -4,17 +4,15 @@ # SPDX-License-Identifier: Apache-2.0 from benchpark.caliper import Caliper -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class PyScaffold( Experiment, - CudaExperiment, - ROCmExperiment, + ProgrammingModel(ProgrammingModelType.Cuda, ProgrammingModelType.Rocm), Scaling(ScalingMode.Strong, ScalingMode.Weak), Caliper, ): diff --git a/experiments/quicksilver/experiment.py b/experiments/quicksilver/experiment.py index 76f340345..7a6b85b43 100644 --- a/experiments/quicksilver/experiment.py +++ b/experiments/quicksilver/experiment.py @@ -6,15 +6,13 @@ from benchpark.caliper import Caliper from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class Quicksilver( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, + ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp), Scaling(ScalingMode.Strong, ScalingMode.Weak), Caliper, ): diff --git a/experiments/qws/experiment.py b/experiments/qws/experiment.py index 66c59f561..ea4d7bc03 100644 --- a/experiments/qws/experiment.py +++ b/experiments/qws/experiment.py @@ -6,15 +6,13 @@ from benchpark.caliper import Caliper from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class Qws( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, + ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp), Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput), Caliper, ): diff --git a/experiments/raja-perf/experiment.py b/experiments/raja-perf/experiment.py index e171abbf3..0a32ecbf0 100644 --- a/experiments/raja-perf/experiment.py +++ b/experiments/raja-perf/experiment.py @@ -6,21 +6,20 @@ import math from benchpark.caliper import Caliper -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class RajaPerf( Experiment, - MpiOnlyExperiment, - CudaExperiment, - ROCmExperiment, - OpenMPExperiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Openmp, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput), Caliper, ): diff --git a/experiments/remhos/experiment.py b/experiments/remhos/experiment.py index f2cb34b13..ee2d7fc9a 100644 --- a/experiments/remhos/experiment.py +++ b/experiments/remhos/experiment.py @@ -4,19 +4,19 @@ # SPDX-License-Identifier: Apache-2.0 from benchpark.caliper import Caliper -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType from benchpark.scaling import Scaling, ScalingMode class Remhos( Experiment, - MpiOnlyExperiment, - CudaExperiment, - ROCmExperiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), Scaling(ScalingMode.Strong), Caliper, ): diff --git a/experiments/salmon-tddft/experiment.py b/experiments/salmon-tddft/experiment.py index f6ef86494..1bf7d3b01 100644 --- a/experiments/salmon-tddft/experiment.py +++ b/experiments/salmon-tddft/experiment.py @@ -5,11 +5,13 @@ from benchpark.directives import variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType -class SalmonTddft(Experiment, MpiOnlyExperiment, OpenMPExperiment): +class SalmonTddft( + Experiment, + ProgrammingModel(ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp), +): variant( "workload", default="gs", diff --git a/experiments/smb/experiment.py b/experiments/smb/experiment.py index 2c3b8c7a2..97364ae85 100644 --- a/experiments/smb/experiment.py +++ b/experiments/smb/experiment.py @@ -5,10 +5,13 @@ from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType -class Smb(Experiment, MpiOnlyExperiment): +class Smb( + Experiment, + ProgrammingModel(ProgrammingModelType.Mpionly), +): variant( "workload", default="mpi_overhead", diff --git a/experiments/sparta-snl/experiment.py b/experiments/sparta-snl/experiment.py index 3a58d863b..9820edd57 100644 --- a/experiments/sparta-snl/experiment.py +++ b/experiments/sparta-snl/experiment.py @@ -3,20 +3,19 @@ # # SPDX-License-Identifier: Apache-2.0 -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType class SpartaSnl( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, - CudaExperiment, - ROCmExperiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Openmp, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), ): variant( "workload", diff --git a/experiments/stream/experiment.py b/experiments/stream/experiment.py index f817c1dc1..996ec833e 100644 --- a/experiments/stream/experiment.py +++ b/experiments/stream/experiment.py @@ -6,12 +6,12 @@ from benchpark.caliper import Caliper from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType class Stream( Experiment, - MpiOnlyExperiment, + ProgrammingModel(ProgrammingModelType.Mpionly), Caliper, ): variant( diff --git a/experiments/test/saxpy/experiment.py b/experiments/test/saxpy/experiment.py index 1c13face7..be062cdbd 100644 --- a/experiments/test/saxpy/experiment.py +++ b/experiments/test/saxpy/experiment.py @@ -5,20 +5,19 @@ from benchpark.caliper import Caliper -from benchpark.cuda import CudaExperiment from benchpark.directives import maintainers, variant from benchpark.experiment import Experiment -from benchpark.mpi import MpiOnlyExperiment -from benchpark.openmp import OpenMPExperiment -from benchpark.rocm import ROCmExperiment +from benchpark.programming_model import ProgrammingModel, ProgrammingModelType class Saxpy( Experiment, - MpiOnlyExperiment, - OpenMPExperiment, - CudaExperiment, - ROCmExperiment, + ProgrammingModel( + ProgrammingModelType.Mpionly, + ProgrammingModelType.Openmp, + ProgrammingModelType.Cuda, + ProgrammingModelType.Rocm, + ), Caliper, ): variant( diff --git a/lib/benchpark/accounting.py b/lib/benchpark/accounting.py index e8a289fee..2ff508d9f 100644 --- a/lib/benchpark/accounting.py +++ b/lib/benchpark/accounting.py @@ -11,10 +11,10 @@ PROGRAMMING_MODEL_CATEGORY = "programming_model" SCALING_CATEGORY = "scaling" EXP_DICT = { - "OpenMPExperiment": (PROGRAMMING_MODEL_CATEGORY, "openmp"), - "CudaExperiment": (PROGRAMMING_MODEL_CATEGORY, "cuda"), - "ROCmExperiment": (PROGRAMMING_MODEL_CATEGORY, "rocm"), - "MpiOnlyExperiment": (PROGRAMMING_MODEL_CATEGORY, "mpi"), + "ProgrammingModelType.Openmp": (PROGRAMMING_MODEL_CATEGORY, "openmp"), + "ProgrammingModelType.Cuda": (PROGRAMMING_MODEL_CATEGORY, "cuda"), + "ProgrammingModelType.Rocm": (PROGRAMMING_MODEL_CATEGORY, "rocm"), + "ProgrammingModelType.Mpionly": (PROGRAMMING_MODEL_CATEGORY, "mpi"), "ScalingMode.Strong": (SCALING_CATEGORY, "strong"), "ScalingMode.Weak": (SCALING_CATEGORY, "weak"), "ScalingMode.Throughput": (SCALING_CATEGORY, "throughput"), diff --git a/lib/benchpark/cmd/info.py b/lib/benchpark/cmd/info.py index ecb043b1e..541ac9591 100644 --- a/lib/benchpark/cmd/info.py +++ b/lib/benchpark/cmd/info.py @@ -148,7 +148,7 @@ def _info_ramble_name(experiment_class): msg = ( f"'{experiment_spec_str}' must be a valid experiment spec;" " some experiments require specifying additional variants" - " (e.g. experiments not inheriting MpiOnlyExperiment must" + " (e.g. experiments not inheriting ProgrammingModelType.Mpionly must" " set +rocm or +cuda)." ) raise ValueError(msg) from e diff --git a/lib/benchpark/cuda.py b/lib/benchpark/cuda.py deleted file mode 100644 index 97434db4c..000000000 --- a/lib/benchpark/cuda.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - - -from benchpark.directives import requires, variant -from benchpark.experiment import ExperimentHelper - - -class CudaExperiment: - requires("cuda", when="+cuda") - variant("cuda", default=False, description="Build and run with CUDA") - - def __init__(self): - super().__init__() - if self.spec.variants["cuda"][0]: - self.device_type = "gpu" - self.programming_models.append("cuda") - - class Helper(ExperimentHelper): - def get_helper_name_prefix(self): - return "cuda" if self.spec.satisfies("+cuda") else "" - - def get_spack_variants(self): - return ( - "+cuda cuda_arch={cuda_arch}" - if self.spec.satisfies("+cuda") - else "~cuda" - ) diff --git a/lib/benchpark/experiment.py b/lib/benchpark/experiment.py index 77f427c25..da2bbbc08 100644 --- a/lib/benchpark/experiment.py +++ b/lib/benchpark/experiment.py @@ -255,32 +255,35 @@ def __init__(self, spec): self.package_specs = {} + # Set available programming models for checks + models = set() + for cls in self.__class__.mro(): + models.update(getattr(cls, "_available_programming_models", ())) + self.programming_models = list(models) + # Explicitly ordered list. "mpi" first models = ["mpi"] + ["openmp", "cuda", "rocm"] + valid_models = [] invalid_models = [] for model in models: - # Experiment specifying model in add_package_spec that it doesn't implement - if ( - self.spec.satisfies("+" + model) - and model not in self.programming_models - ): - invalid_models.append(model) + if self.spec.satisfies("+" + model): + valid_models.append(model) + # Experiment specifying model in add_package_spec that it doesn't implement + if model not in self.programming_models: + invalid_models.append(model) + # MPI is always valid if with another programming model, even if no mpionly + if "mpi" in invalid_models and len(valid_models) > 1: + invalid_models.remove("mpi") # Case where there are no experiments specified in experiment.py if len(self.programming_models) == 0: - raise NotImplementedError( - f"Please specify a programming model in your {self.name}/experiment.py (e.g. MpiOnlyExperiment, OpenMPExperiment, CudaExperiment, ROCmExperiment). See other experiments for examples." + raise BenchparkError( + f"Please specify a programming model in your {self.name}/experiment.py (e.g. ProgrammingModelType.Mpionly, ProgrammingModelType.Openmp, ProgrammingModelType.Cuda, ProgrammingModelType.Rocm). See other experiments for examples." ) elif len(invalid_models) > 0: - raise NotImplementedError( + print(self.spec) + raise BenchparkError( f'{invalid_models} are not valid programming models for "{self.name}". Choose from {self.programming_models}.' ) - # Check if experiment is trying to run in MpiOnly mode without being an MpiOnlyExperiment - elif "mpi" not in str(self.spec) and not any( - self.spec.satisfies("+" + model) for model in models[1:] - ): - raise NotImplementedError( - f'"{self.name}" cannot run with MPI only without inheriting from MpiOnlyExperiment. Choose from {self.programming_models}' - ) if ( sum([self.spec.satisfies(s) for s in ["+strong", "+weak", "+throughput"]]) @@ -469,7 +472,10 @@ def compute_applications_section_wrapper(self): for cls in self.helpers: helper_prefix = cls.get_helper_name_prefix() if helper_prefix: - expr_helper_list.append(helper_prefix) + if isinstance(helper_prefix, list): + expr_helper_list.extend(helper_prefix) + else: + expr_helper_list.append(helper_prefix) expr_name_suffix = "_".join(expr_helper_list + self.expr_var_names) self.check_required_variables() diff --git a/lib/benchpark/mpi.py b/lib/benchpark/mpi.py deleted file mode 100644 index b9273c5ad..000000000 --- a/lib/benchpark/mpi.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - - -from benchpark.directives import requires, variant -from benchpark.experiment import ExperimentHelper - - -class MpiOnlyExperiment: - requires("mpi") - variant("mpi", default=True, description="Run with MPI only") - - def __init__(self): - super().__init__() - if self.spec.variants["mpi"][0]: - self.device_type = "cpu" - self.programming_models.append("mpi") - - class Helper(ExperimentHelper): - def get_helper_name_prefix(self): - return "mpi" if self.spec.satisfies("+mpi") else "" diff --git a/lib/benchpark/openmp.py b/lib/benchpark/openmp.py deleted file mode 100644 index 00c40355a..000000000 --- a/lib/benchpark/openmp.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - - -from benchpark.directives import requires, variant -from benchpark.experiment import ExperimentHelper - - -class OpenMPExperiment: - requires("openmp", when="+openmp") - variant("openmp", default=False, description="Build and run with OpenMP") - - def __init__(self): - super().__init__() - if self.spec.variants["openmp"][0]: - self.device_type = "cpu" - self.programming_models.append("openmp") - - class Helper(ExperimentHelper): - def get_helper_name_prefix(self): - return "openmp" if self.spec.satisfies("+openmp") else "" - - def get_spack_variants(self): - return "+openmp" if self.spec.satisfies("+openmp") else "~openmp" diff --git a/lib/benchpark/programming_model.py b/lib/benchpark/programming_model.py new file mode 100644 index 000000000..bbd1b407a --- /dev/null +++ b/lib/benchpark/programming_model.py @@ -0,0 +1,85 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + + +from enum import Enum + +from benchpark.directives import requires, variant +from benchpark.experiment import ExperimentHelper + + +class ProgrammingModelType(Enum): + Mpionly = "mpi" + Openmp = "openmp" + Cuda = "cuda" + Rocm = "rocm" + + +def ProgrammingModel(*types): + for ty in types: + if not isinstance(ty, ProgrammingModelType): + raise ValueError(f"Invalid programming model: {ty}") + + # Normalize once so we can reuse + _available = tuple(t.value for t in types) + + class BaseModel: + requires("mpi", when="+mpi") + requires("rocm", when="+rocm") + requires("cuda", when="+cuda") + requires("openmp", when="+openmp") + + variant("mpi", default=True, description="Run with MPI") + variant("rocm", default=False, description="Build and run with ROCm") + variant("cuda", default=False, description="Build and run with CUDA") + variant("openmp", default=False, description="Build and run with OpenMP") + + # Class-level list of supported models for any class that includes this mixin + _available_programming_models = _available + + # Handy instance-level property (works on Experiment instances) + @property + def available_programming_models(self): + # If multiple mixins contribute, merge them from MRO at runtime + models = set() + for cls in type(self).mro(): + models.update(getattr(cls, "_available_programming_models", ())) + return tuple(sorted(models)) + + # Quick check helper + @staticmethod + def supports_model(name: str) -> bool: + return name in _available + + # Helper class (unchanged except for optional new method) + class Helper(ExperimentHelper): + def get_helper_name_prefix(self): + models = [] + for s in [ + ProgrammingModelType.Mpionly.value, + ProgrammingModelType.Openmp.value, + ProgrammingModelType.Cuda.value, + ProgrammingModelType.Rocm.value, + ]: + if self.spec.satisfies("+" + s): + models.append(s) + if len(models) > 0: + return models + return "no_model" + + # Optional: expose *available* (not selected) models via helper, too + def get_available_models(self): + models = set() + for cls in type(self).__mro__: + models.update(getattr(cls, "_available_programming_models", ())) + return tuple(sorted(models)) + + return type( + "ProgrammingModelType", + (BaseModel,), + { + "Helper": Helper, + }, + ) diff --git a/lib/benchpark/rocm.py b/lib/benchpark/rocm.py deleted file mode 100644 index b0c60d887..000000000 --- a/lib/benchpark/rocm.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - - -from benchpark.directives import requires, variant -from benchpark.experiment import ExperimentHelper - - -class ROCmExperiment: - requires("rocm", when="+rocm") - variant("rocm", default=False, description="Build and run with ROCm") - - def __init__(self): - super().__init__() - if self.spec.variants["rocm"][0]: - self.device_type = "gpu" - self.programming_models.append("rocm") - - class Helper(ExperimentHelper): - def get_helper_name_prefix(self): - return "rocm" if self.spec.satisfies("+rocm") else "" - - def get_spack_variants(self): - return ( - "+rocm amdgpu_target={rocm_arch}" - if self.spec.satisfies("+rocm") - else "~rocm" - ) diff --git a/lib/benchpark/test/experiment_errors.py b/lib/benchpark/test/experiment_errors.py new file mode 100644 index 000000000..e0ab6998d --- /dev/null +++ b/lib/benchpark/test/experiment_errors.py @@ -0,0 +1,32 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +import benchpark.spec +from benchpark.error import BenchparkError + + +def test_programming_model_checks(): + # babelstream mpi-only not valid + with pytest.raises( + BenchparkError, match=r"mpi.*are not valid programming models for.*babelstream" + ): + spec = benchpark.spec.ExperimentSpec("babelstream").concretize() + experiment = spec.experiment # noqa: F841 + + # stream+openmp not valid + with pytest.raises( + Exception, match=r"openmp.*are not valid programming models for.*stream.*mpi" + ): + spec = benchpark.spec.ExperimentSpec( + "stream+openmp workload=stream" + ).concretize() + experiment = spec.experiment # noqa: F841 + + # Multiple scaling options not valid + with pytest.raises(BenchparkError, match="cannot specify multiple scaling options"): + spec = benchpark.spec.ExperimentSpec("kripke+strong+weak").concretize() + experiment = spec.experiment # noqa: F841 diff --git a/lib/main.py b/lib/main.py index 95adde548..df3749bd4 100755 --- a/lib/main.py +++ b/lib/main.py @@ -374,7 +374,15 @@ def benchpark_tags_handler(args): """ source_dir = paths.benchpark_root ramble_exe = paths.benchpark_home / "ramble/bin/ramble" - subprocess.run([ramble_exe, "repo", "add", "--scope=site", f"{source_dir}/repo"]) + subprocess.run( + [ + ramble_exe, + "repo", + "add", + "--scope=site", + f"{source_dir}/repos/ramble_applications", + ] + ) benchmarks = benchpark_benchmarks() if args.tag: diff --git a/modifiers/caliper/modifier.py b/modifiers/caliper/modifier.py index 938830b24..010d770b5 100644 --- a/modifiers/caliper/modifier.py +++ b/modifiers/caliper/modifier.py @@ -157,5 +157,5 @@ def _build_metadata(self, workspace, app_inst): f.write(json.dumps(cali_metadata)) with when("package_manager_family=spack"): - software_spec("caliper", pkg_spec="caliper", package_manager="spack") + software_spec("caliper", pkg_spec="caliper") required_package("caliper") diff --git a/repos/ramble_applications/ad/application.py b/repos/ramble_applications/ad/application.py index c2f9e1d04..a452523ec 100644 --- a/repos/ramble_applications/ad/application.py +++ b/repos/ramble_applications/ad/application.py @@ -17,7 +17,7 @@ class Ad(ExecutableApplication): executable('c_interface_test', 'c_interface_test', use_mpi=True) workload('ad', executables=['c_interface_test']) - + #figure_of_merit('Photons per Second', # log_file='{experiment_run_dir}/{experiment_name}.out', # fom_regex=r'Photons Per Second \(FOM\):\s+(?P[0-9]+\.[0-9]*([0-9]*)?e\+[0-9]*)', diff --git a/repos/ramble_applications/commbench/application.py b/repos/ramble_applications/commbench/application.py index 45aa7a789..44f7c0cd7 100644 --- a/repos/ramble_applications/commbench/application.py +++ b/repos/ramble_applications/commbench/application.py @@ -14,7 +14,8 @@ class Commbench(ExecutableApplication): maintainers("arhag23") - software_spec("commbench", pkg_spec="commbench", package_manager="spack*") + with when("package_manager_family=spack"): + software_spec("commbench", pkg_spec="commbench") executable("execute-bench", "CommBench --use {lib} --pattern {pat}", use_mpi=True) # executable("execute-test", "mpiexec -n {n_ranks} CommBench --use {lib} --pattern {pat} --validate", use_mpi=True) diff --git a/repos/ramble_applications/gromacs/application.py b/repos/ramble_applications/gromacs/application.py index 97baeb883..dde03dcf7 100644 --- a/repos/ramble_applications/gromacs/application.py +++ b/repos/ramble_applications/gromacs/application.py @@ -17,9 +17,10 @@ class Gromacs(ExecutableApplication): tags('molecular-dynamics') - define_compiler('gcc9', pkg_spec='gcc@9.3.0', package_manager='spack*') - software_spec('impi2018', pkg_spec='intel-mpi@2018.4.274', package_manager='spack*') - software_spec('gromacs', pkg_spec='gromacs@2023.3', compiler='gcc12', package_manager='spack*') + with when("package_manager_family=spack"): + define_compiler('gcc9', pkg_spec='gcc@9.3.0') + software_spec('impi2018', pkg_spec='intel-mpi@2018.4.274') + software_spec('gromacs', pkg_spec='gromacs@2023.3', compiler='gcc12') executable('pre-process', 'gmx_mpi grompp ' + '-f {input_path}/{type}.mdp ' + diff --git a/repos/ramble_applications/stream/application.py b/repos/ramble_applications/stream/application.py index 4f5494250..559004bd1 100644 --- a/repos/ramble_applications/stream/application.py +++ b/repos/ramble_applications/stream/application.py @@ -22,10 +22,10 @@ class Stream(ExecutableApplication): 'high-memory-bandwidth','regular-memory-access', 'mpi','c','fortran'] - software_spec('stream', - pkg_spec='stream@5.10 +openmp cflags="-O3 -DSTREAM_ARRAY_SIZE=80000000 -DNTIMES=20"', - compiler='gcc12', - package_manager='spack*') + with when("package_manager_family=spack"): + software_spec('stream', + pkg_spec='stream@5.10 +openmp cflags="-O3 -DSTREAM_ARRAY_SIZE=80000000 -DNTIMES=20"', + compiler='gcc12') required_package('stream')