diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 238b27b58..57cc59f2d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,10 @@ on: branches: [ master ] merge_group: +permissions: + contents: write # Required to push to the repository + pages: write + jobs: build: name: Build (Java ${{ matrix.java }} - ${{ matrix.os }}) @@ -84,3 +88,43 @@ jobs: uses: docker/build-push-action@v6 with: file: opendc-web/opendc-web-runner/Dockerfile + benchmark: + name: Benchmark + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Validate Gradle wrapper + uses: gradle/actions/wrapper-validation@v3 + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: 'zulu' + java-version: 21 + - name: Run benchmark + uses: gradle/actions/setup-gradle@v3 + with: + arguments: jmh + - name: Combine JMH benchmark results + run: | + mkdir -p combined-results + echo "[]" > combined-results/results.json + find . -name "results.json" -exec cat {} + > all-results.tmp.json + jq -s 'add' all-results.tmp.json > combined-results/results.json + rm all-results.tmp.json + - name: Store benchmark results + uses: benchmark-action/github-action-benchmark@v1 + with: + name: OpenDC benchmarks + tool: 'jmh' + output-file-path: combined-results/results.json + # Access token to deploy GitHub Pages branch + github-token: ${{ secrets.GITHUB_TOKEN }} + + comment-on-alert: true + alert-comment-cc-users: '@DanteNiewenhuis' + + # Push and deploy GitHub pages branch automatically + auto-push: true + summary-always: true + benchmark-data-dir-path: site/dev/benchmarks diff --git a/buildSrc/src/main/kotlin/benchmark-conventions.gradle.kts b/buildSrc/src/main/kotlin/benchmark-conventions.gradle.kts index e16733a47..55b8d664a 100644 --- a/buildSrc/src/main/kotlin/benchmark-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/benchmark-conventions.gradle.kts @@ -34,12 +34,14 @@ configure { } jmh { - jmhVersion.set("1.35") + jmhVersion.set("1.37") profilers.add("stack") profilers.add("gc") includeTests.set(false) // Do not include tests by default + + resultFormat = "JSON" } tasks.named("jmh", JMHTask::class) { diff --git a/buildSrc/src/main/kotlin/testing-conventions.gradle.kts b/buildSrc/src/main/kotlin/testing-conventions.gradle.kts index b374d0ffa..7975ae5c0 100644 --- a/buildSrc/src/main/kotlin/testing-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/testing-conventions.gradle.kts @@ -42,21 +42,9 @@ dependencies { testRuntimeOnly(versionCatalog["junit.jupiter.engine"]) } -tasks.register("testsOn18") { +tasks.register("testsOn21") { javaLauncher.set(javaToolchains.launcherFor { - languageVersion.set(JavaLanguageVersion.of(18)) - }) - - useJUnitPlatform() - - minHeapSize = "512m" - maxHeapSize = "1024m" - jvmArgs = listOf("-XX:MaxMetaspaceSize=512m") -} - -tasks.register("testsOn19") { - javaLauncher.set(javaToolchains.launcherFor { - languageVersion.set(JavaLanguageVersion.of(19)) + languageVersion.set(JavaLanguageVersion.of(21)) }) useJUnitPlatform() diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 44cd4003e..eb18f0e64 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -10,7 +10,7 @@ jackson = "2.16.1" jandex-gradle = "1.1.0" java = "21" jline = "3.25.1" -jmh-gradle = "0.7.0" +jmh-gradle = "0.7.3" jakarta = "3.0.2" junit-jupiter = "5.10.2" kotlin = "1.9.22" diff --git a/opendc-experiments/opendc-experiments-base/build.gradle.kts b/opendc-experiments/opendc-experiments-base/build.gradle.kts index 8a77f0c45..5a5cddbc3 100644 --- a/opendc-experiments/opendc-experiments-base/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-base/build.gradle.kts @@ -1,3 +1,7 @@ +import me.champeau.jmh.JMHTask +import org.gradle.kotlin.dsl.assign +import org.gradle.kotlin.dsl.named + /* * Copyright (c) 2022 AtLarge Research * @@ -27,6 +31,7 @@ plugins { `kotlin-library-conventions` `testing-conventions` `jacoco-conventions` + `benchmark-conventions` distribution kotlin("plugin.serialization") version "1.9.22" } @@ -58,6 +63,17 @@ val createScenarioApp by tasks.creating(CreateStartScripts::class) { outputDir = layout.buildDirectory.dir("scripts").get().asFile } +//tasks.named("jmh", JMHTask::class) { +// outputs.upToDateWhen { false } // XXX Do not cache the output of this task +// jvmArgs = listOf("-Djmh.outputFormat=json", "-Djmh.output=/reports/jmh/results-${project.name}.json") +// testRuntimeClasspath.setFrom() // XXX Clear test runtime classpath to eliminate duplicate dependencies on classpath +//} + +//jmh { +// resultFormat = "JSON" // <--- Key: This sets the output format +// resultsFile = file("$buildDir/reports/jmh/results-${project.name}.json") // <--- Sets the output file +//} + // Create custom Scenario distribution distributions { main { diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/base/BasicBenchmark.kt b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/base/BasicBenchmark.kt new file mode 100644 index 000000000..91d77e0ec --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/base/BasicBenchmark.kt @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2025 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.base + +import org.opendc.compute.workload.Task +import org.opendc.simulator.compute.workload.trace.TraceFragment +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Fork +import org.openjdk.jmh.annotations.Level +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.Setup +import org.openjdk.jmh.annotations.State +import org.openjdk.jmh.annotations.Warmup +import java.util.ArrayList +import java.util.concurrent.TimeUnit + +@BenchmarkMode(Mode.AverageTime) // or Mode.AverageTime +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Warmup(iterations = 2, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@State(Scope.Thread) +@Fork(1) +class BasicBenchmark { + private lateinit var list: MutableList + + @Setup(Level.Iteration) + fun setUp() { + list = mutableListOf() + } + + /** + * FlowDistributor test 13: 1000 tasks. This tests the performance + * In this test, two tasks are scheduled, and they can both fit. + * However, task 0 increases its demand which overloads the FlowDistributor. + * This test shows how the FlowDistributor handles transition from fitting to overloading when multiple tasks are running. + * We check if both the host and the Tasks show the correct cpu usage and demand during the two fragments. + */ + @Benchmark + fun benchmarkFlowDistributor1() { + val workload: ArrayList = + arrayListOf().apply { + repeat(1000) { + this.add( + createBenchmarkTask( + name = "0", + fragments = + arrayListOf(TraceFragment(10 * 60 * 1000, 2000.0, 1)), + ), + ) + } + } + val topology = createTopology("single_1_2000.json") + + val monitor = runBenchmark(topology, workload) + } +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/base/BenchmarkingUtils.kt b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/base/BenchmarkingUtils.kt new file mode 100644 index 000000000..103f31802 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/base/BenchmarkingUtils.kt @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2025 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.base + +import org.opendc.compute.simulator.provisioner.Provisioner +import org.opendc.compute.simulator.provisioner.registerComputeMonitor +import org.opendc.compute.simulator.provisioner.setupComputeService +import org.opendc.compute.simulator.provisioner.setupHosts +import org.opendc.compute.simulator.scheduler.ComputeScheduler +import org.opendc.compute.simulator.scheduler.FilterScheduler +import org.opendc.compute.simulator.scheduler.filters.ComputeFilter +import org.opendc.compute.simulator.scheduler.filters.RamFilter +import org.opendc.compute.simulator.scheduler.filters.VCpuFilter +import org.opendc.compute.simulator.scheduler.weights.CoreRamWeigher +import org.opendc.compute.simulator.service.ComputeService +import org.opendc.compute.simulator.telemetry.ComputeMonitor +import org.opendc.compute.simulator.telemetry.table.host.HostTableReader +import org.opendc.compute.simulator.telemetry.table.powerSource.PowerSourceTableReader +import org.opendc.compute.simulator.telemetry.table.service.ServiceTableReader +import org.opendc.compute.simulator.telemetry.table.task.TaskTableReader +import org.opendc.compute.topology.clusterTopology +import org.opendc.compute.topology.specs.ClusterSpec +import org.opendc.compute.workload.Task +import org.opendc.experiments.base.experiment.specs.FailureModelSpec +import org.opendc.experiments.base.runner.replay +import org.opendc.simulator.compute.workload.trace.TraceFragment +import org.opendc.simulator.compute.workload.trace.TraceWorkload +import org.opendc.simulator.compute.workload.trace.scaling.NoDelayScaling +import org.opendc.simulator.compute.workload.trace.scaling.ScalingPolicy +import org.opendc.simulator.kotlin.runSimulation +import java.time.Duration +import java.time.LocalDateTime +import java.time.ZoneOffset +import java.util.UUID +import kotlin.collections.ArrayList + +/** + * Obtain the topology factory for the Benchmark. + */ +fun createTopology(name: String): List { + val stream = checkNotNull(object {}.javaClass.getResourceAsStream("/topologies/$name")) + return stream.use { clusterTopology(stream) } +} + +fun createBenchmarkTask( + name: String, + memCapacity: Long = 0L, + submissionTime: String = "1970-01-01T00:00", + duration: Long = 0L, + fragments: ArrayList, + checkpointInterval: Long = 0L, + checkpointDuration: Long = 0L, + checkpointIntervalScaling: Double = 1.0, + scalingPolicy: ScalingPolicy = NoDelayScaling(), +): Task { + return Task( + UUID.nameUUIDFromBytes(name.toByteArray()), + name, + fragments.maxOf { it.coreCount }, + fragments.maxOf { it.cpuUsage }, + memCapacity, + 1800000.0, + LocalDateTime.parse(submissionTime).toInstant(ZoneOffset.UTC).toEpochMilli(), + duration, + "", + -1, + TraceWorkload( + fragments, + checkpointInterval, + checkpointDuration, + checkpointIntervalScaling, + scalingPolicy, + name, + ), + ) +} + +fun runBenchmark( + topology: List, + workload: ArrayList, + failureModelSpec: FailureModelSpec? = null, + computeScheduler: ComputeScheduler = + FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(1.0), RamFilter(1.0)), + weighers = listOf(CoreRamWeigher(multiplier = 1.0)), + ), +): BenchmarkComputeMonitor { + val monitor = BenchmarkComputeMonitor() + + runSimulation { + val seed = 0L + Provisioner(dispatcher, seed).use { provisioner -> + + val startTimeLong = workload.minOf { it.submissionTime } + val startTime = Duration.ofMillis(startTimeLong) + + provisioner.runSteps( + setupComputeService(serviceDomain = "compute.opendc.org", { computeScheduler }), + registerComputeMonitor(serviceDomain = "compute.opendc.org", monitor, exportInterval = Duration.ofMinutes(1), startTime), + setupHosts(serviceDomain = "compute.opendc.org", topology, startTimeLong), + ) + + val service = provisioner.registry.resolve("compute.opendc.org", ComputeService::class.java)!! + service.setTasksExpected(workload.size) + service.setMetricReader(provisioner.getMonitor()) + + service.replay(timeSource, ArrayDeque(workload), failureModelSpec = failureModelSpec) + } + } + + return monitor +} + +class BenchmarkComputeMonitor : ComputeMonitor { + var taskCpuDemands = mutableMapOf>() + var taskCpuSupplied = mutableMapOf>() + + override fun record(reader: TaskTableReader) { + val taskName: String = reader.taskInfo.name + + if (taskName in taskCpuDemands) { + taskCpuDemands[taskName]?.add(reader.cpuDemand) + taskCpuSupplied[taskName]?.add(reader.cpuUsage) + } else { + taskCpuDemands[taskName] = arrayListOf(reader.cpuDemand) + taskCpuSupplied[taskName] = arrayListOf(reader.cpuUsage) + } + } + + var attemptsSuccess = 0 + var attemptsFailure = 0 + var attemptsError = 0 + var tasksPending = 0 + var tasksActive = 0 + var tasksTerminated = 0 + var tasksCompleted = 0 + + var timestamps = ArrayList() + var absoluteTimestamps = ArrayList() + + var maxTimestamp = 0L + + override fun record(reader: ServiceTableReader) { + attemptsSuccess = reader.attemptsSuccess + attemptsFailure = reader.attemptsFailure + attemptsError = 0 + tasksPending = reader.tasksPending + tasksActive = reader.tasksActive + tasksTerminated = reader.tasksTerminated + tasksCompleted = reader.tasksCompleted + + timestamps.add(reader.timestamp.toEpochMilli()) + absoluteTimestamps.add(reader.timestampAbsolute.toEpochMilli()) + maxTimestamp = reader.timestamp.toEpochMilli() + } + + var hostIdleTimes = mutableMapOf>() + var hostActiveTimes = mutableMapOf>() + var hostStealTimes = mutableMapOf>() + var hostLostTimes = mutableMapOf>() + + var hostCpuDemands = mutableMapOf>() + var hostCpuSupplied = mutableMapOf>() + var hostPowerDraws = mutableMapOf>() + var hostEnergyUsages = mutableMapOf>() + + override fun record(reader: HostTableReader) { + val hostName: String = reader.hostInfo.name + + if (!(hostName in hostCpuDemands)) { + hostIdleTimes[hostName] = ArrayList() + hostActiveTimes[hostName] = ArrayList() + hostStealTimes[hostName] = ArrayList() + hostLostTimes[hostName] = ArrayList() + + hostCpuDemands[hostName] = ArrayList() + hostCpuSupplied[hostName] = ArrayList() + hostPowerDraws[hostName] = ArrayList() + hostEnergyUsages[hostName] = ArrayList() + } + + hostIdleTimes[hostName]?.add(reader.cpuIdleTime) + hostActiveTimes[hostName]?.add(reader.cpuActiveTime) + hostStealTimes[hostName]?.add(reader.cpuStealTime) + hostLostTimes[hostName]?.add(reader.cpuLostTime) + + hostCpuDemands[hostName]?.add(reader.cpuDemand) + hostCpuSupplied[hostName]?.add(reader.cpuUsage) + hostPowerDraws[hostName]?.add(reader.powerDraw) + hostEnergyUsages[hostName]?.add(reader.energyUsage) + } + + var powerDraws = ArrayList() + var energyUsages = ArrayList() + + var carbonIntensities = ArrayList() + var carbonEmissions = ArrayList() + + override fun record(reader: PowerSourceTableReader) { + powerDraws.add(reader.powerDraw) + energyUsages.add(reader.energyUsage) + + carbonIntensities.add(reader.carbonIntensity) + carbonEmissions.add(reader.carbonEmission) + } +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt deleted file mode 100644 index c3408226e..000000000 --- a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin - -import org.opendc.compute.service.ComputeService -import org.opendc.compute.service.scheduler.FilterScheduler -import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.filters.RamFilter -import org.opendc.compute.service.scheduler.filters.VCpuFilter -import org.opendc.compute.service.scheduler.weights.CoreRamWeigher -import org.opendc.experiments.capelin.topology.clusterTopology -import org.opendc.experiments.compute.ComputeWorkloadLoader -import org.opendc.experiments.compute.VirtualMachine -import org.opendc.experiments.compute.replay -import org.opendc.experiments.compute.setupComputeService -import org.opendc.experiments.compute.setupHosts -import org.opendc.experiments.compute.topology.HostSpec -import org.opendc.experiments.compute.trace -import org.opendc.common.experiments.provisioner.Provisioner -import org.opendc.simulator.kotlin.runSimulation -import org.openjdk.jmh.annotations.Benchmark -import org.openjdk.jmh.annotations.Fork -import org.openjdk.jmh.annotations.Measurement -import org.openjdk.jmh.annotations.Param -import org.openjdk.jmh.annotations.Scope -import org.openjdk.jmh.annotations.Setup -import org.openjdk.jmh.annotations.State -import org.openjdk.jmh.annotations.Warmup -import java.io.File -import java.util.Random -import java.util.concurrent.TimeUnit - -/** - * Benchmark suite for the Capelin experiments. - */ -@State(Scope.Thread) -@Fork(1) -@Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) -class CapelinBenchmarks { - private lateinit var vms: List - private lateinit var topology: List - - @Param("true", "false") - private var isOptimized: Boolean = false - - @Setup - fun setUp() { - val loader = ComputeWorkloadLoader(File("src/test/resources/trace")) - vms = trace("bitbrains-small").resolve(loader, Random(1L)) - topology = checkNotNull(object {}.javaClass.getResourceAsStream("/topology.txt")).use { clusterTopology(it) } - } - - @Benchmark - fun benchmarkCapelin() = runSimulation { - val serviceDomain = "compute.opendc.org" - - Provisioner(dispatcher, seed = 0).use { provisioner -> - val computeScheduler = FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), - weighers = listOf(CoreRamWeigher(multiplier = 1.0)) - ) - - provisioner.runSteps( - setupComputeService(serviceDomain, { computeScheduler }), - setupHosts(serviceDomain, topology, optimize = isOptimized) - ) - - val service = provisioner.registry.resolve(serviceDomain, ComputeService::class.java)!! - service.replay(timeSource, vms, 0L, interference = true) - } - } -} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_2022-12-31_BE.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_2022-12-31_BE.parquet new file mode 100644 index 000000000..ab2b5f8b9 Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_2022-12-31_BE.parquet differ diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_2022-12-31_DE.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_2022-12-31_DE.parquet new file mode 100644 index 000000000..213e24a4a Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_2022-12-31_DE.parquet differ diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_2022-12-31_FR.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_2022-12-31_FR.parquet new file mode 100644 index 000000000..a2d64d8f3 Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_2022-12-31_FR.parquet differ diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_2022-12-31_NL.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_2022-12-31_NL.parquet new file mode 100644 index 000000000..6b4a05add Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_2022-12-31_NL.parquet differ diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_single_100.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_single_100.parquet new file mode 100644 index 000000000..195a340bc Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_single_100.parquet differ diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_two_80_120.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_two_80_120.parquet new file mode 100644 index 000000000..a7b2b63f1 Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/carbonTraces/2022-01-01_two_80_120.parquet differ diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/failureTraces/single_failure.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/failureTraces/single_failure.parquet new file mode 100644 index 000000000..44804ffaa Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/failureTraces/single_failure.parquet differ diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/failureTraces/single_failure_2.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/failureTraces/single_failure_2.parquet new file mode 100644 index 000000000..7dae482a1 Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/failureTraces/single_failure_2.parquet differ diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/failureTraces/two_failures.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/failureTraces/two_failures.parquet new file mode 100644 index 000000000..9f0fea0f0 Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/failureTraces/two_failures.parquet differ diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/batteries/experiment1.json b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/batteries/experiment1.json new file mode 100644 index 000000000..8835faeb2 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/batteries/experiment1.json @@ -0,0 +1,40 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 1, + "coreSpeed": 2000 + }, + "memory": { + "memorySize": 140457600000 + }, + "powerModel": { + "modelType": "linear", + "power": 400.0, + "idlePower": 100.0, + "maxPower": 200.0 + } + } + ], + "powerSource": { + "carbonTracePath": "src/test/resources/carbonTraces/2022-01-01_single_100.parquet" + }, + "battery": { + "capacity": 0.1, + "chargingSpeed": 1000, + "batteryPolicy": + { + "type": "single", + "carbonThreshold": 90 + } + } + } + ] +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/batteries/experiment2.json b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/batteries/experiment2.json new file mode 100644 index 000000000..8882af092 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/batteries/experiment2.json @@ -0,0 +1,40 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 1, + "coreSpeed": 2000 + }, + "memory": { + "memorySize": 140457600000 + }, + "powerModel": { + "modelType": "linear", + "power": 400.0, + "idlePower": 100.0, + "maxPower": 200.0 + } + } + ], + "powerSource": { + "carbonTracePath": "src/test/resources/carbonTraces/2022-01-01_single_100.parquet" + }, + "battery": { + "capacity": 0.1, + "chargingSpeed": 1000, + "batteryPolicy": + { + "type": "single", + "carbonThreshold": 120 + } + } + } + ] +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/batteries/experiment3.json b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/batteries/experiment3.json new file mode 100644 index 000000000..d78626f12 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/batteries/experiment3.json @@ -0,0 +1,40 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 1, + "coreSpeed": 2000 + }, + "memory": { + "memorySize": 140457600000 + }, + "powerModel": { + "modelType": "linear", + "power": 400.0, + "idlePower": 100.0, + "maxPower": 200.0 + } + } + ], + "powerSource": { + "carbonTracePath": "src/test/resources/carbonTraces/2022-01-01_two_80_120.parquet" + }, + "battery": { + "capacity": 0.02, + "chargingSpeed": 1000, + "batteryPolicy": + { + "type": "single", + "carbonThreshold": 100 + } + } + } + ] +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/batteries/experiment4.json b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/batteries/experiment4.json new file mode 100644 index 000000000..cb0ef4e54 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/batteries/experiment4.json @@ -0,0 +1,31 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 1, + "coreSpeed": 2000 + }, + "memory": { + "memorySize": 140457600000 + }, + "powerModel": { + "modelType": "linear", + "power": 400.0, + "idlePower": 100.0, + "maxPower": 200.0 + } + } + ], + "powerSource": { + "carbonTracePath": "src/test/resources/carbonTraces/2022-01-01_two_80_120.parquet" + } + } + ] +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000.json b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000.json new file mode 100644 index 000000000..ac9a30821 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000.json @@ -0,0 +1,28 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 1, + "coreSpeed": 2000 + }, + "memory": { + "memorySize": 140457600000 + }, + "powerModel": { + "modelType": "linear", + "power": 400.0, + "idlePower": 100.0, + "maxPower": 200.0 + } + } + ] + } + ] +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000_BE.json b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000_BE.json new file mode 100644 index 000000000..3a04b275f --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000_BE.json @@ -0,0 +1,31 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 1, + "coreSpeed": 2000 + }, + "memory": { + "memorySize": 140457600000 + }, + "powerModel": { + "modelType": "linear", + "power": 400.0, + "idlePower": 100.0, + "maxPower": 200.0 + } + } + ], + "powerSource": { + "carbonTracePath": "src/test/resources/carbonTraces/2022-01-01_2022-12-31_BE.parquet" + } + } + ] +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000_DE.json b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000_DE.json new file mode 100644 index 000000000..651e8b546 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000_DE.json @@ -0,0 +1,31 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 1, + "coreSpeed": 2000 + }, + "memory": { + "memorySize": 140457600000 + }, + "powerModel": { + "modelType": "linear", + "power": 400.0, + "idlePower": 100.0, + "maxPower": 200.0 + } + } + ], + "powerSource": { + "carbonTracePath": "src/test/resources/carbonTraces/2022-01-01_2022-12-31_DE.parquet" + } + } + ] +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000_FR.json b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000_FR.json new file mode 100644 index 000000000..fed097e91 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000_FR.json @@ -0,0 +1,31 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 1, + "coreSpeed": 2000 + }, + "memory": { + "memorySize": 140457600000 + }, + "powerModel": { + "modelType": "linear", + "power": 400.0, + "idlePower": 100.0, + "maxPower": 200.0 + } + } + ], + "powerSource": { + "carbonTracePath": "src/test/resources/carbonTraces/2022-01-01_2022-12-31_FR.parquet" + } + } + ] +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000_NL.json b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000_NL.json new file mode 100644 index 000000000..05805c886 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_1_2000_NL.json @@ -0,0 +1,31 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 1, + "coreSpeed": 2000 + }, + "memory": { + "memorySize": 140457600000 + }, + "powerModel": { + "modelType": "linear", + "power": 400.0, + "idlePower": 100.0, + "maxPower": 200.0 + } + } + ], + "powerSource": { + "carbonTracePath": "src/test/resources/carbonTraces/2022-01-01_2022-12-31_NL.parquet" + } + } + ] +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_2_2000.json b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_2_2000.json new file mode 100644 index 000000000..24ab0bcd8 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_2_2000.json @@ -0,0 +1,28 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 2, + "coreSpeed": 2000 + }, + "memory": { + "memorySize": 140457600000 + }, + "powerModel": { + "modelType": "linear", + "power": 400.0, + "idlePower": 100.0, + "maxPower": 200.0 + } + } + ] + } + ] +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_5000_2000.json b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_5000_2000.json new file mode 100644 index 000000000..9f1e418f8 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/resources/topologies/single_5000_2000.json @@ -0,0 +1,23 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 1, + "coreSpeed": 2000 + }, + "memory": { + "memorySize": 140457600000 + }, + "count": 5000 + } + ] + } + ] +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/bitbrains-small/fragments.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/bitbrains-small/fragments.parquet new file mode 100644 index 000000000..240f58e3e Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/bitbrains-small/fragments.parquet differ diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/bitbrains-small/interference-model.json b/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/bitbrains-small/interference-model.json new file mode 100644 index 000000000..51fc6366e --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/bitbrains-small/interference-model.json @@ -0,0 +1,21 @@ +[ + { + "vms": [ + "141", + "379", + "851", + "116" + ], + "minServerLoad": 0.0, + "performanceScore": 0.8830158730158756 + }, + { + "vms": [ + "205", + "116", + "463" + ], + "minServerLoad": 0.0, + "performanceScore": 0.7133055555552751 + } +] diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/bitbrains-small/tasks.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/bitbrains-small/tasks.parquet new file mode 100644 index 000000000..8e9dcea77 Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/bitbrains-small/tasks.parquet differ diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/single_task/fragments.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/single_task/fragments.parquet new file mode 100644 index 000000000..94a2d69ee Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/single_task/fragments.parquet differ diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/single_task/tasks.parquet b/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/single_task/tasks.parquet new file mode 100644 index 000000000..2a7da2eb6 Binary files /dev/null and b/opendc-experiments/opendc-experiments-base/src/jmh/resources/workloadTraces/single_task/tasks.parquet differ diff --git a/opendc-simulator/opendc-simulator-compute/build.gradle.kts b/opendc-simulator/opendc-simulator-compute/build.gradle.kts index 246fa5e8a..31a54797a 100644 --- a/opendc-simulator/opendc-simulator-compute/build.gradle.kts +++ b/opendc-simulator/opendc-simulator-compute/build.gradle.kts @@ -24,7 +24,6 @@ description = "Library for simulating computing workloads" plugins { `kotlin-library-conventions` - `benchmark-conventions` } dependencies { diff --git a/opendc-simulator/opendc-simulator-flow/build.gradle.kts b/opendc-simulator/opendc-simulator-flow/build.gradle.kts index 4f04bdc19..12ed93847 100644 --- a/opendc-simulator/opendc-simulator-flow/build.gradle.kts +++ b/opendc-simulator/opendc-simulator-flow/build.gradle.kts @@ -33,6 +33,4 @@ dependencies { testImplementation(projects.opendcSimulator.opendcSimulatorCore) testImplementation(libs.slf4j.simple) - - jmhImplementation(projects.opendcSimulator.opendcSimulatorCore) } diff --git a/opendc-simulator/opendc-simulator-flow/src/jmh/kotlin/org/opendc/simulator/engine/engine/PlaceholderBenchmark.kt b/opendc-simulator/opendc-simulator-flow/src/jmh/kotlin/org/opendc/simulator/engine/engine/PlaceholderBenchmark.kt new file mode 100644 index 000000000..50f0ee14a --- /dev/null +++ b/opendc-simulator/opendc-simulator-flow/src/jmh/kotlin/org/opendc/simulator/engine/engine/PlaceholderBenchmark.kt @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2025 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.simulator.engine.engine + +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Fork +import org.openjdk.jmh.annotations.Level +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.Setup +import org.openjdk.jmh.annotations.State +import org.openjdk.jmh.annotations.Warmup +import java.util.ArrayList +import java.util.concurrent.TimeUnit + +@BenchmarkMode(Mode.AverageTime) // or Mode.AverageTime +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@Warmup(iterations = 2, time = 1, timeUnit = TimeUnit.MILLISECONDS) +@State(Scope.Thread) +@Fork(1) +class PlaceholderBenchmark { + private lateinit var list: MutableList + + @Setup(Level.Iteration) + fun setUp() { + list = mutableListOf() + } + + /** + * FlowDistributor test 13: 1000 tasks. This tests the performance + * In this test, two tasks are scheduled, and they can both fit. + * However, task 0 increases its demand which overloads the FlowDistributor. + * This test shows how the FlowDistributor handles transition from fitting to overloading when multiple tasks are running. + * We check if both the host and the Tasks show the correct cpu usage and demand during the two fragments. + */ + @Benchmark + fun PlaceHolder() { + for (i in 0..1000) { + list.add(i) + } + } +}