Merge pull request #55 from OpenVADL/feature/run-benchmarks-on-master

ci: Setup benchmarking lcb

Author: Jozott00

.github/workflows/benchmark-lcb.yml

@@ -0,0 +1,61 @@
+name: benchmark LCB
+
+on:
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: read
+  actions: write
+  checks: write
+
+jobs:
+  build:
+    runs-on: self-hosted
+    steps:
+      - uses: actions/checkout@v4
+      - uses: graalvm/setup-graalvm@v1
+        with:
+          java-version: '21'
+          distribution: 'graalvm'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build
+        run: ./gradlew assemble testClasses -PFailOnWarnings
+  test:
+    needs: build
+    runs-on: self-hosted-x64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: graalvm/setup-graalvm@v1
+        with:
+          java-version: '21'
+          distribution: 'graalvm'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cache: 'gradle'
+      - name: Run tests
+        run: |
+          ./gradlew test --tests EmbenchBenchmarkNumberOfExecutedInstructionsSpikeRiscv32SimulationTest -DincludeTags=BenchmarkTest
+        env:
+          isCI: true
+          GATE_KEEPER_EMBENCH_NUMBER_EXECUTED_INSTRUCTIONS: true
+          EMBENCH_BENCHMARK_RESULT_HOST_PATH: ${{ github.workspace }}/riscv32.csv
+          EMBENCH_BENCHMARK_RESULT_GUEST_PATH: /src/embench/result/executed_instructions_relative.csv
+      - name: Read
+        run: cat ${{ github.workspace }}/riscv32.csv
+      - name: Update csv
+        run: awk -v hash="${{ github.sha }}" -v date="$(date --rfc-3339=date)" -F',' '{print $0 "," hash "," date}' ${{ github.workspace }}/riscv32.csv > ${{ github.workspace }}/${{ github.sha }}.csv
+      - name: Read Updated
+        run: cat ${{ github.workspace }}/${{ github.sha }}.csv
+      - name: Pushes csv
+        uses: dmnemec/copy_file_to_another_repo_action@main
+        env:
+          API_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_RESSELPARK }}
+        with:
+          source_file: ${{ github.workspace }}/${{ github.sha }}.csv
+          destination_repo: 'openVADL/resselpark'
+          destination_branch: 'master'
+          destination_folder: 'data/lcb/riscv32'
+          user_email: 'ci@openvadl.org'
+          user_name: 'ci'
+          commit_message: 'data: Pushed LCB data'

.run/Benchmark Number Executed Instructions RV32IM.run.xml

@@ -7,17 +7,17 @@
         </map>
       </option>
       <option name="executionName" />
-      <option name="externalProjectPath" value="$PROJECT_DIR$/.." />
+      <option name="externalProjectPath" value="$PROJECT_DIR$" />
       <option name="externalSystemIdString" value="GRADLE" />
-      <option name="scriptParameters" value="-PopenVadl.tests=EmbenchBenchmarkNumberOfExecutedInstructionsSpikeRiscv32SimulationTest -PnoXtendCompile -i" />
+      <option name="scriptParameters" value="-i" />
       <option name="taskDescriptions">
         <list />
       </option>
       <option name="taskNames">
         <list>
           <option value="test" />
           <option value="--tests" />
-          <option value="at.ac.tuwien.complang.vadl.tests.OpenVADLSpecificTests" />
+          <option value="vadl.lcb.riscv.EmbenchBenchmarkNumberOfExecutedInstructionsSpikeRiscv32SimulationTest" />
         </list>
       </option>
       <option name="vmOptions" />

vadl/test/vadl/DockerExecutionTest.java

@@ -32,6 +32,7 @@
 import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -201,18 +202,21 @@ protected void runContainerAndCopyInputIntoContainer(ImageFromDockerfile image,
    * It will write the given {@code content} into a temporary file. The
    * temporary file requires a {@code prefix} and {@code suffix}.
    * Copies the data from {@code copyMappings}. Additionally, it will
-   * set environment variables based on {@code environmentMappings}.
+   * set environment variables based on {@code environmentMappings}. The {@code copyFromContainerToHost}
+   * can be used to define mappings between host and guest system.
    *
-   * @param image               is the docker image for the {@link GenericContainer}.
-   * @param copyMappings        is a list where each {@link Pair} indicates what should be copied
-   *                            from the host to the container.
-   * @param environmentMappings is a list where each entry defines an environment variable which
-   *                            will be set in the container.
-   * @param cmd                 is the command which is executed.
+   * @param image                   is the docker image for the {@link GenericContainer}.
+   * @param copyMappings            is a list where each {@link Pair} indicates what should be copied
+   *                                from the host to the container.
+   * @param environmentMappings     is a list where each entry defines an environment variable which
+   *                                will be set in the container.
+   * @param copyFromContainerToHost copies from guest to host.
+   * @param cmd                     is the command which is executed.
    */
   protected void runContainerAndCopyInputIntoContainer(ImageFromDockerfile image,
                                                        List<Pair<Path, String>> copyMappings,
                                                        Map<String, String> environmentMappings,
+                                                       List<Pair<String, String>> copyFromContainerToHost,
                                                        String cmd) {
     runContainer(image, (container) -> {
       for (var mapping : copyMappings) {
@@ -233,9 +237,37 @@ protected void runContainerAndCopyInputIntoContainer(ImageFromDockerfile image,
 
       return container;
     }, (container) -> {
+      for (var mapping : copyFromContainerToHost) {
+        container.copyFileFromContainer(mapping.left(), mapping.right());
+      }
     });
   }
 
+  /**
+   * Starts a container and checks the status code for the exited container.
+   * It will write the given {@code content} into a temporary file. The
+   * temporary file requires a {@code prefix} and {@code suffix}.
+   * Copies the data from {@code copyMappings}. Additionally, it will
+   * set environment variables based on {@code environmentMappings}.
+   *
+   * @param image               is the docker image for the {@link GenericContainer}.
+   * @param copyMappings        is a list where each {@link Pair} indicates what should be copied
+   *                            from the host to the container.
+   * @param environmentMappings is a list where each entry defines an environment variable which
+   *                            will be set in the container.
+   * @param cmd                 is the command which is executed.
+   */
+  protected void runContainerAndCopyInputIntoContainer(ImageFromDockerfile image,
+                                                       List<Pair<Path, String>> copyMappings,
+                                                       Map<String, String> environmentMappings,
+                                                       String cmd) {
+    runContainerAndCopyInputIntoContainer(image,
+        copyMappings,
+        environmentMappings,
+        Collections.emptyList(),
+        cmd);
+  }
+
   /**
    * Starts a container and checks the status code for the exited container.
    * It will assert that the status code is zero. If the check takes longer

vadl/test/vadl/lcb/riscv/EmbenchBenchmarkNumberOfExecutedInstructionsSpikeRiscv32SimulationTest.java

@@ -23,19 +23,21 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
 import org.testcontainers.images.builder.ImageFromDockerfile;
 import org.testcontainers.shaded.org.apache.commons.io.FileUtils;
 import vadl.configuration.LcbConfiguration;
-import vadl.gcb.valuetypes.ProcessorName;
 import vadl.gcb.valuetypes.TargetName;
 import vadl.lcb.AbstractLcbTest;
 import vadl.pass.exception.DuplicatedPassKeyException;
 import vadl.utils.Pair;
 
+@Tag("benchmarkTest")
 public class EmbenchBenchmarkNumberOfExecutedInstructionsSpikeRiscv32SimulationTest
     extends AbstractLcbTest {
 
@@ -71,17 +73,31 @@ void testEmbench() throws IOException, DuplicatedPassKeyException {
     }
 
     var redisCache = getRunningRedisCache();
-    var image = redisCache.setupEnv(new ImageFromDockerfile("tc_embench_spike_riscv32")
+    var image = redisCache.setupEnv(new ImageFromDockerfile("tc_embench_spike_riscv32", false)
         .withDockerfile(Paths.get(configuration.outputPath() + "/lcb/Dockerfile"))
         .withBuildArg("TARGET", target)
         .withBuildArg("UPSTREAM_BUILD_TARGET", upstreamBuildTarget));
 
+    var volumeMapping = volumeMappings();
+
     runContainerAndCopyInputIntoContainer(image,
         List.of(Pair.of(Path.of("test/resources/llvm/riscv/spike"),
             "/src/inputs")),
         Map.of(
             "LLVM_PARALLEL_COMPILE_JOBS", "4",
             "LLVM_PARALLEL_LINK_JOBS", "2"),
+        volumeMapping,
         "sh /src/embench/benchmark-extras/get-number-executed-instructions-spike-clang-lcb.sh");
   }
+
+  private List<Pair<String, String>> volumeMappings() {
+    var embenchHostOutput = System.getenv().get("EMBENCH_BENCHMARK_RESULT_HOST_PATH");
+    var embenchGuestInput = System.getenv().get("EMBENCH_BENCHMARK_RESULT_GUEST_PATH");
+
+    if (embenchGuestInput != null && embenchHostOutput != null) {
+      return List.of(Pair.of(embenchGuestInput, embenchHostOutput));
+    }
+
+    return Collections.emptyList();
+  }
 }