Skip to content

Commit a7d9dc4

Browse files
authored
Add very simple benchmarking against reactor-c (#140)
* Remove any platform specific code in code-generated CMakeLists * Fix FlexPRET * Dump working changes * Minimum reaction and event queue size is 1 * Ironing out the final wrinkles * Add support for running in fast mode, simply return immediatly from any call to wait_until. * Typo * Update examples * Fix preamble handling * Add two simple benchmarks * Run benchmarks in CI * Report benchmark result units * Post benchmark results to the PR * Try to output something from a step * CI * Fix colorized output * Remove the need for dynamically sized array on the stack * Add a newline * Another newline
1 parent 86c9fad commit a7d9dc4

File tree

17 files changed

+744
-13
lines changed

17 files changed

+744
-13
lines changed

.github/workflows/benchmark.yml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: "Benchmarks"
2+
3+
permissions:
4+
contents: write
5+
pull-requests: write
6+
7+
on:
8+
pull_request:
9+
10+
jobs:
11+
ci:
12+
name: Run benchmarks
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Checkout
16+
uses: actions/checkout@v3
17+
with:
18+
submodules: recursive
19+
20+
- name: Setup java and gradle for compiling lfc
21+
uses: ./.github/actions/lingua-franca
22+
23+
- name: Install lfc
24+
run: curl -Ls https://install.lf-lang.org | bash -s cli
25+
- name: Run benchmarks
26+
id: run_benchmarks
27+
run: |
28+
source env.bash
29+
cd benchmarks
30+
./runAll.sh
31+
32+
# This in conjunction with create-or-update-comment allows us to only
33+
# comment once and update it after
34+
- name: Find Comment
35+
uses: peter-evans/find-comment@v3
36+
id: fc
37+
with:
38+
issue-number: ${{ github.event.pull_request.number }}
39+
comment-author: 'github-actions[bot]'
40+
body-includes: Benchmark results
41+
42+
- name: Create or update comment
43+
uses: peter-evans/create-or-update-comment@v4
44+
with:
45+
comment-id: ${{ steps.fc.outputs.comment-id }}
46+
issue-number: ${{ github.event.pull_request.number }}
47+
body-path: benchmarks/benchmark_results.md
48+
edit-mode: replace
49+

.github/workflows/memory.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ jobs:
5050
with:
5151
issue-number: ${{ github.event.pull_request.number }}
5252
comment-author: 'github-actions[bot]'
53-
body-includes: Memory report
53+
body-includes: Memory usage after merging this PR
5454

5555
- name: Create or update comment
5656
uses: peter-evans/create-or-update-comment@v4

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
**/build/
22
**/src-gen/
33
**/bin/
4+
benchmarks/include
45
.vscode/
56
cmake-build-debug
67
cmake-build-release

CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ target_compile_options(reactor-uc PRIVATE -Wall -Wextra -Werror)
9393

9494
# Disable selected warnings
9595
if (CMAKE_C_COMPILER_ID STREQUAL "GNU")
96-
target_compile_options(reactor-uc PUBLIC -Wno-zero-length-bounds -Wno-stack-usage)
96+
target_compile_options(reactor-uc PRIVATE -Wno-zero-length-bounds)
9797
endif()
9898

9999
add_compile_options (-fdiagnostics-color=always)

benchmarks/runAll.sh

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/bin/env bash
2+
set -e
3+
4+
LFC=lfc
5+
LFCG=${REACTOR_UC_PATH}/lfc/bin/lfc-dev
6+
7+
$LFC src/PingPongC.lf
8+
$LFCG src/PingPongUc.lf
9+
10+
$LFC src/ReactionLatencyC.lf
11+
$LFCG src/ReactionLatencyUc.lf
12+
13+
echo "Running benchmarks..."
14+
15+
ping_pong_c_result=$(bin/PingPongC | grep -E "Time: *.")
16+
ping_pong_uc_result=$(bin/PingPongUc | grep -E "Time: *.")
17+
latency_c_result=$(bin/ReactionLatencyC | grep -E " latency: *.")
18+
latency_uc_result=$(bin/ReactionLatencyUc | grep -E "latency: *.")
19+
20+
21+
# Create or clear the output file
22+
output_file="benchmark_results.md"
23+
: > "$output_file"
24+
25+
# Print and dump the results into the file
26+
echo "Benchmark results after merging this PR: " >> "$output_file"
27+
echo "<details><summary>Benchmark results</summary>" >> "$output_file"
28+
echo "" >> "$output_file"
29+
echo "## Performance:" >> "$output_file"
30+
echo "" >> "$output_file"
31+
32+
benchmarks=("PingPongUc" "PingPongC" "ReactionLatencyUc" "ReactionLatencyC")
33+
results=("$ping_pong_uc_result" "$ping_pong_c_result" "$latency_uc_result" "$latency_c_result")
34+
echo $latency_uc_result >> test.md
35+
36+
for i in "${!benchmarks[@]}"; do
37+
echo "${benchmarks[$i]}:" >> "$output_file"
38+
echo "${results[$i]}" >> "$output_file"
39+
echo "" >> "$output_file"
40+
done
41+
42+
echo "## Memory usage:" >> "$output_file"
43+
for benchmark in PingPongUc PingPongC ReactionLatencyUc ReactionLatencyC;
44+
do
45+
echo "$benchmark:" >> "$output_file"
46+
echo "$(size -d bin/$benchmark)" >> "$output_file"
47+
echo "" >> "$output_file"
48+
done
49+
50+
cat "$output_file"

benchmarks/src/BenchmarkRunnerC.lf

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
target C
2+
3+
/**
4+
* Reactor that starts the kernel of a benchmark, measures its runtime and outputs the results for a
5+
* given number of iterations.
6+
*
7+
* This reactor is instantiated by the main reactor of a benchmark and the startup reaction of this
8+
* reactor is the starting point for that benchmark. The reactor runs a given number of iterations
9+
* of the benchmark, measures the runtime of each iteration and outputs them. The benchmark itself
10+
* is responsible to reset its state between the iterations. A benchmark can have an optional
11+
* initialization phase that is run once before the first iteration and is not measured. A benchmark
12+
* can have an optional cleanup phase after each iteration before the next iteration start which is
13+
* not considered in the runtime measurement.
14+
*
15+
* How to use:
16+
* - Instantiate this reactor in the main reactor of the benchmark.
17+
* - Connect the ports start, finish with the appropriate reactors of the benchmark.
18+
* - Create a startup reaction in the main reactor that calls printBenchmarkInfo(),
19+
*
20+
* Prototype startup reaction in the main reactor of a benchmark: runner = new
21+
* BenchmarkRunner(num_iterations=num_iterations); reaction(startup) {=
22+
* printBenchmarkInfo("ThreadRingReactorLFCppBenchmark"); printSystemInfo();
23+
* =}
24+
*
25+
* @param num_iterations How many times to execute the kernel of the benchmark to measure.
26+
*
27+
* @author Hannes Klein
28+
* @author Shaokai Lin
29+
* @author Matt Chorlian
30+
* @author Arthur Deng
31+
*/
32+
preamble {=
33+
#include <stdio.h>
34+
=}
35+
36+
reactor BenchmarkRunner(num_iterations: size_t = 12) {
37+
/** Signal to start execution. Set this input from a startup reaction in the main reactor. */
38+
input inStart: bool
39+
40+
/** Signals for starting and finishing the kernel and runtime measurement. */
41+
output start: bool
42+
input finish: bool
43+
44+
/** Events to switch between the phases of running the iterations. */
45+
logical action nextIteration
46+
logical action done
47+
48+
/** Number of iterations already executed. */
49+
state count: unsigned = 0
50+
51+
/** Start time for runtime measurement. */
52+
state startTime: instant_t
53+
54+
/** Runtime measurements. */
55+
state measuredTimes: interval_t*
56+
57+
preamble {=
58+
static double toMS(interval_t t) {
59+
return t / 1000000.0;
60+
}
61+
62+
int comp (const void * elem1, const void * elem2) {
63+
int f = *((double*)elem1);
64+
int s = *((double*)elem2);
65+
if (f > s) return 1;
66+
if (f < s) return -1;
67+
return 0;
68+
}
69+
70+
static double median(double* execTimes, int size) {
71+
if (size == 0) {
72+
return 0.0;
73+
}
74+
75+
int middle = size / 2;
76+
if(size % 2 == 1) {
77+
return execTimes[middle];
78+
} else {
79+
return (execTimes[middle-1] + execTimes[middle]) / 2;
80+
}
81+
}
82+
83+
static double* getMSMeasurements(interval_t* measured_times, int num_iterations) {
84+
85+
double* msMeasurements = (double *) calloc(num_iterations, sizeof(double));
86+
for (int i = 0; i < num_iterations; i++) {
87+
msMeasurements[i] = toMS(measured_times[i]);
88+
}
89+
90+
return msMeasurements;
91+
}
92+
=}
93+
94+
preamble {=
95+
void printBenchmarkInfo(char* benchmarkId) {
96+
printf("Benchmark: %s\n", benchmarkId);
97+
}
98+
99+
void printSystemInfo() {
100+
101+
printf("System information\n");
102+
printf("O/S Name: ");
103+
104+
#ifdef _WIN32
105+
printf("Windows 32-bit");
106+
#elif _WIN64
107+
printf("Windows 64-bit");
108+
#elif __APPLE__ || __MACH__
109+
printf("Mac OSX");
110+
#elif __linux__
111+
printf("Linux");
112+
#elif __FreeBSD__
113+
printf("FreeBSD");
114+
#elif __unix || __unix__
115+
printf("Unix");
116+
#else
117+
printf("Other");
118+
#endif
119+
120+
printf("\n");
121+
}
122+
=}
123+
124+
reaction(startup) -> nextIteration {=
125+
// Initialize an array of interval_t
126+
self->measuredTimes = (interval_t *) calloc(self->num_iterations, sizeof(interval_t));
127+
lf_schedule(nextIteration, 0);
128+
=}
129+
130+
reaction(nextIteration) -> start, done {=
131+
if (self->count < self->num_iterations) {
132+
self->startTime = lf_time_physical();
133+
lf_set(start, true);
134+
} else {
135+
lf_schedule(done, 0);
136+
}
137+
=}
138+
139+
reaction(finish) -> nextIteration {=
140+
interval_t end_time = lf_time_physical();
141+
interval_t duration = end_time - self->startTime;
142+
self->measuredTimes[self->count] = duration;
143+
self->count += 1;
144+
145+
printf("Iteration %d - %.3f ms\n", self->count, toMS(duration));
146+
147+
lf_schedule(nextIteration, 0);
148+
=}
149+
150+
reaction(done) {=
151+
double* measuredMSTimes = getMSMeasurements(self->measuredTimes, self->num_iterations);
152+
qsort(measuredMSTimes, self->num_iterations, sizeof(double), comp);
153+
154+
printf("Execution - Summary:\n");
155+
printf("Best Time:\t %.3f msec\n", measuredMSTimes[0]);
156+
printf("Worst Time:\t %.3f msec\n", measuredMSTimes[self->num_iterations - 1]);
157+
printf("Median Time:\t %.3f msec\n", median(measuredMSTimes, self->num_iterations));
158+
lf_request_stop();
159+
=}
160+
}

0 commit comments

Comments
 (0)