Skip to content

Commit b4b2160

Browse files
triceotriceo
authored
ci: enable perf testing (#25)
1 parent b6b34a6 commit b4b2160

File tree

11 files changed

+242
-43
lines changed

11 files changed

+242
-43
lines changed

‎.github/workflows/nightly_performance_score_director.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: Schedule a nightly run of the Score Director performance benchmark
2+
3+
on:
4+
schedule:
5+
- cron: '59 23 * * 1-5' # Every workday at the end of the day.
6+
7+
jobs:
8+
trigger:
9+
runs-on: ubuntu-latest
10+
steps:
11+
- name: Checkout timefold-solver
12+
uses: actions/checkout@v4
13+
with:
14+
repository: TimefoldAI/timefold-solver
15+
- name: Schedule the other workflow
16+
shell: bash
17+
run: |
18+
if git log --since="24 hours ago" --oneline | grep -q .; then
19+
echo '{}' | gh workflow run performance_score_director.yml --json
20+
echo "Launched nightly perf tests." >> $GITHUB_STEP_SUMMARY
21+
else
22+
# Don't waste money.
23+
echo "No commits in the past 24 hours." >> $GITHUB_STEP_SUMMARY
24+
fi

‎.github/workflows/performance_score_director.yml‎

Lines changed: 181 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,211 @@
1-
name: Performance - Score Director
1+
# - Runs entirely on a single machine.
2+
# - The baseline is established first, then the branch under test is measured.
3+
# - Each benchmark gives a 99.9 % confidence interval.
4+
# - The confidence intervals are compared to determine if the branch under test is a regression or an improvement.
5+
# - The error threshold is expected to be below +/- 2.5 %.
6+
# We have yet to see an error of over +/- 4 %.
7+
# With the error so high, the impact is that small regressions are not considered statistically significant.
8+
name: Performance Regression Test - Score Director
29

310
on:
411
workflow_dispatch:
512
inputs:
613
jdk:
7-
description: 'JDK version (17, 21, 23, ...)'
8-
default: '23'
14+
description: 'JDK version'
15+
default: '21'
916
required: true
1017
baseline:
1118
description: 'Timefold Solver release'
1219
default: '1.14.0'
1320
required: true
1421
branch:
15-
description: 'Development branch to test against'
22+
description: 'Branch to benchmark (needs to use 999-SNAPSHOT)'
1623
default: 'main'
1724
required: true
1825
branch_owner:
1926
description: 'User owning the branch'
2027
default: 'TimefoldAI'
2128
required: true
29+
async_profiler_version:
30+
description: 'async-profiler version'
31+
default: '3.0'
32+
required: true
2233

2334
jobs:
2435

25-
test:
26-
concurrency:
27-
group: perf-score-director-${{ matrix.example }}
28-
cancel-in-progress: true
29-
runs-on: ubuntu-latest
36+
benchmark:
37+
runs-on: perf-linux-x64-2cores
3038
strategy:
39+
fail-fast: false # Jobs fail if the benchmark error is over predefined thresholds; other benchmarks continue.
3140
matrix:
32-
example: [cloudbalancing, conferencescheduling, curriculumcourse, examination, machinereassignment, meetingscheduling, nurserostering, pas, taskassigning, travelingtournament, tsp, vehiclerouting]
41+
example: [cloud_balancing, conference_scheduling, curriculum_course, examination, machine_reassignment, meeting_scheduling, nurse_rostering, patient_admission_scheduling, task_assigning, traveling_tournament, tsp, vehicle_routing]
42+
env:
43+
MVN_USERNAME: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_USERNAME }}'
44+
MVN_PASSWORD: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_TOKEN }}'
3345
steps:
34-
- uses: sdkman/sdkman-action@v1
46+
- name: Phase 0 - Checkout timefold-solver-benchmarks
47+
uses: actions/checkout@v4
3548
with:
36-
candidate: java
37-
version: ${{ github.event.inputs.jdk }}-tem
38-
- uses: actions/setup-java@v4
49+
repository: TimefoldAI/timefold-solver-benchmarks
50+
path: ./timefold-solver-benchmarks
51+
52+
- name: Phase 0 - Setup JDK and Maven
53+
uses: actions/setup-java@v4
3954
with:
40-
distribution: 'jdkfile'
4155
java-version: ${{ github.event.inputs.jdk }}
42-
jdkFile: ${{ steps.sdkman.outputs.file }}
43-
- name: Checkout timefold-solver-benchmarks
56+
distribution: 'temurin'
57+
cache: 'maven'
58+
server-id: 'timefold-solver-enterprise'
59+
server-username: 'MVN_USERNAME'
60+
server-password: 'MVN_PASSWORD'
61+
62+
- name: Phase 0 - Setup Async Profiler
63+
working-directory: ./timefold-solver-benchmarks
64+
run: |
65+
export FILENAME=async-profiler-${{ github.event.inputs.async_profiler_version }}-linux-x64.tar.gz
66+
wget https://github.com/async-profiler/async-profiler/releases/download/v${{ github.event.inputs.async_profiler_version }}/$FILENAME
67+
tar -xzf $FILENAME
68+
ls -l
69+
70+
# Fine-tuned for stability on GHA.
71+
- name: Phase 0 - Configure the benchmark
72+
working-directory: ./timefold-solver-benchmarks
73+
shell: bash
74+
run: |
75+
echo "forks=20" > scoredirector-benchmark.properties
76+
echo "warmup_iterations=10" >> scoredirector-benchmark.properties
77+
echo "measurement_iterations=5" >> scoredirector-benchmark.properties
78+
echo "relative_score_error_threshold=0.025" >> scoredirector-benchmark.properties
79+
echo "score_director_type=cs" >> scoredirector-benchmark.properties
80+
echo "example=${{ matrix.example }}" >> scoredirector-benchmark.properties
81+
cat scoredirector-benchmark.properties
82+
chmod +x run-scoredirector.sh
83+
84+
- name: Phase 1 - Compile the benchmark
85+
working-directory: ./timefold-solver-benchmarks
86+
shell: bash
87+
run: mvn clean install -B -Dquickly -Dversion.ai.timefold.solver=${{ github.event.inputs.baseline }} -Dversion.tools.provider="${{ github.event.inputs.async_profiler_version }}"
88+
89+
- name: Phase 1 - Run the baseline configuration
90+
working-directory: ./timefold-solver-benchmarks
91+
id: benchmark_baseline
92+
env:
93+
RUN_ID: ${{ github.event.inputs.baseline }}
94+
shell: bash
95+
run: |
96+
./run-scoredirector.sh
97+
echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]|round' results/scoredirector/${{ github.event.inputs.baseline }}/results.json)" >> "$GITHUB_OUTPUT"
98+
echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]|round' results/scoredirector/${{ github.event.inputs.baseline }}/results.json)" >> "$GITHUB_OUTPUT"
99+
echo "RANGE_MID=$(jq '.[0].primaryMetric.score|round' results/scoredirector/${{ github.event.inputs.baseline }}/results.json)" >> "$GITHUB_OUTPUT"
100+
101+
- name: Phase 2 - Checkout timefold-solver
44102
uses: actions/checkout@v4
45103
with:
46-
repository: TimefoldAI/timefold-solver-benchmarks
47-
path: ./timefold-solver-benchmarks
48-
- name: Compile the benchmarks
104+
repository: ${{ github.event.inputs.branch_owner }}/timefold-solver
105+
ref: ${{ github.event.inputs.branch }}
106+
path: ./timefold-solver
107+
108+
- name: Phase 2 - Quickly build timefold-solver
109+
working-directory: ./timefold-solver
110+
shell: bash
111+
run: mvn -B -Dquickly clean install
112+
113+
# Clone timefold-solver-enterprise
114+
- name: Phase 2 - Checkout timefold-solver-enterprise (Specified)
115+
id: checkout-solver-enterprise
116+
uses: actions/checkout@v4
117+
continue-on-error: true
118+
with:
119+
repository: TimefoldAI/timefold-solver-enterprise
120+
ref: ${{ github.event.inputs.branch }}
121+
token: ${{ secrets.BENCHMARK_PUBLISH_TOKEN }}
122+
path: ./timefold-solver-enterprise
123+
- name: Phase 2 - Checkout timefold-solver-enterprise (Fallback)
124+
if: steps.checkout-solver-enterprise.outcome != 'success'
125+
uses: actions/checkout@v4
126+
with:
127+
repository: TimefoldAI/timefold-solver-enterprise
128+
ref: main
129+
token: ${{ secrets.BENCHMARK_PUBLISH_TOKEN }}
130+
path: ./timefold-solver-enterprise
131+
132+
- name: Phase 2 - Quickly build timefold-solver-enterprise
133+
working-directory: ./timefold-solver-enterprise
134+
shell: bash
135+
run: mvn -B -Dquickly clean install
136+
137+
- name: Phase 2 - Compile the benchmarks
138+
working-directory: ./timefold-solver-benchmarks
139+
shell: bash
140+
run: mvn clean install -B -Dquickly -Dversion.tools.provider="${{ github.event.inputs.async_profiler_version }}"
141+
142+
- name: Phase 2 - Run the benchmark on the new code
143+
id: benchmark_new
144+
working-directory: ./timefold-solver-benchmarks
145+
env:
146+
RUN_ID: ${{ github.event.inputs.branch }}
147+
shell: bash
148+
run: |
149+
./run-scoredirector.sh
150+
echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]|round' results/scoredirector/${{ github.event.inputs.branch }}/results.json)" >> "$GITHUB_OUTPUT"
151+
echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]|round' results/scoredirector/${{ github.event.inputs.branch }}/results.json)" >> "$GITHUB_OUTPUT"
152+
echo "RANGE_MID=$(jq '.[0].primaryMetric.score|round' results/scoredirector/${{ github.event.inputs.branch }}/results.json)" >> "$GITHUB_OUTPUT"
153+
154+
- name: Phase 3 - Archive benchmark data
155+
uses: actions/upload-artifact@v4
156+
with:
157+
name: results-${{ matrix.example }}-${{ github.event.inputs.baseline }}_vs_${{ github.event.inputs.branch }}
158+
path: |
159+
./timefold-solver-benchmarks/results/scoredirector
160+
161+
- name: Phase 3 - Report results
49162
working-directory: ./timefold-solver-benchmarks
163+
env:
164+
OLD_RANGE_START: ${{ steps.benchmark_baseline.outputs.RANGE_START }}
165+
OLD_RANGE_MID: ${{ steps.benchmark_baseline.outputs.RANGE_MID }}
166+
OLD_RANGE_END: ${{ steps.benchmark_baseline.outputs.RANGE_END }}
167+
NEW_RANGE_START: ${{ steps.benchmark_new.outputs.RANGE_START }}
168+
NEW_RANGE_MID: ${{ steps.benchmark_new.outputs.RANGE_MID }}
169+
NEW_RANGE_END: ${{ steps.benchmark_new.outputs.RANGE_END }}
50170
shell: bash
51-
run: mvn clean install -Dai.timefold.solver.version=${{ github.event.inputs.baseline }}
171+
run: |
172+
export DIFF_START=$(echo "scale=2; ($OLD_RANGE_START / $NEW_RANGE_START) * 100" | bc)
173+
export DIFF_MID=$(echo "scale=2; ($OLD_RANGE_MID / $NEW_RANGE_MID) * 100" | bc)
174+
export DIFF_END=$(echo "scale=2; ($OLD_RANGE_END / $NEW_RANGE_END) * 100" | bc)
175+
export FAIL=false
176+
177+
if (( $(echo "$DIFF_MID >= 98.00" | bc -l) && $(echo "$DIFF_MID <= 102.00"|bc -l) )); then
178+
# Ignore differences of up to 2 %.
179+
echo "### Performance unchanged" >> $GITHUB_STEP_SUMMARY
180+
echo "(Decided to ignore a very small difference of under 2 %.)" >> $GITHUB_STEP_SUMMARY
181+
else
182+
if [ "$NEW_RANGE_START" -le "$OLD_RANGE_END" ] && [ "$NEW_RANGE_END" -ge "$OLD_RANGE_START" ]; then
183+
if [ "$NEW_RANGE_START" -ge "$OLD_RANGE_MID" ]; then
184+
echo "### 🍀 Possible improvement 🍀" >> $GITHUB_STEP_SUMMARY
185+
elif [ "$OLD_RANGE_END" -le "$NEW_RANGE_MID" ]; then
186+
echo "### ⚠️ Possible regression ⚠️" >> $GITHUB_STEP_SUMMARY
187+
else
188+
echo "### Performance unchanged " >> $GITHUB_STEP_SUMMARY
189+
fi
190+
elif [ "$NEW_RANGE_START" -gt "$OLD_RANGE_END" ]; then
191+
echo "### 🚀🚀🚀 Statistically significant improvement 🚀🚀🚀" >> $GITHUB_STEP_SUMMARY
192+
else
193+
echo "### ‼️‼️‼️ Statistically significant regression ‼️‼️‼️" >> $GITHUB_STEP_SUMMARY
194+
export FAIL=true
195+
fi
196+
fi
197+
198+
echo "| | **Ref** | **Min** | **Mean** | **Max** |" >> $GITHUB_STEP_SUMMARY
199+
echo "|:------:|:-----------:|:-----------------:|:-----------------:|:-----------------:|" >> $GITHUB_STEP_SUMMARY
200+
echo "| _Old_ | [v${{ github.event.inputs.baseline }}](https://github.com/TimefoldAI/timefold-solver/releases/tag/v${{ github.event.inputs.baseline }}) | ${OLD_RANGE_START} | ${OLD_RANGE_MID} | ${OLD_RANGE_END} |" >> $GITHUB_STEP_SUMMARY
201+
echo "| _New_ | [${{ github.event.inputs.branch_owner }}'s ${{ github.event.inputs.branch }}](https://github.com/${{ github.event.inputs.branch_owner }}/timefold-solver/tree/${{ github.event.inputs.branch }}) | ${NEW_RANGE_START} | ${NEW_RANGE_MID} | ${NEW_RANGE_END} |" >> $GITHUB_STEP_SUMMARY
202+
echo "| _Diff_ | | ${DIFF_START} % | ${DIFF_MID} % | ${DIFF_END} % |" >> $GITHUB_STEP_SUMMARY
203+
204+
echo "" >> $GITHUB_STEP_SUMMARY
205+
echo "Min and max define a 99.9 % confidence interval." >> $GITHUB_STEP_SUMMARY
206+
echo "Min and max are in operations per second. Higher is better." >> $GITHUB_STEP_SUMMARY
207+
echo "Diff under 100 % represents an improvement, over 100 % a regression." >> $GITHUB_STEP_SUMMARY
208+
209+
if [ "$FAIL" = true ]; then
210+
exit 1
211+
fi

‎.github/workflows/turtle.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: Turtle Tests
22

33
on:
44
schedule:
5-
- cron: '0 2 * * *' # Every day at 2am UTC
5+
- cron: '0 3 * * *' # Every day at 3am UTC
66

77
jobs:
88
test:

‎pom.xml‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@
3434
<dependencyManagement>
3535
<dependencies>
3636
<dependency>
37-
<groupId>ai.timefold.solver</groupId>
38-
<artifactId>timefold-solver-build-parent</artifactId>
37+
<groupId>ai.timefold.solver.enterprise</groupId>
38+
<artifactId>timefold-solver-enterprise-build-parent</artifactId>
3939
<version>${version.ai.timefold.solver}</version>
4040
<type>pom</type>
4141
<scope>import</scope>
@@ -58,7 +58,6 @@
5858
<dependency>
5959
<groupId>ai.timefold.solver.enterprise</groupId>
6060
<artifactId>timefold-solver-enterprise-core</artifactId>
61-
<version>${version.ai.timefold.solver}</version>
6261
</dependency>
6362
<dependency>
6463
<groupId>ai.timefold.solver</groupId>
@@ -124,7 +123,6 @@
124123
<dependency>
125124
<groupId>ai.timefold.solver</groupId>
126125
<artifactId>timefold-solver-core</artifactId>
127-
<version>${version.ai.timefold.solver}</version>
128126
<type>test-jar</type>
129127
<scope>test</scope>
130128
</dependency>
@@ -167,6 +165,13 @@
167165
<version>3.13.0</version>
168166
<configuration>
169167
<release>${java.release}</release>
168+
<annotationProcessorPaths>
169+
<path>
170+
<groupId>org.openjdk.jmh</groupId>
171+
<artifactId>jmh-generator-annprocess</artifactId>
172+
<version>${jmh.version}</version>
173+
</path>
174+
</annotationProcessorPaths>
170175
</configuration>
171176
</plugin>
172177
<plugin>
@@ -207,7 +212,7 @@
207212
<dependency>
208213
<groupId>ai.timefold.solver</groupId>
209214
<artifactId>timefold-solver-ide-config</artifactId>
210-
<version>${project.version}</version>
215+
<version>${version.ai.timefold.solver}</version>
211216
</dependency>
212217
</dependencies>
213218
<executions>
@@ -242,6 +247,18 @@
242247
</build>
243248

244249
<profiles>
250+
<profile>
251+
<id>quickly</id>
252+
<activation>
253+
<property>
254+
<name>quickly</name>
255+
</property>
256+
</activation>
257+
<properties>
258+
<spotless.skip>true</spotless.skip>
259+
<skipTests>true</skipTests>
260+
</properties>
261+
</profile>
245262
<profile>
246263
<id>jmh</id>
247264
<activation>

‎run-coldstart.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
#!/bin/bash
22
sudo -i sysctl kernel.perf_event_paranoid=1
33
sudo -i sysctl kernel.kptr_restrict=0
4-
nohup taskset -c 0 java -cp target/benchmarks.jar ai.timefold.solver.benchmarks.micro.coldstart.Main > target/nohup.out 2>&1 &
4+
java -cp target/benchmarks.jar ai.timefold.solver.benchmarks.micro.coldstart.Main

‎run-scoredirector.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
#!/bin/bash
22
sudo -i sysctl kernel.perf_event_paranoid=1
33
sudo -i sysctl kernel.kptr_restrict=0
4-
nohup taskset -c 0 java -cp target/benchmarks.jar ai.timefold.solver.benchmarks.micro.scoredirector.Main > target/nohup.out 2>&1 &
4+
java -cp target/benchmarks.jar ai.timefold.solver.benchmarks.micro.scoredirector.Main

‎src/main/java/ai/timefold/solver/benchmarks/examples/pas/persistence/PatientAdmissionScheduleImporter.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -147,9 +147,7 @@ private void readDepartmentListAndDepartmentSpecialismList() throws IOException
147147
List<Department> departmentList =
148148
new ArrayList<>(departmentListSize);
149149
idToDepartmentMap = new HashMap<>(departmentListSize);
150-
List<DepartmentSpecialism> departmentSpecialismList =
151-
new ArrayList<>(
152-
departmentListSize * 5);
150+
List<DepartmentSpecialism> departmentSpecialismList = new ArrayList<>(departmentListSize * 5);
153151
long departmentSpecialismId = 0L;
154152
for (int i = 0; i < departmentListSize; i++) {
155153
String line = bufferedReader.readLine();
@@ -229,8 +227,7 @@ private void readRoomListAndRoomSpecialismListAndRoomEquipmentList() throws IOEx
229227
String line = bufferedReader.readLine();
230228
String[] lineTokens = splitByPipelineAndTrim(line, 6);
231229
String[] roomTokens = splitBySpace(lineTokens[0], 2);
232-
Department department = idToDepartmentMap.get(
233-
Long.parseLong(lineTokens[2]));
230+
Department department = idToDepartmentMap.get(Long.parseLong(lineTokens[2]));
234231
Room room =
235232
new Room(Long.parseLong(roomTokens[0]), roomTokens[1],
236233
department, Integer.parseInt(lineTokens[1]),

‎src/main/java/ai/timefold/solver/benchmarks/micro/coldstart/Main.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@
3838
import ai.timefold.solver.benchmarks.micro.coldstart.jmh.TimeToSolverFactoryBenchmark;
3939
import ai.timefold.solver.benchmarks.micro.common.AbstractMain;
4040

41-
import org.openjdk.jmh.results.Result;
4241
import org.openjdk.jmh.runner.Runner;
4342
import org.openjdk.jmh.runner.RunnerException;
4443
import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
@@ -76,7 +75,7 @@ public static void main(String[] args) throws RunnerException, IOException {
7675
var relativeScoreErrorThreshold = configuration.getRelativeScoreErrorThreshold();
7776
var thresholdForPrint = ((int) Math.round(relativeScoreErrorThreshold * 10_000)) / 100.0D;
7877
runResults.forEach(result -> {
79-
Result<?> primaryResult = result.getPrimaryResult();
78+
var primaryResult = result.getPrimaryResult();
8079
var score = primaryResult.getScore();
8180
var scoreError = primaryResult.getScoreError();
8281
var relativeScoreError = scoreError / score;

0 commit comments

Comments
 (0)