Skip to content

Commit 84f12c7

Browse files
Add NodeJs bindings benchmarks (#7004)
1 parent 7802f86 commit 84f12c7

File tree

5 files changed

+258
-7
lines changed

5 files changed

+258
-7
lines changed

.github/workflows/osrm-backend.yml

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -672,6 +672,11 @@ jobs:
672672
key: v1-data-osm-pbf
673673
restore-keys: |
674674
v1-data-osm-pbf
675+
- name: Use Node 20
676+
if: ${{ matrix.NODE_PACKAGE_TESTS_ONLY == 'ON' }}
677+
uses: actions/setup-node@v4
678+
with:
679+
node-version: 20
675680
- name: Enable compiler cache
676681
uses: actions/cache@v4
677682
with:
@@ -722,18 +727,24 @@ jobs:
722727
path: base
723728
- name: Build Base Branch
724729
run: |
730+
cd base
731+
npm ci --ignore-scripts
732+
cd ..
725733
mkdir base/build
726734
cd base/build
727-
cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release ..
735+
cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_NODE_BINDINGS=ON ..
728736
make -j$(nproc)
729737
make -j$(nproc) benchmarks
730738
cd ..
731739
make -C test/data
732740
- name: Build PR Branch
733741
run: |
742+
cd pr
743+
npm ci --ignore-scripts
744+
cd ..
734745
mkdir -p pr/build
735746
cd pr/build
736-
cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release ..
747+
cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_NODE_BINDINGS=ON ..
737748
make -j$(nproc)
738749
make -j$(nproc) benchmarks
739750
cd ..
@@ -745,6 +756,7 @@ jobs:
745756
run: |
746757
sudo mount -t tmpfs -o size=4g none /opt/benchmarks
747758
cp -rf pr/build /opt/benchmarks/build
759+
cp -rf pr/lib /opt/benchmarks/lib
748760
mkdir -p /opt/benchmarks/test
749761
cp -rf pr/test/data /opt/benchmarks/test/data
750762
cp -rf pr/profiles /opt/benchmarks/profiles
@@ -755,6 +767,7 @@ jobs:
755767
run: |
756768
sudo mount -t tmpfs -o size=4g none /opt/benchmarks
757769
cp -rf base/build /opt/benchmarks/build
770+
cp -rf base/lib /opt/benchmarks/lib
758771
mkdir -p /opt/benchmarks/test
759772
cp -rf base/test/data /opt/benchmarks/test/data
760773
cp -rf base/profiles /opt/benchmarks/profiles

package-lock.json

Lines changed: 12 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
"private": false,
55
"description": "The Open Source Routing Machine is a high performance routing engine written in C++ designed to run on OpenStreetMap data.",
66
"dependencies": {
7-
"@mapbox/node-pre-gyp": "^1.0.11"
7+
"@mapbox/node-pre-gyp": "^1.0.11",
8+
"seedrandom": "^3.0.5"
89
},
910
"browserify": {
1011
"transform": [
@@ -57,19 +58,21 @@
5758
"jsonpath": "^1.1.1",
5859
"mkdirp": "^0.5.6",
5960
"node-addon-api": "^5.0.0",
61+
"node-cmake": "^2.5.1",
6062
"node-timeout": "0.0.4",
6163
"polyline": "^0.2.0",
6264
"request": "^2.88.2",
6365
"rimraf": "^2.7.1",
6466
"tape": "^4.16.0",
6567
"turf": "^3.0.14",
6668
"uglify-js": "^3.17.0",
67-
"xmlbuilder": "^4.2.1",
68-
"node-cmake": "^2.5.1"
69+
"xmlbuilder": "^4.2.1"
6970
},
7071
"main": "lib/index.js",
7172
"binary": {
72-
"napi_versions": [8],
73+
"napi_versions": [
74+
8
75+
],
7376
"module_name": "node_osrm",
7477
"module_path": "./lib/binding_napi_v{napi_build_version}/",
7578
"host": "https://github.com",

scripts/ci/bench.js

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
const fs = require('fs');
2+
const path = require('path');
3+
const readline = require('readline');
4+
const seedrandom = require('seedrandom');
5+
6+
7+
let RNG;
8+
9+
class GPSData {
10+
constructor(gpsTracesFilePath) {
11+
this.tracks = {};
12+
this.coordinates = [];
13+
this.trackIds = [];
14+
this._loadGPSTraces(gpsTracesFilePath);
15+
}
16+
17+
_loadGPSTraces(gpsTracesFilePath) {
18+
const expandedPath = path.resolve(gpsTracesFilePath);
19+
const data = fs.readFileSync(expandedPath, 'utf-8');
20+
const lines = data.split('\n');
21+
const headers = lines[0].split(',');
22+
23+
const latitudeIndex = headers.indexOf('Latitude');
24+
const longitudeIndex = headers.indexOf('Longitude');
25+
const trackIdIndex = headers.indexOf('TrackID');
26+
27+
for (let i = 1; i < lines.length; i++) {
28+
if (lines[i].trim() === '') continue;
29+
const row = lines[i].split(',');
30+
31+
const latitude = parseFloat(row[latitudeIndex]);
32+
const longitude = parseFloat(row[longitudeIndex]);
33+
const trackId = row[trackIdIndex];
34+
35+
const coord = [longitude, latitude];
36+
this.coordinates.push(coord);
37+
38+
if (!this.tracks[trackId]) {
39+
this.tracks[trackId] = [];
40+
}
41+
this.tracks[trackId].push(coord);
42+
}
43+
44+
this.trackIds = Object.keys(this.tracks);
45+
}
46+
47+
getRandomCoordinate() {
48+
const randomIndex = Math.floor(RNG() * this.coordinates.length);
49+
return this.coordinates[randomIndex];
50+
}
51+
52+
getRandomTrack() {
53+
const randomIndex = Math.floor(RNG() * this.trackIds.length);
54+
const trackId = this.trackIds[randomIndex];
55+
return this.tracks[trackId];
56+
}
57+
};
58+
59+
async function runOSRMMethod(osrm, method, coordinates) {
60+
const time = await new Promise((resolve, reject) => {
61+
const startTime = process.hrtime();
62+
osrm[method]({coordinates}, (err, result) => {
63+
if (err) {
64+
if (['NoSegment', 'NoMatch', 'NoRoute', 'NoTrips'].includes(err.message)) {
65+
resolve(null);
66+
} else {
67+
68+
reject(err);
69+
}
70+
} else {
71+
const endTime = process.hrtime(startTime);
72+
resolve(endTime[0] + endTime[1] / 1e9);
73+
}
74+
});
75+
});
76+
return time;
77+
}
78+
79+
async function nearest(osrm, gpsData) {
80+
const times = [];
81+
for (let i = 0; i < 1000; i++) {
82+
const coord = gpsData.getRandomCoordinate();
83+
times.push(await runOSRMMethod(osrm, 'nearest', [coord]));
84+
}
85+
return times;
86+
}
87+
88+
async function route(osrm, gpsData) {
89+
const times = [];
90+
for (let i = 0; i < 1000; i++) {
91+
const from = gpsData.getRandomCoordinate();
92+
const to = gpsData.getRandomCoordinate();
93+
94+
95+
times.push(await runOSRMMethod(osrm, 'route', [from, to]));
96+
}
97+
return times;
98+
}
99+
100+
async function table(osrm, gpsData) {
101+
const times = [];
102+
for (let i = 0; i < 250; i++) {
103+
const numPoints = Math.floor(RNG() * 3) + 15;
104+
const coordinates = [];
105+
for (let i = 0; i < numPoints; i++) {
106+
coordinates.push(gpsData.getRandomCoordinate());
107+
}
108+
109+
110+
times.push(await runOSRMMethod(osrm, 'table', coordinates));
111+
}
112+
return times;
113+
}
114+
115+
async function match(osrm, gpsData) {
116+
const times = [];
117+
for (let i = 0; i < 1000; i++) {
118+
const numPoints = Math.floor(RNG() * 50) + 50;
119+
const coordinates = gpsData.getRandomTrack().slice(0, numPoints);
120+
121+
122+
times.push(await runOSRMMethod(osrm, 'match', coordinates));
123+
}
124+
return times;
125+
}
126+
127+
async function trip(osrm, gpsData) {
128+
const times = [];
129+
for (let i = 0; i < 250; i++) {
130+
const numPoints = Math.floor(RNG() * 2) + 5;
131+
const coordinates = [];
132+
for (let i = 0; i < numPoints; i++) {
133+
coordinates.push(gpsData.getRandomCoordinate());
134+
}
135+
136+
137+
times.push(await runOSRMMethod(osrm, 'trip', coordinates));
138+
}
139+
return times;
140+
}
141+
142+
function bootstrapConfidenceInterval(data, numSamples = 1000, confidenceLevel = 0.95) {
143+
let means = [];
144+
let dataLength = data.length;
145+
146+
for (let i = 0; i < numSamples; i++) {
147+
let sample = [];
148+
for (let j = 0; j < dataLength; j++) {
149+
let randomIndex = Math.floor(RNG() * dataLength);
150+
sample.push(data[randomIndex]);
151+
}
152+
let sampleMean = sample.reduce((a, b) => a + b, 0) / sample.length;
153+
means.push(sampleMean);
154+
}
155+
156+
means.sort((a, b) => a - b);
157+
let lowerBoundIndex = Math.floor((1 - confidenceLevel) / 2 * numSamples);
158+
let upperBoundIndex = Math.floor((1 + confidenceLevel) / 2 * numSamples);
159+
let mean = means.reduce((a, b) => a + b, 0) / means.length;
160+
let lowerBound = means[lowerBoundIndex];
161+
let upperBound = means[upperBoundIndex];
162+
163+
return { mean: mean, lowerBound: lowerBound, upperBound: upperBound };
164+
}
165+
166+
function calculateConfidenceInterval(data) {
167+
let { mean, lowerBound, upperBound } = bootstrapConfidenceInterval(data);
168+
let bestValue = Math.max(...data);
169+
let errorMargin = (upperBound - lowerBound) / 2;
170+
171+
return { mean, errorMargin, bestValue };
172+
}
173+
174+
async function main() {
175+
const args = process.argv.slice(2);
176+
177+
const {OSRM} = require(args[0]);
178+
const path = args[1];
179+
const algorithm = args[2].toUpperCase();
180+
const method = args[3];
181+
const gpsTracesFilePath = args[4];
182+
const iterations = parseInt(args[5]);
183+
184+
const gpsData = new GPSData(gpsTracesFilePath);
185+
const osrm = new OSRM({path, algorithm});
186+
187+
188+
const functions = {
189+
route: route,
190+
table: table,
191+
nearest: nearest,
192+
match: match,
193+
trip: trip
194+
};
195+
const func = functions[method];
196+
if (!func) {
197+
throw new Error('Unknown method');
198+
}
199+
const allTimes = [];
200+
for (let i = 0; i < iterations; i++) {
201+
RNG = seedrandom(42);
202+
allTimes.push((await func(osrm, gpsData)).filter(t => t !== null));
203+
}
204+
205+
const opsPerSec = allTimes.map(times => times.length / times.reduce((a, b) => a + b, 0));
206+
const { mean, errorMargin, bestValue } = calculateConfidenceInterval(opsPerSec);
207+
console.log(`Ops: ${mean.toFixed(1)} ± ${errorMargin.toFixed(1)} ops/s. Best: ${bestValue.toFixed(1)} ops/s`);
208+
209+
}
210+
211+
main();

scripts/ci/run_benchmarks.sh

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ function run_benchmarks_for_folder {
5353
mkdir -p $RESULTS_FOLDER
5454

5555
BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks"
56+
5657
echo "Running match-bench MLD"
5758
$BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench"
5859
echo "Running match-bench CH"
@@ -81,6 +82,18 @@ function run_benchmarks_for_folder {
8182
echo "Running osrm-contract"
8283
measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench"
8384

85+
86+
for ALGORITHM in ch mld; do
87+
for BENCH in nearest table trip route match; do
88+
echo "Running node $BENCH $ALGORITHM"
89+
START=$(date +%s.%N)
90+
node $SCRIPTS_FOLDER/scripts/ci/bench.js $FOLDER/lib/binding/node_osrm.node $FOLDER/data.osrm $ALGORITHM $BENCH $GPS_TRACES > "$RESULTS_FOLDER/node_${BENCH}_${ALGORITHM}.bench" 5
91+
END=$(date +%s.%N)
92+
DIFF=$(echo "$END - $START" | bc)
93+
echo "Took: ${DIFF}s"
94+
done
95+
done
96+
8497
for ALGORITHM in ch mld; do
8598
for BENCH in nearest table trip route match; do
8699
echo "Running random $BENCH $ALGORITHM"

0 commit comments

Comments
 (0)