Skip to content

Commit 255e351

Browse files
jobo322jobo322
committed
chore: add benchmarks for cardinality and size comparisons
1 parent 80686cd commit 255e351

File tree

4 files changed

+360
-3
lines changed

4 files changed

+360
-3
lines changed

‎benchmark/benchmarkLinePlot.js‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@ lineplot(() => {
2525
const B = new SparseMatrix(randomMatrix(size, size, density));
2626
// Benchmark the multiplication
2727
yield () => do_not_optimize(A.mmul(B));
28-
}).args('size', sizes); // 16, 32, 64, 128, 256
28+
})
29+
.gc('inner')
30+
.args('size', sizes); // 16, 32, 64, 128, 256
2931

3032
bench('SparseOld.mmul($size)', function* (ctx) {
3133
const size = ctx.get('size');
@@ -36,7 +38,9 @@ lineplot(() => {
3638

3739
// Benchmark the multiplication
3840
yield () => do_not_optimize(AOld.mmul(BOld));
39-
}).args('size', sizes);
41+
})
42+
.gc('inner')
43+
.args('size', sizes);
4044

4145
// bench('Dense.mmul($size)', function* (ctx) {
4246
// const size = ctx.get('size');
@@ -49,7 +53,7 @@ lineplot(() => {
4953

5054
// // Benchmark the multiplication
5155
// yield () => do_not_optimize(ADense.mmul(BDense));
52-
// }).range('size', min, max, multiplier);
56+
// }).gc('inner').range('size', min, max, multiplier);
5357
});
5458

5559
await run();

‎benchmark/combinationOfCardinalityAndSize.js‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
import { run, bench, group, do_not_optimize } from 'mitata';
2+
import { xSequentialFillFromStep } from 'ml-spectra-processing';
3+
4+
import { SparseMatrix } from '../src/index.js';
5+
6+
import { randomMatrix } from './utils/randomMatrix.js';
7+
import { writeFile } from 'node:fs/promises';
8+
import path from 'node:path';
9+
10+
/* eslint
11+
func-names: 0
12+
camelcase: 0
13+
*/
14+
// Prepare matrices once
15+
const cardinalities = Array.from(
16+
xSequentialFillFromStep({ from: 10, step: 5, size: 25 }),
17+
);
18+
19+
// const dimensions = Array.from(
20+
// xSequentialFillFromStep({ from: 700, step: 100, size: 13 }),
21+
// );
22+
23+
const dimensions = [512];
24+
group('comparation internal method', () => {
25+
bench('hibrid($cardinality,$dimension)', function* (ctx) {
26+
const cardinality = ctx.get('cardinality');
27+
const size = ctx.get('dimension');
28+
// Prepare matrices once
29+
let A = new SparseMatrix(randomMatrix(size, size, cardinality));
30+
let B = new SparseMatrix(randomMatrix(size, size, cardinality));
31+
32+
// Benchmark the multiplication
33+
yield () => do_not_optimize(A.mmul(B));
34+
do_not_optimize(A);
35+
do_not_optimize(B);
36+
})
37+
.gc('inner')
38+
.args('cardinality', cardinalities) //.range('size', 32, 1024, 2); //.args('size', sizes);
39+
.args('dimension', dimensions);
40+
41+
bench('small($cardinality,$dimension)', function* (ctx) {
42+
const cardinality = ctx.get('cardinality');
43+
const size = ctx.get('dimension');
44+
// Prepare matrices once
45+
let A = new SparseMatrix(randomMatrix(size, size, cardinality));
46+
let B = new SparseMatrix(randomMatrix(size, size, cardinality));
47+
48+
// Benchmark the multiplication
49+
yield () => do_not_optimize(A._mmulSmall(B));
50+
// Explicit cleanup
51+
do_not_optimize(A);
52+
do_not_optimize(B);
53+
})
54+
.gc('inner')
55+
.args('cardinality', cardinalities) //.range('size', 32, 1024, 2); //.args('size', sizes);
56+
.args('dimension', dimensions);
57+
58+
bench('low($cardinality,$dimension)', function* (ctx) {
59+
const cardinality = ctx.get('cardinality');
60+
const size = ctx.get('dimension');
61+
// Prepare matrices once
62+
let A = new SparseMatrix(randomMatrix(size, size, cardinality));
63+
let B = new SparseMatrix(randomMatrix(size, size, cardinality));
64+
65+
// Benchmark the multiplication
66+
yield () => do_not_optimize(A._mmulLowDensity(B));
67+
// Explicit cleanup
68+
do_not_optimize(A);
69+
do_not_optimize(B);
70+
})
71+
.gc('inner')
72+
.args('cardinality', cardinalities) //.range('size', 32, 1024, 2); //.args('size', sizes);
73+
.args('dimension', dimensions);
74+
75+
bench('medium($cardinality,$dimension)', function* (ctx) {
76+
const cardinality = ctx.get('cardinality');
77+
const size = ctx.get('dimension');
78+
// Prepare matrices once
79+
let A = new SparseMatrix(randomMatrix(size, size, cardinality));
80+
let B = new SparseMatrix(randomMatrix(size, size, cardinality));
81+
82+
// Benchmark the multiplication
83+
yield () => {
84+
do_not_optimize(A._mmulMediumDensity(B));
85+
};
86+
87+
// Explicit cleanup
88+
do_not_optimize(A);
89+
do_not_optimize(B);
90+
})
91+
.gc('inner')
92+
.args('cardinality', cardinalities) //.range('size', 32, 1024, 2); //.args('size', sizes);
93+
.args('dimension', dimensions);
94+
});
95+
96+
// Run benchmarks and capture results
97+
const results = await run({
98+
// Force GC between every benchmark
99+
gc: true,
100+
// // More samples for statistical significance
101+
// min_samples: 20,
102+
// max_samples: 200,
103+
// // Longer warmup to stabilize CPU state
104+
// warmup_samples: 10,
105+
// warmup_threshold: 100, // ms
106+
// Longer minimum time for stable measurements
107+
min_cpu_time: 2000, // 2 seconds minimum
108+
// Batch settings to reduce variance
109+
batch_samples: 5,
110+
batch_threshold: 10, // ms
111+
// Enable colors
112+
colors: true,
113+
});
114+
115+
// Process and store results
116+
const processedResults = [];
117+
118+
for (const benchmark of results.benchmarks) {
119+
for (const run of benchmark.runs) {
120+
if (run.stats) {
121+
processedResults.push({
122+
name: benchmark.alias,
123+
cardinality: run.args.cardinality,
124+
dimension: run.args.dimension,
125+
avg: run.stats.avg,
126+
min: run.stats.min,
127+
max: run.stats.max,
128+
p50: run.stats.p50,
129+
p75: run.stats.p75,
130+
p99: run.stats.p99,
131+
samples: run.stats.samples.length,
132+
ticks: run.stats.ticks,
133+
});
134+
}
135+
}
136+
}
137+
138+
// Save results to JSON file
139+
await writeFile(
140+
path.join(import.meta.dirname, `benchmark-results.json`),
141+
JSON.stringify(processedResults, null, 2),
142+
);

‎benchmark/densityDecreaseWhenSizeIncrease.js‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import { run, bench, group, do_not_optimize } from 'mitata';
2+
// import { Matrix } from 'ml-matrix';
3+
4+
import { SparseMatrix as SparseMatrixOld } from './class/SparseMatrixOld.js';
5+
import { randomMatrix } from './utils/randomMatrix.js';
6+
import { SparseMatrix } from '../src/index.js';
7+
8+
/* eslint
9+
func-names: 0
10+
camelcase: 0
11+
*/
12+
13+
const sizes = [8, 16, 32, 256, 512, 1024];
14+
const densities = [0.125, 0.0625, 0.03125, 0.0039, 0.00197, 0.001];
15+
16+
for (let i = 0; i < sizes.length; i++) {
17+
const size = sizes[i];
18+
const density = densities[i];
19+
const denseA = randomMatrix(size, size, density);
20+
const denseB = randomMatrix(size, size, density);
21+
const AOld = new SparseMatrixOld(denseA);
22+
const BOld = new SparseMatrixOld(denseB);
23+
24+
const A = new SparseMatrix(denseA);
25+
const B = new SparseMatrix(denseB);
26+
27+
group(`size:${size}-density:${density}`, () => {
28+
bench('mmulNew', () => {
29+
do_not_optimize(A.mmul(B));
30+
}).gc('inner');
31+
bench('mmul', () => {
32+
do_not_optimize(AOld.mmul(BOld));
33+
}).gc('inner');
34+
});
35+
}
36+
37+
await run({ silent: false });

‎benchmark/plot.html‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="UTF-8">
5+
<title>Benchmark Results Plot</title>
6+
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
7+
<style>
8+
body { font-family: sans-serif; }
9+
.charts-grid {
10+
display: flex;
11+
flex-wrap: wrap;
12+
gap: 24px;
13+
}
14+
.chart-container {
15+
min-width: 320px;
16+
box-sizing: border-box;
17+
margin-bottom: 40px;
18+
/* width will be set dynamically */
19+
}
20+
@media (max-width: 1000px) {
21+
.chart-container { min-width: 100% !important; max-width: 100% !important; }
22+
}
23+
#fileInput { margin: 20px 0; }
24+
#error { color: red; margin: 20px 0; }
25+
</style>
26+
</head>
27+
<body>
28+
<h1>Benchmark Results: avg vs cardinality</h1>
29+
<input type="file" id="fileInput" accept=".json" style="display:none" />
30+
<div id="error"></div>
31+
<div id="charts" class="charts-grid"></div>
32+
<script>
33+
async function loadData() {
34+
try {
35+
const response = await fetch('benchmark-results4.json');
36+
if (!response.ok) throw new Error('Fetch failed');
37+
return await response.json();
38+
} catch (e) {
39+
return null;
40+
}
41+
}
42+
43+
function groupByDimensionAndMethod(data) {
44+
// { dimension: { method: [entries] } }
45+
const grouped = {};
46+
for (const entry of data) {
47+
const { name, dimension } = entry;
48+
if (!grouped[dimension]) grouped[dimension] = {};
49+
if (!grouped[dimension][name]) grouped[dimension][name] = [];
50+
grouped[dimension][name].push(entry);
51+
}
52+
// Sort by cardinality for each group
53+
for (const dim in grouped) {
54+
for (const method in grouped[dim]) {
55+
grouped[dim][method].sort((a, b) => a.cardinality - b.cardinality);
56+
}
57+
}
58+
return grouped;
59+
}
60+
61+
function makeColor(idx) {
62+
// Generate visually distinct colors
63+
const colors = [
64+
'#e6194b','#3cb44b','#ffe119','#4363d8','#f58231','#911eb4','#46f0f0',
65+
'#f032e6','#bcf60c','#fabebe','#008080','#e6beff','#9a6324','#fffac8',
66+
'#800000','#aaffc3','#808000','#ffd8b1','#000075','#808080'
67+
];
68+
return colors[idx % colors.length];
69+
}
70+
71+
function createChart(container, dimension, methodData, methodNames) {
72+
const canvas = document.createElement('canvas');
73+
container.appendChild(canvas);
74+
const ctx = canvas.getContext('2d');
75+
const allCardinalities = Array.from(
76+
new Set(
77+
Object.values(methodData)
78+
.flat()
79+
.map(d => d.cardinality)
80+
)
81+
).sort((a, b) => a - b);
82+
83+
const datasets = methodNames.map((method, idx) => {
84+
const dataMap = {};
85+
(methodData[method] || []).forEach(d => { dataMap[d.cardinality] = d.avg; });
86+
return {
87+
label: method,
88+
data: allCardinalities.map(c => dataMap[c] !== undefined ? dataMap[c] : null),
89+
borderColor: makeColor(idx),
90+
backgroundColor: makeColor(idx) + '33',
91+
fill: false,
92+
tension: 0.1,
93+
pointRadius: 3
94+
};
95+
});
96+
97+
new Chart(ctx, {
98+
type: 'line',
99+
data: {
100+
labels: allCardinalities,
101+
datasets
102+
},
103+
options: {
104+
responsive: true,
105+
plugins: {
106+
title: {
107+
display: true,
108+
text: `dimension=${dimension}`
109+
},
110+
legend: { display: true }
111+
},
112+
scales: {
113+
x: { title: { display: true, text: 'cardinality' } },
114+
y: { title: { display: true, text: 'avg' }, beginAtZero: false }
115+
}
116+
}
117+
});
118+
}
119+
120+
function renderCharts(data) {
121+
const grouped = groupByDimensionAndMethod(data);
122+
const chartsDiv = document.getElementById('charts');
123+
chartsDiv.innerHTML = '';
124+
const methodNames = Array.from(
125+
new Set(data.map(d => d.name))
126+
);
127+
const dimensions = Object.keys(grouped);
128+
const n = dimensions.length;
129+
let widthPercent = "100%";
130+
if (n === 1) widthPercent = "100%";
131+
else if (n === 2) widthPercent = "48%";
132+
else widthPercent = "32%";
133+
dimensions.forEach(dimension => {
134+
const methodData = grouped[dimension];
135+
const chartDiv = document.createElement('div');
136+
chartDiv.className = 'chart-container';
137+
chartDiv.style.maxWidth = widthPercent;
138+
chartDiv.style.flexBasis = widthPercent;
139+
chartsDiv.appendChild(chartDiv);
140+
createChart(chartDiv, dimension, methodData, methodNames);
141+
});
142+
}
143+
144+
async function main() {
145+
let data = await loadData();
146+
if (data) {
147+
renderCharts(data);
148+
} else {
149+
// Show file input for manual loading
150+
document.getElementById('fileInput').style.display = '';
151+
document.getElementById('error').textContent =
152+
"Could not load 'benchmark-results.json'. Please select the file manually.";
153+
document.getElementById('fileInput').addEventListener('change', function (e) {
154+
const file = e.target.files[0];
155+
if (!file) return;
156+
const reader = new FileReader();
157+
reader.onload = function (evt) {
158+
try {
159+
const json = JSON.parse(evt.target.result);
160+
document.getElementById('error').textContent = '';
161+
renderCharts(json);
162+
} catch (err) {
163+
document.getElementById('error').textContent = 'Invalid JSON file.';
164+
}
165+
};
166+
reader.readAsText(file);
167+
});
168+
}
169+
}
170+
171+
main();
172+
</script>
173+
</body>
174+
</html>

0 commit comments

Comments
 (0)