diff --git a/bin/perf-test-results/compare-multiple.js b/bin/perf-test-results/compare-multiple.js new file mode 100755 index 0000000000..bd5b97782a --- /dev/null +++ b/bin/perf-test-results/compare-multiple.js @@ -0,0 +1,59 @@ +#!/usr/bin/env node +/* eslint-disable curly,max-len */ + +const { loadResultFile, printComparisonReport, SUITE_FOR } = require('./lib'); + +const [ , , ...files ] = process.argv; + +const rawResults = files.map(loadResultFile); + +const adapters = []; +const testSuites = {}; +const resultsByAdapter = {}; + +const clientFilter = process.env.CLIENT; +const adapterFilter = process.env.ADAPTERS && process.env.ADAPTERS.split(','); +const gitFilter = process.env.COMMITS && process.env.COMMITS.split(',').map(commit => commit.substring(0, 7)); + +rawResults.forEach(({ adapter, client, results }) => { + if (adapterFilter) { + if (!adapterFilter.includes(adapter.split(':')[0])) return; + } + if (gitFilter) { + if (!gitFilter.includes(adapter.split(':')[1])) return; + } + if (clientFilter) { + if (client.toLowerCase() !== clientFilter.toLowerCase()) return; + } + + if (!adapters.includes(adapter)) { + adapters.push(adapter); + resultsByAdapter[adapter] = {}; + } + + Object.entries(results).forEach(([ t, { median } ]) => { + const suite = SUITE_FOR[t] || 'TODO:suite-mapping'; + + if (!testSuites[suite]) testSuites[suite] = []; + if (!testSuites[suite].includes(t)) testSuites[suite].push(t); + + if (!resultsByAdapter[adapter][suite]) resultsByAdapter[adapter][suite] = {}; + if (!resultsByAdapter[adapter][suite][t]) resultsByAdapter[adapter][suite][t] = { numIterations:0, min:Number.MAX_VALUE }; + + resultsByAdapter[adapter][suite][t].min = Math.min(resultsByAdapter[adapter][suite][t].min, median); + resultsByAdapter[adapter][suite][t].numIterations++; + }); +}); + +if (adapters.length < 2) { + console.log('!!! At least 2 different adapters are required to make comparisons!'); + process.exit(1); +} + +adapters.sort(); + +const sortedResults = adapters.map(adapter => ({ + adapter, results:resultsByAdapter[adapter] +})); + +printComparisonReport({ useStat:'min' }, ...sortedResults); diff --git a/bin/perf-test-results/compare-two.js b/bin/perf-test-results/compare-two.js new file mode 100755 index 0000000000..b192ff3c20 --- /dev/null +++ b/bin/perf-test-results/compare-two.js @@ -0,0 +1,24 @@ +#!/usr/bin/env node +/* eslint-disable curly */ + +const { loadResultFile, printComparisonReport, SUITE_FOR } = require('./lib'); + +const [ , , ...files ] = process.argv; +if (files.length !== 2) throw new Error('Can currently only compare 2 results.'); + +const [ a, b ] = files.map(loadResultFile); + +printComparisonReport({ useStat:'median' }, remap(a), remap(b)); + +function remap({ adapter, results }) { + const suiteResults = {}; + for (const [ name, { median, numIterations } ] of Object.entries(results)) { + const suiteName = SUITE_FOR[name]; + if (!suiteResults[suiteName]) suiteResults[suiteName] = {}; + suiteResults[suiteName][name] = { median, numIterations }; + } + return { + adapter, + results: suiteResults, + }; +} diff --git a/bin/perf-test-results/lib.js b/bin/perf-test-results/lib.js new file mode 100644 index 0000000000..e7b6822e58 --- /dev/null +++ b/bin/perf-test-results/lib.js @@ -0,0 +1,127 @@ +/* eslint-disable curly */ + +// Current JSON reporter does not save suite names. These are hardcoded from +// the test files. +const SUITE_FOR = { + 'basic-inserts': 'basics', + 'bulk-inserts': 'basics', + 'bulk-inserts-large-docs': 'basics', + 'bulk-inserts-massive-docs': 'basics', + 'basic-updates': 'basics', + 'basic-gets': 'basics', + 'all-docs-skip-limit': 'basics', + 'all-docs-startkey-endkey': 'basics', + 'all-docs-keys': 'basics', + 'all-docs-include-docs': 'basics', + 'pull-replication-one-generation': 'basics', + 'pull-replication-two-generation': 'basics', + 'temp-views': 'views', + 'build-secondary-index': 'views', + 'persisted-views': 'views', + 'persisted-views-stale-ok': 'views', + 'create-index': 'find', + 'simple-find-query': 'find', + 'simple-find-query-no-index': 'find', + 'complex-find-query': 'find', + 'complex-find-query-no-index': 'find', + 'multi-field-query': 'find', + 'basic-attachments': 'attachments', + 'many-attachments-base64': 'attachments', + 'many-attachments-binary': 'attachments', +}; + +module.exports = { + loadResultFile, + printComparisonReport, + SUITE_FOR, // TODO instead of exporting here, do remapping inside printComparisonReport() +}; + +const fs = require('node:fs'); + +function loadResultFile(file) { + console.error(`[compare-perf-results.lib]`, 'Loading file:', file, '...'); + const { adapter, client, srcRoot, tests:results } = JSON.parse(fs.readFileSync(file, { encoding:'utf8' })); + + const gitMatch = srcRoot.match(/^\.\.\/\.\.\/dist-bundles\/([0-9a-f]{40})$/); + const description = (gitMatch && gitMatch[1].substr(0,7)) || srcRoot; + + const browserName = client.browser.name; + + return { adapter:`${adapter}:${description}`, client:browserName, results }; +} + +function report(...args) { console.log(' ', ...args); } + +const colFormat = idx => { + switch (idx) { + case 0: return { width:11, pad:'padEnd' }; + case 1: return { width:31, pad:'padEnd' }; + } + if (idx & 1) return { width:17, pad:'padStart' }; + else return { width:17, pad:'padStart' }; +}; +function reportTableRow(...cols) { + report(cols.map((c, i) => { + const { width, pad } = colFormat(i); + return ((c && c.toString()) || '-')[pad](width, ' '); + }).join(' | ')); +} +function reportTableDivider(results) { + let totalWidth = -3; + for (let i=(results.length*2)+1; i>=0; --i) { + totalWidth += colFormat(i).width + 3; + } + report(''.padStart(totalWidth, '-')); +} + +function forHumans(n) { + return n != null ? n.toFixed(2) : null; +} + +function printComparisonReport({ useStat }, ...results) { + report(); + report('Using stat:', useStat); + report('Comparing adapters:'); + results.map(({ adapter }) => report(' *', adapter)); + report(); + reportTableRow('', '', ...results.map(r => [ r.adapter, r.adapter ]).flat()); + reportTableRow('', '', ...results.map(() => [ 'iterations', useStat ]).flat()); + + const [ a, ...others ] = results; + Object.entries(a.results) + .forEach(([ suite, suiteResults ]) => { + Object.entries(suiteResults) + .forEach(([ test, testResults ], idx) => { + if (!idx) reportTableDivider(results); + const suiteName = idx ? '' : suite; + + const resA = testResults[useStat]; + const iterationsA = testResults.numIterations; + + const resOthers = others.map(b => b.results[suite][test][useStat]); + const iterationsOther = others.map(b => b.results[suite][test].numIterations); + reportTableRow(suiteName, test, + iterationsA, + forHumans(resA) + isBetter(resA, ...resOthers), + ...others.map((_, i) => [ + iterationsOther[i], + forHumans(resOthers[i]) + isBetter(resOthers[i], resA, ...arrWithout(resOthers, i)), + ]).flat(), + ); + }); + }); + report(); +} + +function isBetter(a, ...others) { + const minOthers = Math.min(...others); + + if (Math.abs(a - minOthers) / a < 0.05) return ' ~'; // less than 5 percent different - is it significant? do we care? + if (a < minOthers) return ' !'; + if (a === minOthers) return ' !'; + return ' '; +} + +function arrWithout(arr, idx) { + return [ ...arr.slice(0, idx), ...arr.slice(idx+1) ]; +}