diff --git a/Benchmarks/README.md b/Benchmarks/README.md index eeafc395..f7264b6b 100644 --- a/Benchmarks/README.md +++ b/Benchmarks/README.md @@ -27,4 +27,11 @@ node run.js --adaptive --output=stable-results.json # Run benchmarks and compare with previous results node run.js --baseline=previous-results.json + +# Run only a subset of benchmarks +# Substring match +node run.js --filter=Call +# Regex (with flags) +node run.js --filter=/^Property access\// +node run.js --filter=/string/i ``` diff --git a/Benchmarks/Sources/Benchmarks.swift b/Benchmarks/Sources/Benchmarks.swift index 155acae1..55b9f3eb 100644 --- a/Benchmarks/Sources/Benchmarks.swift +++ b/Benchmarks/Sources/Benchmarks.swift @@ -14,7 +14,7 @@ class Benchmark { body() return .undefined } - benchmarkRunner("\(title)/\(name)", jsBody) + try! benchmarkRunner("\(title)/\(name)", jsBody) } } @@ -24,13 +24,13 @@ class Benchmark { call.testSuite("JavaScript function call through Wasm import") { for _ in 0..<20_000_000 { - benchmarkHelperNoop() + try! benchmarkHelperNoop() } } call.testSuite("JavaScript function call through Wasm import with int") { for _ in 0..<10_000_000 { - benchmarkHelperNoopWithNumber(42) + try! benchmarkHelperNoopWithNumber(42) } } diff --git a/Benchmarks/run.js b/Benchmarks/run.js index 2305373a..17b19214 100644 --- a/Benchmarks/run.js +++ b/Benchmarks/run.js @@ -21,6 +21,32 @@ function updateProgress(current, total, label = '', width) { process.stdout.write(`${label} [${bar}] ${current}/${total}`); } +/** + * Create a name filter function from CLI argument + * - Supports substring match (default) + * - Supports /regex/flags syntax + * @param {string|undefined} arg + * @returns {(name: string) => boolean} + */ +function createNameFilter(arg) { + if (!arg) { + return () => true; + } + if (arg.startsWith('/') && arg.lastIndexOf('/') > 0) { + const lastSlash = arg.lastIndexOf('/'); + const pattern = arg.slice(1, lastSlash); + const flags = arg.slice(lastSlash + 1); + try { + const re = new RegExp(pattern, flags); + return (name) => re.test(name); + } catch (e) { + console.error('Invalid regular expression for --filter:', e.message); + process.exit(1); + } + } + return (name) => name.includes(arg); +} + /** * Calculate coefficient of variation (relative standard deviation) * @param {Array} values - Array of measurement values @@ -238,24 +264,28 @@ function saveJsonResults(filePath, data) { * @param {Object} results - Results object to store benchmark data * @returns {Promise} */ -async function singleRun(results) { +async function singleRun(results, nameFilter) { const options = await defaultNodeSetup({}) + const benchmarkRunner = (name, body) => { + if (nameFilter && !nameFilter(name)) { + return; + } + const startTime = performance.now(); + body(); + const endTime = performance.now(); + const duration = endTime - startTime; + if (!results[name]) { + results[name] = [] + } + results[name].push(duration) + } const { exports } = await instantiate({ ...options, - imports: { + getImports: () => ({ benchmarkHelperNoop: () => { }, benchmarkHelperNoopWithNumber: (n) => { }, - benchmarkRunner: (name, body) => { - const startTime = performance.now(); - body(); - const endTime = performance.now(); - const duration = endTime - startTime; - if (!results[name]) { - results[name] = [] - } - results[name].push(duration) - } - } + benchmarkRunner: benchmarkRunner + }) }); exports.run(); } @@ -266,7 +296,7 @@ async function singleRun(results) { * @param {Object} options - Adaptive sampling options * @returns {Promise} */ -async function runUntilStable(results, options, width) { +async function runUntilStable(results, options, width, nameFilter, filterArg) { const { minRuns = 5, maxRuns = 50, @@ -285,9 +315,14 @@ async function runUntilStable(results, options, width) { // Update progress with estimated completion updateProgress(runs, maxRuns, "Benchmark Progress:", width); - await singleRun(results); + await singleRun(results, nameFilter); runs++; + if (runs === 1 && Object.keys(results).length === 0) { + console.error(`\nNo benchmarks matched filter: ${filterArg}`); + process.exit(1); + } + // Check if we've reached minimum runs if (runs < minRuns) continue; @@ -349,6 +384,7 @@ Options: --min-runs=NUMBER Minimum runs for adaptive sampling (default: 5) --max-runs=NUMBER Maximum runs for adaptive sampling (default: 50) --target-cv=NUMBER Target coefficient of variation % (default: 5) + --filter=PATTERN Filter benchmarks by name (substring or /regex/flags) --help Show this help message `); } @@ -363,7 +399,8 @@ async function main() { adaptive: { type: 'boolean', default: false }, 'min-runs': { type: 'string', default: '5' }, 'max-runs': { type: 'string', default: '50' }, - 'target-cv': { type: 'string', default: '5' } + 'target-cv': { type: 'string', default: '5' }, + filter: { type: 'string' } } }); @@ -374,6 +411,8 @@ async function main() { const results = {}; const width = 30; + const filterArg = args.values.filter; + const nameFilter = createNameFilter(filterArg); if (args.values.adaptive) { // Adaptive sampling mode @@ -388,7 +427,7 @@ async function main() { console.log(`Results will be saved to: ${args.values.output}`); } - await runUntilStable(results, options, width); + await runUntilStable(results, options, width, nameFilter, filterArg); } else { // Fixed number of runs mode const runs = parseInt(args.values.runs, 10); @@ -410,7 +449,12 @@ async function main() { console.log("\nOverall Progress:"); for (let i = 0; i < runs; i++) { updateProgress(i, runs, "Benchmark Runs:", width); - await singleRun(results); + await singleRun(results, nameFilter); + if (i === 0 && Object.keys(results).length === 0) { + process.stdout.write("\n"); + console.error(`No benchmarks matched filter: ${filterArg}`); + process.exit(1); + } } updateProgress(runs, runs, "Benchmark Runs:", width); console.log("\n");