From a99df25234812c2f6eb08297efaad32d1d61fbfa Mon Sep 17 00:00:00 2001 From: Jason Date: Sat, 5 Jul 2025 15:45:13 -0700 Subject: [PATCH 1/2] Support async functions in tests. Not all code that needs benchmarks is synchronous. --- README.md | 9 +++++++++ src/index.js | 20 ++++++++++++++++---- test/index.js | 41 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 63 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0228915..fea47f9 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,15 @@ benchmarks.add( { setup } ); +benchmarks.add( + 'asynchronous', + async (client, { histogram }) => { + const result = await lookup(); + return result.name; + }, + { setup } +); + benchmarks.run().catch(err => { console.error(err.stack); process.exit(1); diff --git a/src/index.js b/src/index.js index e80df80..055e737 100644 --- a/src/index.js +++ b/src/index.js @@ -27,7 +27,6 @@ function createRegressionBenchmark(baseModule, comparisonModules = []) { for (const bucketName of Object.keys(buckets)) { const bucket = buckets[bucketName]; - for (const benchmarkName of Object.keys(bucket)) { const { fn, opts: { setup, teardown } } = bucket[benchmarkName]; @@ -36,9 +35,21 @@ function createRegressionBenchmark(baseModule, comparisonModules = []) { for (const testModule of testModules) { const name = `${bucketName} ➭ ${benchmarkName} ➭ ${testModule.name}`; - const ctx = await Promise.resolve(setup(testModule.module)); - const result = await new Promise((resolve, reject) => { - const bench = new benchmark.Benchmark(name, () => fn(testModule.module, ctx)); + + let ctx; + + const result = await new Promise(async (resolve, reject) => { + const bench = new benchmark.Benchmark({ + name, + defer: true, + fn: async (done) => { + try { + done.resolve(await fn(testModule.module, ctx)); + } catch (err) { + done.resolve(err); + } + } + }); bench.on('complete', (event) => { if (event.target.error) { @@ -76,6 +87,7 @@ function createRegressionBenchmark(baseModule, comparisonModules = []) { } }); + ctx = await Promise.resolve(setup(testModule.module)); bench.run(); }); diff --git a/test/index.js b/test/index.js index 5ca7768..ae51ae0 100644 --- a/test/index.js +++ b/test/index.js @@ -1,5 +1,6 @@ 'use strict'; +const assert = require('assert'); const createRegressionBenchmark = require('..'); const benchmarks = createRegressionBenchmark( @@ -29,11 +30,39 @@ benchmarks.add( { setup } ); -benchmarks.run().catch(err => { - console.error(err.stack); - process.exit(1); +const selftest = createRegressionBenchmark( + {}, [] +); + +selftest.suite('async support', (suite) => { + suite.add( + 'async setup', + (client, { completed }) => assert(completed), + { setup: asyncSetup }, + ); + + suite.add( + 'async fn', + (client, ctx) => { + assert(ctx.running !== true); + ctx.running = true; + + return new Promise(resolve => setTimeout(() => { + ctx.running = false; + resolve(); + }, 100)); + }, + { setup: asyncSetup }, + ); }); +benchmarks.run() + .then(() => selftest.run()) + .catch(err => { + console.error(err.stack); + process.exit(1); + }); + function setup(client) { const registry = new client.Registry(); @@ -48,3 +77,9 @@ function setup(client) { return {registry, histogram}; } + +async function asyncSetup() { + await new Promise(resolve => setTimeout(resolve, 300)); + + return { completed: {} }; +} From cff1a9369e456afba99947bab49675ec4d13ab5f Mon Sep 17 00:00:00 2001 From: Jason Date: Sat, 5 Jul 2025 17:37:37 -0700 Subject: [PATCH 2/2] Support skipping tests. This will mostly be useful for adding benchmarks that cover features that only exist in certain versions. However it was first conceived when trying to write tests for cluster mode, where you only want the child process to run the same tests as the parent process is currently running. --- README.md | 12 ++++++++++++ src/index.js | 16 ++++++++++++++-- test/index.js | 13 +++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fea47f9..38b1345 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,18 @@ benchmarks.add( { setup } ); + +benchmarks.add( + 'skippable', + (client, { histogram }) => histogram.observe(1, { a: 1, b: 1 }), + { + setup, + start: (event) => (event.target.name !== 'prom-client@10.1.3') // function not supported or broken in this version + } +); + + + benchmarks.run().catch(err => { console.error(err.stack); process.exit(1); diff --git a/src/index.js b/src/index.js index 055e737..47c11cb 100644 --- a/src/index.js +++ b/src/index.js @@ -28,14 +28,13 @@ function createRegressionBenchmark(baseModule, comparisonModules = []) { const bucket = buckets[bucketName]; for (const benchmarkName of Object.keys(bucket)) { - const { fn, opts: { setup, teardown } } = bucket[benchmarkName]; + const { fn, opts: { setup, teardown, start = undefined } } = bucket[benchmarkName]; const fastest = { time: -Infinity }; const slowest = { time: +Infinity }; for (const testModule of testModules) { const name = `${bucketName} ➭ ${benchmarkName} ➭ ${testModule.name}`; - let ctx; const result = await new Promise(async (resolve, reject) => { @@ -87,6 +86,19 @@ function createRegressionBenchmark(baseModule, comparisonModules = []) { } }); + + if (start !== undefined) { + // fake a start() call early + bench.on('start', start); + const cancelled = (bench.emit('start') === false); + bench.off('start', start); + + if (cancelled) { + bench.emit('complete'); + return; + } + } + ctx = await Promise.resolve(setup(testModule.module)); bench.run(); }); diff --git a/test/index.js b/test/index.js index ae51ae0..be29139 100644 --- a/test/index.js +++ b/test/index.js @@ -54,6 +54,19 @@ selftest.suite('async support', (suite) => { }, { setup: asyncSetup }, ); + + suite.add( + 'start callback', + () => { + assert.fail('run should have been skipped'); + }, + { + setup: () => assert.fail('Setup should have been skipped'), + start: (event) => { + return (event.target.name !== 'async support ➭ start callback ➭ current'); + } + }, + ); }); benchmarks.run()