refactor(plugins): enable support for memory & fixes on types

Author: H4ad

README.md

@@ -69,19 +69,32 @@ See the [examples folder](./examples/) for more common usage examples.
 
 ## Table of Contents
 
-1. [Class `Suite`](#class-suite)
-    1. [`suite.add()`](#suiteaddname-options-fn)
-    2. [`suite.run()`](#suiterun)
-2. [Plugins](#plugins)
-3. [Using Reporter](#using-reporter)
-    1. [Text Reporter](#textreport-default)
-    2. [Chart Reporter](#chartreport)
-    3. [Custom Reporter](#custom-reporter) 
-4. [Setup and Teardown](#setup-and-teardown)
-    1. [Managed Benchmarks](#managed-benchmarks)
-5. [Benchmark Modes](#benchmark-modes)
-    1. [Operations Mode (Default)](#operations-mode)
-    2. [Time Mode](#time-mode)
+- [Install](#install)
+- [Usage](#usage)
+- [Table of Contents](#table-of-contents)
+- [Sponsors](#sponsors)
+- [Class: `Suite`](#class-suite)
+  - [`new Suite([options])`](#new-suiteoptions)
+  - [`suite.add(name[, options], fn)`](#suiteaddname-options-fn)
+  - [`suite.run()`](#suiterun)
+- [Plugins](#plugins)
+  - [Plugin Methods](#plugin-methods)
+  - [Example Plugin](#example-plugin)
+- [Using Reporter](#using-reporter)
+  - [`textReport` (Default)](#textreport-default)
+  - [`chartReport`](#chartreport)
+  - [`htmlReport`](#htmlreport)
+  - [`jsonReport`](#jsonreport)
+  - [CSV Reporter](#csv-reporter)
+  - [Pretty Reporter](#pretty-reporter)
+  - [Custom Reporter](#custom-reporter)
+- [Setup and Teardown](#setup-and-teardown)
+  - [Managed Benchmarks](#managed-benchmarks)
+  - [Worker Threads](#worker-threads)
+- [Benchmark Modes](#benchmark-modes)
+  - [Operations Mode](#operations-mode)
+  - [Time Mode](#time-mode)
+- [Writing JavaScript Mistakes](#writing-javascript-mistakes)
 
 ## Sponsors
 
@@ -171,8 +184,11 @@ See [Plugins](./doc/Plugins.md) for details.
   * `Wrapper` {string} (optional) Function to wrap the benchmark function.
 - **`afterClockTemplate(varNames)`**: Injects code after the benchmark finishes. Returns an array with:
   * `Code` {string} JavaScript code to execute.
-- **`onCompleteBenchmark(result)`**: Called when the benchmark completes, allowing plugins to process results.
+- **`onCompleteBenchmark(result, bench)`**: Called when the benchmark completes, allowing plugins to process results.
 - **`toString()`**: Returns a string identifier for the plugin.
+- **`getReport(benchmarkName)`**: Returns a string to be displayed in the benchmark result line.
+- **`getResult(benchmarkName)`**: Returns the data that can be used by the reporter.
+- **`reset()`**: Resets the plugin state, to avoid carrying over data between benchmarks.
 
 ### Example Plugin
 

examples/plugins/all.js

@@ -3,13 +3,15 @@ const {
   V8GetOptimizationStatus,
   V8NeverOptimizePlugin,
   V8OptimizeOnNextCallPlugin,
+  MemoryPlugin,
 } = require('../../lib');
 
 const suite = new Suite({
   plugins: [
     new V8GetOptimizationStatus(),
     new V8NeverOptimizePlugin(),
-    // new V8OptimizeOnNextCallPlugin(),
+    new MemoryPlugin(),
+    new V8OptimizeOnNextCallPlugin(),
   ],
 });
 

examples/plugins/memory.js

@@ -0,0 +1,24 @@
+const { Suite, MemoryPlugin} = require('../../lib');
+
+const suite = new Suite({
+    plugins: [new MemoryPlugin()],
+});
+
+suite
+    .add(`new Uint32Array(1024)`, function () {
+        return new Uint32Array(1024);
+    })
+    .add(`[Managed] new Uint32Array(1024)`, function (timer) {
+        const assert = require('node:assert');
+
+        let r;
+
+        timer.start();
+        for (let i = 0; i < timer.count; i++) {
+            r = new Uint32Array(1024);
+        }
+        timer.end(timer.count);
+
+        assert.ok(r);
+    })
+    .run();

index.d.ts

@@ -4,11 +4,36 @@
 import type { Histogram } from "node:perf_hooks";
 
 export declare namespace BenchNode {
+	class Benchmark {
+		name: string;
+		fn: any;
+		minTime: number;
+		maxTime: number;
+		plugins: Plugin[];
+		repeatSuite: number;
+		minSamples: number;
+		baseline: boolean;
+
+		constructor(
+			name: string,
+			fn: any,
+			minTime: number,
+			maxTime: number,
+			plugins: Plugin[],
+			repeatSuite: number,
+			minSamples: number,
+			baseline?: boolean,
+		);
+
+		serializeBenchmark(): void;
+	}
+
 	interface PluginHookVarNames {
 		awaitOrEmpty: string;
-		bench: any; // Can be string during validation, object with 'fn' property during actual run
-		context: any;
-		timer: any;
+		bench: string;
+		context: string;
+		timer: string;
+		managed: boolean;
 	}
 
 	interface BenchmarkResult {
@@ -44,15 +69,28 @@ export declare namespace BenchNode {
 		count: number;
 	}) => void | Promise<void>;
 
+	type OnCompleteBenchmarkResult = [
+		duration: number,
+		count: number,
+		context: Record<string, any>,
+	];
+	type PluginResult = {
+		type: string;
+		[key: string]: any;
+	};
+
 	interface Plugin {
 		isSupported?(): boolean;
 		beforeClockTemplate?(varNames: PluginHookVarNames): string[];
 		afterClockTemplate?(varNames: PluginHookVarNames): string[];
-		onCompleteClock?(result: BenchmarkResult): string[];
-		onCompleteBenchmark?(result: BenchmarkResult): void;
+		onCompleteBenchmark?(
+			result: OnCompleteBenchmarkResult,
+			bench: Benchmark,
+		): void;
 		toString?(): string;
-		getReport?(): string;
-		getResult?(benchmarkName: string): Record<string, any>;
+		getReport?(benchmarkName: string): string;
+		getResult?(benchmarkName: string): PluginResult;
+		reset?(): void;
 	}
 
 	class Suite {
@@ -66,16 +104,16 @@ export declare namespace BenchNode {
 		isSupported(): boolean;
 		beforeClockTemplate(varNames: PluginHookVarNames): string[];
 		toString(): string;
-		getReport(): string;
+		getReport(benchmarkName: string): string;
 	}
 
 	class V8GetOptimizationStatus implements Plugin {
 		isSupported(): boolean;
 		afterClockTemplate(varNames: PluginHookVarNames): string[];
-		onCompleteBenchmark(result: BenchmarkResult): void;
+		onCompleteBenchmark(result: OnCompleteBenchmarkResult): void;
 		toString(): string;
-		getReport(): string;
-		getResult?(benchmarkName: string): Record<string, any>;
+		getReport(benchmarkName: string): string;
+		getResult?(benchmarkName: string): PluginResult;
 	}
 
 	class V8OptimizeOnNextCallPlugin implements Plugin {
@@ -89,7 +127,9 @@ export declare namespace BenchNode {
 		isSupported(): boolean;
 		beforeClockTemplate(varNames: PluginHookVarNames): string[];
 		afterClockTemplate(varNames: PluginHookVarNames): string[];
-		onCompleteBenchmark?(result: BenchmarkResult): void;
+		onCompleteBenchmark(result: OnCompleteBenchmarkResult): void;
+		getReport(benchmarkName: string): string;
+		getResult(benchmarkName: string): PluginResult;
 		toString(): string;
 	}
 }

lib/clock.js

@@ -102,6 +102,7 @@ function createRunUnmanagedBenchmark(bench, awaitOrEmpty) {
 		timer: "timer",
 		context: "context",
 		bench: "bench",
+		managed: false,
 	};
 
 	let code = `
@@ -150,6 +151,7 @@ function createRunManagedBenchmark(bench, awaitOrEmpty) {
 		timer: "timer",
 		context: "context",
 		bench: "bench",
+		managed: true,
 	};
 
 	let code = `

lib/histogram.js

@@ -12,6 +12,7 @@ class StatisticalHistogram {
 	mean;
 	cv;
 	stddev;
+	finished = false;
 
 	/**
 	 * @returns {number[]}
@@ -49,6 +50,9 @@ class StatisticalHistogram {
 	}
 
 	finish() {
+		if (this.finished) return;
+
+		this.finished = true;
 		this.removeOutliers();
 
 		this.calculateMinMax();

lib/lifecycle.js

@@ -219,6 +219,11 @@ async function runBenchmark(
 		);
 	}
 
+	// since the instance is shared across benchmarks, reset it after use
+	for (const plugin of bench.plugins) {
+		plugin.reset?.();
+	}
+
 	return result;
 }
 

lib/plugins/memory.js

@@ -1,8 +1,4 @@
-const {
-	kStatisticalHistogramRecord,
-	StatisticalHistogram,
-	kStatisticalHistogramFinish,
-} = require("../histogram");
+const { StatisticalHistogram } = require("../histogram");
 
 /**
  * Formats a byte value into a human-readable string with appropriate units (B, Kb, MB, GB)
@@ -34,17 +30,13 @@ class MemoryPlugin {
 	/**
 	 * @type {StatisticalHistogram}
 	 */
-	#heapUsedHistogram;
-
-	constructor() {
-		this.reset();
-	}
+	#heapUsedHistogram = new StatisticalHistogram();
 
 	isSupported() {
 		return typeof globalThis.gc === "function";
 	}
 
-	beforeClockTemplate({ managed, globalThisVar, contextVar }) {
+	beforeClockTemplate({ managed, context }) {
 		if (managed && !MemoryPlugin.#WARNING_REPORTED) {
 			MemoryPlugin.#WARNING_REPORTED = true;
 			process.emitWarning(
@@ -54,22 +46,21 @@ class MemoryPlugin {
 
 		let code = "";
 
-		code += `${contextVar}.${MemoryPlugin.MEMORY_BEFORE_RUN} = 0;\n`;
-		code += `${contextVar}.${MemoryPlugin.MEMORY_AFTER_RUN} = 0;\n`;
-		code += `${globalThisVar}.gc();\n`;
-		code += `${contextVar}.${MemoryPlugin.MEMORY_BEFORE_RUN} = ${globalThisVar}.process.memoryUsage();\n`;
+		code += `${context}.${MemoryPlugin.MEMORY_BEFORE_RUN} = 0;\n`;
+		code += `${context}.${MemoryPlugin.MEMORY_AFTER_RUN} = 0;\n`;
+		code += "globalThis.gc();\n";
+		code += `${context}.${MemoryPlugin.MEMORY_BEFORE_RUN} = globalThis.process.memoryUsage();\n`;
 
-		return code;
+		return [code];
 	}
 
-	afterClockTemplate({ globalThisVar, contextVar }) {
-		return `${contextVar}.${MemoryPlugin.MEMORY_AFTER_RUN} = ${globalThisVar}.process.memoryUsage();\n`;
+	afterClockTemplate({ context }) {
+		return [
+			`${context}.${MemoryPlugin.MEMORY_AFTER_RUN} = globalThis.process.memoryUsage();\n`,
+		];
 	}
 
-	onCompleteClock(result) {
-		const realIterations = result[1];
-		const context = result[2];
-
+	onCompleteBenchmark([, realIterations, context]) {
 		const heapUsed =
 			context[MemoryPlugin.MEMORY_AFTER_RUN].heapUsed -
 			context[MemoryPlugin.MEMORY_BEFORE_RUN].heapUsed;
@@ -80,20 +71,16 @@ class MemoryPlugin {
 		const memoryAllocated = (heapUsed + externalUsed) / realIterations;
 
 		// below 0, we just coerce to be zero
-		this.#heapUsedHistogram[kStatisticalHistogramRecord](
-			Math.max(0, memoryAllocated),
-		);
-	}
-
-	onCompleteBenchmark() {
-		this.#heapUsedHistogram[kStatisticalHistogramFinish]();
+		this.#heapUsedHistogram.record(Math.max(0, memoryAllocated));
 	}
 
 	toString() {
 		return "MemoryPlugin";
 	}
 
 	getReport() {
+		this.#heapUsedHistogram.finish();
+
 		return `heap usage=${formatBytes(this.#heapUsedHistogram.mean)} (${formatBytes(this.#heapUsedHistogram.min)} ... ${formatBytes(this.#heapUsedHistogram.max)})`;
 	}
 
@@ -104,6 +91,10 @@ class MemoryPlugin {
 			histogram: this.#heapUsedHistogram,
 		};
 	}
+
+	reset() {
+		this.#heapUsedHistogram = new StatisticalHistogram();
+	}
 }
 
 module.exports = {

lib/plugins/v8-never-opt.js

@@ -9,7 +9,7 @@ class V8NeverOptimizePlugin {
 		}
 	}
 
-	beforeClockTemplate(_varNames) {
+	beforeClockTemplate() {
 		let code = "";
 
 		code += `

lib/plugins/v8-print-status.js

@@ -97,6 +97,10 @@ class V8GetOptimizationStatus {
 			optimizationStatuses: translateStatus(allAvailableStatus),
 		};
 	}
+
+	reset() {
+		this.#optimizationStatuses = [];
+	}
 }
 
 module.exports = {