perf: use pull architecture and leaner streams (#95)

Author: ayuhito

README.md

@@ -156,7 +156,9 @@ const fileStream = createWriteStream('./project.tar');
 await pipeline(tarStream, fileStream);
 
 // Extract a tar file to a directory
-const tarReadStream = createReadStream('./project.tar');
+const tarReadStream = createReadStream('./project.tar', {
+	highWaterMark: 256 * 1024 // 256 KB for optimal performance
+});
 const extractStream = unpackTar('./output/directory');
 await pipeline(tarReadStream, extractStream);
 ```
@@ -176,7 +178,9 @@ const packStream = packTar('./my/project', {
 });
 
 // Unpack with advanced options
-const sourceStream = createReadStream('./archive.tar');
+const sourceStream = createReadStream('./archive.tar', {
+	highWaterMark: 256 * 1024 // 256 KB for optimal performance
+});
 const extractStream = unpackTar('./output', {
 	// Core options
 	strip: 1, // Remove first directory level
@@ -187,8 +191,7 @@ const extractStream = unpackTar('./output', {
 	fmode: 0o644, // Override file permissions
 	dmode: 0o755, // Override directory permissions
 	maxDepth: 50,  // Limit extraction depth for security (default: 1024)
-	concurrency: 8, // Limit concurrent filesystem operations (default: CPU cores)
-	streamTimeout: 10000 // Timeout after 10 seconds of inactivity (default: 5000ms)
+	concurrency: 8 // Limit concurrent filesystem operations (default: CPU cores)
 });
 
 await pipeline(sourceStream, extractStream);
@@ -205,7 +208,6 @@ import { pipeline } from 'node:stream/promises';
 const sources: TarSource[] = [
   { type: 'file', source: './package.json', target: 'project/package.json' },
   { type: 'directory', source: './src', target: 'project/src' },
-  
   { type: 'content', content: 'Hello World!', target: 'project/hello.txt' },
   { type: 'content', content: '#!/bin/bash\necho "Executable"', target: 'bin/script.sh', mode: 0o755 },
   { type: 'stream', content: createReadStream('./large-file.bin'), target: 'project/data.bin', size: 1048576 },
@@ -229,14 +231,22 @@ const tarStream = packTar('./my/project');
 await pipeline(tarStream, createGzip(), createWriteStream('./project.tar.gz'));
 
 // Decompress and extract .tar.gz
-const gzipStream = createReadStream('./project.tar.gz');
+const gzipStream = createReadStream('./project.tar.gz', {
+	highWaterMark: 256 * 1024 // 256 KB for optimal performance
+});
 await pipeline(gzipStream, createGunzip(), unpackTar('./output'));
 ```
 
 ## API Reference
 
 See the [API Reference](./REFERENCE.md).
 
+# Benchmarks
+
+Current benchmarks indicate we're much faster than other popular tar libraries for small file archives (packing and unpacking). On the other hand, larger files hit an I/O bottleneck resulting in similar performance between libraries.
+
+See the [Results](./benchmarks/README.md).
+
 ## Compatibility
 
 The core library uses the [Web Streams API](https://caniuse.com/streams) and requires:

REFERENCE.md

@@ -149,12 +149,13 @@ import { unpackTar } from 'modern-tar/fs';
 import { createReadStream } from 'node:fs';
 import { pipeline } from 'node:stream/promises';
 
-const tarStream = createReadStream('backup.tar');
+const tarStream = createReadStream('backup.tar', {
+  highWaterMark: 256 * 1024 // 256 KB for optimal performance
+});
 const extractStream = unpackTar('/restore/location', {
   strip: 1,
   fmode: 0o644, // Set consistent file permissions
   strict: true, // Enable strict validation
-  streamTimeout: 10000, // Timeout after 10 seconds of inactivity
 });
 await pipeline(tarStream, extractStream);
 ```
@@ -215,12 +216,6 @@ interface UnpackOptions extends DecoderOptions {
   filter?: (header: TarHeader) => boolean;
   /** Transform function to modify tar headers before extraction */
   map?: (header: TarHeader) => TarHeader;
-  /**
-   * The number of milliseconds of inactivity before a stream is considered stalled.
-   * Prevents hangs when processing corrupted or incomplete archives.
-   * @default 5000
-   */
-  streamTimeout?: number;
 }
 ```
 
@@ -291,8 +286,7 @@ interface UnpackOptionsFS extends UnpackOptions {
   filter?: (header: TarHeader) => boolean;
   /** Transform function to modify headers before extraction */
   map?: (header: TarHeader) => TarHeader;
-  /** Stream timeout in milliseconds for detecting stalled streams */
-  streamTimeout?: number;
+
 
   // Filesystem-specific options:
   /** Default mode for created directories (e.g., 0o755). Overrides tar header mode */

benchmarks/README.md

@@ -68,28 +68,28 @@ These benchmarks were run on an Apple M3 Pro and can include a lot of noise. Res
 ┌─────────┬────────────────────────────────────────────────────┬─────────────────────┬────────────────────────┬────────────────────────┬────────────────────────┬─────────┐
 │ (index) │ Task name                                          │ Latency avg (ns)    │ Latency med (ns)       │ Throughput avg (ops/s) │ Throughput med (ops/s) │ Samples │
 ├─────────┼────────────────────────────────────────────────────┼─────────────────────┼────────────────────────┼────────────────────────┼────────────────────────┼─────────┤
-│ 0       │ 'modern-tar: Unpack Many Small Files (2500 x 1KB)' │ '231053373 ± 3.03%' │ '228506875 ± 23751375' │ '4 ± 3.03%'            │ '4 ± 0'                │ 65      │
-│ 1       │ 'node-tar: Unpack Many Small Files (2500 x 1KB)'   │ '232562313 ± 3.26%' │ '222701583 ± 17677541' │ '4 ± 3.17%'            │ '4 ± 0'                │ 65      │
-│ 2       │ 'tar-fs: Unpack Many Small Files (2500 x 1KB)'     │ '524874342 ± 2.59%' │ '518207771 ± 22493479' │ '2 ± 2.45%'            │ '2 ± 0'                │ 30      │
+│ 0       │ 'modern-tar: Unpack Many Small Files (2500 x 1KB)' │ '135197180 ± 1.09%' │ '134176333 ± 4318458'  │ '7 ± 1.04%'            │ '7 ± 0'                │ 111     │
+│ 1       │ 'node-tar: Unpack Many Small Files (2500 x 1KB)'   │ '194336629 ± 1.10%' │ '192998833 ± 7543895'  │ '5 ± 1.09%'            │ '5 ± 0'                │ 78      │
+│ 2       │ 'tar-fs: Unpack Many Small Files (2500 x 1KB)'     │ '524907015 ± 2.02%' │ '536602646 ± 18800583' │ '2 ± 2.09%'            │ '2 ± 0'                │ 30      │
 └─────────┴────────────────────────────────────────────────────┴─────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴─────────┘
 
 --- Many Small Nested Files (2500 x 1KB) ---
 ┌─────────┬───────────────────────────────────────────────────────────┬─────────────────────┬────────────────────────┬────────────────────────┬────────────────────────┬─────────┐
 │ (index) │ Task name                                                 │ Latency avg (ns)    │ Latency med (ns)       │ Throughput avg (ops/s) │ Throughput med (ops/s) │ Samples │
 ├─────────┼───────────────────────────────────────────────────────────┼─────────────────────┼────────────────────────┼────────────────────────┼────────────────────────┼─────────┤
-│ 0       │ 'modern-tar: Unpack Many Small Nested Files (2500 x 1KB)' │ '210424280 ± 1.32%' │ '209016354 ± 8327583'  │ '5 ± 1.30%'            │ '5 ± 0'                │ 72      │
-│ 1       │ 'node-tar: Unpack Many Small Nested Files (2500 x 1KB)'   │ '375473636 ± 2.64%' │ '365691875 ± 16068167' │ '3 ± 2.50%'            │ '3 ± 0'                │ 41      │
-│ 2       │ 'tar-fs: Unpack Many Small Nested Files (2500 x 1KB)'     │ '656863267 ± 1.94%' │ '654005562 ± 25563875' │ '2 ± 1.94%'            │ '2 ± 0'                │ 30      │
+│ 0       │ 'modern-tar: Unpack Many Small Nested Files (2500 x 1KB)' │ '166566736 ± 0.76%' │ '165969875 ± 4058125'  │ '6 ± 0.75%'            │ '6 ± 0'                │ 91      │
+│ 1       │ 'node-tar: Unpack Many Small Nested Files (2500 x 1KB)'   │ '364342735 ± 1.64%' │ '359822521 ± 12994563' │ '3 ± 1.60%'            │ '3 ± 0'                │ 42      │
+│ 2       │ 'tar-fs: Unpack Many Small Nested Files (2500 x 1KB)'     │ '567124578 ± 1.46%' │ '566608812 ± 14051833' │ '2 ± 1.47%'            │ '2 ± 0'                │ 30      │
 └─────────┴───────────────────────────────────────────────────────────┴─────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴─────────┘
 
 --- Few Large Files (5 x 20MB) ---
 ┌─────────┬─────────────────────────────────────────────────┬────────────────────┬──────────────────────┬────────────────────────┬────────────────────────┬─────────┐
 │ (index) │ Task name                                       │ Latency avg (ns)   │ Latency med (ns)     │ Throughput avg (ops/s) │ Throughput med (ops/s) │ Samples │
 ├─────────┼─────────────────────────────────────────────────┼────────────────────┼──────────────────────┼────────────────────────┼────────────────────────┼─────────┤
-│ 0       │ 'modern-tar: Unpack Few Large Files (5 x 20MB)' │ '47089232 ± 4.02%' │ '44317458 ± 1487584' │ '22 ± 1.20%'           │ '23 ± 1'               │ 319     │
-│ 1       │ 'node-tar: Unpack Few Large Files (5 x 20MB)'   │ '29503568 ± 7.99%' │ '22717625 ± 2264667' │ '40 ± 2.29%'           │ '44 ± 5'               │ 509     │
-│ 2       │ 'tar-fs: Unpack Few Large Files (5 x 20MB)'     │ '47116490 ± 8.18%' │ '39027917 ± 2082917' │ '24 ± 2.30%'           │ '26 ± 1'               │ 319     │
+│ 0       │ 'modern-tar: Unpack Few Large Files (5 x 20MB)' │ '35877987 ± 4.11%' │ '29541167 ± 2477751' │ '31 ± 2.45%'           │ '34 ± 3'               │ 419     │
+│ 1       │ 'node-tar: Unpack Few Large Files (5 x 20MB)'   │ '39709622 ± 4.50%' │ '34362438 ± 904791'  │ '27 ± 1.97%'           │ '29 ± 1'               │ 378     │
+│ 2       │ 'tar-fs: Unpack Few Large Files (5 x 20MB)'     │ '66414937 ± 7.42%' │ '59902979 ± 8376146' │ '16 ± 2.78%'           │ '17 ± 2'               │ 226     │
 └─────────┴─────────────────────────────────────────────────┴────────────────────┴──────────────────────┴────────────────────────┴────────────────────────┴─────────┘
 ```
 
-`modern-tar` is expected to be slower than alternatives due to its focus on zero dependencies, but there is still plenty of room to improve performance and is generally comparable for many usecases due to work being very I/O bound.
+For large files `modern-tar` and `node-tar` are practically very similar in performance since at this point the bottleneck is I/O. However, for many small files, `modern-tar` shows much better results than other libraries.

benchmarks/unpack.bench.ts

@@ -65,9 +65,12 @@ export async function runUnpackingBenchmarks() {
 			.add(
 				`modern-tar: Unpack ${testCase.name}`,
 				async () => {
-					const readStream = fs.createReadStream(tarballPath);
-					const extractStream = unpackTar(extractDir);
-					await pipeline(readStream, extractStream);
+					const readStream = fs.createReadStream(tarballPath, {
+						// Recommended tuning params for large files.
+						highWaterMark: 256 * 1024, // 256 KB
+					});
+					const unpackStream = unpackTar(extractDir);
+					await pipeline(readStream, unpackStream);
 				},
 				{
 					async beforeEach() {

biome.json

@@ -1,5 +1,5 @@
 {
-	"$schema": "https://biomejs.dev/schemas/2.2.5/schema.json",
+	"$schema": "https://biomejs.dev/schemas/2.3.5/schema.json",
 	"files": {
 		"includes": [
 			"**",
@@ -14,7 +14,12 @@
 		"enabled": true
 	},
 	"linter": {
-		"enabled": true
+		"enabled": true,
+		"rules": {
+			"suspicious": {
+				"noAssignInExpressions": "off"
+			}
+		}
 	},
 	"assist": {
 		"enabled": true