diff --git a/integration-tests/debugger/snapshot-pruning.spec.js b/integration-tests/debugger/snapshot-pruning.spec.js index be81e8b79d7..d3ba92c4e3a 100644 --- a/integration-tests/debugger/snapshot-pruning.spec.js +++ b/integration-tests/debugger/snapshot-pruning.spec.js @@ -1,6 +1,7 @@ 'use strict' -const { assert } = require('chai') +const assert = require('node:assert/strict') + const { setup } = require('./utils') describe('Dynamic Instrumentation', function () { @@ -12,14 +13,12 @@ describe('Dynamic Instrumentation', function () { it('should prune snapshot if payload is too large', function (done) { t.agent.on('debugger-input', ({ payload: [payload] }) => { - assert.isBelow(Buffer.byteLength(JSON.stringify(payload)), 1024 * 1024) // 1MB - assert.notProperty(payload.debugger.snapshot, 'captures') - assert.strictEqual( - payload.debugger.snapshot.captureError, - 'Snapshot was too large (max allowed size is 1 MiB). ' + - 'Consider reducing the capture depth or turn off "Capture Variables" completely, ' + - 'and instead include the variables of interest directly in the message template.' - ) + const payloadSize = Buffer.byteLength(JSON.stringify(payload)) + assert.ok(payloadSize < 1024 * 1024) // 1MB + + const capturesJson = JSON.stringify(payload.debugger.snapshot.captures) + assert.ok(capturesJson.includes('"pruned":true')) + done() }) diff --git a/packages/dd-trace/src/debugger/devtools_client/send.js b/packages/dd-trace/src/debugger/devtools_client/send.js index b0a66c84737..af1352c559d 100644 --- a/packages/dd-trace/src/debugger/devtools_client/send.js +++ b/packages/dd-trace/src/debugger/devtools_client/send.js @@ -10,6 +10,7 @@ const { GIT_COMMIT_SHA, GIT_REPOSITORY_URL } = require('../../plugins/util/tags' const log = require('./log') const { version } = require('../../../../../package.json') const { getEnvironmentVariable } = require('../../config-helper') +const { pruneSnapshot } = require('./snapshot-pruner') module.exports = send @@ -55,14 +56,23 @@ function send (message, logger, dd, snapshot) { let size = Buffer.byteLength(json) if (size > MAX_LOG_PAYLOAD_SIZE_BYTES) { - // TODO: This is a very crude way to handle large payloads. Proper pruning will be implemented later (DEBUG-2624) - delete payload.debugger.snapshot.captures - payload.debugger.snapshot.captureError = - `Snapshot was too large (max allowed size is ${MAX_LOG_PAYLOAD_SIZE_MB} MiB). ` + - 'Consider reducing the capture depth or turn off "Capture Variables" completely, ' + - 'and instead include the variables of interest directly in the message template.' - json = JSON.stringify(payload) - size = Buffer.byteLength(json) + let pruned + try { + pruned = pruneSnapshot(json, size, MAX_LOG_PAYLOAD_SIZE_BYTES) + } catch (err) { + log.error('[debugger:devtools_client] Error pruning snapshot', err) + } + + if (pruned) { + json = pruned + size = Buffer.byteLength(json) + } else { + // Fallback if pruning fails + const line = Object.keys(snapshot.captures.lines)[0] + snapshot.captures.lines[line] = { pruned: true } + json = JSON.stringify(payload) + size = Buffer.byteLength(json) + } } jsonBuffer.write(json, size) diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot-pruner.js b/packages/dd-trace/src/debugger/devtools_client/snapshot-pruner.js new file mode 100644 index 00000000000..e8439e3031a --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot-pruner.js @@ -0,0 +1,473 @@ +'use strict' + +/** + * This module provides functionality to intelligently prune oversized JSON snapshots by selectively removing the + * largest and deepest leaf nodes while preserving the schema structure. + */ + +// The RFC specifies that we should prune nodes at level 5 or deeper, but the Node.js implementation has an extra level +// of depth because it doesn't use the compound key `debugger.snapshot`, but individual `debugger` and `snapshot` keys, +// so we prune at level 6 or deeper. This level contains the `locals` key. +const MIN_PRUNE_LEVEL = 6 +const PRUNED_JSON = '{"pruned":true}' +const PRUNED_JSON_BYTES = Buffer.byteLength(PRUNED_JSON) + +module.exports = { pruneSnapshot } + +/** + * Tree node representing a JSON object in the parsed structure + */ +class TreeNode { + /** @type {number} End position in JSON string (set when object closes) */ + end = -1 + /** @type {TreeNode[]} Child nodes */ + children = [] + /** @type {boolean} Has notCapturedReason: "depth" */ + notCapturedDepth = false + /** @type {boolean} Has any notCapturedReason */ + notCaptured = false + /** @type {number} Cached byte size */ + #sizeCache = -1 + + /** + * @param {number} start - Start position in JSON string + * @param {number} level - Depth in tree (root = 0) + * @param {string} json - Reference to original JSON string + * @param {TreeNode|null} [parent] - Parent node reference + */ + constructor (start, level, json, parent = null) { + /** @type {number} Start position in JSON string */ + this.start = start + /** @type {number} Depth in tree (root = 0) */ + this.level = level + /** @type {string} Reference to original JSON string */ + this.json = json + /** @type {TreeNode|null} Parent node reference */ + this.parent = parent + } + + get size () { + if (this.#sizeCache === -1) { + if (this.end === -1) { + throw new Error('Cannot get size: node.end has not been set yet') + } + this.#sizeCache = Buffer.byteLength(this.json.slice(this.start, this.end + 1)) + } + return this.#sizeCache + } + + get isLeaf () { + return this.children.length === 0 + } + + /** + * Priority key for sorting in queue (higher values = higher priority for pruning). Checks ancestors for + * `notCapturedReason` flags to prioritize children of partially captured objects (where the flag is on the parent). + * + * @returns {[number, number, number, number]} Priority key tuple: [not_captured_depth, level, not_captured, size] + */ + get priorityKey () { + let hasNotCapturedDepth = this.notCapturedDepth + let hasNotCaptured = this.notCaptured + + // Check ancestors for notCapturedReason flags + let ancestor = this.parent + while (ancestor) { + if (ancestor.notCapturedDepth) hasNotCaptured = hasNotCapturedDepth = true + else if (ancestor.notCaptured) hasNotCaptured = true + ancestor = ancestor.parent + } + + return [ + hasNotCapturedDepth ? 1 : 0, + this.level, + hasNotCaptured ? 1 : 0, + this.size + ] + } +} + +/** + * Priority queue implementation using a binary heap. + * Items with higher priority (by priorityKey) are popped first. + */ +class PriorityQueue { + /** @type {TreeNode[]} Binary heap of nodes */ + #heap = [] + + push (node) { + this.#heap.push(node) + this.#bubbleUp(this.#heap.length - 1) + } + + pop () { + if (this.#heap.length === 0) return + if (this.#heap.length === 1) return /** @type {TreeNode} */ (this.#heap.pop()) + + const top = this.#heap[0] + this.#heap[0] = /** @type {TreeNode} */ (this.#heap.pop()) + this.#bubbleDown(0) + return top + } + + get size () { + return this.#heap.length + } + + #bubbleUp (index) { + while (index > 0) { + const parentIndex = Math.floor((index - 1) / 2) + if (this.#compare(this.#heap[index], this.#heap[parentIndex]) <= 0) break + + [this.#heap[index], this.#heap[parentIndex]] = [this.#heap[parentIndex], this.#heap[index]] + index = parentIndex + } + } + + #bubbleDown (index) { + while (true) { + let largest = index + const leftChild = 2 * index + 1 + const rightChild = 2 * index + 2 + + if (leftChild < this.#heap.length && this.#compare(this.#heap[leftChild], this.#heap[largest]) > 0) { + largest = leftChild + } + if (rightChild < this.#heap.length && this.#compare(this.#heap[rightChild], this.#heap[largest]) > 0) { + largest = rightChild + } + + if (largest === index) break + + [this.#heap[index], this.#heap[largest]] = [this.#heap[largest], this.#heap[index]] + index = largest + } + } + + /** + * Compare two nodes by their priority keys. + * + * @param {TreeNode} a - First node to compare + * @param {TreeNode} b - Second node to compare + * @returns {number} - > 0 if a has higher priority, < 0 if b has higher priority, 0 if equal + */ + #compare (a, b) { + const keyA = a.priorityKey + const keyB = b.priorityKey + for (let i = 0; i < 4; i++) { + if (keyA[i] !== keyB[i]) { + return keyA[i] - keyB[i] + } + } + return 0 + } +} + +/** + * Parse JSON string and build a tree of objects with position tracking. + * Also detects notCapturedReason properties to set node flags. + * + * @param {string} json - The JSON string to parse + * @returns {TreeNode|null} The root node of the tree, or null if parsing fails + */ +function parseJsonToTree (json) { + /** @type {TreeNode[]} Stack of nodes */ + const stack = [] + /** @type {TreeNode|null} The root node of the tree, or null if parsing fails */ + let root = null + let depth = 0 + + for (let index = 0; index < json.length; index++) { + switch (json.charCodeAt(index)) { + case 34: // 34: double quote + // Skip strings to avoid false positives + index = skipString(json, index) + break + case 123: { // 123: opening brace + const parentNode = stack.at(-1) + const level = depth + const node = new TreeNode(index, level, json, parentNode) + + if (parentNode) { + parentNode.children.push(node) + } else { + root = node + } + + stack.push(node) + depth++ + break + } + case 125: { // 125: closing brace + const node = stack.pop() + if (node === undefined) throw new SyntaxError('Invalid JSON: unexpected closing brace') + node.end = index + + const notCapturedReason = findNotCapturedReason(node) + if (notCapturedReason) { + node.notCaptured = true + if (notCapturedReason === 'depth') { + node.notCapturedDepth = true + } + } + depth-- + break + } + } + } + + return root +} + +/** + * Skip to the end of a JSON string, properly handling escape sequences. + * + * @param {string} json - The JSON string to skip + * @param {number} startIndex - The index to start skipping from + * @returns {number} The index of the closing quote + */ +function skipString (json, startIndex) { + let index = startIndex + 1 // Skip opening quote + + while (index < json.length) { + const code = json.charCodeAt(index) + + if (code === 92) { // 92: backslash + // Skip the backslash and the next character (whatever it is) + index += 2 + continue + } + + if (code === 34) { // 34: double quote + // Found unescaped closing quote + return index + } + + index++ + } + + return index +} + +/** + * Find notCapturedReason value in a JSON object string. + * + * @param {TreeNode} node - The node to search in + * @returns {string|undefined} The reason value or undefined if not found + */ +function findNotCapturedReason (node) { + let { json, start: index, end, children } = node + let childIndex = 0 + + while (true) { + // Skip children logic: if current position overlaps with next child, jump over it + if (childIndex < children.length) { + const child = children[childIndex] + // If we are at or past the start of the current child + if (index >= child.start) { + // Skip to the end of the child + index = child.end + 1 + childIndex++ + continue + } + } + + // Find the next string + const nextQuote = json.indexOf('"', index) + // If no more strings, stop + if (nextQuote === -1 || nextQuote >= end) return + + // Skip over any children that come before or contain the found quote + let quoteIsInsideChild = false + while (childIndex < children.length) { + const child = children[childIndex] + if (nextQuote > child.end) { + // Quote is after this child, skip it + childIndex++ + continue + } + if (nextQuote >= child.start && nextQuote <= child.end) { + // Quote is inside this child, skip to end of child and restart outer loop + index = child.end + 1 + childIndex++ + quoteIsInsideChild = true + break + } + // Quote is before this child, so it's valid at current level + break + } + + // If the quote was inside a child, restart the outer loop + if (quoteIsInsideChild) continue + + // Valid quote at current level + index = nextQuote + + const stringStart = index + 1 // Skip opening quote + index = skipString(json, index) + + if (json.slice(stringStart, index) === 'notCapturedReason') { + // Found the potential property name, now see if it has a value and if so, return it + index++ // Skip closing quote + + let code + + // Skip whitespace and colon + while (true) { + if (index >= end) return + code = json.charCodeAt(index) + // 32: space, 9: tab, 10: newline, 13: carriage return, 58: colon + if (code === 32 || code === 9 || code === 10 || code === 13 || code === 58) { + index++ + } else { + break + } + } + + // If next character is a quote, `notCapturedReason` must have been a property name, and now we're at the value + if (code === 34) { // 34: double quote + const valueStart = index + 1 // Skip opening quote + index = skipString(json, index) + return json.slice(valueStart, index) + } + } + + index++ + } +} + +/** + * Collect all leaf nodes at MIN_PRUNE_LEVEL or deeper. + * + * @param {TreeNode} root - The root node of the tree + * @returns {TreeNode[]} The array of leaf nodes + */ +function collectPrunableLeaves (root) { + const leaves = [] + + function traverse (node) { + if (!node) return + + if (node.isLeaf && node.level >= MIN_PRUNE_LEVEL) { + leaves.push(node) + } + + for (const child of node.children) { + traverse(child) + } + } + + traverse(root) + return leaves +} + +/** + * Select nodes to prune using the priority queue algorithm. + * + * @param {TreeNode} root - The root node of the tree + * @param {number} bytesToRemove - The number of bytes to remove + * @returns {Set} The set of nodes marked for pruning + */ +function selectNodesToPrune (root, bytesToRemove) { + const queue = new PriorityQueue() + const prunedNodes = new Set() + const promotedParents = new Set() + + // Collect initial leaf nodes + const leaves = collectPrunableLeaves(root) + for (const leaf of leaves) { + queue.push(leaf) + } + + let bytesRemoved = 0 + + while (queue.size > 0 && bytesRemoved < bytesToRemove) { + const node = /** @type {TreeNode} */ (queue.pop()) + + if (prunedNodes.has(node)) continue + prunedNodes.add(node) + + bytesRemoved += node.size - PRUNED_JSON_BYTES + + // Check if parent should be promoted to leaf + const parent = node.parent + if (parent && parent.level >= MIN_PRUNE_LEVEL && !promotedParents.has(parent)) { + // Check if all children are now pruned + const allChildrenPruned = parent.children.every(child => prunedNodes.has(child)) + + if (allChildrenPruned) { + // Unmark all children as pruned (parent will represent them) + for (const child of parent.children) { + prunedNodes.delete(child) + bytesRemoved -= child.size - PRUNED_JSON_BYTES + } + + // Promote parent to leaf by marking it with notCapturedDepth flag + parent.notCaptured = true + parent.notCapturedDepth = true + promotedParents.add(parent) + + // Add parent to queue for potential pruning + queue.push(parent) + } + } + } + + return prunedNodes +} + +/** + * Rebuild JSON string with pruned nodes replaced by {"pruned":true} + * + * @param {string} json - The JSON string to rebuild + * @param {Set} prunedNodes - The set of nodes to replace with {"pruned":true} + * @returns {string} The rebuilt JSON string + */ +function rebuildJson (json, prunedNodes) { + // Convert set to array and sort by start position (descending) + const sortedNodes = [...prunedNodes].sort((a, b) => b.start - a.start) + + // Replace from end to start to maintain position indices + for (const node of sortedNodes) { + const before = json.slice(0, node.start) + const after = json.slice(node.end + 1) + json = before + PRUNED_JSON + after + } + + return json +} + +/** + * Main pruning function + * + * @param {string} json - The JSON string to prune + * @param {number} originalSize - Size of the original JSON string in bytes + * @param {number} maxSize - Maximum allowed size in bytes + * @returns {string|undefined} - Pruned JSON string, or undefined if pruning fails + */ +function pruneSnapshot (json, originalSize, maxSize) { + const bytesToRemove = originalSize - maxSize + + if (bytesToRemove <= 0) return json // No pruning needed + + let prunedSize = originalSize + let attempts = 0 + const maxAttempts = 6 + + while (prunedSize > maxSize && attempts < maxAttempts) { + attempts++ + + const root = parseJsonToTree(json) + if (!root) break + + const targetBytesToRemove = prunedSize - maxSize + const prunedNodes = selectNodesToPrune(root, targetBytesToRemove) + if (prunedNodes.size === 0) break + + json = rebuildJson(json, prunedNodes) + prunedSize = Buffer.byteLength(json) + } + + // If pruning didn't help, return undefined + return prunedSize >= originalSize ? undefined : json +} diff --git a/packages/dd-trace/test/debugger/devtools_client/send.spec.js b/packages/dd-trace/test/debugger/devtools_client/send.spec.js index a223d8d34cd..a1aedc3a13f 100644 --- a/packages/dd-trace/test/debugger/devtools_client/send.spec.js +++ b/packages/dd-trace/test/debugger/devtools_client/send.spec.js @@ -1,19 +1,18 @@ 'use strict' const assert = require('node:assert/strict') -require('../../setup/mocha') +const { hostname: getHostname } = require('node:os') -const { expect } = require('chai') const { afterEach, beforeEach, describe, it } = require('mocha') const proxyquire = require('proxyquire') const sinon = require('sinon') -const { hostname: getHostname } = require('node:os') - const { getRequestOptions } = require('./utils') const JSONBuffer = require('../../../src/debugger/devtools_client/json-buffer') const { version } = require('../../../../../package.json') +require('../../setup/mocha') + process.env.DD_ENV = 'my-env' process.env.DD_VERSION = 'my-version' const service = 'my-service' @@ -36,6 +35,8 @@ describe('input message http requests', function () { let request /** @type {sinon.SinonSpy} */ let jsonBufferWrite + /** @type {sinon.SinonStub} */ + let pruneSnapshotStub beforeEach(function () { clock = sinon.useFakeTimers({ @@ -45,6 +46,9 @@ describe('input message http requests', function () { request = sinon.spy() request['@noCallThru'] = true + pruneSnapshotStub = sinon.stub() + pruneSnapshotStub['@noCallThru'] = true + class JSONBufferSpy extends JSONBuffer { constructor (...args) { super(...args) @@ -65,7 +69,8 @@ describe('input message http requests', function () { '@noCallThru': true }, './json-buffer': JSONBufferSpy, - '../../exporters/common/request': request + '../../exporters/common/request': request, + './snapshot-pruner': { pruneSnapshot: pruneSnapshotStub }, }) }) @@ -95,8 +100,7 @@ describe('input message http requests', function () { const opts = getRequestOptions(request) assert.strictEqual(opts.method, 'POST') - expect(opts).to.have.property( - 'path', + assert.strictEqual(opts.path, '/debugger/v1/input?ddtags=' + `env%3A${process.env.DD_ENV}%2C` + `version%3A${process.env.DD_VERSION}%2C` + @@ -108,13 +112,90 @@ describe('input message http requests', function () { done() }) + + describe('snapshot pruning', function () { + const largeSnapshot = { + id: '123', + stack: [{ function: 'test' }], + captures: { + lines: { + 10: { + locals: { + largeData: { type: 'string', value: 'x'.repeat(2 * 1024 * 1024) } + } + } + } + } + } + const prunedPayload = { + ...getPayload(message), + debugger: { + snapshot: { + id: '123', + stack: [{ function: 'test' }], + captures: { + lines: { + 10: { + locals: { + largeData: { pruned: true } + } + } + } + } + } + } + } + + it('should not attempt to prune if payload is under size limit', function () { + send(message, logger, dd, snapshot) + sinon.assert.notCalled(pruneSnapshotStub) + }) + + it('should attempt to prune if payload exceeds 1MB', function () { + const prunedJson = JSON.stringify(getPayload(message, largeSnapshot)) + pruneSnapshotStub.returns(prunedJson) + + send(message, logger, dd, largeSnapshot) + + sinon.assert.calledOnce(pruneSnapshotStub) + const call = pruneSnapshotStub.getCall(0) + assert.strictEqual(typeof call.args[0], 'string') // json + assert.strictEqual(typeof call.args[1], 'number') // currentSize + assert.strictEqual(call.args[2], 1024 * 1024) // maxSize + }) + + it('should use pruned snapshot if pruning succeeds', function () { + const prunedJson = JSON.stringify(prunedPayload) + pruneSnapshotStub.returns(prunedJson) + + send(message, logger, dd, largeSnapshot) + + sinon.assert.calledOnce(pruneSnapshotStub) + sinon.assert.calledOnceWithMatch(jsonBufferWrite, prunedJson) + }) + + it('should fall back to deleting captures if pruning fails', function () { + pruneSnapshotStub.returns(undefined) + + send(message, logger, dd, largeSnapshot) + + sinon.assert.calledOnce(pruneSnapshotStub) + + // Should write fallback payload without captures + const writtenJson = jsonBufferWrite.getCall(0).args[0] + const written = JSON.parse(writtenJson) + + assert.deepStrictEqual(written.debugger.snapshot.captures.lines[10], { pruned: true }) + }) + }) }) /** * @param {object} [_message] - The message to get the payload for. Defaults to the {@link message} object. + * @param {object} [_snapshot] - The snapshot to get the payload for. Defaults to the {@link snapshot} object. * @returns {object} - The payload. */ -function getPayload (_message = message) { +function getPayload (_message = message, _snapshot = snapshot) { return { ddsource, hostname, @@ -122,6 +203,6 @@ function getPayload (_message = message) { message: _message, logger, dd, - debugger: { snapshot } + debugger: { snapshot: _snapshot } } } diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot-pruner.spec.js b/packages/dd-trace/test/debugger/devtools_client/snapshot-pruner.spec.js new file mode 100644 index 00000000000..8bdc34b466b --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot-pruner.spec.js @@ -0,0 +1,557 @@ +'use strict' + +const assert = require('node:assert/strict') + +const { describe, it } = require('mocha') + +const { pruneSnapshot } = require('../../../src/debugger/devtools_client/snapshot-pruner') + +require('../../setup/mocha') + +describe('snapshot-pruner', function () { + describe('pruneSnapshot', function () { + let locals, snapshot + + beforeEach(() => { + locals = {} + snapshot = { + service: 'my-service', + hostname: 'my-host', + message: 'my-message', + logger: { + name: 'test.js', + method: 'testMethod', + version: '1.0.0' + }, + dd: { service: 'my-service' }, + debugger: { + snapshot: { + id: '12345', + timestamp: 123456789, + probe: { id: 'probe-1', version: 1 }, + stack: [ + { function: 'test', fileName: 'test.js', lineNumber: 10 } + ], + language: 'javascript', + captures: { + lines: { + 10: { + locals + } + } + } + } + } + } + }) + + it('should return original JSON if already under size limit', function () { + Object.assign(locals, { + smallVar: { type: 'number', value: '42' } + }) + + const json = JSON.stringify(snapshot) + const size = Buffer.byteLength(json) + const maxSize = size + 1000 + + const result = pruneSnapshot(json, size, maxSize) + + assert.strictEqual(result, json) + }) + + it('should return undefined if JSON cannot be parsed', function () { + const invalidJson = '{ invalid json' + const result = pruneSnapshot(invalidJson, 100, 50) + + assert.strictEqual(result, undefined) + }) + + it('should handle empty captures gracefully', function () { + const json = JSON.stringify(snapshot) + const size = Buffer.byteLength(json) + const maxSize = size + + const result = pruneSnapshot(json, size, maxSize) + + assert.notStrictEqual(result, undefined, 'Expected pruneSnapshot() to successfully prune') + assert.strictEqual(result, json) + }) + + it('should prune large leaf nodes to reduce size', function () { + assertPrunedSnapshot(-100, { + smallVar1: { type: 'number', value: '1' }, + largeVar: { type: 'string', value: 'x'.repeat(500) }, + smallVar2: { type: 'number', value: '2' }, + }, { + smallVar1: { type: 'number', value: '1' }, + largeVar: { pruned: true }, + smallVar2: { type: 'number', value: '2' }, + }) + }) + + it('should preserve schema fields at levels 0-5', function () { + assertPrunedSnapshot(400, { + data: { type: 'string', value: 'x'.repeat(1000) } + }, { + pruned: true + }) + }) + + it('should prioritize pruning nodes with notCapturedReason="depth"', function () { + // We want to set maxSize such that pruning deepObj1+deepObj2 is sufficient, + // or at least that the algorithm chooses them first. + // Using size - 40 ensures we need to prune at least ~40 bytes. + // Pruning deepObj1 gives ~26 bytes. Not enough. + // Pruning deepObj2 gives ~26 bytes. Total 52. Enough. + // So it should prune both deep objects and stop, preserving normal objects. + assertPrunedSnapshot(-40, { + deepObj1: { + type: 'object', + notCapturedReason: 'depth' + }, + deepObj2: { + type: 'object', + notCapturedReason: 'depth' + }, + normalObj1: { type: 'object', fields: { z: { type: 'string', value: '3'.repeat(100) } } }, + normalObj2: { type: 'object', fields: { w: { type: 'string', value: '4'.repeat(100) } } } + }, { + // Objects with notCapturedReason="depth" should be pruned first + deepObj1: { pruned: true }, + deepObj2: { pruned: true }, + // Normal objects should be preserved if possible + normalObj1: { type: 'object', fields: { z: { type: 'string', value: '3'.repeat(100) } } }, + normalObj2: { type: 'object', fields: { w: { type: 'string', value: '4'.repeat(100) } } }, + }) + }) + + it('should prioritize pruning nodes with generic notCapturedReason over normal nodes', function () { + assertPrunedSnapshot(-1, { + objWithReason: { + type: 'object', + notCapturedReason: 'timeout' // Generic reason (not "depth") + }, + normalObj: { + type: 'string', + value: 'x'.repeat(150) + } + }, { + // Object with generic notCapturedReason should be pruned first + objWithReason: { pruned: true }, + // Normal object should be preserved if possible + normalObj: { type: 'string', value: 'x'.repeat(150) } + }) + }) + + it('should prioritize pruning children of nodes with notCapturedReason="collectionSize"', function () { + assertPrunedSnapshot(-1, { + normalList1: { + type: 'array', + elements: [ + { type: 'string', value: 'a'.repeat(100) }, + { type: 'string', value: 'b'.repeat(100) } + ] + }, + truncatedList: { + type: 'array', + notCapturedReason: 'collectionSize', + elements: [ + { type: 'string', value: 'x'.repeat(100) }, + { type: 'string', value: 'y'.repeat(100) } + ] + }, + normalList2: { + type: 'array', + elements: [ + { type: 'string', value: 'a'.repeat(100) }, + { type: 'string', value: 'b'.repeat(100) } + ] + } + }, { + normalList1: { + type: 'array', + elements: [ + { type: 'string', value: 'a'.repeat(100) }, + { type: 'string', value: 'b'.repeat(100) } + ] + }, + truncatedList: { + type: 'array', + notCapturedReason: 'collectionSize', + elements: [ + { pruned: true }, + { type: 'string', value: 'y'.repeat(100) } + ] + }, + normalList2: { + type: 'array', + elements: [ + { type: 'string', value: 'a'.repeat(100) }, + { type: 'string', value: 'b'.repeat(100) } + ] + } + }) + }) + + it('should prioritize pruning children of nodes with notCapturedReason="fieldCount"', function () { + assertPrunedSnapshot(-1, { + normalObj1: { + type: 'object', + fields: { + c: { type: 'string', value: 'a'.repeat(100) }, + d: { type: 'string', value: 'b'.repeat(100) } + } + }, + truncatedObj1: { + type: 'object', + notCapturedReason: 'fieldCount', + fields: { + a: { type: 'string', value: 'x'.repeat(100) }, + b: { type: 'string', value: 'y'.repeat(100) } + } + }, + normalObj2: { + type: 'object', + fields: { + c: { type: 'string', value: 'a'.repeat(100) }, + d: { type: 'string', value: 'b'.repeat(100) } + } + } + }, { + normalObj1: { + type: 'object', + fields: { + c: { type: 'string', value: 'a'.repeat(100) }, + d: { type: 'string', value: 'b'.repeat(100) } + } + }, + truncatedObj1: { + type: 'object', + notCapturedReason: 'fieldCount', + fields: { + a: { pruned: true }, + b: { type: 'string', value: 'y'.repeat(100) } + } + }, + normalObj2: { + type: 'object', + fields: { + c: { type: 'string', value: 'a'.repeat(100) }, + d: { type: 'string', value: 'b'.repeat(100) } + } + } + }) + }) + + it('should prioritize larger nodes when level and capture reason are equal', function () { + assertPrunedSnapshot(-1, { + largeObj: { type: 'string', value: 'x'.repeat(500) }, + smallObj: { type: 'string', value: 'y'.repeat(100) } + }, { + largeObj: { pruned: true }, + smallObj: { type: 'string', value: 'y'.repeat(100) } + }) + }) + + it('should prune deeper nested objects before shallower ones', function () { + assertPrunedSnapshot(-1, { + shallowObj: { type: 'object', fields: { data: { type: 'string', value: 'x'.repeat(200) } } }, + deeperObj: { + type: 'object', + fields: { nested: { type: 'object', fields: { deepData: { type: 'string', value: 'y'.repeat(100) } } } } + }, + }, { + shallowObj: { type: 'object', fields: { data: { type: 'string', value: 'x'.repeat(200) } } }, + deeperObj: { pruned: true } + }) + }) + + it('should return undefined if no prunable nodes are available', function () { + delete snapshot.debugger.snapshot.captures + + const json = JSON.stringify(snapshot) + const size = Buffer.byteLength(json) + // Set maxSize impossibly low - all content is schema + const maxSize = 10 + + const result = pruneSnapshot(json, size, maxSize) + + // Should return undefined because there's nothing to prune at level 5+ + assert.strictEqual(result, undefined) + }) + + it('should handle complex nested structures with multiple levels', function () { + assertPrunedSnapshot(-100, { + complexObj: { + type: 'object', + fields: { + level1: { + type: 'object', + fields: { + level2: { + type: 'object', + fields: { + level3: { + type: 'object', + fields: { + level4: { + type: 'object', + fields: { + level5: { + type: 'object', + fields: { + level6a: { + type: 'string', + value: 'x'.repeat(500) + }, + // Add an extra random field to make sure we don't prune the parent when all children + // are pruned + level6b: { + type: 'number', + value: 42 + } + } + } + } + } + } + } + } + } + } + } + } + } + }, { + complexObj: { + type: 'object', + fields: { + level1: { + type: 'object', + fields: { + level2: { + type: 'object', + fields: { + level3: { + type: 'object', + fields: { + level4: { + type: 'object', + fields: { + level5: { + type: 'object', + fields: { + level6a: { pruned: true }, + level6b: { + type: 'number', + value: 42 + } + } + } + } + } + } + } + } + } + } + } + } + } + }) + }) + + it('should handle stringified JSON in keys and values', function () { + assertPrunedSnapshot(-100, { + // Value contains stringified JSON that looks like notCapturedReason + strWithNotCaptured: { type: 'string', value: '{"notCapturedReason":"depth","type":"object"}' }, + // Value contains stringified JSON that looks like pruned marker + strWithPruned: { type: 'string', value: '{"pruned":true}' }, + // Normal large value that should actually be pruned + actualLargeValue: { type: 'string', value: 'x'.repeat(500) }, + smallValue: { type: 'number', value: '42' } + }, { + strWithNotCaptured: { type: 'string', value: '{"notCapturedReason":"depth","type":"object"}' }, + strWithPruned: { type: 'string', value: '{"pruned":true}' }, + actualLargeValue: { pruned: true }, + smallValue: { type: 'number', value: '42' } + }) + }) + + it('should handle escape sequences in string values', function () { + assertPrunedSnapshot(-100, { + // String ending with escaped backslash - this is the tricky case + // JSON: "test\\" represents the string: test\ + escapedBackslash: { type: 'string', value: 'test\\' }, + // String with escaped quote in the middle + escapedQuote: { type: 'string', value: 'test"quote' }, + // String with multiple escape sequences + multipleEscapes: { type: 'string', value: 'line1\\nline2\\"quoted\\"' }, + // Large value to force pruning + largeValue: { type: 'string', value: 'x'.repeat(500) }, + smallValue: { type: 'number', value: '1' } + }, { + escapedBackslash: { type: 'string', value: 'test\\' }, + escapedQuote: { type: 'string', value: 'test"quote' }, + multipleEscapes: { type: 'string', value: 'line1\\nline2\\"quoted\\"' }, + largeValue: { pruned: true }, + smallValue: { type: 'number', value: '1' } + }) + }) + + it('should handle very large snapshots efficiently', function () { + for (let i = 0; i < 100; i++) { + locals[`var${i}`] = { type: 'string', value: 'x'.repeat(1000) } + } + + const json = JSON.stringify(snapshot) + const size = Buffer.byteLength(json) + const maxSize = 5000 + + const start = process.hrtime.bigint() + const result = pruneSnapshot(json, size, maxSize) + const elapsed = Number(process.hrtime.bigint() - start) / 1_000_000 + + assert.notStrictEqual(result, undefined, 'Expected pruneSnapshot() to successfully prune') + + // The algorithm tries to prune to target but may not always hit exactly + // Just verify significant reduction happened + const reduction = size - Buffer.byteLength(/** @type {string} */(result)) + assert.ok(reduction > size * 0.9) // At least 90% reduction + + // Should complete in reasonable time + assert.ok(elapsed < 30, `Expected elapsed time to be less than 30ms, but got ${elapsed}ms`) + }) + + it('should promote parent to leaf when all children are pruned', function () { + // At 674, all the children are pruned, but the parent is not, at 675 one of the children is not pruned + assertPrunedSnapshot(674, { + smallVar: { type: 'number', value: '42' }, + parent: { + type: 'object', + fields: { + child1: { + type: 'object', + fields: { + data: { type: 'string', value: 'x'.repeat(100) } + } + }, + child2: { + type: 'object', + fields: { + data: { type: 'string', value: 'y'.repeat(100) } + } + } + } + } + }, { + smallVar: { type: 'number', value: '42' }, + parent: { pruned: true } + }) + }) + + it('should handle multi-byte characters correctly', function () { + // All objects have the same character length (100 chars + wrapper). + // But emojiStr is much larger in bytes (400 bytes vs 100 bytes). + // Algorithm should prioritize pruning the larger one (emojiStr) if levels are equal. + // Set limit to force pruning of the larger one only. + assertPrunedSnapshot(-200, { + normalStr1: { type: 'string', value: 'x'.repeat(100) }, + emojiStr: { type: 'string', value: '🔒'.repeat(100) }, // Emoji is 4 bytes + normalStr2: { type: 'string', value: 'x'.repeat(100) }, + }, { + normalStr1: { type: 'string', value: 'x'.repeat(100) }, + emojiStr: { pruned: true }, + normalStr2: { type: 'string', value: 'x'.repeat(100) }, + }) + }) + + it('should handle objects within arrays', function () { + assertPrunedSnapshot(-1, { + list: { + type: 'array', + elements: [ + { + type: 'object', + fields: { id: { type: 'number', value: '1' }, data: { type: 'string', value: 'x'.repeat(200) } } + }, + { + type: 'object', + fields: { id: { type: 'number', value: '2' }, data: { type: 'string', value: 'y'.repeat(10) } } + } + ] + } + }, { + list: { + type: 'array', + elements: [ + { + type: 'object', + fields: { id: { type: 'number', value: '1' }, data: { pruned: true } } + }, + { + type: 'object', + fields: { id: { type: 'number', value: '2' }, data: { type: 'string', value: 'y'.repeat(10) } } + } + ] + } + }) + }) + + it('should prune objects inside Map entries (arrays of arrays)', function () { + assertPrunedSnapshot(-1, { + myMap: { + type: 'map', + entries: [ + [ + { type: 'string', value: 'key1' }, + { type: 'string', value: 'x'.repeat(500) } // Should be pruned + ], + [ + { type: 'string', value: 'key2' }, + { type: 'string', value: 'small' } + ] + ] + } + }, { + myMap: { + type: 'map', + entries: [ + [ + { type: 'string', value: 'key1' }, + { pruned: true } + ], + [ + { type: 'string', value: 'key2' }, + { type: 'string', value: 'small' } + ] + ] + } + }) + }) + + /** + * Assert that the pruneSnapshot function successfully prunes the snapshot and returns the expected locals. + * @param {number} maxSize - Used to define the max allowed size of the snapshot. If positive, it's the absolute max + * size value. If negative, it's redacted from the actual size. + * @param {Object} originalLocals - The locals to use for the snapshot. + * @param {Object} expectedLocals - The expected locals after pruning. + */ + function assertPrunedSnapshot (maxSize, originalLocals, expectedLocals) { + Object.assign(locals, originalLocals) + + const json = JSON.stringify(snapshot) + const size = Buffer.byteLength(json) + maxSize = maxSize < 0 ? size + maxSize : maxSize + + const result = pruneSnapshot(json, size, maxSize) + + assert.notStrictEqual(result, undefined, 'Expected pruneSnapshot() to successfully prune') + + const parsed = JSON.parse(/** @type {string} */(result)) + const parsedLocals = parsed.debugger.snapshot.captures.lines['10'].locals + + assert.deepStrictEqual(parsedLocals, expectedLocals) + } + }) +})