Skip to content

feat(Aggregate Node): add group by functionality to aggregate node #22058

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pemontto wants to merge 2 commits into
base: master
Choose a base branch
from
+1,164 −15
Open

feat(Aggregate Node): add group by functionality to aggregate node #22058

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c6ea2bf
feat(Aggregate Node): add group by functionality to aggregate node
pemontto Nov 19, 2025
a93b338
feat(Aggregate): improve conflict detection for group by fields
pemontto Nov 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
268 changes: 253 additions & 15 deletions packages/nodes-base/nodes/Transform/Aggregate/Aggregate.node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,9 +171,19 @@ export class Aggregate implements INodeType {
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Field',
placeholder: 'Add Option',
default: {},
options: [
{
displayName: 'Group By Fields',
name: 'groupByFields',
type: 'string',
default: '',
placeholder: 'e.g. category, type',
description:
'Comma-separated list of fields to group by. Creates one output item per unique combination of these field values. Dot-notation is supported to access nested fields.',
requiresDataPath: 'multiple',
},
{
displayName: 'Disable Dot Notation',
name: 'disableDotNotation',
Expand Down Expand Up @@ -244,6 +254,8 @@ export class Aggregate implements INodeType {
const notFoundedFields: { [key: string]: boolean[] } = {};

const aggregate = this.getNodeParameter('aggregate', 0, '') as string;
const groupByFieldsParam = this.getNodeParameter('options.groupByFields', 0, '') as string;
const groupByFields = prepareFieldsArray(groupByFieldsParam, 'Group By Fields');

if (aggregate === 'aggregateIndividualFields') {
const disableDotNotation = this.getNodeParameter(
Expand Down Expand Up @@ -405,20 +417,6 @@ export class Aggregate implements INodeType {
returnData = output;
}

const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean;

if (includeBinaries) {
const pairedItems = (returnData.pairedItem || []) as IPairedItemData[];

const aggregatedItems = pairedItems.map((item) => {
return items[item.item];
});

const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean;

addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique);
}

if (Object.keys(notFoundedFields).length) {
const hints: NodeExecutionHint[] = [];

Expand All @@ -436,6 +434,246 @@ export class Aggregate implements INodeType {
}
}

// Apply grouping if specified
if (groupByFields.length > 0) {
const groups: {
[key: string]: { items: IDataObject[]; pairedItems: number[]; values: any[] };
} = {};

// Group original items by the specified fields
items.forEach((item, index) => {
const groupValues = groupByFields.map((field) => {
const value = get(item.json, field);
if (typeof value === 'object' && value !== null) {
throw new NodeOperationError(
this.getNode(),
`Field "${field}" has a non-scalar value which cannot be used for grouping.`,
{
description:
'Only scalar values (string, number, boolean, null) can be used in "Group By Fields".',
},
);
}
return value;
});
const groupKey = JSON.stringify(groupValues);

if (!groups[groupKey]) {
groups[groupKey] = { items: [], pairedItems: [], values: groupValues };
}

groups[groupKey].items.push(item.json);
groups[groupKey].pairedItems.push(index);
});

// Create output items for each group
const result: INodeExecutionData[] = [];
const groupPairedItems: number[][] = []; // Track all items in each group for binary aggregation

Object.values(groups).forEach((groupData) => {
const groupItem: IDataObject = {};

// Add the grouping field values to the output
groupByFields.forEach((field, index) => {
const value = groupData.values[index];
set(groupItem, field, value);
});

if (aggregate === 'aggregateAllItemData') {
const destinationFieldName = this.getNodeParameter('destinationFieldName', 0) as string;

// Check for conflicts with group-by field names
if (groupByFields.includes(destinationFieldName)) {
throw new NodeOperationError(
this.getNode(),
`The destination field '${destinationFieldName}' conflicts with a Group By field`,
{ description: 'Please choose a different destination field name or Group By field' },
);
}

const fieldsToExclude = prepareFieldsArray(
this.getNodeParameter('fieldsToExclude', 0, '') as string,
'Fields To Exclude',
);
const fieldsToInclude = prepareFieldsArray(
this.getNodeParameter('fieldsToInclude', 0, '') as string,
'Fields To Include',
);

let groupedAndFilteredItems = groupData.items;

if (fieldsToExclude.length || fieldsToInclude.length) {
groupedAndFilteredItems = groupedAndFilteredItems
.map((item) => {
const newItem: IDataObject = {};
let outputFields = Object.keys(item);

if (fieldsToExclude.length) {
outputFields = outputFields.filter((key) => !fieldsToExclude.includes(key));
}
if (fieldsToInclude.length) {
outputFields = outputFields.filter((key) =>
fieldsToInclude.length ? fieldsToInclude.includes(key) : true,
);
}

outputFields.forEach((key) => {
newItem[key] = item[key];
});
return newItem;
})
.filter((item) => !isEmpty(item));
}

groupItem[destinationFieldName] = groupedAndFilteredItems;
} else {
// For aggregateIndividualFields mode, recompute aggregation for the group
const fieldsToAggregate = this.getNodeParameter(
'fieldsToAggregate.fieldToAggregate',
0,
[],
) as [{ fieldToAggregate: string; renameField: boolean; outputFieldName: string }];
const disableDotNotation = this.getNodeParameter(
'options.disableDotNotation',
0,
false,
) as boolean;
const mergeLists = this.getNodeParameter('options.mergeLists', 0, false) as boolean;
const keepMissing = this.getNodeParameter('options.keepMissing', 0, false) as boolean;

// Validate that fields are specified
if (!fieldsToAggregate.length) {
throw new NodeOperationError(this.getNode(), 'No fields specified', {
description: 'Please add a field to aggregate',
});
}

// Validate unique output field names and detect conflicts with group-by fields
const outputFields: string[] = [];
for (const { fieldToAggregate, outputFieldName, renameField } of fieldsToAggregate) {
const getFieldToAggregate = () =>
!disableDotNotation && fieldToAggregate.includes('.')
? fieldToAggregate.split('.').pop()
: fieldToAggregate;

const field = renameField ? outputFieldName : (getFieldToAggregate() as string);

if (outputFields.includes(field)) {
throw new NodeOperationError(
this.getNode(),
`The '${field}' output field is used more than once`,
{ description: 'Please make sure each output field name is unique' },
);
}

// Check for conflicts with group-by field names
if (groupByFields.includes(field)) {
throw new NodeOperationError(
this.getNode(),
`The output field '${field}' conflicts with a Group By field`,
{
description:
'Please rename the aggregated field or choose a different Group By field',
},
);
}

outputFields.push(field);
}

const values: { [key: string]: any } = {};

for (const { fieldToAggregate, outputFieldName } of fieldsToAggregate) {
const getFieldToAggregate = () =>
!disableDotNotation && fieldToAggregate.includes('.')
? fieldToAggregate.split('.').pop()
: fieldToAggregate;

const _outputFieldName = outputFieldName
? outputFieldName
: (getFieldToAggregate() as string);

if (fieldToAggregate !== '') {
values[_outputFieldName] = [];
for (const itemJson of groupData.items) {
// Track missing fields
if (notFoundedFields[fieldToAggregate] === undefined) {
notFoundedFields[fieldToAggregate] = [];
}

let value;
if (!disableDotNotation) {
value = get(itemJson, fieldToAggregate);
} else {
value = itemJson[fieldToAggregate];
}

notFoundedFields[fieldToAggregate].push(value === undefined ? false : true);

if (!keepMissing) {
if (Array.isArray(value)) {
// Align with standard flow: filter only null, not undefined
value = value.filter((entry) => entry !== null);
} else if (value === null || value === undefined) {
continue;
}
}

if (Array.isArray(value) && mergeLists) {
values[_outputFieldName].push(...value);
} else {
values[_outputFieldName].push(value);
}
}
}
}

for (const key of Object.keys(values)) {
if (!disableDotNotation) {
set(groupItem, key, values[key]);
} else {
groupItem[key] = values[key];
}
}
}

// Store all paired items for binary aggregation
groupPairedItems.push(groupData.pairedItems);

result.push({
json: groupItem,
pairedItem: { item: groupData.pairedItems[0] },
});
});

const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean;
if (includeBinaries) {
const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean;
for (let i = 0; i < result.length; i++) {
const resultItem = result[i];
const pairedItemIndices = groupPairedItems[i];
const aggregatedItems = pairedItemIndices.map((itemIndex) => items[itemIndex]);
addBinariesToItem(resultItem, aggregatedItems, keepOnlyUnique);
}
}

return [result];
}

const includeBinaries = this.getNodeParameter('options.includeBinaries', 0, false) as boolean;

if (includeBinaries) {
const pairedItems = (returnData.pairedItem || []) as IPairedItemData[];

const aggregatedItems = pairedItems.map((item) => {
return items[item.item];
});

const keepOnlyUnique = this.getNodeParameter('options.keepOnlyUnique', 0, false) as boolean;

addBinariesToItem(returnData, aggregatedItems, keepOnlyUnique);
}

return [[returnData]];
}
}
Loading