Merge pull request #1896 from rocket-admin/backend_bulk_delete_optimisation

feat: implement bulk deletion for tables with composite primary keys

Author: Artuomka

backend/src/entities/table-logs/table-logs.service.ts

@@ -1,7 +1,6 @@
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { findSensitiveValues, scrub } from '@zapier/secret-scrubber';
-import PQueue from 'p-queue';
 import { Repository } from 'typeorm';
 import { LogOperationTypeEnum } from '../../enums/log-operation-type.enum.js';
 import { OperationResultStatusEnum } from '../../enums/operation-result-status.enum.js';
@@ -17,6 +16,9 @@ import { TableLogsEntity } from './table-logs.entity.js';
 import { buildCreatedLogRecord } from './utils/build-created-log-record.js';
 import { buildTableLogsEntity } from './utils/build-table-logs-entity.js';
 
+/** Rows per INSERT when a bulk operation writes its audit log; keeps one statement well under Postgres' bind limit. */
+const TABLE_LOGS_INSERT_CHUNK_SIZE = 500;
+
 @Injectable()
 export class TableLogsService {
 	constructor(
@@ -220,16 +222,13 @@ export class TableLogsService {
 			const newLogRecord = buildTableLogsEntity(loggedData, email);
 			tableLogsEntities.push(newLogRecord);
 		}
-		const queue = new PQueue({ concurrency: 2 });
-		const createdLogs: Array<CreatedLogRecordDs | undefined> = await Promise.all(
-			tableLogsEntities.map(async (newLogRecord) => {
-				return await queue.add(async () => {
-					const savedLogRecord = await this.tableLogsRepository.save(newLogRecord);
-					return buildCreatedLogRecord(savedLogRecord);
-				});
-			}),
-		);
-		return createdLogs.filter((log) => log) as Array<CreatedLogRecordDs>;
+		if (tableLogsEntities.length === 0) {
+			return [];
+		}
+		const savedLogRecords = await this.tableLogsRepository.save(tableLogsEntities, {
+			chunk: TABLE_LOGS_INSERT_CHUNK_SIZE,
+		});
+		return savedLogRecords.map((savedLogRecord) => buildCreatedLogRecord(savedLogRecord));
 	}
 
 	private async isTableAuditEnabledInConnectionProperties(connectionId: string): Promise<boolean> {

backend/src/entities/table/table-pure-crud-operations/use-cases/pure-delete-row-from-table.use.case.ts

@@ -64,7 +64,7 @@ export class PureDeleteRowFromTableUseCase
 		const availablePrimaryColumns = primaryColumns.map((column) => column.column_name);
 		for (const key in primaryKey) {
 			// eslint-disable-next-line security/detect-object-injection
-			if (!primaryKey[key] && primaryKey[key] !== '') delete primaryKey[key];
+			if (primaryKey[key] === undefined) delete primaryKey[key];
 		}
 		const receivedPrimaryColumns = Object.keys(primaryKey);
 		if (!compareArrayElements(availablePrimaryColumns, receivedPrimaryColumns)) {

backend/src/entities/table/table-pure-crud-operations/use-cases/pure-read-row-from-table.use.case.ts

@@ -64,7 +64,7 @@ export class PureReadRowFromTableUseCase
 		const availablePrimaryColumns = primaryColumns.map((column) => column.column_name);
 		for (const key in primaryKey) {
 			// eslint-disable-next-line security/detect-object-injection
-			if (!primaryKey[key] && primaryKey[key] !== '') delete primaryKey[key];
+			if (primaryKey[key] === undefined) delete primaryKey[key];
 		}
 		const receivedPrimaryColumns = Object.keys(primaryKey);
 		if (!compareArrayElements(availablePrimaryColumns, receivedPrimaryColumns)) {

backend/src/entities/table/table-pure-crud-operations/use-cases/pure-update-row-in-table.use.case.ts

@@ -67,7 +67,7 @@ export class PureUpdateRowInTableUseCase
 		const availablePrimaryColumns = tablePrimaryKeys.map((key) => key.column_name);
 		for (const key in primaryKey) {
 			// eslint-disable-next-line security/detect-object-injection
-			if (!primaryKey[key] && primaryKey[key] !== '') delete primaryKey[key];
+			if (primaryKey[key] === undefined) delete primaryKey[key];
 		}
 		const receivedPrimaryColumns = Object.keys(primaryKey);
 		if (!compareArrayElements(availablePrimaryColumns, receivedPrimaryColumns)) {

backend/src/entities/table/table.controller.ts

@@ -565,6 +565,7 @@ export class TableController {
 	})
 	@ApiQuery({ name: 'tableName', required: true })
 	@UseGuards(TableDeleteGuard)
+	@Timeout(TimeoutDefaults.EXTENDED)
 	@Put('/table/rows/delete/:connectionId')
 	async deleteRowsInTable(
 		@MasterPassword() masterPwd: string,

backend/src/entities/table/use-cases/delete-row-from-table.use.case.ts

@@ -80,7 +80,7 @@ export class DeleteRowFromTableUseCase
 
 		Object.keys(primaryKey).forEach((key) => {
 			// eslint-disable-next-line security/detect-object-injection
-			if (!primaryKey[key] && primaryKey[key] !== '') {
+			if (primaryKey[key] === undefined) {
 				// eslint-disable-next-line security/detect-object-injection
 				delete primaryKey[key];
 			}

backend/src/entities/table/use-cases/delete-rows-from-table.use.case.ts

@@ -17,7 +17,6 @@ import { TableLogsService } from '../../table-logs/table-logs.service.js';
 import { DeleteRowsFromTableDs } from '../application/data-structures/delete-row-from-table.ds.js';
 import { buildCommonTableSettingsInput } from '../utils/build-common-table-settings-input.util.js';
 import { convertHexDataInPrimaryKeyUtil } from '../utils/convert-hex-data-in-primary-key.util.js';
-import { findObjectsWithProperties } from '../utils/find-objects-with-properties.js';
 import { getUserEmailForAgent, validateConnection } from '../utils/validate-connection.util.js';
 import { IDeleteRowsFromTable } from './table-use-cases.interface.js';
 
@@ -87,7 +86,7 @@ export class DeleteRowsFromTableUseCase
 		primaryKeys.forEach((primaryKey) => {
 			Object.keys(primaryKey).forEach((key) => {
 				// eslint-disable-next-line security/detect-object-injection
-				if (!primaryKey[key] && primaryKey[key] !== '') {
+				if (primaryKey[key] === undefined) {
 					// eslint-disable-next-line security/detect-object-injection
 					delete primaryKey[key];
 				}
@@ -124,6 +123,7 @@ export class DeleteRowsFromTableUseCase
 			);
 		}
 
+		const oldRowsByPrimaryKey = indexRowsByPrimaryKey(oldRowsData, availablePrimaryColumns);
 		const deleteOperationsResults: Array<DeleteRowsFromTableResult> = [];
 
 		try {
@@ -132,7 +132,7 @@ export class DeleteRowsFromTableUseCase
 				deleteOperationsResults.push({
 					operationStatusResult: OperationResultStatusEnum.successfully,
 					row: primaryKey,
-					old_data: findObjectsWithProperties(oldRowsData, primaryKey).at(0) ?? {},
+					old_data: oldRowsByPrimaryKey.get(primaryKeyIndexKey(primaryKey, availablePrimaryColumns)) ?? {},
 					error: null,
 					affected_primary_key: primaryKey as unknown as string,
 				});
@@ -143,7 +143,7 @@ export class DeleteRowsFromTableUseCase
 				deleteOperationsResults.push({
 					operationStatusResult: OperationResultStatusEnum.unsuccessfully,
 					row: primaryKey,
-					old_data: findObjectsWithProperties(oldRowsData, primaryKey).at(0) ?? {},
+					old_data: oldRowsByPrimaryKey.get(primaryKeyIndexKey(primaryKey, availablePrimaryColumns)) ?? {},
 					error: getErrorMessage(error),
 					affected_primary_key: primaryKey as unknown as string,
 				});
@@ -166,3 +166,19 @@ export class DeleteRowsFromTableUseCase
 		}
 	}
 }
+
+function primaryKeyIndexKey(row: Record<string, unknown>, primaryColumns: Array<string>): string {
+	// eslint-disable-next-line security/detect-object-injection
+	return primaryColumns.map((column) => String(row[column])).join('\u0000');
+}
+
+function indexRowsByPrimaryKey(
+	rows: Array<Record<string, unknown>>,
+	primaryColumns: Array<string>,
+): Map<string, Record<string, unknown>> {
+	const index = new Map<string, Record<string, unknown>>();
+	for (const row of rows) {
+		index.set(primaryKeyIndexKey(row, primaryColumns), row);
+	}
+	return index;
+}

backend/src/entities/table/use-cases/get-row-by-primary-key.use.case.ts

@@ -100,7 +100,7 @@ export class GetRowByPrimaryKeyUseCase
 		const availablePrimaryColumns: Array<string> = tablePrimaryKeys.map((column) => column.column_name);
 		for (const key in primaryKey) {
 			// eslint-disable-next-line security/detect-object-injection
-			if (!primaryKey[key] && primaryKey[key] !== '') delete primaryKey[key];
+			if (primaryKey[key] === undefined) delete primaryKey[key];
 		}
 		const receivedPrimaryColumns = Object.keys(primaryKey);
 		if (!compareArrayElements(availablePrimaryColumns, receivedPrimaryColumns)) {

backend/src/entities/table/use-cases/update-row-in-table.use.case.ts

@@ -172,7 +172,7 @@ export class UpdateRowInTableUseCase
 		});
 		for (const key in primaryKey) {
 			// eslint-disable-next-line security/detect-object-injection
-			if (!primaryKey[key] && primaryKey[key] !== '') delete primaryKey[key];
+			if (primaryKey[key] === undefined) delete primaryKey[key];
 		}
 		const receivedPrimaryColumns = Object.keys(primaryKey);
 

backend/src/entities/table/utils/add-display-names-for-tables.util.ts

@@ -1,5 +1,5 @@
-import { TableSettingsEntity } from '../../table-settings/common-table-settings/table-settings.entity.js';
 import { ITableAndViewPermissionData } from '../../permission/permission.interface.js';
+import { TableSettingsEntity } from '../../table-settings/common-table-settings/table-settings.entity.js';
 import { FoundTableDs } from '../application/data-structures/found-table.ds.js';
 
 export function addDisplayNamesForTables(