From a6ff1b720a84974fd10b056fe9ce0a7491c90ec4 Mon Sep 17 00:00:00 2001 From: alyssazieli3 Date: Sat, 11 Apr 2026 22:15:35 -0400 Subject: [PATCH 1/8] Add escalation properties to Monitor type --- client/src/Types/Monitor.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/src/Types/Monitor.ts b/client/src/Types/Monitor.ts index 053b517d1d..bfd0bf5827 100644 --- a/client/src/Types/Monitor.ts +++ b/client/src/Types/Monitor.ts @@ -76,6 +76,10 @@ export interface Monitor { geoCheckEnabled?: boolean; geoCheckLocations?: GeoContinent[]; geoCheckInterval?: number; + escalationDelay?: number; + escalationNotifications?: string[]; + downSince?: number; + escalationFired?: boolean; recentChecks: CheckSnapshot[]; createdAt: string; updatedAt: string; From 5542257b5c17d667057a45898de6c4a3f4ca2d9a Mon Sep 17 00:00:00 2001 From: alyssazieli3 Date: Sat, 11 Apr 2026 22:16:49 -0400 Subject: [PATCH 2/8] Add escalationDelay and escalationNotifications fields --- client/src/Validation/monitor.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/src/Validation/monitor.ts b/client/src/Validation/monitor.ts index 9acffe6fed..d9fe286d3a 100644 --- a/client/src/Validation/monitor.ts +++ b/client/src/Validation/monitor.ts @@ -27,6 +27,8 @@ const baseSchema = z.object({ .number() .min(300000, "Interval must be at least 5 minutes") .optional(), + escalationDelay: z.number().optional(), + escalationNotifications: z.array(z.string()).optional(), }); // HTTP monitor schema From 72f19e5543316c894f38edb86a6d2bee7b5526c5 Mon Sep 17 00:00:00 2001 From: alyssazieli3 Date: Sat, 11 Apr 2026 22:18:45 -0400 Subject: [PATCH 3/8] Add escalationDelay and escalationNotifications to defaults --- client/src/Hooks/useMonitorForm.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/src/Hooks/useMonitorForm.ts b/client/src/Hooks/useMonitorForm.ts index 963409fc8a..ecf58915cd 100644 --- a/client/src/Hooks/useMonitorForm.ts +++ b/client/src/Hooks/useMonitorForm.ts @@ -17,6 +17,8 @@ const getBaseDefaults = (data?: Monitor | null) => ({ geoCheckEnabled: data?.geoCheckEnabled ?? false, geoCheckLocations: data?.geoCheckLocations || [], geoCheckInterval: data?.geoCheckInterval || 300000, + escalationDelay: data?.escalationDelay ?? undefined, + escalationNotifications: data?.escalationNotifications || [], }); export const useMonitorForm = ({ From 302af83b031a57a36e210f6bcdba0e67068650a3 Mon Sep 17 00:00:00 2001 From: alyssazieli3 Date: Sat, 11 Apr 2026 22:20:52 -0400 Subject: [PATCH 4/8] Add escalation rules configuration to monitor --- client/src/Pages/CreateMonitor/index.tsx | 91 ++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/client/src/Pages/CreateMonitor/index.tsx b/client/src/Pages/CreateMonitor/index.tsx index 15b76eab36..4f9571b27c 100644 --- a/client/src/Pages/CreateMonitor/index.tsx +++ b/client/src/Pages/CreateMonitor/index.tsx @@ -765,6 +765,97 @@ const CreateMonitorPage = () => { } /> + + ( + + )} + /> + { + const notificationOptions = (notifications ?? []).map((n) => ({ + ...n, + name: n.notificationName, + })); + const selectedNotifications = notificationOptions.filter((n) => + (field.value ?? []).includes(n.id) + ); + return ( + + option.name} + onChange={(_: unknown, newValue: typeof notificationOptions) => { + field.onChange(newValue.map((n) => n.id)); + }} + isOptionEqualToValue={(option, value) => option.id === value.id} + fieldLabel="Escalation notification channels" + /> + {selectedNotifications.length > 0 && ( + + {selectedNotifications.map((notification, index) => ( + + + {notification.notificationName} + + { + field.onChange( + (field.value ?? []).filter( + (id: string) => id !== notification.id + ) + ); + }} + aria-label="Remove escalation notification" + > + + + {index < selectedNotifications.length - 1 && } + + ))} + + )} + + ); + }} + /> + + } + /> + {(watchedType === "http" || watchedType === "grpc" || watchedType === "websocket") && ( From 7796fbb140edbd069233ed60b4eb36c5e95158c6 Mon Sep 17 00:00:00 2001 From: alyssazieli3 Date: Sat, 11 Apr 2026 22:22:32 -0400 Subject: [PATCH 5/8] Add escalation properties to monitor type Added escalation properties to monitor type. --- server/src/types/monitor.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/types/monitor.ts b/server/src/types/monitor.ts index f29ce75d78..c276b393ab 100644 --- a/server/src/types/monitor.ts +++ b/server/src/types/monitor.ts @@ -53,6 +53,10 @@ export interface Monitor { geoCheckEnabled?: boolean; geoCheckLocations?: GeoContinent[]; geoCheckInterval?: number; + escalationDelay?: number; + escalationNotifications?: string[]; + downSince?: number; + escalationFired?: boolean; recentChecks: CheckSnapshot[]; createdAt: string; updatedAt: string; From 1fb4b1f37308521caf1775623f2c033a8f781c53 Mon Sep 17 00:00:00 2001 From: alyssazieli3 Date: Sat, 11 Apr 2026 22:23:33 -0400 Subject: [PATCH 6/8] Add 'escalation' type to NotificationType --- server/src/types/notificationMessage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/types/notificationMessage.ts b/server/src/types/notificationMessage.ts index f06ff1bd9a..eb62f8762c 100644 --- a/server/src/types/notificationMessage.ts +++ b/server/src/types/notificationMessage.ts @@ -3,7 +3,7 @@ * Part of notification system unification effort */ -export type NotificationType = "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "test"; +export type NotificationType = "monitor_down" | "monitor_up" | "threshold_breach" | "threshold_resolved" | "test" | "escalation"; export type NotificationSeverity = "critical" | "warning" | "info" | "success"; From 5fc471dcb52f0005ac56e8f0b6b16a741460e1b8 Mon Sep 17 00:00:00 2001 From: alyssazieli3 Date: Sat, 11 Apr 2026 22:24:39 -0400 Subject: [PATCH 7/8] Add escalationDelay and escalationNotifications fields --- server/src/validation/monitorValidation.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/validation/monitorValidation.ts b/server/src/validation/monitorValidation.ts index df000ecef2..7ea81242dd 100644 --- a/server/src/validation/monitorValidation.ts +++ b/server/src/validation/monitorValidation.ts @@ -78,6 +78,8 @@ export const createMonitorBodyValidation = z.object({ geoCheckEnabled: z.boolean().optional(), geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), + escalationDelay: z.number().optional(), + escalationNotifications: z.array(z.string()).optional(), }); export const editMonitorBodyValidation = z.object({ @@ -107,6 +109,8 @@ export const editMonitorBodyValidation = z.object({ geoCheckEnabled: z.boolean().optional(), geoCheckLocations: z.array(z.enum(GeoContinents)).optional(), geoCheckInterval: z.number().min(300000).optional(), + escalationDelay: z.number().optional(), + escalationNotifications: z.array(z.string()).optional(), }); export const pauseMonitorParamValidation = z.object({ From fa1872f32efec319889480869fc37cee9a6c1116 Mon Sep 17 00:00:00 2001 From: alyssazieli3 Date: Sat, 11 Apr 2026 22:36:32 -0400 Subject: [PATCH 8/8] added escalation notifications --- server/src/db/models/Monitor.ts | 23 ++++++- .../monitors/MongoMonitorsRepository.ts | 10 +++ .../SuperSimpleQueueHelper.ts | 35 +++++++++- .../notificationMessageBuilder.ts | 51 ++++++++++++++- .../notificationProviders/email.ts | 4 +- .../infrastructure/notificationsService.ts | 42 +++++++++++- .../service/infrastructure/statusService.ts | 65 ++++++++----------- 7 files changed, 187 insertions(+), 43 deletions(-) diff --git a/server/src/db/models/Monitor.ts b/server/src/db/models/Monitor.ts index 036aeadad6..84d250b930 100644 --- a/server/src/db/models/Monitor.ts +++ b/server/src/db/models/Monitor.ts @@ -18,11 +18,12 @@ type CheckSnapshotDocument = Omit & { createdAt: Dat type MonitorDocumentBase = Omit< Monitor, - "id" | "userId" | "teamId" | "notifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt" + "id" | "userId" | "teamId" | "notifications" | "escalationNotifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt" > & { statusWindow: boolean[]; recentChecks: CheckSnapshotDocument[]; notifications: Types.ObjectId[]; + escalationNotifications: Types.ObjectId[]; selectedDisks: string[]; matchMethod?: MonitorMatchMethod; }; @@ -351,6 +352,24 @@ const MonitorSchema = new Schema( type: Number, default: 300000, }, + escalationDelay: { + type: Number, + default: undefined, + }, + escalationNotifications: [ + { + type: Schema.Types.ObjectId, + ref: "Notification", + }, + ], + downSince: { + type: Number, + default: undefined, + }, + escalationFired: { + type: Boolean, + default: false, + }, recentChecks: { type: [checkSnapshotSchema], default: [], @@ -367,4 +386,4 @@ const MonitorModel = model("Monitor", MonitorSchema); export type { MonitorDocument, CheckSnapshotDocument }; export { MonitorModel }; -export default MonitorModel; +export default MonitorModel; \ No newline at end of file diff --git a/server/src/repositories/monitors/MongoMonitorsRepository.ts b/server/src/repositories/monitors/MongoMonitorsRepository.ts index b2d7594483..ba73463e96 100644 --- a/server/src/repositories/monitors/MongoMonitorsRepository.ts +++ b/server/src/repositories/monitors/MongoMonitorsRepository.ts @@ -351,6 +351,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { }; const notificationIds = (doc.notifications ?? []).map((notification) => toStringId(notification)); + const escalationNotificationIds = (doc.escalationNotifications ?? []).map((n) => toStringId(n)); return { id: toStringId(doc._id), @@ -391,6 +392,10 @@ class MongoMonitorsRepository implements IMonitorsRepository { geoCheckEnabled: doc.geoCheckEnabled ?? false, geoCheckLocations: doc.geoCheckLocations ?? [], geoCheckInterval: doc.geoCheckInterval ?? 300000, + escalationDelay: doc.escalationDelay ?? undefined, + escalationNotifications: escalationNotificationIds, + downSince: doc.downSince ?? undefined, + escalationFired: doc.escalationFired ?? false, createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), }; @@ -410,6 +415,7 @@ class MongoMonitorsRepository implements IMonitorsRepository { }; const notificationIds = (doc.notifications ?? []).map((notification: unknown) => toStringId(notification)); + const escalationNotificationIds = (doc.escalationNotifications ?? []).map((n: unknown) => toStringId(n)); return { id: toStringId(doc._id), @@ -450,6 +456,10 @@ class MongoMonitorsRepository implements IMonitorsRepository { geoCheckEnabled: doc.geoCheckEnabled ?? false, geoCheckLocations: doc.geoCheckLocations ?? [], geoCheckInterval: doc.geoCheckInterval ?? 300000, + escalationDelay: doc.escalationDelay ?? undefined, + escalationNotifications: escalationNotificationIds, + downSince: doc.downSince ?? undefined, + escalationFired: doc.escalationFired ?? false, createdAt: toDateString(doc.createdAt), updatedAt: toDateString(doc.updatedAt), }; diff --git a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts index b6908127b2..537e32c038 100644 --- a/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts +++ b/server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts @@ -37,6 +37,7 @@ export interface MonitorActionDecision { shouldCreateIncident: boolean; shouldResolveIncident: boolean; shouldSendNotification: boolean; + shouldSendEscalation: boolean; incidentReason: "status_down" | "threshold_breach" | null; notificationReason: "status_change" | "threshold_breach" | null; thresholdBreaches?: { @@ -168,6 +169,22 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { }); } + // Step 6b. Handle escalation notifications if delay threshold met + if (decision.shouldSendEscalation) { + // Mark escalation as fired before sending to prevent duplicate escalations + await this.monitorsRepository.updateById(statusChangeResult.monitor.id, statusChangeResult.monitor.teamId, { escalationFired: true }); + this.notificationsService + .handleEscalationNotifications(statusChangeResult.monitor, status, decision) + .catch((error: unknown) => { + this.logger.error({ + message: `Error sending escalation notifications for job ${statusChangeResult.monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`, + service: SERVICE_NAME, + method: "getMonitorJob", + stack: error instanceof Error ? error.stack : undefined, + }); + }); + } + // Step 7. Handle incidents (best effort, don't wait) this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code, decision, status).catch((error: unknown) => { this.logger.warn({ @@ -426,10 +443,26 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { shouldCreateIncident: false, shouldResolveIncident: false, shouldSendNotification: false, + shouldSendEscalation: false, incidentReason: null, notificationReason: null, }; + // Check for escalation: monitor is currently down, has been for longer than escalationDelay, + // has escalation channels configured, and hasn't already fired escalation this downtime + if ( + monitor.status === "down" && + monitor.escalationDelay != null && + monitor.escalationDelay > 0 && + monitor.escalationNotifications && + monitor.escalationNotifications.length > 0 && + !monitor.escalationFired && + monitor.downSince != null && + Date.now() - monitor.downSince >= monitor.escalationDelay + ) { + decision.shouldSendEscalation = true; + } + if (!statusChanged) { return decision; } @@ -455,4 +488,4 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper { return decision; } -} +} \ No newline at end of file diff --git a/server/src/service/infrastructure/notificationMessageBuilder.ts b/server/src/service/infrastructure/notificationMessageBuilder.ts index 934163b2a9..6511f693ae 100644 --- a/server/src/service/infrastructure/notificationMessageBuilder.ts +++ b/server/src/service/infrastructure/notificationMessageBuilder.ts @@ -15,6 +15,12 @@ export interface INotificationMessageBuilder { decision: MonitorActionDecision, clientHost: string ): NotificationMessage; + buildEscalationMessage( + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + clientHost: string + ): NotificationMessage; extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[]; } @@ -52,6 +58,49 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { }; } + buildEscalationMessage( + monitor: Monitor, + monitorStatusResponse: MonitorStatusResponse, + decision: MonitorActionDecision, + clientHost: string + ): NotificationMessage { + const delayMinutes = monitor.escalationDelay ? Math.round(monitor.escalationDelay / 60000) : 0; + const downSinceStr = monitor.downSince ? new Date(monitor.downSince).toUTCString() : "unknown"; + + const content: NotificationContent = { + title: `Escalation Alert: ${monitor.name} Still Down`, + summary: `Monitor "${monitor.name}" has been down for more than ${delayMinutes} minute${delayMinutes !== 1 ? "s" : ""} and requires immediate attention.`, + details: [ + `URL: ${monitor.url}`, + `Status: Down`, + `Type: ${monitor.type}`, + `Down Since: ${downSinceStr}`, + `Escalation Delay: ${delayMinutes} minute${delayMinutes !== 1 ? "s" : ""}`, + ...(monitorStatusResponse.code ? [`Response Code: ${monitorStatusResponse.code}`] : []), + ...(monitorStatusResponse.message ? [`Error: ${monitorStatusResponse.message}`] : []), + ], + timestamp: new Date(), + }; + + return { + type: "escalation", + severity: "critical", + monitor: { + id: monitor.id, + name: monitor.name, + url: monitor.url, + type: monitor.type, + status: monitor.status, + }, + content, + clientHost, + metadata: { + teamId: monitor.teamId, + notificationReason: "status_change", + }, + }; + } + private determineNotificationType(decision: MonitorActionDecision, monitor: Monitor): NotificationType { // Down status has highest priority (critical) if (monitor.status === "down") { @@ -271,4 +320,4 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder { return breaches; } -} +} \ No newline at end of file diff --git a/server/src/service/infrastructure/notificationProviders/email.ts b/server/src/service/infrastructure/notificationProviders/email.ts index b3686651cc..4e3d34936c 100644 --- a/server/src/service/infrastructure/notificationProviders/email.ts +++ b/server/src/service/infrastructure/notificationProviders/email.ts @@ -81,6 +81,8 @@ export class EmailProvider implements INotificationProvider { switch (message.type) { case "monitor_down": return `Monitor ${message.monitor.name} is down`; + case "escalation": + return `Monitor ${message.monitor.name} is still down`; case "monitor_up": return `Monitor ${message.monitor.name} is back up`; case "threshold_breach": @@ -127,4 +129,4 @@ export class EmailProvider implements INotificationProvider { }; return colorMap[severity] ?? "#3b82f6"; } -} +} \ No newline at end of file diff --git a/server/src/service/infrastructure/notificationsService.ts b/server/src/service/infrastructure/notificationsService.ts index c75477c88c..cb99400799 100644 --- a/server/src/service/infrastructure/notificationsService.ts +++ b/server/src/service/infrastructure/notificationsService.ts @@ -14,6 +14,7 @@ export interface INotificationsService { updateById(id: string, teamId: string, updateData: Partial): Promise; deleteById: (id: string, teamId: string) => Promise; handleNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise; + handleEscalationNotifications: (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => Promise; sendTestNotification: (notification: Partial) => Promise; testAllNotifications: (notificationIds: string[]) => Promise; @@ -33,6 +34,7 @@ export class NotificationsService implements INotificationsService { private pagerDutyProvider: INotificationProvider; private matrixProvider: INotificationProvider; private teamsProvider: INotificationProvider; + private telegramProvider: INotificationProvider; private logger: ILogger; private settingsService: ISettingsService; private notificationMessageBuilder: INotificationMessageBuilder; @@ -47,6 +49,7 @@ export class NotificationsService implements INotificationsService { pagerDutyProvider: INotificationProvider, matrixProvider: INotificationProvider, teamsProvider: INotificationProvider, + telegramProvider: INotificationProvider, settingsService: ISettingsService, logger: ILogger, notificationMessageBuilder: INotificationMessageBuilder @@ -60,6 +63,7 @@ export class NotificationsService implements INotificationsService { this.pagerDutyProvider = pagerDutyProvider; this.matrixProvider = matrixProvider; this.teamsProvider = teamsProvider; + this.telegramProvider = telegramProvider; this.settingsService = settingsService; this.logger = logger; this.notificationMessageBuilder = notificationMessageBuilder; @@ -97,6 +101,8 @@ export class NotificationsService implements INotificationsService { return await this.emailProvider.sendMessage!(notification, notificationMessage); case "teams": return await this.teamsProvider.sendMessage!(notification, notificationMessage); + case "telegram": + return await this.telegramProvider.sendMessage!(notification, notificationMessage); default: this.logger.warn({ message: `Unknown notification type: ${notification.type}`, @@ -141,6 +147,38 @@ export class NotificationsService implements INotificationsService { return await this.sendNotifications(monitor, monitorStatusResponse, decision); }; + handleEscalationNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse, decision: MonitorActionDecision) => { + if (!decision.shouldSendEscalation) { + return false; + } + + const escalationIds = monitor.escalationNotifications ?? []; + if (escalationIds.length === 0) { + return false; + } + + const notifications = await this.notificationsRepository.findNotificationsByIds(escalationIds); + const settings = this.settingsService.getSettings(); + const clientHost = settings.clientHost || "Host not defined"; + const notificationMessage = this.notificationMessageBuilder.buildEscalationMessage(monitor, monitorStatusResponse, decision, clientHost); + + const tasks = notifications.map((notification) => + this.send(notification, monitor, monitorStatusResponse, decision, notificationMessage) + ); + + const outcomes = await Promise.all(tasks); + const succeeded = outcomes.filter(Boolean).length; + const failed = outcomes.length - succeeded; + if (failed > 0) { + this.logger.warn({ + message: `Escalation notification send completed with ${succeeded} success, ${failed} failure(s)`, + service: SERVICE_NAME, + method: "handleEscalationNotifications", + }); + } + return succeeded === notifications.length; + }; + sendTestNotification = async (notification: Partial) => { switch (notification.type) { case "email": @@ -157,6 +195,8 @@ export class NotificationsService implements INotificationsService { return await this.webhookProvider.sendTestAlert(notification); case "teams": return await this.teamsProvider.sendTestAlert(notification); + case "telegram": + return await this.telegramProvider.sendTestAlert(notification); default: return false; } @@ -197,4 +237,4 @@ export class NotificationsService implements INotificationsService { await this.monitorsRepository.removeNotificationFromMonitors(id); return deleted; }; -} +} \ No newline at end of file diff --git a/server/src/service/infrastructure/statusService.ts b/server/src/service/infrastructure/statusService.ts index ef725b80af..4b06a4efdf 100755 --- a/server/src/service/infrastructure/statusService.ts +++ b/server/src/service/infrastructure/statusService.ts @@ -147,13 +147,7 @@ export class StatusService implements IStatusService { } // Calculate uptime percentage - let uptimePercentage; - if (stats.totalChecks > 0) { - uptimePercentage = stats.totalUpChecks / stats.totalChecks; - } else { - uptimePercentage = status === true ? 100 : 0; - } - stats.uptimePercentage = uptimePercentage; + stats.uptimePercentage = stats.totalUpChecks / stats.totalChecks; // latest check stats.lastCheckTimestamp = new Date().getTime(); @@ -239,7 +233,8 @@ export class StatusService implements IStatusService { // Return early if not enough data points if (monitor.statusWindow.length < monitor.statusWindowSize) { monitor.status = newStatus; - const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor); + const { escalationNotifications, downSince, escalationFired, ...earlyPatch } = monitor; + const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, earlyPatch); return { monitor: updated, statusChanged: false, @@ -257,18 +252,22 @@ export class StatusService implements IStatusService { if (failureRate >= monitor.statusWindowThreshold && monitor.status !== "down") { newStatus = "down"; statusChanged = true; + monitor.downSince = Date.now(); + monitor.escalationFired = false; } // If the failure rate is below the threshold and the monitor is down, recover: else if (failureRate < monitor.statusWindowThreshold && monitor.status === "down") { newStatus = "up"; statusChanged = true; + monitor.downSince = undefined; + monitor.escalationFired = false; } // Evaluate hardware threshold breaches (only for hardware monitors) let thresholdBreaches: { cpu: boolean; memory: boolean; disk: boolean; temp: boolean } | undefined; if (monitor.type === "hardware" && statusResponse.payload) { const payload = statusResponse.payload as HardwareStatusPayload; - const metrics = payload?.data; + const metrics = payload.data; if (metrics) { // Evaluate threshold breaches @@ -278,9 +277,11 @@ export class StatusService implements IStatusService { const memoryUsage = metrics.memory?.usage_percent ?? -1; const memoryBreach = memoryUsage !== -1 && memoryUsage > monitor.memoryAlertThreshold / 100; - const diskBreach = - metrics.disk?.some((d: CheckDiskInfo) => typeof d?.usage_percent === "number" && d.usage_percent > monitor.diskAlertThreshold / 100) ?? - false; + const diskBreach = metrics.disk + ? metrics.disk.some( + (d: CheckDiskInfo) => d != null && typeof d.usage_percent === "number" && d.usage_percent > monitor.diskAlertThreshold / 100 + ) + : false; const temps = metrics.cpu?.temperature ?? []; const tempBreach = temps.some((temp: number) => temp > monitor.tempAlertThreshold); @@ -348,7 +349,20 @@ export class StatusService implements IStatusService { // Apply the final status monitor.status = newStatus; - const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor); + // Build a targeted patch that excludes fields managed separately to avoid + // overwriting escalationFired set by the heartbeat job, and avoids + // Mongoose ObjectId casting issues with escalationNotifications string arrays. + const { escalationNotifications, downSince, escalationFired, ...monitorPatch } = monitor; + const patch: Partial = { ...monitorPatch }; + + // Only include escalation tracking fields when they are explicitly changing + // (i.e. on status transitions, where we set them above on the monitor object) + if (statusChanged) { + patch.downSince = monitor.downSince; + patch.escalationFired = monitor.escalationFired; + } + + const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, patch); return { monitor: updated, @@ -366,27 +380,4 @@ export class StatusService implements IStatusService { }); } }; - - insertCheck = async (check: Check) => { - try { - if (typeof check === "undefined") { - this.logger.warn({ - message: "Failed to build check", - service: SERVICE_NAME, - method: "insertCheck", - }); - return false; - } - this.buffer.addToBuffer(check); - return true; - } catch (error: unknown) { - this.logger.error({ - message: error instanceof Error ? error.message : "Unknown error", - service: SERVICE_NAME, - method: "insertCheck", - details: { msg: `Error inserting check for monitor: ${check?.metadata.monitorId}` }, - stack: error instanceof Error ? error.stack : undefined, - }); - } - }; -} +} \ No newline at end of file