Escalation Notifications

client/src/Hooks/useMonitorForm.ts

@@ -17,6 +17,8 @@ const getBaseDefaults = (data?: Monitor | null) => ({
 	geoCheckEnabled: data?.geoCheckEnabled ?? false,
 	geoCheckLocations: data?.geoCheckLocations || [],
 	geoCheckInterval: data?.geoCheckInterval || 300000,
+	escalationDelay: data?.escalationDelay ?? undefined,
+	escalationNotifications: data?.escalationNotifications || [],
 });
 
 export const useMonitorForm = ({

client/src/Pages/CreateMonitor/index.tsx

@@ -765,6 +765,97 @@ const CreateMonitorPage = () => {
 				}
 			/>
 
+			<ConfigBox
+				title="Escalation Rules"
+				subtitle="If the monitor stays down for the specified time, notify additional channels."
+				rightContent={
+					<Stack spacing={theme.spacing(LAYOUT.MD)}>
+						<Controller
+							name="escalationDelay"
+							control={control}
+							render={({ field }) => (
+								<Select
+									{...field}
+									value={field.value ?? undefined}
+									onChange={(e) => field.onChange(Number(e.target.value))}
+									fieldLabel="Escalate after (minutes)"
+								>
+									{[1, 2, 5, 10, 15, 20, 25, 30].map((m) => (
+										<MenuItem
+											key={m}
+											value={m * 60000}
+										>
+											{m} {m === 1 ? "minute" : "minutes"}
+										</MenuItem>
+									))}
+								</Select>
+							)}
+						/>
+						<Controller
+							name="escalationNotifications"
+							control={control}
+							render={({ field }) => {
+								const notificationOptions = (notifications ?? []).map((n) => ({
+									...n,
+									name: n.notificationName,
+								}));
+								const selectedNotifications = notificationOptions.filter((n) =>
+									(field.value ?? []).includes(n.id)
+								);
+								return (
+									<Stack spacing={theme.spacing(LAYOUT.MD)}>
+										<Autocomplete
+											multiple
+											options={notificationOptions}
+											value={selectedNotifications}
+											getOptionLabel={(option) => option.name}
+											onChange={(_: unknown, newValue: typeof notificationOptions) => {
+												field.onChange(newValue.map((n) => n.id));
+											}}
+											isOptionEqualToValue={(option, value) => option.id === value.id}
+											fieldLabel="Escalation notification channels"
+										/>
+										{selectedNotifications.length > 0 && (
+											<Stack
+												flex={1}
+												width="100%"
+											>
+												{selectedNotifications.map((notification, index) => (
+													<Stack
+														direction="row"
+														alignItems="center"
+														key={notification.id}
+														width="100%"
+													>
+														<Typography flexGrow={1}>
+															{notification.notificationName}
+														</Typography>
+														<IconButton
+															size="small"
+															onClick={() => {
+																field.onChange(
+																	(field.value ?? []).filter(
+																		(id: string) => id !== notification.id
+																	)
+																);
+															}}
+															aria-label="Remove escalation notification"
+														>
+															<Trash2 size={16} />
+														</IconButton>
+														{index < selectedNotifications.length - 1 && <Divider />}
+													</Stack>
+												))}
+											</Stack>
+										)}
+									</Stack>
+								);
+							}}
+						/>
+					</Stack>
+				}
+			/>
+
 			{(watchedType === "http" ||
 				watchedType === "grpc" ||
 				watchedType === "websocket") && (

client/src/Types/Monitor.ts

@@ -76,6 +76,10 @@ export interface Monitor {
 	geoCheckEnabled?: boolean;
 	geoCheckLocations?: GeoContinent[];
 	geoCheckInterval?: number;
+	escalationDelay?: number;
+	escalationNotifications?: string[];
+	downSince?: number;
+	escalationFired?: boolean;
 	recentChecks: CheckSnapshot[];
 	createdAt: string;
 	updatedAt: string;

client/src/Validation/monitor.ts

@@ -27,6 +27,8 @@ const baseSchema = z.object({
 		.number()
 		.min(300000, "Interval must be at least 5 minutes")
 		.optional(),
+	escalationDelay: z.number().optional(),
+	escalationNotifications: z.array(z.string()).optional(),
 });
 
 // HTTP monitor schema

server/src/db/models/Monitor.ts

@@ -18,11 +18,12 @@ type CheckSnapshotDocument = Omit<CheckSnapshot, "createdAt"> & { createdAt: Dat
 
 type MonitorDocumentBase = Omit<
 	Monitor,
-	"id" | "userId" | "teamId" | "notifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt"
+	"id" | "userId" | "teamId" | "notifications" | "escalationNotifications" | "selectedDisks" | "statusWindow" | "recentChecks" | "createdAt" | "updatedAt"
 > & {
 	statusWindow: boolean[];
 	recentChecks: CheckSnapshotDocument[];
 	notifications: Types.ObjectId[];
+	escalationNotifications: Types.ObjectId[];
 	selectedDisks: string[];
 	matchMethod?: MonitorMatchMethod;
 };
@@ -351,6 +352,24 @@ const MonitorSchema = new Schema<MonitorDocument>(
 			type: Number,
 			default: 300000,
 		},
+		escalationDelay: {
+			type: Number,
+			default: undefined,
+		},
+		escalationNotifications: [
+			{
+				type: Schema.Types.ObjectId,
+				ref: "Notification",
+			},
+		],
+		downSince: {
+			type: Number,
+			default: undefined,
+		},
+		escalationFired: {
+			type: Boolean,
+			default: false,
+		},
 		recentChecks: {
 			type: [checkSnapshotSchema],
 			default: [],
@@ -367,4 +386,4 @@ const MonitorModel = model<MonitorDocument>("Monitor", MonitorSchema);
 
 export type { MonitorDocument, CheckSnapshotDocument };
 export { MonitorModel };
-export default MonitorModel;
+export default MonitorModel;

server/src/repositories/monitors/MongoMonitorsRepository.ts

@@ -351,6 +351,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 		};
 
 		const notificationIds = (doc.notifications ?? []).map((notification) => toStringId(notification));
+		const escalationNotificationIds = (doc.escalationNotifications ?? []).map((n) => toStringId(n));
 
 		return {
 			id: toStringId(doc._id),
@@ -391,6 +392,10 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			geoCheckEnabled: doc.geoCheckEnabled ?? false,
 			geoCheckLocations: doc.geoCheckLocations ?? [],
 			geoCheckInterval: doc.geoCheckInterval ?? 300000,
+			escalationDelay: doc.escalationDelay ?? undefined,
+			escalationNotifications: escalationNotificationIds,
+			downSince: doc.downSince ?? undefined,
+			escalationFired: doc.escalationFired ?? false,
 			createdAt: toDateString(doc.createdAt),
 			updatedAt: toDateString(doc.updatedAt),
 		};
@@ -410,6 +415,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 		};
 
 		const notificationIds = (doc.notifications ?? []).map((notification: unknown) => toStringId(notification));
+		const escalationNotificationIds = (doc.escalationNotifications ?? []).map((n: unknown) => toStringId(n));
 
 		return {
 			id: toStringId(doc._id),
@@ -450,6 +456,10 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			geoCheckEnabled: doc.geoCheckEnabled ?? false,
 			geoCheckLocations: doc.geoCheckLocations ?? [],
 			geoCheckInterval: doc.geoCheckInterval ?? 300000,
+			escalationDelay: doc.escalationDelay ?? undefined,
+			escalationNotifications: escalationNotificationIds,
+			downSince: doc.downSince ?? undefined,
+			escalationFired: doc.escalationFired ?? false,
 			createdAt: toDateString(doc.createdAt),
 			updatedAt: toDateString(doc.updatedAt),
 		};

server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts

@@ -37,6 +37,7 @@ export interface MonitorActionDecision {
 	shouldCreateIncident: boolean;
 	shouldResolveIncident: boolean;
 	shouldSendNotification: boolean;
+	shouldSendEscalation: boolean;
 	incidentReason: "status_down" | "threshold_breach" | null;
 	notificationReason: "status_change" | "threshold_breach" | null;
 	thresholdBreaches?: {
@@ -168,6 +169,22 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper {
 					});
 				}
 
+				// Step 6b. Handle escalation notifications if delay threshold met
+				if (decision.shouldSendEscalation) {
+					// Mark escalation as fired before sending to prevent duplicate escalations
+					await this.monitorsRepository.updateById(statusChangeResult.monitor.id, statusChangeResult.monitor.teamId, { escalationFired: true });
+					this.notificationsService
+						.handleEscalationNotifications(statusChangeResult.monitor, status, decision)
+						.catch((error: unknown) => {
+							this.logger.error({
+								message: `Error sending escalation notifications for job ${statusChangeResult.monitor.id}: ${error instanceof Error ? error.message : "Unknown error"}`,
+								service: SERVICE_NAME,
+								method: "getMonitorJob",
+								stack: error instanceof Error ? error.stack : undefined,
+							});
+						});
+				}
+
 				// Step 7. Handle incidents (best effort, don't wait)
 				this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code, decision, status).catch((error: unknown) => {
 					this.logger.warn({
@@ -426,10 +443,26 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper {
 			shouldCreateIncident: false,
 			shouldResolveIncident: false,
 			shouldSendNotification: false,
+			shouldSendEscalation: false,
 			incidentReason: null,
 			notificationReason: null,
 		};
 
+		// Check for escalation: monitor is currently down, has been for longer than escalationDelay,
+		// has escalation channels configured, and hasn't already fired escalation this downtime
+		if (
+			monitor.status === "down" &&
+			monitor.escalationDelay != null &&
+			monitor.escalationDelay > 0 &&
+			monitor.escalationNotifications &&
+			monitor.escalationNotifications.length > 0 &&
+			!monitor.escalationFired &&
+			monitor.downSince != null &&
+			Date.now() - monitor.downSince >= monitor.escalationDelay
+		) {
+			decision.shouldSendEscalation = true;
+		}
+
 		if (!statusChanged) {
 			return decision;
 		}
@@ -455,4 +488,4 @@ export class SuperSimpleQueueHelper implements ISuperSimpleQueueHelper {
 
 		return decision;
 	}
-}
+}

server/src/service/infrastructure/notificationMessageBuilder.ts

@@ -15,6 +15,12 @@ export interface INotificationMessageBuilder {
 		decision: MonitorActionDecision,
 		clientHost: string
 	): NotificationMessage;
+	buildEscalationMessage(
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision,
+		clientHost: string
+	): NotificationMessage;
 	extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[];
 }
 
@@ -52,6 +58,49 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
 		};
 	}
 
+	buildEscalationMessage(
+		monitor: Monitor,
+		monitorStatusResponse: MonitorStatusResponse,
+		decision: MonitorActionDecision,
+		clientHost: string
+	): NotificationMessage {
+		const delayMinutes = monitor.escalationDelay ? Math.round(monitor.escalationDelay / 60000) : 0;
+		const downSinceStr = monitor.downSince ? new Date(monitor.downSince).toUTCString() : "unknown";
+
+		const content: NotificationContent = {
+			title: `Escalation Alert: ${monitor.name} Still Down`,
+			summary: `Monitor "${monitor.name}" has been down for more than ${delayMinutes} minute${delayMinutes !== 1 ? "s" : ""} and requires immediate attention.`,
+			details: [
+				`URL: ${monitor.url}`,
+				`Status: Down`,
+				`Type: ${monitor.type}`,
+				`Down Since: ${downSinceStr}`,
+				`Escalation Delay: ${delayMinutes} minute${delayMinutes !== 1 ? "s" : ""}`,
+				...(monitorStatusResponse.code ? [`Response Code: ${monitorStatusResponse.code}`] : []),
+				...(monitorStatusResponse.message ? [`Error: ${monitorStatusResponse.message}`] : []),
+			],
+			timestamp: new Date(),
+		};
+
+		return {
+			type: "escalation",
+			severity: "critical",
+			monitor: {
+				id: monitor.id,
+				name: monitor.name,
+				url: monitor.url,
+				type: monitor.type,
+				status: monitor.status,
+			},
+			content,
+			clientHost,
+			metadata: {
+				teamId: monitor.teamId,
+				notificationReason: "status_change",
+			},
+		};
+	}
+
 	private determineNotificationType(decision: MonitorActionDecision, monitor: Monitor): NotificationType {
 		// Down status has highest priority (critical)
 		if (monitor.status === "down") {
@@ -271,4 +320,4 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
 
 		return breaches;
 	}
-}
+}

server/src/service/infrastructure/notificationProviders/email.ts

@@ -81,6 +81,8 @@ export class EmailProvider implements INotificationProvider {
 		switch (message.type) {
 			case "monitor_down":
 				return `Monitor ${message.monitor.name} is down`;
+			case "escalation":
+				return `Monitor ${message.monitor.name} is still down`;
 			case "monitor_up":
 				return `Monitor ${message.monitor.name} is back up`;
 			case "threshold_breach":
@@ -127,4 +129,4 @@ export class EmailProvider implements INotificationProvider {
 		};
 		return colorMap[severity] ?? "#3b82f6";
 	}
-}
+}