diff --git a/Common/Models/DatabaseModels/AlertEpisode.ts b/Common/Models/DatabaseModels/AlertEpisode.ts index 63c361a569..bc3350146e 100644 --- a/Common/Models/DatabaseModels/AlertEpisode.ts +++ b/Common/Models/DatabaseModels/AlertEpisode.ts @@ -543,6 +543,31 @@ export default class AlertEpisode extends BaseModel { }) public resolvedAt?: Date = undefined; + @ColumnAccessControl({ + create: [], + read: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.ReadAlertEpisode, + Permission.ReadAllProjectResources, + ], + update: [], + }) + @Index() + @TableColumn({ + type: TableColumnType.Date, + title: "All Alerts Resolved At", + description: + "When all alerts in this episode were first detected as resolved. Used for resolve delay calculation.", + }) + @Column({ + type: ColumnType.Date, + nullable: true, + unique: false, + }) + public allAlertsResolvedAt?: Date = undefined; + @ColumnAccessControl({ create: [ Permission.ProjectOwner, diff --git a/Common/Models/DatabaseModels/IncidentEpisode.ts b/Common/Models/DatabaseModels/IncidentEpisode.ts index 3c91998796..cdc5a067cb 100644 --- a/Common/Models/DatabaseModels/IncidentEpisode.ts +++ b/Common/Models/DatabaseModels/IncidentEpisode.ts @@ -542,6 +542,31 @@ export default class IncidentEpisode extends BaseModel { }) public resolvedAt?: Date = undefined; + @ColumnAccessControl({ + create: [], + read: [ + Permission.ProjectOwner, + Permission.ProjectAdmin, + Permission.ProjectMember, + Permission.ReadIncidentEpisode, + Permission.ReadAllProjectResources, + ], + update: [], + }) + @Index() + @TableColumn({ + type: TableColumnType.Date, + title: "All Incidents Resolved At", + description: + "When all incidents in this episode were first detected as resolved. Used for resolve delay calculation.", + }) + @Column({ + type: ColumnType.Date, + nullable: true, + unique: false, + }) + public allIncidentsResolvedAt?: Date = undefined; + @ColumnAccessControl({ create: [ Permission.ProjectOwner, diff --git a/Common/Server/Services/AlertEpisodeMemberService.ts b/Common/Server/Services/AlertEpisodeMemberService.ts index b464f825b0..21e0d2863e 100644 --- a/Common/Server/Services/AlertEpisodeMemberService.ts +++ b/Common/Server/Services/AlertEpisodeMemberService.ts @@ -94,26 +94,18 @@ export class Service extends DatabaseService { }); // Update episode's alertCount and lastAlertAddedAt - Promise.resolve() - .then(async () => { - try { - await AlertEpisodeService.updateAlertCount( - createdItem.alertEpisodeId!, - ); - await AlertEpisodeService.updateLastAlertAddedAt( - createdItem.alertEpisodeId!, - ); - } catch (error) { - logger.error( - `Error updating episode counts in AlertEpisodeMemberService.onCreateSuccess: ${error}`, - ); - } - }) - .catch((error: Error) => { - logger.error( - `Critical error in AlertEpisodeMemberService.onCreateSuccess: ${error}`, - ); - }); + try { + await AlertEpisodeService.updateAlertCount( + createdItem.alertEpisodeId!, + ); + await AlertEpisodeService.updateLastAlertAddedAt( + createdItem.alertEpisodeId!, + ); + } catch (error) { + logger.error( + `Error updating episode counts in AlertEpisodeMemberService.onCreateSuccess: ${error}`, + ); + } // Get alert details for feed const alert: Alert | null = await AlertService.findOneById({ diff --git a/Common/Server/Services/AlertEpisodeService.ts b/Common/Server/Services/AlertEpisodeService.ts index fcc2d05134..4d6f05061e 100644 --- a/Common/Server/Services/AlertEpisodeService.ts +++ b/Common/Server/Services/AlertEpisodeService.ts @@ -815,11 +815,12 @@ export class Service extends DatabaseService { }, }); - // Clear resolved timestamp + // Clear resolved timestamp and allAlertsResolvedAt await this.updateOneById({ id: episodeId, data: { resolvedAt: undefined as any, + allAlertsResolvedAt: undefined as any, }, props: { isRoot: true, diff --git a/Common/Server/Services/AlertGroupingEngineService.ts b/Common/Server/Services/AlertGroupingEngineService.ts index 566ce722c4..343ce94f0f 100644 --- a/Common/Server/Services/AlertGroupingEngineService.ts +++ b/Common/Server/Services/AlertGroupingEngineService.ts @@ -688,6 +688,7 @@ class AlertGroupingEngineServiceClass { newEpisode.alertGroupingRuleId = rule.id!; newEpisode.groupingKey = groupingKey; newEpisode.isManuallyCreated = false; + newEpisode.lastAlertAddedAt = OneUptimeDate.getCurrentDate(); // Set severity from alert if (alert.alertSeverityId) { diff --git a/Common/Server/Services/IncidentEpisodeMemberService.ts b/Common/Server/Services/IncidentEpisodeMemberService.ts index 10d0936edd..8411f87984 100644 --- a/Common/Server/Services/IncidentEpisodeMemberService.ts +++ b/Common/Server/Services/IncidentEpisodeMemberService.ts @@ -96,26 +96,18 @@ export class Service extends DatabaseService { }); // Update episode's incidentCount and lastIncidentAddedAt - Promise.resolve() - .then(async () => { - try { - await IncidentEpisodeService.updateIncidentCount( - createdItem.incidentEpisodeId!, - ); - await IncidentEpisodeService.updateLastIncidentAddedAt( - createdItem.incidentEpisodeId!, - ); - } catch (error) { - logger.error( - `Error updating episode counts in IncidentEpisodeMemberService.onCreateSuccess: ${error}`, - ); - } - }) - .catch((error: Error) => { - logger.error( - `Critical error in IncidentEpisodeMemberService.onCreateSuccess: ${error}`, - ); - }); + try { + await IncidentEpisodeService.updateIncidentCount( + createdItem.incidentEpisodeId!, + ); + await IncidentEpisodeService.updateLastIncidentAddedAt( + createdItem.incidentEpisodeId!, + ); + } catch (error) { + logger.error( + `Error updating episode counts in IncidentEpisodeMemberService.onCreateSuccess: ${error}`, + ); + } // Get incident details for feed const incident: Incident | null = await IncidentService.findOneById({ diff --git a/Common/Server/Services/IncidentEpisodeService.ts b/Common/Server/Services/IncidentEpisodeService.ts index e3353a8ec1..c059ec4dd8 100644 --- a/Common/Server/Services/IncidentEpisodeService.ts +++ b/Common/Server/Services/IncidentEpisodeService.ts @@ -609,6 +609,17 @@ export class Service extends DatabaseService { }, cascadeToIncidents: cascadeToIncidents, }); + + // Clear allIncidentsResolvedAt when episode is reopened + await this.updateOneById({ + id: episodeId, + data: { + allIncidentsResolvedAt: undefined as any, + }, + props: { + isRoot: true, + }, + }); } @CaptureSpan() diff --git a/Common/Server/Services/IncidentGroupingEngineService.ts b/Common/Server/Services/IncidentGroupingEngineService.ts index b2932c6748..7538f2b9c0 100644 --- a/Common/Server/Services/IncidentGroupingEngineService.ts +++ b/Common/Server/Services/IncidentGroupingEngineService.ts @@ -757,6 +757,7 @@ class IncidentGroupingEngineServiceClass { newEpisode.incidentGroupingRuleId = rule.id!; newEpisode.groupingKey = groupingKey; newEpisode.isManuallyCreated = false; + newEpisode.lastIncidentAddedAt = OneUptimeDate.getCurrentDate(); // Set severity from incident if (incident.incidentSeverityId) { diff --git a/Worker/Jobs/AlertEpisode/AutoResolve.ts b/Worker/Jobs/AlertEpisode/AutoResolve.ts index 0b41b646db..b63654566b 100644 --- a/Worker/Jobs/AlertEpisode/AutoResolve.ts +++ b/Worker/Jobs/AlertEpisode/AutoResolve.ts @@ -38,6 +38,7 @@ RunCron( projectId: true, alertGroupingRuleId: true, lastAlertAddedAt: true, + allAlertsResolvedAt: true, }, props: { isRoot: true, @@ -134,7 +135,6 @@ const checkAndResolveEpisode: CheckAndResolveEpisodeFunction = async ( // Check if all alerts are in resolved state or higher let allResolved: boolean = true; - let lastResolvedAt: Date | null = null; for (const alertId of alertIds) { const alert: Alert | null = await AlertService.findOneById({ @@ -143,7 +143,6 @@ const checkAndResolveEpisode: CheckAndResolveEpisodeFunction = async ( currentAlertState: { order: true, }, - updatedAt: true, }, props: { isRoot: true, @@ -160,31 +159,58 @@ const checkAndResolveEpisode: CheckAndResolveEpisodeFunction = async ( allResolved = false; break; } - - // Track the latest resolved time among alerts - if (alert.updatedAt) { - if (!lastResolvedAt || alert.updatedAt > lastResolvedAt) { - lastResolvedAt = alert.updatedAt; - } - } } if (!allResolved) { + // If any alert is unresolved, clear allAlertsResolvedAt + if (episode.allAlertsResolvedAt) { + await AlertEpisodeService.updateOneById({ + id: episode.id, + data: { + allAlertsResolvedAt: undefined as any, + }, + props: { + isRoot: true, + }, + }); + } + logger.debug( `AlertEpisode:AutoResolve - Episode ${episode.id} has unresolved alerts`, ); return; } - // All alerts are resolved. Check if resolve delay has passed (only if enabled) - if (enableResolveDelay && resolveDelayMinutes > 0 && lastResolvedAt) { - const timeSinceLastResolved: number = + // All alerts are resolved. Set allAlertsResolvedAt if not already set. + if (!episode.allAlertsResolvedAt) { + await AlertEpisodeService.updateOneById({ + id: episode.id, + data: { + allAlertsResolvedAt: OneUptimeDate.getCurrentDate(), + }, + props: { + isRoot: true, + }, + }); + + // If resolve delay is enabled, return and wait for the delay + if (enableResolveDelay && resolveDelayMinutes > 0) { + logger.debug( + `AlertEpisode:AutoResolve - Episode ${episode.id} all alerts resolved, starting resolve delay (${resolveDelayMinutes} minutes)`, + ); + return; + } + } + + // Check if resolve delay has passed (only if enabled) + if (enableResolveDelay && resolveDelayMinutes > 0 && episode.allAlertsResolvedAt) { + const timeSinceAllResolved: number = OneUptimeDate.getDifferenceInMinutes( - lastResolvedAt, + episode.allAlertsResolvedAt, OneUptimeDate.getCurrentDate(), ); - if (timeSinceLastResolved < resolveDelayMinutes) { + if (timeSinceAllResolved < resolveDelayMinutes) { logger.debug( `AlertEpisode:AutoResolve - Episode ${episode.id} waiting for resolve delay (${resolveDelayMinutes} minutes)`, ); diff --git a/Worker/Jobs/IncidentEpisode/AutoResolve.ts b/Worker/Jobs/IncidentEpisode/AutoResolve.ts index 8e4bbc0659..1635138075 100644 --- a/Worker/Jobs/IncidentEpisode/AutoResolve.ts +++ b/Worker/Jobs/IncidentEpisode/AutoResolve.ts @@ -38,6 +38,7 @@ RunCron( projectId: true, incidentGroupingRuleId: true, lastIncidentAddedAt: true, + allIncidentsResolvedAt: true, }, props: { isRoot: true, @@ -137,7 +138,6 @@ const checkAndResolveEpisode: CheckAndResolveEpisodeFunction = async ( // Check if all incidents are in resolved state or higher let allResolved: boolean = true; - let lastResolvedAt: Date | null = null; for (const incidentId of incidentIds) { const incident: Incident | null = await IncidentService.findOneById({ @@ -146,7 +146,6 @@ const checkAndResolveEpisode: CheckAndResolveEpisodeFunction = async ( currentIncidentState: { order: true, }, - updatedAt: true, }, props: { isRoot: true, @@ -163,31 +162,58 @@ const checkAndResolveEpisode: CheckAndResolveEpisodeFunction = async ( allResolved = false; break; } - - // Track the latest resolved time among incidents - if (incident.updatedAt) { - if (!lastResolvedAt || incident.updatedAt > lastResolvedAt) { - lastResolvedAt = incident.updatedAt; - } - } } if (!allResolved) { + // If any incident is unresolved, clear allIncidentsResolvedAt + if (episode.allIncidentsResolvedAt) { + await IncidentEpisodeService.updateOneById({ + id: episode.id, + data: { + allIncidentsResolvedAt: undefined as any, + }, + props: { + isRoot: true, + }, + }); + } + logger.debug( `IncidentEpisode:AutoResolve - Episode ${episode.id} has unresolved incidents`, ); return; } - // All incidents are resolved. Check if resolve delay has passed (only if enabled) - if (enableResolveDelay && resolveDelayMinutes > 0 && lastResolvedAt) { - const timeSinceLastResolved: number = + // All incidents are resolved. Set allIncidentsResolvedAt if not already set. + if (!episode.allIncidentsResolvedAt) { + await IncidentEpisodeService.updateOneById({ + id: episode.id, + data: { + allIncidentsResolvedAt: OneUptimeDate.getCurrentDate(), + }, + props: { + isRoot: true, + }, + }); + + // If resolve delay is enabled, return and wait for the delay + if (enableResolveDelay && resolveDelayMinutes > 0) { + logger.debug( + `IncidentEpisode:AutoResolve - Episode ${episode.id} all incidents resolved, starting resolve delay (${resolveDelayMinutes} minutes)`, + ); + return; + } + } + + // Check if resolve delay has passed (only if enabled) + if (enableResolveDelay && resolveDelayMinutes > 0 && episode.allIncidentsResolvedAt) { + const timeSinceAllResolved: number = OneUptimeDate.getDifferenceInMinutes( - lastResolvedAt, + episode.allIncidentsResolvedAt, OneUptimeDate.getCurrentDate(), ); - if (timeSinceLastResolved < resolveDelayMinutes) { + if (timeSinceAllResolved < resolveDelayMinutes) { logger.debug( `IncidentEpisode:AutoResolve - Episode ${episode.id} waiting for resolve delay (${resolveDelayMinutes} minutes)`, );