Add configuration options to disable automatic alert and incident creation

This commit is contained in:
Simon Larsen
2024-12-16 09:57:24 +00:00
parent ca3855d109
commit 88cf2c3cb0
9 changed files with 45 additions and 37 deletions

View File

@@ -84,74 +84,62 @@ export const ClusterKey: ObjectID = new ObjectID(
export const HasClusterKey: boolean = Boolean(process.env["ONEUPTIME_SECRET"]);
export const AppApiHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_APP_HOSTNAME"] || "localhost"}:${
process.env["APP_PORT"] || 80
`${process.env["SERVER_APP_HOSTNAME"] || "localhost"}:${process.env["APP_PORT"] || 80
}`,
);
export const ProbeIngestHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_PROBE_INGEST_HOSTNAME"] || "localhost"}:${
process.env["PROBE_INGEST_PORT"] || 80
`${process.env["SERVER_PROBE_INGEST_HOSTNAME"] || "localhost"}:${process.env["PROBE_INGEST_PORT"] || 80
}`,
);
export const OpenTelemetryIngestHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME"] || "localhost"}:${
process.env["OPEN_TELEMETRY_INGEST_PORT"] || 80
`${process.env["SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME"] || "localhost"}:${process.env["OPEN_TELEMETRY_INGEST_PORT"] || 80
}`,
);
export const IncomingRequestIngestHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_INCOMING_REQUEST_INGEST_HOSTNAME"] || "localhost"}:${
process.env["INCOMING_REQUEST_INGEST_PORT"] || 80
`${process.env["SERVER_INCOMING_REQUEST_INGEST_HOSTNAME"] || "localhost"}:${process.env["INCOMING_REQUEST_INGEST_PORT"] || 80
}`,
);
export const FluentIngestHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_FLUENT_INGEST_HOSTNAME"] || "localhost"}:${
process.env["FLUENT_INGEST_PORT"] || 80
`${process.env["SERVER_FLUENT_INGEST_HOSTNAME"] || "localhost"}:${process.env["FLUENT_INGEST_PORT"] || 80
}`,
);
export const IsolatedVMHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_ISOLATED_VM_HOSTNAME"] || "localhost"}:${
process.env["ISOLATED_VM_PORT"] || 80
`${process.env["SERVER_ISOLATED_VM_HOSTNAME"] || "localhost"}:${process.env["ISOLATED_VM_PORT"] || 80
}`,
);
export const WorkerHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_WORKER_HOSTNAME"] || "localhost"}:${
process.env["WORKER_PORT"] || 80
`${process.env["SERVER_WORKER_HOSTNAME"] || "localhost"}:${process.env["WORKER_PORT"] || 80
}`,
);
export const WorkflowHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_WORKFLOW_HOSTNAME"] || "localhost"}:${
process.env["WORKFLOW_PORT"] || 80
`${process.env["SERVER_WORKFLOW_HOSTNAME"] || "localhost"}:${process.env["WORKFLOW_PORT"] || 80
}`,
);
export const HomeHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_HOME_HOSTNAME"] || "localhost"}:${
process.env["HOME_PORT"] || 80
`${process.env["SERVER_HOME_HOSTNAME"] || "localhost"}:${process.env["HOME_PORT"] || 80
}`,
);
export const AccountsHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_ACCOUNTS_HOSTNAME"] || "localhost"}:${
process.env["ACCOUNTS_PORT"] || 80
`${process.env["SERVER_ACCOUNTS_HOSTNAME"] || "localhost"}:${process.env["ACCOUNTS_PORT"] || 80
}`,
);
export const DashboardHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_DASHBOARD_HOSTNAME"] || "localhost"}:${
process.env["DASHBOARD_PORT"] || 80
`${process.env["SERVER_DASHBOARD_HOSTNAME"] || "localhost"}:${process.env["DASHBOARD_PORT"] || 80
}`,
);
export const AdminDashboardHostname: Hostname = Hostname.fromString(
`${process.env["SERVER_ADMIN_DASHBOARD_HOSTNAME"] || "localhost"}:${
process.env["ADMIN_DASHBOARD_PORT"] || 80
`${process.env["SERVER_ADMIN_DASHBOARD_HOSTNAME"] || "localhost"}:${process.env["ADMIN_DASHBOARD_PORT"] || 80
}`,
);
@@ -198,6 +186,9 @@ export const AnalyticsHost: string = process.env["ANALYTICS_HOST"] || "";
export const DisableAutomaticIncidentCreation: boolean =
process.env["DISABLE_AUTOMATIC_INCIDENT_CREATION"] === "true";
export const DisableAutomaticAlertCreation: boolean =
process.env["DISABLE_AUTOMATIC_ALERT_CREATION"] === "true";
export const ClickhouseHost: Hostname = Hostname.fromString(
process.env["CLICKHOUSE_HOST"] || "clickhouse",
);
@@ -256,8 +247,8 @@ export const AllowedActiveMonitorCountInFreePlan: number = process.env[
"ALLOWED_ACTIVE_MONITOR_COUNT_IN_FREE_PLAN"
]
? parseInt(
process.env["ALLOWED_ACTIVE_MONITOR_COUNT_IN_FREE_PLAN"].toString(),
)
process.env["ALLOWED_ACTIVE_MONITOR_COUNT_IN_FREE_PLAN"].toString(),
)
: 10;
export const AllowedStatusPageCountInFreePlan: number = process.env[

View File

@@ -12,6 +12,7 @@ import MonitorCriteriaInstance from "../../../Types/Monitor/MonitorCriteriaInsta
import ObjectID from "../../../Types/ObjectID";
import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
import { TelemetryQuery } from "../../../Types/Telemetry/TelemetryQuery";
import { DisableAutomaticAlertCreation } from "../../EnvironmentConfig";
import AlertService from "../../Services/AlertService";
import AlertSeverityService from "../../Services/AlertSeverityService";
import AlertStateTimelineService from "../../Services/AlertStateTimelineService";
@@ -193,6 +194,10 @@ export default class MonitorAlert {
alert.remediationNotes = criteriaAlert.remediationNotes;
}
if (DisableAutomaticAlertCreation) {
return;
}
await AlertService.create({
data: alert,
props: {

View File

@@ -12,6 +12,7 @@ import MonitorCriteriaInstance from "../../../Types/Monitor/MonitorCriteriaInsta
import ObjectID from "../../../Types/ObjectID";
import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
import { TelemetryQuery } from "../../../Types/Telemetry/TelemetryQuery";
import { DisableAutomaticIncidentCreation } from "../../EnvironmentConfig";
import IncidentService from "../../Services/IncidentService";
import IncidentSeverityService from "../../Services/IncidentSeverityService";
import IncidentStateTimelineService from "../../Services/IncidentStateTimelineService";
@@ -108,9 +109,9 @@ export default class MonitorIncident {
(incident: Incident) => {
return (
incident.createdCriteriaId ===
input.criteriaInstance.data?.id.toString() &&
input.criteriaInstance.data?.id.toString() &&
incident.createdIncidentTemplateId ===
criteriaIncident.id.toString()
criteriaIncident.id.toString()
);
},
);
@@ -203,6 +204,10 @@ export default class MonitorIncident {
incident.remediationNotes = criteriaIncident.remediationNotes;
}
if (DisableAutomaticIncidentCreation) {
return;
}
await IncidentService.create({
data: incident,
props: {
@@ -217,9 +222,9 @@ export default class MonitorIncident {
openIncident: Incident;
rootCause: string;
dataToProcess:
| ProbeMonitorResponse
| IncomingMonitorRequest
| DataToProcess;
| ProbeMonitorResponse
| IncomingMonitorRequest
| DataToProcess;
}): Promise<void> {
const resolvedStateId: ObjectID =
await IncidentStateTimelineService.getResolvedStateIdForProject(
@@ -279,7 +284,7 @@ export default class MonitorIncident {
if (
input.autoResolveCriteriaInstanceIdIncidentIdsDictionary[
input.openIncident.createdCriteriaId?.toString()
input.openIncident.createdCriteriaId?.toString()
]
) {
if (

View File

@@ -109,6 +109,7 @@ The following table lists the configurable parameters of the OneUptime chart and
| `internalSmtp.name` | Name to send emails from | `nil` | |
| `logLevel` | Can be one of the following - INFO, WARN, ERROR, DEBUG | `INFO` | |
| `incidents.disableAutomaticCreation` | Disable incident creation (use this when your team is overloaded with incidents or in emergencies) | `false` | |
| `alerts.disableAutomaticCreation` | Disable alert creation (use this when your team is overloaded with alerts or in emergencies) | `false` | |
| `podSecurityContext` | Pod Security Context. Please refer to Kubernetes docuemntation to set these. This chart depends on other bitnami charts. You will have to set security context for those as well | `{}` | |
| `conatinerSecurityContext` | Container Security Context. Please refer to kubernetes documentation to set these. This chart depends on other bitnami charts. You will have to set security context for those as well | `{}` | |
| `nodeSelector` | Node Selector. Please refer to Kubernetes documentation on how to use them. | `{}` | |

View File

@@ -432,6 +432,9 @@ Usage:
- name: DISABLE_AUTOMATIC_INCIDENT_CREATION
value: {{ $.Values.incidents.disableAutomaticCreation | squote }}
- name: DISABLE_AUTOMATIC_ALERT_CREATION
value: {{ $.Values.alert.disableAutomaticCreation | squote }}
- name: WORKFLOW_SCRIPT_TIMEOUT_IN_MS
value: {{ $.Values.script.workflowScriptTimeoutInMs | squote }}

View File

@@ -179,6 +179,9 @@ internalSmtp:
incidents:
disableAutomaticCreation: false
alerts:
disableAutomaticCreation: false
# If you would like to attach status page to custom domains use this setting.
# For example, lets say you would like the status page to be hosted on status.yourcompany.com, then
# 1. Create a A record in your DNS provider with the name "oneuptime.yourcompany.com" and value to Public IP of the server oneuptime is deployed on.

View File

@@ -229,11 +229,6 @@ router.post(
next: NextFunction,
): Promise<void> => {
try {
if (DisableAutomaticIncidentCreation) {
return Response.sendJsonObjectResponse(req, res, {
message: "Automatic incident creation is disabled.",
});
}
const probeResponse: ProbeMonitorResponse = JSONFunctions.deserialize(
req.body["probeMonitorResponse"],

View File

@@ -206,6 +206,9 @@ BILLING_PRIVATE_KEY=
# Use this when you want to disable incident creation.
DISABLE_AUTOMATIC_INCIDENT_CREATION=false
# Use this when you want to disable incident creation.
DISABLE_AUTOMATIC_ALERT_CREATION=false
# If you're using an extrenal open telemetry collector, you can set the endpoint here - both server and client endpoint can be the same in this case.
# You can set the env var to http://otel-collector:4318 if you want instrumentation to be sent to otel collector.

View File

@@ -111,6 +111,8 @@ x-common-server-variables: &common-server-variables
DISABLE_AUTOMATIC_INCIDENT_CREATION: ${DISABLE_AUTOMATIC_INCIDENT_CREATION}
DISABLE_AUTOMATIC_ALERT_CREATION: ${DISABLE_AUTOMATIC_ALERT_CREATION}
# Notification Webhooks
NOTIFICATION_WEBHOOK_ON_CREATED_USER: ${NOTIFICATION_WEBHOOK_ON_CREATED_USER}