From 5f7dcf7433c8db476ed575d1af64ef9e670746e6 Mon Sep 17 00:00:00 2001 From: Simon Larsen Date: Wed, 22 May 2024 20:06:32 +0100 Subject: [PATCH] refactor: Update timeout handling in monitor scripts This commit updates the timeout handling in the monitor scripts to use the `WorkflowScriptTimeoutInMS` and `PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS` values from the environment configuration. This ensures that the monitor scripts have consistent and configurable timeout values, improving the reliability and performance of the monitoring system. --- CommonServer/EnvironmentConfig.ts | 6 ++++++ .../Types/Workflow/Components/JavaScript.ts | 2 ++ .../Probe/Criteria/CustomCodeMonitorCriteria.ts | 6 +++--- CommonServer/Utils/VM/VMAPI.ts | 1 + .../Public/oneuptime/templates/_helpers.tpl | 3 +++ HelmChart/Public/oneuptime/templates/probe.yaml | 5 +++++ HelmChart/Public/oneuptime/values.yaml | 9 ++++++++- IsolatedVM/API/VM.ts | 2 +- Probe/Config.ts | 17 +++++++++++++++++ .../Monitors/MonitorTypes/CustomCodeMonitor.ts | 3 ++- .../Monitors/MonitorTypes/SyntheticMonitor.ts | 3 ++- config.example.env | 10 ++++++++++ docker-compose.base.yml | 7 +++++++ 13 files changed, 67 insertions(+), 7 deletions(-) diff --git a/CommonServer/EnvironmentConfig.ts b/CommonServer/EnvironmentConfig.ts index dea9490dc2..41c51e0521 100644 --- a/CommonServer/EnvironmentConfig.ts +++ b/CommonServer/EnvironmentConfig.ts @@ -173,3 +173,9 @@ export const HttpProtocol: Protocol = process.env['HTTP_PROTOCOL'] === 'https' ? Protocol.HTTPS : Protocol.HTTP; export const Host: string = process.env['HOST'] || ''; + +export const WorkflowScriptTimeoutInMS: number = process.env[ + 'WORKFLOW_SCRIPT_TIMEOUT_IN_MS' +] + ? parseInt(process.env['WORKFLOW_SCRIPT_TIMEOUT_IN_MS'].toString()) + : 5000; diff --git a/CommonServer/Types/Workflow/Components/JavaScript.ts b/CommonServer/Types/Workflow/Components/JavaScript.ts index 82e70a615e..46f871bceb 100644 --- a/CommonServer/Types/Workflow/Components/JavaScript.ts +++ b/CommonServer/Types/Workflow/Components/JavaScript.ts @@ -6,6 +6,7 @@ import JavaScriptComponents from 'Common/Types/Workflow/Components/JavaScript'; import ComponentCode, { RunOptions, RunReturnType } from '../ComponentCode'; import VMUtil from '../../../Utils/VM/VMAPI'; import ReturnResult from 'Common/Types/IsolatedVM/ReturnResult'; +import { WorkflowScriptTimeoutInMS } from '../../../EnvironmentConfig'; export default class JavaScriptCode extends ComponentCode { public constructor() { @@ -69,6 +70,7 @@ export default class JavaScriptCode extends ComponentCode { code, options: { args: scriptArgs as JSONObject, + timeout: WorkflowScriptTimeoutInMS, }, }); diff --git a/CommonServer/Utils/Probe/Criteria/CustomCodeMonitorCriteria.ts b/CommonServer/Utils/Probe/Criteria/CustomCodeMonitorCriteria.ts index fb13883992..badd61b952 100644 --- a/CommonServer/Utils/Probe/Criteria/CustomCodeMonitorCriteria.ts +++ b/CommonServer/Utils/Probe/Criteria/CustomCodeMonitorCriteria.ts @@ -31,7 +31,7 @@ export default class CustomCodeMonitoringCriteria { if (input.criteriaFilter.checkOn === CheckOn.Error) { const emptyNotEmptyResult: string | null = CompareCriteria.compareEmptyAndNotEmpty({ - value: syntheticMonitorResponse.result, + value: syntheticMonitorResponse.scriptError, criteriaFilter: input.criteriaFilter, }); @@ -41,11 +41,11 @@ export default class CustomCodeMonitoringCriteria { if ( threshold && - typeof syntheticMonitorResponse.result === 'string' + typeof syntheticMonitorResponse.scriptError === 'string' ) { const result: string | null = CompareCriteria.compareCriteriaStrings({ - value: syntheticMonitorResponse.result, + value: syntheticMonitorResponse.scriptError!, threshold: threshold.toString(), criteriaFilter: input.criteriaFilter, }); diff --git a/CommonServer/Utils/VM/VMAPI.ts b/CommonServer/Utils/VM/VMAPI.ts index 512e039444..696c2ec0fe 100644 --- a/CommonServer/Utils/VM/VMAPI.ts +++ b/CommonServer/Utils/VM/VMAPI.ts @@ -14,6 +14,7 @@ export default class VMUtil { code: string; options: { args?: JSONObject | undefined; + timeout?: number | undefined; }; }): Promise { const returnResultHttpResponse: diff --git a/HelmChart/Public/oneuptime/templates/_helpers.tpl b/HelmChart/Public/oneuptime/templates/_helpers.tpl index b1f63322a3..4580ad90f3 100644 --- a/HelmChart/Public/oneuptime/templates/_helpers.tpl +++ b/HelmChart/Public/oneuptime/templates/_helpers.tpl @@ -173,6 +173,9 @@ Usage: - name: DISABLE_AUTOMATIC_INCIDENT_CREATION value: {{ $.Values.incidents.disableAutomaticCreation | squote }} +- name: WORKFLOW_SCRIPT_TIMEOUT_IN_MS + value: {{ $.Values.script.workflowScriptTimeoutInMs | squote }} + {{- end }} {{- define "oneuptime.env.pod" }} diff --git a/HelmChart/Public/oneuptime/templates/probe.yaml b/HelmChart/Public/oneuptime/templates/probe.yaml index 376f9d6a6f..904f303037 100644 --- a/HelmChart/Public/oneuptime/templates/probe.yaml +++ b/HelmChart/Public/oneuptime/templates/probe.yaml @@ -57,6 +57,11 @@ spec: value: {{ $val.description }} - name: PROBE_MONITORING_WORKERS value: {{ $val.monitoringWorkers | squote }} + # syntheticMonitorScriptTimeoutInMs + - name: PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS + value: {{ $val.syntheticMonitorScriptTimeoutInMs | squote }} + - name: PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS + value: {{ $val.customCodeMonitorScriptTimeoutInMs | squote }} - name: PROBE_KEY {{- if $val.key }} value: {{ $val.key }} diff --git a/HelmChart/Public/oneuptime/values.yaml b/HelmChart/Public/oneuptime/values.yaml index 00ff067544..616159f080 100644 --- a/HelmChart/Public/oneuptime/values.yaml +++ b/HelmChart/Public/oneuptime/values.yaml @@ -143,7 +143,9 @@ probes: monitoringWorkers: 3 monitorFetchLimit: 10 key: - replicaCount: 1 + replicaCount: 1 + syntheticMonitorScriptTimeoutInMs: 60000 + customCodeMonitorScriptTimeoutInMs: 60000 # Feel free to leave this blank if you're not integrating this with OpenTelemetry Backend. openTelemetryExporter: headers: @@ -154,6 +156,8 @@ probes: # monitorFetchLimit: 10 # key: # replicaCount: 1 + # syntheticMonitorScriptTimeoutInMs: 60000 + # customCodeMonitorScriptTimeoutInMs: 60000 # openTelemetryExporter: # headers: @@ -237,6 +241,9 @@ oneuptimeIngress: # secretName: "oneuptime-tls +script: + workflowScriptTimeoutInMs: 5000 + # extraTemplates -- Array of extra objects to deploy with the release. Strings # are evaluated as a template and can use template expansions and functions. All # other objects are used as yaml. diff --git a/IsolatedVM/API/VM.ts b/IsolatedVM/API/VM.ts index 00abd86b9e..18784acff4 100644 --- a/IsolatedVM/API/VM.ts +++ b/IsolatedVM/API/VM.ts @@ -40,7 +40,7 @@ router.post( result = await VMRunner.runCodeInSandbox({ code: req.body.code, options: { - timeout: 5000, + timeout: req.body?.['options']?.['timeout'] || 5000, args: req.body?.['options']?.['args'] || {}, }, }); diff --git a/Probe/Config.ts b/Probe/Config.ts index d4690262a6..bb390f58d6 100644 --- a/Probe/Config.ts +++ b/Probe/Config.ts @@ -58,3 +58,20 @@ if (typeof monitorFetchLimit === 'string') { export const PROBE_MONITOR_FETCH_LIMIT: number = monitorFetchLimit; export const HOSTNAME: string = process.env['HOSTNAME'] || 'localhost'; + +export const PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS: number = process.env[ + 'PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS' +] + ? parseInt( + process.env['PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS'].toString() + ) + : 60000; + +export const PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS: number = process + .env['PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS'] + ? parseInt( + process.env[ + 'PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS' + ].toString() + ) + : 60000; diff --git a/Probe/Utils/Monitors/MonitorTypes/CustomCodeMonitor.ts b/Probe/Utils/Monitors/MonitorTypes/CustomCodeMonitor.ts index cc1a8494ad..26ac683047 100644 --- a/Probe/Utils/Monitors/MonitorTypes/CustomCodeMonitor.ts +++ b/Probe/Utils/Monitors/MonitorTypes/CustomCodeMonitor.ts @@ -3,6 +3,7 @@ import logger from 'CommonServer/Utils/Logger'; import VMRunner from 'CommonServer/Utils/VM/VMRunner'; import ReturnResult from 'Common/Types/IsolatedVM/ReturnResult'; import CustomCodeMonitorResponse from 'Common/Types/Monitor/CustomCodeMonitor/CustomCodeMonitorResponse'; +import { PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS } from '../../../Config'; export interface CustomCodeMonitorOptions { monitorId?: ObjectID | undefined; @@ -37,7 +38,7 @@ export default class CustomCodeMonitor { result = await VMRunner.runCodeInSandbox({ code: options.script, options: { - timeout: 120000, // 2 minutes + timeout: PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS, args: {}, }, }); diff --git a/Probe/Utils/Monitors/MonitorTypes/SyntheticMonitor.ts b/Probe/Utils/Monitors/MonitorTypes/SyntheticMonitor.ts index 1dbef5a6db..ba25dfde20 100644 --- a/Probe/Utils/Monitors/MonitorTypes/SyntheticMonitor.ts +++ b/Probe/Utils/Monitors/MonitorTypes/SyntheticMonitor.ts @@ -7,6 +7,7 @@ import VMRunner from 'CommonServer/Utils/VM/VMRunner'; import ReturnResult from 'Common/Types/IsolatedVM/ReturnResult'; import { Browser, firefox, webkit, chromium, Page } from 'playwright'; import BadDataException from 'Common/Types/Exception/BadDataException'; +import { PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS } from '../../../Config'; export interface SyntheticMonitorOptions { monitorId?: ObjectID | undefined; @@ -88,7 +89,7 @@ export default class SyntheticMonitor { result = await VMRunner.runCodeInSandbox({ code: options.script, options: { - timeout: 120000, // 2 minutes + timeout: PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS, args: {}, context: { page: pageAndBrowser.page, diff --git a/config.example.env b/config.example.env index a904c7b1de..6d7c0d7275 100644 --- a/config.example.env +++ b/config.example.env @@ -151,12 +151,17 @@ GLOBAL_PROBE_1_DESCRIPTION="Global probe to monitor oneuptime resources" GLOBAL_PROBE_1_MONITORING_WORKERS=5 GLOBAL_PROBE_1_MONITOR_FETCH_LIMIT=10 GLOBAL_PROBE_1_ONEUPTIME_URL=http://ingestor:3400 +GLOBAL_PROBE_1_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS=60000 +GLOBAL_PROBE_1_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS=60000 + GLOBAL_PROBE_2_NAME="Probe-2" GLOBAL_PROBE_2_DESCRIPTION="Global probe to monitor oneuptime resources" GLOBAL_PROBE_2_MONITORING_WORKERS=5 GLOBAL_PROBE_2_MONITOR_FETCH_LIMIT=10 GLOBAL_PROBE_2_ONEUPTIME_URL=http://ingestor:3400 +GLOBAL_PROBE_2_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS=60000 +GLOBAL_PROBE_2_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS=60000 SMS_DEFAULT_COST_IN_CENTS= CALL_DEFAULT_COST_IN_CENTS_PER_MINUTE= @@ -204,6 +209,11 @@ E2E_TEST_STATUS_PAGE_URL= E2E_TESTS_FAILED_WEBHOOK_URL= +# This is the timeout for the workflow script in milliseconds. +WORKFLOW_SCRIPT_TIMEOUT_IN_MS=5000 + + + # Lets encrypt notification email. This email will be used when certs are about to expire LETS_ENCRYPT_NOTIFICATION_EMAIL= # Generate a private key via openssl, encode it to base64 and paste it here. diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 5b6b95fd0c..242c3816d5 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -54,6 +54,7 @@ x-common-server-variables: &common-server-variables <<: *common-variables ONEUPTIME_SECRET: ${ONEUPTIME_SECRET} + DATABASE_PORT: ${DATABASE_PORT} DATABASE_USERNAME: ${DATABASE_USERNAME} DATABASE_PASSWORD: ${DATABASE_PASSWORD} @@ -88,6 +89,8 @@ x-common-server-variables: &common-server-variables IS_SERVER: "true" + WORKFLOW_SCRIPT_TIMEOUT_IN_MS: ${WORKFLOW_SCRIPT_TIMEOUT_IN_MS} + DISABLE_AUTOMATIC_INCIDENT_CREATION: ${DISABLE_AUTOMATIC_INCIDENT_CREATION} @@ -265,6 +268,8 @@ services: PROBE_DESCRIPTION: ${GLOBAL_PROBE_1_DESCRIPTION} PROBE_MONITORING_WORKERS: ${GLOBAL_PROBE_1_MONITORING_WORKERS} PROBE_KEY: ${GLOBAL_PROBE_1_KEY} + PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS: ${GLOBAL_PROBE_1_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS} + PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS: ${GLOBAL_PROBE_1_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS} ONEUPTIME_URL: ${GLOBAL_PROBE_1_ONEUPTIME_URL} PROBE_MONITOR_FETCH_LIMIT: ${GLOBAL_PROBE_1_MONITOR_FETCH_LIMIT} OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT: ${SERVER_OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT} @@ -286,6 +291,8 @@ services: PROBE_MONITORING_WORKERS: ${GLOBAL_PROBE_2_MONITORING_WORKERS} PROBE_KEY: ${GLOBAL_PROBE_2_KEY} ONEUPTIME_URL: ${GLOBAL_PROBE_2_ONEUPTIME_URL} + PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS: ${GLOBAL_PROBE_2_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS} + PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS: ${GLOBAL_PROBE_2_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS} PROBE_MONITOR_FETCH_LIMIT: ${GLOBAL_PROBE_2_MONITOR_FETCH_LIMIT} OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT: ${SERVER_OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT} OPENTELEMETRY_EXPORTER_OTLP_HEADERS: ${PROBE_OPENTELEMETRY_EXPORTER_OTLP_HEADERS}