github/oneuptime
1
1
Fork 0
mirror of https://github.com/OneUptime/oneuptime.git synced 2026-04-06 00:32:12 +02:00
Code Issues 193 Packages Projects Releases Wiki Activity

refactor: Update timeout handling in monitor scripts

Browse Source 
This commit updates the timeout handling in the monitor scripts to use the `WorkflowScriptTimeoutInMS` and `PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS` values from the environment configuration. This ensures that the monitor scripts have consistent and configurable timeout values, improving the reliability and performance of the monitoring system.
This commit is contained in:
Simon Larsen
2024-05-22 20:06:32 +01:00
parent 39407795a2
commit 5f7dcf7433
13 changed files with 67 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
CommonServer/EnvironmentConfig.ts
View File

@@ -173,3 +173,9 @@ export const HttpProtocol: Protocol =
process.env['HTTP_PROTOCOL'] === 'https' ? Protocol.HTTPS : Protocol.HTTP;
export const Host: string = process.env['HOST'] || '';
export const WorkflowScriptTimeoutInMS: number = process.env[
'WORKFLOW_SCRIPT_TIMEOUT_IN_MS'
]
? parseInt(process.env['WORKFLOW_SCRIPT_TIMEOUT_IN_MS'].toString())
: 5000;

2
CommonServer/Types/Workflow/Components/JavaScript.ts
View File

@@ -6,6 +6,7 @@ import JavaScriptComponents from 'Common/Types/Workflow/Components/JavaScript';
import ComponentCode, { RunOptions, RunReturnType } from '../ComponentCode';
import VMUtil from '../../../Utils/VM/VMAPI';
import ReturnResult from 'Common/Types/IsolatedVM/ReturnResult';
import { WorkflowScriptTimeoutInMS } from '../../../EnvironmentConfig';
export default class JavaScriptCode extends ComponentCode {
public constructor() {
@@ -69,6 +70,7 @@ export default class JavaScriptCode extends ComponentCode {
code,
options: {
args: scriptArgs as JSONObject,
timeout: WorkflowScriptTimeoutInMS,
},
});

6
CommonServer/Utils/Probe/Criteria/CustomCodeMonitorCriteria.ts
View File

@@ -31,7 +31,7 @@ export default class CustomCodeMonitoringCriteria {
if (input.criteriaFilter.checkOn === CheckOn.Error) {
const emptyNotEmptyResult: string | null =
CompareCriteria.compareEmptyAndNotEmpty({
value: syntheticMonitorResponse.result,
value: syntheticMonitorResponse.scriptError,
criteriaFilter: input.criteriaFilter,
});
@@ -41,11 +41,11 @@ export default class CustomCodeMonitoringCriteria {
if (
threshold &&
typeof syntheticMonitorResponse.result === 'string'
typeof syntheticMonitorResponse.scriptError === 'string'
) {
const result: string | null =
CompareCriteria.compareCriteriaStrings({
value: syntheticMonitorResponse.result,
value: syntheticMonitorResponse.scriptError!,
threshold: threshold.toString(),
criteriaFilter: input.criteriaFilter,
});

1
CommonServer/Utils/VM/VMAPI.ts
View File

@@ -14,6 +14,7 @@ export default class VMUtil {
code: string;
options: {
args?: JSONObject | undefined;
timeout?: number | undefined;
};
}): Promise<ReturnResult> {
const returnResultHttpResponse:

3
HelmChart/Public/oneuptime/templates/_helpers.tpl
View File

@@ -173,6 +173,9 @@ Usage:
- name: DISABLE_AUTOMATIC_INCIDENT_CREATION
value: {{ $.Values.incidents.disableAutomaticCreation | squote }}
- name: WORKFLOW_SCRIPT_TIMEOUT_IN_MS
value: {{ $.Values.script.workflowScriptTimeoutInMs | squote }}
{{- end }}
{{- define "oneuptime.env.pod" }}

5
HelmChart/Public/oneuptime/templates/probe.yaml
View File

@@ -57,6 +57,11 @@ spec:
value: {{ $val.description }}
- name: PROBE_MONITORING_WORKERS
value: {{ $val.monitoringWorkers | squote }}
# syntheticMonitorScriptTimeoutInMs
- name: PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS
value: {{ $val.syntheticMonitorScriptTimeoutInMs | squote }}
- name: PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS
value: {{ $val.customCodeMonitorScriptTimeoutInMs | squote }}
- name: PROBE_KEY
{{- if $val.key }}
value: {{ $val.key }}

9
HelmChart/Public/oneuptime/values.yaml
View File

@@ -143,7 +143,9 @@ probes:
monitoringWorkers: 3
monitorFetchLimit: 10
key:
replicaCount: 1
replicaCount: 1
syntheticMonitorScriptTimeoutInMs: 60000
customCodeMonitorScriptTimeoutInMs: 60000
# Feel free to leave this blank if you're not integrating this with OpenTelemetry Backend.
openTelemetryExporter:
headers:
@@ -154,6 +156,8 @@ probes:
# monitorFetchLimit: 10
# key:
# replicaCount: 1
# syntheticMonitorScriptTimeoutInMs: 60000
# customCodeMonitorScriptTimeoutInMs: 60000
# openTelemetryExporter:
# headers:
@@ -237,6 +241,9 @@ oneuptimeIngress:
# secretName: "oneuptime-tls
script:
workflowScriptTimeoutInMs: 5000
# extraTemplates -- Array of extra objects to deploy with the release. Strings
# are evaluated as a template and can use template expansions and functions. All
# other objects are used as yaml.

2
IsolatedVM/API/VM.ts
View File

@@ -40,7 +40,7 @@ router.post(
result = await VMRunner.runCodeInSandbox({
code: req.body.code,
options: {
timeout: 5000,
timeout: req.body?.['options']?.['timeout'] || 5000,
args: req.body?.['options']?.['args'] || {},
},
});

17
Probe/Config.ts
View File

@@ -58,3 +58,20 @@ if (typeof monitorFetchLimit === 'string') {
export const PROBE_MONITOR_FETCH_LIMIT: number = monitorFetchLimit;
export const HOSTNAME: string = process.env['HOSTNAME'] || 'localhost';
export const PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS: number = process.env[
'PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS'
]
? parseInt(
process.env['PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS'].toString()
)
: 60000;
export const PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS: number = process
.env['PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS']
? parseInt(
process.env[
'PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS'
].toString()
)
: 60000;

3
Probe/Utils/Monitors/MonitorTypes/CustomCodeMonitor.ts
View File

@@ -3,6 +3,7 @@ import logger from 'CommonServer/Utils/Logger';
import VMRunner from 'CommonServer/Utils/VM/VMRunner';
import ReturnResult from 'Common/Types/IsolatedVM/ReturnResult';
import CustomCodeMonitorResponse from 'Common/Types/Monitor/CustomCodeMonitor/CustomCodeMonitorResponse';
import { PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS } from '../../../Config';
export interface CustomCodeMonitorOptions {
monitorId?: ObjectID | undefined;
@@ -37,7 +38,7 @@ export default class CustomCodeMonitor {
result = await VMRunner.runCodeInSandbox({
code: options.script,
options: {
timeout: 120000, // 2 minutes
timeout: PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS,
args: {},
},
});

3
Probe/Utils/Monitors/MonitorTypes/SyntheticMonitor.ts
View File

@@ -7,6 +7,7 @@ import VMRunner from 'CommonServer/Utils/VM/VMRunner';
import ReturnResult from 'Common/Types/IsolatedVM/ReturnResult';
import { Browser, firefox, webkit, chromium, Page } from 'playwright';
import BadDataException from 'Common/Types/Exception/BadDataException';
import { PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS } from '../../../Config';
export interface SyntheticMonitorOptions {
monitorId?: ObjectID | undefined;
@@ -88,7 +89,7 @@ export default class SyntheticMonitor {
result = await VMRunner.runCodeInSandbox({
code: options.script,
options: {
timeout: 120000, // 2 minutes
timeout: PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS,
args: {},
context: {
page: pageAndBrowser.page,

10
config.example.env
View File

@@ -151,12 +151,17 @@ GLOBAL_PROBE_1_DESCRIPTION="Global probe to monitor oneuptime resources"
GLOBAL_PROBE_1_MONITORING_WORKERS=5
GLOBAL_PROBE_1_MONITOR_FETCH_LIMIT=10
GLOBAL_PROBE_1_ONEUPTIME_URL=http://ingestor:3400
GLOBAL_PROBE_1_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS=60000
GLOBAL_PROBE_1_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS=60000
GLOBAL_PROBE_2_NAME="Probe-2"
GLOBAL_PROBE_2_DESCRIPTION="Global probe to monitor oneuptime resources"
GLOBAL_PROBE_2_MONITORING_WORKERS=5
GLOBAL_PROBE_2_MONITOR_FETCH_LIMIT=10
GLOBAL_PROBE_2_ONEUPTIME_URL=http://ingestor:3400
GLOBAL_PROBE_2_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS=60000
GLOBAL_PROBE_2_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS=60000
SMS_DEFAULT_COST_IN_CENTS=
CALL_DEFAULT_COST_IN_CENTS_PER_MINUTE=
@@ -204,6 +209,11 @@ E2E_TEST_STATUS_PAGE_URL=
E2E_TESTS_FAILED_WEBHOOK_URL=
# This is the timeout for the workflow script in milliseconds.
WORKFLOW_SCRIPT_TIMEOUT_IN_MS=5000
# Lets encrypt notification email. This email will be used when certs are about to expire
LETS_ENCRYPT_NOTIFICATION_EMAIL=
# Generate a private key via openssl, encode it to base64 and paste it here.

7
docker-compose.base.yml
View File

@@ -54,6 +54,7 @@ x-common-server-variables: &common-server-variables
<<: *common-variables
ONEUPTIME_SECRET: ${ONEUPTIME_SECRET}
DATABASE_PORT: ${DATABASE_PORT}
DATABASE_USERNAME: ${DATABASE_USERNAME}
DATABASE_PASSWORD: ${DATABASE_PASSWORD}
@@ -88,6 +89,8 @@ x-common-server-variables: &common-server-variables
IS_SERVER: "true"
WORKFLOW_SCRIPT_TIMEOUT_IN_MS: ${WORKFLOW_SCRIPT_TIMEOUT_IN_MS}
DISABLE_AUTOMATIC_INCIDENT_CREATION: ${DISABLE_AUTOMATIC_INCIDENT_CREATION}
@@ -265,6 +268,8 @@ services:
PROBE_DESCRIPTION: ${GLOBAL_PROBE_1_DESCRIPTION}
PROBE_MONITORING_WORKERS: ${GLOBAL_PROBE_1_MONITORING_WORKERS}
PROBE_KEY: ${GLOBAL_PROBE_1_KEY}
PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS: ${GLOBAL_PROBE_1_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS}
PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS: ${GLOBAL_PROBE_1_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS}
ONEUPTIME_URL: ${GLOBAL_PROBE_1_ONEUPTIME_URL}
PROBE_MONITOR_FETCH_LIMIT: ${GLOBAL_PROBE_1_MONITOR_FETCH_LIMIT}
OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT: ${SERVER_OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT}
@@ -286,6 +291,8 @@ services:
PROBE_MONITORING_WORKERS: ${GLOBAL_PROBE_2_MONITORING_WORKERS}
PROBE_KEY: ${GLOBAL_PROBE_2_KEY}
ONEUPTIME_URL: ${GLOBAL_PROBE_2_ONEUPTIME_URL}
PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS: ${GLOBAL_PROBE_2_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS}
PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS: ${GLOBAL_PROBE_2_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS}
PROBE_MONITOR_FETCH_LIMIT: ${GLOBAL_PROBE_2_MONITOR_FETCH_LIMIT}
OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT: ${SERVER_OPENTELEMETRY_EXPORTER_OTLP_ENDPOINT}
OPENTELEMETRY_EXPORTER_OTLP_HEADERS: ${PROBE_OPENTELEMETRY_EXPORTER_OTLP_HEADERS}