mirror of
https://github.com/OneUptime/oneuptime.git
synced 2026-04-06 00:32:12 +02:00
feat: Implement VMRunner for executing code in isolated VM with enhanced logging and context management
refactor: Update SyntheticMonitor to utilize VMRunner for script execution and remove legacy worker implementation chore: Bump version to 10.0.16
This commit is contained in:
Nawaz Dhandala
@@ -1,13 +1,71 @@
|
||||
import ReturnResult from "../../../Types/IsolatedVM/ReturnResult";
|
||||
import { JSONObject } from "../../../Types/JSON";
|
||||
import { JSONObject, JSONValue } from "../../../Types/JSON";
|
||||
import axios, { AxiosResponse } from "axios";
|
||||
import crypto from "crypto";
|
||||
import http from "http";
|
||||
import https from "https";
|
||||
import ivm from "isolated-vm";
|
||||
import CaptureSpan from "../Telemetry/CaptureSpan";
|
||||
import Dictionary from "../../../Types/Dictionary";
|
||||
import GenericObject from "../../../Types/GenericObject";
|
||||
import vm, { Context } from "vm";
|
||||
|
||||
|
||||
export default class VMRunner {
|
||||
|
||||
@CaptureSpan()
|
||||
public static async runCodeInNodeVM(data: {
|
||||
code: string;
|
||||
options: {
|
||||
timeout?: number;
|
||||
args?: JSONObject | undefined;
|
||||
context?: Dictionary<GenericObject | string> | undefined;
|
||||
};
|
||||
}): Promise<ReturnResult> {
|
||||
const { code, options } = data;
|
||||
|
||||
const logMessages: string[] = [];
|
||||
|
||||
let sandbox: Context = {
|
||||
process: {},
|
||||
console: {
|
||||
log: (...args: JSONValue[]) => {
|
||||
logMessages.push(args.join(" "));
|
||||
},
|
||||
},
|
||||
http: http,
|
||||
https: https,
|
||||
axios: axios,
|
||||
crypto: crypto,
|
||||
setTimeout: setTimeout,
|
||||
clearTimeout: clearTimeout,
|
||||
setInterval: setInterval,
|
||||
...options.context,
|
||||
};
|
||||
|
||||
if (options.args) {
|
||||
sandbox = {
|
||||
...sandbox,
|
||||
args: options.args,
|
||||
};
|
||||
}
|
||||
|
||||
vm.createContext(sandbox); // Contextify the object.
|
||||
|
||||
const script: string = `(async()=>{
|
||||
${code}
|
||||
})()`;
|
||||
|
||||
const returnVal: any = await vm.runInContext(script, sandbox, {
|
||||
timeout: options.timeout || 5000,
|
||||
}); // run the script
|
||||
|
||||
return {
|
||||
returnValue: returnVal,
|
||||
logMessages,
|
||||
};
|
||||
}
|
||||
|
||||
@CaptureSpan()
|
||||
public static async runCodeInSandbox(data: {
|
||||
code: string;
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
import { PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS } from "../../../Config";
|
||||
import ProxyConfig from "../../ProxyConfig";
|
||||
import SyntheticMonitorSemaphore from "../../SyntheticMonitorSemaphore";
|
||||
import SyntheticMonitorWorkerPool from "../../SyntheticMonitorWorkerPool";
|
||||
import BadDataException from "Common/Types/Exception/BadDataException";
|
||||
import ReturnResult from "Common/Types/IsolatedVM/ReturnResult";
|
||||
import BrowserType from "Common/Types/Monitor/SyntheticMonitors/BrowserType";
|
||||
import ScreenSizeType from "Common/Types/Monitor/SyntheticMonitors/ScreenSizeType";
|
||||
import SyntheticMonitorResponse from "Common/Types/Monitor/SyntheticMonitors/SyntheticMonitorResponse";
|
||||
import ObjectID from "Common/Types/ObjectID";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
import VMRunner from "Common/Server/Utils/VM/VMRunner";
|
||||
import { Browser, BrowserContext, Page, chromium, firefox } from "playwright";
|
||||
import LocalFile from "Common/Server/Utils/LocalFile";
|
||||
|
||||
export interface SyntheticMonitorOptions {
|
||||
monitorId?: ObjectID | undefined;
|
||||
@@ -16,33 +19,24 @@ export interface SyntheticMonitorOptions {
|
||||
retryCountOnError?: number | undefined;
|
||||
}
|
||||
|
||||
interface WorkerConfig {
|
||||
script: string;
|
||||
browserType: BrowserType;
|
||||
screenSizeType: ScreenSizeType;
|
||||
timeout: number;
|
||||
proxy?:
|
||||
| {
|
||||
server: string;
|
||||
username?: string | undefined;
|
||||
password?: string | undefined;
|
||||
}
|
||||
| undefined;
|
||||
interface BrowserLaunchOptions {
|
||||
executablePath?: string;
|
||||
proxy?: {
|
||||
server: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
bypass?: string;
|
||||
};
|
||||
args?: string[];
|
||||
headless?: boolean;
|
||||
devtools?: boolean;
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
interface WorkerResult {
|
||||
logMessages: string[];
|
||||
scriptError?: string | undefined;
|
||||
result?: unknown | undefined;
|
||||
screenshots: Record<string, string>;
|
||||
executionTimeInMS: number;
|
||||
}
|
||||
|
||||
type ExecuteWithRetrySkipReason = "deduplication" | "infrastructure";
|
||||
|
||||
interface ExecuteWithRetryResult {
|
||||
response: SyntheticMonitorResponse | null;
|
||||
skipReason?: ExecuteWithRetrySkipReason | undefined;
|
||||
interface BrowserSession {
|
||||
browser: Browser;
|
||||
context: BrowserContext;
|
||||
page: Page;
|
||||
}
|
||||
|
||||
export default class SyntheticMonitor {
|
||||
@@ -50,63 +44,29 @@ export default class SyntheticMonitor {
|
||||
options: SyntheticMonitorOptions,
|
||||
): Promise<Array<SyntheticMonitorResponse> | null> {
|
||||
const results: Array<SyntheticMonitorResponse> = [];
|
||||
let totalExecutions: number = 0;
|
||||
let dedupSkips: number = 0;
|
||||
let infraSkips: number = 0;
|
||||
|
||||
for (const browserType of options.browserTypes || []) {
|
||||
for (const screenSizeType of options.screenSizeTypes || []) {
|
||||
totalExecutions++;
|
||||
|
||||
logger.debug(
|
||||
`Running Synthetic Monitor: ${options?.monitorId?.toString()}, Screen Size: ${screenSizeType}, Browser: ${browserType}`,
|
||||
);
|
||||
|
||||
const retryResult: ExecuteWithRetryResult = await this.executeWithRetry(
|
||||
{
|
||||
const result: SyntheticMonitorResponse | null =
|
||||
await this.executeWithRetry({
|
||||
script: options.script,
|
||||
browserType: browserType,
|
||||
screenSizeType: screenSizeType,
|
||||
retryCountOnError: options.retryCountOnError || 0,
|
||||
monitorId: options.monitorId,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
if (retryResult.response) {
|
||||
retryResult.response.browserType = browserType;
|
||||
retryResult.response.screenSizeType = screenSizeType;
|
||||
results.push(retryResult.response);
|
||||
} else if (retryResult.skipReason === "deduplication") {
|
||||
dedupSkips++;
|
||||
} else if (retryResult.skipReason === "infrastructure") {
|
||||
infraSkips++;
|
||||
if (result) {
|
||||
result.browserType = browserType;
|
||||
result.screenSizeType = screenSizeType;
|
||||
results.push(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* If we attempted executions but got zero results, return null to skip
|
||||
* this entire check cycle. This prevents the monitor from flapping to
|
||||
* the default status when the probe infrastructure is under load but the
|
||||
* monitored service may be perfectly healthy.
|
||||
*/
|
||||
if (totalExecutions > 0 && results.length === 0) {
|
||||
if (dedupSkips > 0 && infraSkips === 0) {
|
||||
/*
|
||||
* All skips were due to deduplication — another worker is already
|
||||
* processing this monitor, which is normal and expected.
|
||||
*/
|
||||
logger.debug(
|
||||
`Synthetic Monitor ${options?.monitorId?.toString()}: all ${totalExecutions} executions skipped (already being processed by another worker), skipping this check cycle`,
|
||||
);
|
||||
} else {
|
||||
logger.warn(
|
||||
`Synthetic Monitor ${options?.monitorId?.toString()}: all ${totalExecutions} executions were skipped due to infrastructure issues (${infraSkips} infrastructure, ${dedupSkips} deduplication), skipping this check cycle`,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -116,60 +76,8 @@ export default class SyntheticMonitor {
|
||||
screenSizeType: ScreenSizeType;
|
||||
retryCountOnError: number;
|
||||
currentRetry?: number;
|
||||
monitorId?: ObjectID | undefined;
|
||||
}): Promise<ExecuteWithRetryResult> {
|
||||
const maxRetries: number = options.retryCountOnError;
|
||||
const monitorIdStr: string | undefined =
|
||||
options.monitorId?.toString() || undefined;
|
||||
|
||||
// Acquire semaphore once for all retries so retries reuse the same slot
|
||||
let acquired: boolean = false;
|
||||
|
||||
try {
|
||||
acquired = await SyntheticMonitorSemaphore.acquire(monitorIdStr);
|
||||
} catch (err: unknown) {
|
||||
/*
|
||||
* Semaphore errors (queue full, timeout waiting for slot) are infrastructure
|
||||
* issues, not script failures. Skip this check cycle so the monitor stays in
|
||||
* its last known state instead of flapping to offline.
|
||||
*/
|
||||
logger.error(
|
||||
`Synthetic monitor semaphore acquire failed (skipping this cycle): ${(err as Error)?.message}`,
|
||||
);
|
||||
return { response: null, skipReason: "infrastructure" };
|
||||
}
|
||||
|
||||
if (!acquired) {
|
||||
// This monitor is already running or queued — skip duplicate execution
|
||||
return { response: null, skipReason: "deduplication" };
|
||||
}
|
||||
|
||||
try {
|
||||
const response: SyntheticMonitorResponse | null =
|
||||
await this.executeWithRetryInner({
|
||||
script: options.script,
|
||||
browserType: options.browserType,
|
||||
screenSizeType: options.screenSizeType,
|
||||
retryCountOnError: maxRetries,
|
||||
currentRetry: options.currentRetry || 0,
|
||||
});
|
||||
return {
|
||||
response,
|
||||
skipReason: response ? undefined : "infrastructure",
|
||||
};
|
||||
} finally {
|
||||
SyntheticMonitorSemaphore.release(monitorIdStr);
|
||||
}
|
||||
}
|
||||
|
||||
private static async executeWithRetryInner(options: {
|
||||
script: string;
|
||||
browserType: BrowserType;
|
||||
screenSizeType: ScreenSizeType;
|
||||
retryCountOnError: number;
|
||||
currentRetry: number;
|
||||
}): Promise<SyntheticMonitorResponse | null> {
|
||||
const currentRetry: number = options.currentRetry;
|
||||
const currentRetry: number = options.currentRetry || 0;
|
||||
const maxRetries: number = options.retryCountOnError;
|
||||
|
||||
const result: SyntheticMonitorResponse | null =
|
||||
@@ -190,7 +98,7 @@ export default class SyntheticMonitor {
|
||||
setTimeout(resolve, 1000);
|
||||
});
|
||||
|
||||
return this.executeWithRetryInner({
|
||||
return this.executeWithRetry({
|
||||
script: options.script,
|
||||
browserType: options.browserType,
|
||||
screenSizeType: options.screenSizeType,
|
||||
@@ -202,42 +110,13 @@ export default class SyntheticMonitor {
|
||||
return result;
|
||||
}
|
||||
|
||||
private static getProxyConfig(): WorkerConfig["proxy"] | undefined {
|
||||
if (!ProxyConfig.isProxyConfigured()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const httpsProxyUrl: string | null = ProxyConfig.getHttpsProxyUrl();
|
||||
const httpProxyUrl: string | null = ProxyConfig.getHttpProxyUrl();
|
||||
const proxyUrl: string | null = httpsProxyUrl || httpProxyUrl;
|
||||
|
||||
if (!proxyUrl) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const proxyConfig: WorkerConfig["proxy"] = {
|
||||
server: proxyUrl,
|
||||
};
|
||||
|
||||
try {
|
||||
const parsedUrl: globalThis.URL = new URL(proxyUrl);
|
||||
if (parsedUrl.username && parsedUrl.password) {
|
||||
proxyConfig.username = parsedUrl.username;
|
||||
proxyConfig.password = parsedUrl.password;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to parse proxy URL for authentication: ${error}`);
|
||||
}
|
||||
|
||||
return proxyConfig;
|
||||
}
|
||||
|
||||
private static async executeByBrowserAndScreenSize(options: {
|
||||
script: string;
|
||||
browserType: BrowserType;
|
||||
screenSizeType: ScreenSizeType;
|
||||
}): Promise<SyntheticMonitorResponse | null> {
|
||||
if (!options) {
|
||||
// this should never happen
|
||||
options = {
|
||||
script: "",
|
||||
browserType: BrowserType.Chromium,
|
||||
@@ -255,39 +134,378 @@ export default class SyntheticMonitor {
|
||||
screenSizeType: options.screenSizeType,
|
||||
};
|
||||
|
||||
const timeout: number = PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS;
|
||||
|
||||
const workerConfig: WorkerConfig = {
|
||||
script: options.script,
|
||||
browserType: options.browserType,
|
||||
screenSizeType: options.screenSizeType,
|
||||
timeout: timeout,
|
||||
proxy: this.getProxyConfig(),
|
||||
};
|
||||
let browserSession: BrowserSession | null = null;
|
||||
|
||||
try {
|
||||
const workerResult: WorkerResult =
|
||||
await SyntheticMonitorWorkerPool.execute(workerConfig, timeout);
|
||||
let result: ReturnResult | null = null;
|
||||
|
||||
scriptResult.logMessages = workerResult.logMessages;
|
||||
scriptResult.scriptError = workerResult.scriptError;
|
||||
scriptResult.result = workerResult.result as typeof scriptResult.result;
|
||||
scriptResult.screenshots = workerResult.screenshots;
|
||||
scriptResult.executionTimeInMS = workerResult.executionTimeInMS;
|
||||
} catch (err: unknown) {
|
||||
/*
|
||||
* Errors thrown by the worker pool are always infrastructure issues (worker
|
||||
* timeout, OOM kill, process crash, IPC failure) — NOT script failures.
|
||||
* Actual script errors are returned inside WorkerResult.scriptError without
|
||||
* throwing. Skip this check cycle so the monitor stays in its last known
|
||||
* state instead of flapping between online and offline.
|
||||
*/
|
||||
logger.error(
|
||||
`Synthetic monitor infrastructure error (skipping this cycle): ${(err as Error)?.message || String(err)}`,
|
||||
const startTime: [number, number] = process.hrtime();
|
||||
|
||||
browserSession = await SyntheticMonitor.getPageByBrowserType({
|
||||
browserType: options.browserType,
|
||||
screenSizeType: options.screenSizeType,
|
||||
});
|
||||
|
||||
if (!browserSession) {
|
||||
throw new BadDataException(
|
||||
"Could not create Playwright browser session",
|
||||
);
|
||||
}
|
||||
|
||||
result = await VMRunner.runCodeInNodeVM({
|
||||
code: options.script,
|
||||
options: {
|
||||
timeout: PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS,
|
||||
args: {},
|
||||
context: {
|
||||
browser: browserSession.browser,
|
||||
page: browserSession.page,
|
||||
screenSizeType: options.screenSizeType,
|
||||
browserType: options.browserType,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const endTime: [number, number] = process.hrtime(startTime);
|
||||
|
||||
const executionTimeInMS: number = Math.ceil(
|
||||
(endTime[0] * 1000000000 + endTime[1]) / 1000000,
|
||||
);
|
||||
return null;
|
||||
|
||||
scriptResult.executionTimeInMS = executionTimeInMS;
|
||||
|
||||
scriptResult.logMessages = result.logMessages;
|
||||
|
||||
if (result.returnValue?.screenshots) {
|
||||
if (!scriptResult.screenshots) {
|
||||
scriptResult.screenshots = {};
|
||||
}
|
||||
|
||||
for (const screenshotName in result.returnValue.screenshots) {
|
||||
if (!result.returnValue.screenshots[screenshotName]) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// check if this is of type Buffer. If it is not, continue.
|
||||
|
||||
if (
|
||||
!(result.returnValue.screenshots[screenshotName] instanceof Buffer)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const screenshotBuffer: Buffer = result.returnValue.screenshots[
|
||||
screenshotName
|
||||
] as Buffer;
|
||||
scriptResult.screenshots[screenshotName] =
|
||||
screenshotBuffer.toString("base64"); // convert screenshots to base 64
|
||||
}
|
||||
}
|
||||
|
||||
scriptResult.result = result?.returnValue?.data;
|
||||
} catch (err: unknown) {
|
||||
logger.error(err);
|
||||
scriptResult.scriptError =
|
||||
(err as Error)?.message || (err as Error).toString();
|
||||
} finally {
|
||||
// Always dispose browser session to prevent zombie processes
|
||||
await SyntheticMonitor.disposeBrowserSession(browserSession);
|
||||
}
|
||||
|
||||
return scriptResult;
|
||||
}
|
||||
|
||||
private static getViewportHeightAndWidth(options: {
|
||||
screenSizeType: ScreenSizeType;
|
||||
}): {
|
||||
height: number;
|
||||
width: number;
|
||||
} {
|
||||
let viewPortHeight: number = 0;
|
||||
let viewPortWidth: number = 0;
|
||||
|
||||
switch (options.screenSizeType) {
|
||||
case ScreenSizeType.Desktop:
|
||||
viewPortHeight = 1080;
|
||||
viewPortWidth = 1920;
|
||||
break;
|
||||
case ScreenSizeType.Mobile:
|
||||
viewPortHeight = 640;
|
||||
viewPortWidth = 360;
|
||||
break;
|
||||
case ScreenSizeType.Tablet:
|
||||
viewPortHeight = 768;
|
||||
viewPortWidth = 1024;
|
||||
break;
|
||||
default:
|
||||
viewPortHeight = 1080;
|
||||
viewPortWidth = 1920;
|
||||
break;
|
||||
}
|
||||
|
||||
return { height: viewPortHeight, width: viewPortWidth };
|
||||
}
|
||||
|
||||
public static async getChromeExecutablePath(): Promise<string> {
|
||||
const doesDirectoryExist: boolean = await LocalFile.doesDirectoryExist(
|
||||
"/root/.cache/ms-playwright",
|
||||
);
|
||||
if (!doesDirectoryExist) {
|
||||
throw new BadDataException("Chrome executable path not found.");
|
||||
}
|
||||
|
||||
// get list of files in the directory
|
||||
const directories: string[] = await LocalFile.getListOfDirectories(
|
||||
"/root/.cache/ms-playwright",
|
||||
);
|
||||
|
||||
if (directories.length === 0) {
|
||||
throw new BadDataException("Chrome executable path not found.");
|
||||
}
|
||||
|
||||
const chromeInstallationName: string | undefined = directories.find(
|
||||
(directory: string) => {
|
||||
return directory.includes("chromium");
|
||||
},
|
||||
);
|
||||
|
||||
if (!chromeInstallationName) {
|
||||
throw new BadDataException("Chrome executable path not found.");
|
||||
}
|
||||
|
||||
const chromeExecutableCandidates: Array<string> = [
|
||||
`/root/.cache/ms-playwright/${chromeInstallationName}/chrome-linux/chrome`,
|
||||
`/root/.cache/ms-playwright/${chromeInstallationName}/chrome-linux64/chrome`,
|
||||
`/root/.cache/ms-playwright/${chromeInstallationName}/chrome64/chrome`,
|
||||
`/root/.cache/ms-playwright/${chromeInstallationName}/chrome/chrome`,
|
||||
];
|
||||
|
||||
for (const executablePath of chromeExecutableCandidates) {
|
||||
if (await LocalFile.doesFileExist(executablePath)) {
|
||||
return executablePath;
|
||||
}
|
||||
}
|
||||
|
||||
throw new BadDataException("Chrome executable path not found.");
|
||||
}
|
||||
|
||||
public static async getFirefoxExecutablePath(): Promise<string> {
|
||||
const doesDirectoryExist: boolean = await LocalFile.doesDirectoryExist(
|
||||
"/root/.cache/ms-playwright",
|
||||
);
|
||||
if (!doesDirectoryExist) {
|
||||
throw new BadDataException("Firefox executable path not found.");
|
||||
}
|
||||
|
||||
// get list of files in the directory
|
||||
const directories: string[] = await LocalFile.getListOfDirectories(
|
||||
"/root/.cache/ms-playwright",
|
||||
);
|
||||
|
||||
if (directories.length === 0) {
|
||||
throw new BadDataException("Firefox executable path not found.");
|
||||
}
|
||||
|
||||
const firefoxInstallationName: string | undefined = directories.find(
|
||||
(directory: string) => {
|
||||
return directory.includes("firefox");
|
||||
},
|
||||
);
|
||||
|
||||
if (!firefoxInstallationName) {
|
||||
throw new BadDataException("Firefox executable path not found.");
|
||||
}
|
||||
|
||||
const firefoxExecutableCandidates: Array<string> = [
|
||||
`/root/.cache/ms-playwright/${firefoxInstallationName}/firefox/firefox`,
|
||||
`/root/.cache/ms-playwright/${firefoxInstallationName}/firefox-linux64/firefox`,
|
||||
`/root/.cache/ms-playwright/${firefoxInstallationName}/firefox64/firefox`,
|
||||
`/root/.cache/ms-playwright/${firefoxInstallationName}/firefox-64/firefox`,
|
||||
];
|
||||
|
||||
for (const executablePath of firefoxExecutableCandidates) {
|
||||
if (await LocalFile.doesFileExist(executablePath)) {
|
||||
return executablePath;
|
||||
}
|
||||
}
|
||||
|
||||
throw new BadDataException("Firefox executable path not found.");
|
||||
}
|
||||
|
||||
private static async getPageByBrowserType(data: {
|
||||
browserType: BrowserType;
|
||||
screenSizeType: ScreenSizeType;
|
||||
}): Promise<BrowserSession> {
|
||||
const viewport: {
|
||||
height: number;
|
||||
width: number;
|
||||
} = SyntheticMonitor.getViewportHeightAndWidth({
|
||||
screenSizeType: data.screenSizeType,
|
||||
});
|
||||
|
||||
// Prepare browser launch options with proxy support
|
||||
const baseOptions: BrowserLaunchOptions = {};
|
||||
|
||||
// Configure proxy if available
|
||||
if (ProxyConfig.isProxyConfigured()) {
|
||||
const httpsProxyUrl: string | null = ProxyConfig.getHttpsProxyUrl();
|
||||
const httpProxyUrl: string | null = ProxyConfig.getHttpProxyUrl();
|
||||
|
||||
// Prefer HTTPS proxy, fall back to HTTP proxy
|
||||
const proxyUrl: string | null = httpsProxyUrl || httpProxyUrl;
|
||||
|
||||
if (proxyUrl) {
|
||||
baseOptions.proxy = {
|
||||
server: proxyUrl,
|
||||
};
|
||||
|
||||
// Extract username and password if present in proxy URL
|
||||
try {
|
||||
const parsedUrl: globalThis.URL = new URL(proxyUrl);
|
||||
if (parsedUrl.username && parsedUrl.password) {
|
||||
baseOptions.proxy.username = parsedUrl.username;
|
||||
baseOptions.proxy.password = parsedUrl.password;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to parse proxy URL for authentication: ${error}`);
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
`Synthetic Monitor using proxy: ${proxyUrl} (HTTPS: ${Boolean(httpsProxyUrl)}, HTTP: ${Boolean(httpProxyUrl)})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (data.browserType === BrowserType.Chromium) {
|
||||
const browser: Browser = await chromium.launch({
|
||||
executablePath: await this.getChromeExecutablePath(),
|
||||
...baseOptions,
|
||||
});
|
||||
|
||||
const context: BrowserContext = await browser.newContext({
|
||||
viewport: {
|
||||
width: viewport.width,
|
||||
height: viewport.height,
|
||||
},
|
||||
});
|
||||
|
||||
const page: Page = await context.newPage();
|
||||
|
||||
return {
|
||||
browser,
|
||||
context,
|
||||
page,
|
||||
};
|
||||
}
|
||||
|
||||
if (data.browserType === BrowserType.Firefox) {
|
||||
const browser: Browser = await firefox.launch({
|
||||
executablePath: await this.getFirefoxExecutablePath(),
|
||||
...baseOptions,
|
||||
});
|
||||
|
||||
let context: BrowserContext | null = null;
|
||||
|
||||
try {
|
||||
context = await browser.newContext({
|
||||
viewport: {
|
||||
width: viewport.width,
|
||||
height: viewport.height,
|
||||
},
|
||||
});
|
||||
|
||||
const page: Page = await context.newPage();
|
||||
|
||||
return {
|
||||
browser,
|
||||
context,
|
||||
page,
|
||||
};
|
||||
} catch (error) {
|
||||
await SyntheticMonitor.safeCloseBrowserContext(context);
|
||||
await SyntheticMonitor.safeCloseBrowser(browser);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
throw new BadDataException("Invalid Browser Type.");
|
||||
}
|
||||
|
||||
private static async disposeBrowserSession(
|
||||
session: BrowserSession | null,
|
||||
): Promise<void> {
|
||||
if (!session) {
|
||||
return;
|
||||
}
|
||||
|
||||
await SyntheticMonitor.safeClosePage(session.page);
|
||||
await SyntheticMonitor.safeCloseBrowserContexts({
|
||||
browser: session.browser,
|
||||
});
|
||||
await SyntheticMonitor.safeCloseBrowser(session.browser);
|
||||
}
|
||||
|
||||
private static async safeClosePage(page?: Page | null): Promise<void> {
|
||||
if (!page) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (!page.isClosed()) {
|
||||
await page.close();
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`Failed to close Playwright page: ${(error as Error)?.message || error}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private static async safeCloseBrowserContext(
|
||||
context?: BrowserContext | null,
|
||||
): Promise<void> {
|
||||
if (!context) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await context.close();
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`Failed to close Playwright browser context: ${(error as Error)?.message || error}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private static async safeCloseBrowser(
|
||||
browser?: Browser | null,
|
||||
): Promise<void> {
|
||||
if (!browser) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (browser.isConnected()) {
|
||||
await browser.close();
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`Failed to close Playwright browser: ${(error as Error)?.message || error}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private static async safeCloseBrowserContexts(data: {
|
||||
browser: Browser;
|
||||
}): Promise<void> {
|
||||
if (!data.browser || !data.browser.contexts) {
|
||||
return;
|
||||
}
|
||||
|
||||
const contexts: Array<BrowserContext> = data.browser.contexts();
|
||||
|
||||
for (const context of contexts) {
|
||||
await SyntheticMonitor.safeCloseBrowserContext(context);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,759 +0,0 @@
|
||||
/*
|
||||
* Long-lived worker process for synthetic monitors.
|
||||
* Launched via child_process.fork() by SyntheticMonitorWorkerPool.
|
||||
* Keeps a warm Playwright browser between executions to save ~150MB + ~300ms per run.
|
||||
* Supports the new IPC protocol (execute/shutdown) and legacy protocol (plain WorkerConfig)
|
||||
* for backward compatibility.
|
||||
*/
|
||||
|
||||
import BrowserType from "Common/Types/Monitor/SyntheticMonitors/BrowserType";
|
||||
import ScreenSizeType from "Common/Types/Monitor/SyntheticMonitors/ScreenSizeType";
|
||||
import BrowserUtil from "Common/Server/Utils/Browser";
|
||||
import axios from "axios";
|
||||
import crypto from "crypto";
|
||||
import vm, { Context } from "node:vm";
|
||||
import { Browser, BrowserContext, Page, chromium, firefox } from "playwright";
|
||||
|
||||
interface WorkerConfig {
|
||||
script: string;
|
||||
browserType: BrowserType;
|
||||
screenSizeType: ScreenSizeType;
|
||||
timeout: number;
|
||||
proxy?:
|
||||
| {
|
||||
server: string;
|
||||
username?: string | undefined;
|
||||
password?: string | undefined;
|
||||
}
|
||||
| undefined;
|
||||
}
|
||||
|
||||
interface WorkerResult {
|
||||
logMessages: string[];
|
||||
scriptError?: string | undefined;
|
||||
result?: unknown | undefined;
|
||||
screenshots: Record<string, string>;
|
||||
executionTimeInMS: number;
|
||||
}
|
||||
|
||||
interface ProxyOptions {
|
||||
server: string;
|
||||
username?: string | undefined;
|
||||
password?: string | undefined;
|
||||
}
|
||||
|
||||
// IPC messages: parent → worker
|
||||
interface ExecuteMessage {
|
||||
type: "execute";
|
||||
id: string;
|
||||
config: WorkerConfig;
|
||||
}
|
||||
|
||||
interface ShutdownMessage {
|
||||
type: "shutdown";
|
||||
}
|
||||
|
||||
// IPC messages: worker → parent
|
||||
interface ReadyMessage {
|
||||
type: "ready";
|
||||
browserType?: BrowserType | undefined;
|
||||
}
|
||||
|
||||
interface ResultMessage {
|
||||
type: "result";
|
||||
id: string;
|
||||
data: WorkerResult;
|
||||
}
|
||||
|
||||
interface ErrorMessage {
|
||||
type: "error";
|
||||
id: string;
|
||||
error: string;
|
||||
}
|
||||
|
||||
// Warm browser state
|
||||
let currentBrowser: Browser | null = null;
|
||||
let currentBrowserType: BrowserType | null = null;
|
||||
let currentProxyServer: string | null = null;
|
||||
|
||||
const MAX_BROWSER_LAUNCH_RETRIES: number = 3;
|
||||
const BROWSER_LAUNCH_RETRY_DELAY_MS: number = 2000;
|
||||
|
||||
async function launchBrowserOnly(
|
||||
browserType: BrowserType,
|
||||
proxy?: WorkerConfig["proxy"],
|
||||
): Promise<Browser> {
|
||||
let proxyOptions: ProxyOptions | undefined;
|
||||
|
||||
if (proxy) {
|
||||
proxyOptions = {
|
||||
server: proxy.server,
|
||||
};
|
||||
|
||||
if (proxy.username && proxy.password) {
|
||||
proxyOptions.username = proxy.username;
|
||||
proxyOptions.password = proxy.password;
|
||||
}
|
||||
}
|
||||
|
||||
if (browserType === BrowserType.Chromium) {
|
||||
const launchOptions: Record<string, unknown> = {
|
||||
executablePath: await BrowserUtil.getChromeExecutablePath(),
|
||||
headless: true,
|
||||
args: BrowserUtil.chromiumStabilityArgs,
|
||||
};
|
||||
|
||||
if (proxyOptions) {
|
||||
launchOptions["proxy"] = proxyOptions;
|
||||
}
|
||||
|
||||
return chromium.launch(launchOptions);
|
||||
} else if (browserType === BrowserType.Firefox) {
|
||||
const launchOptions: Record<string, unknown> = {
|
||||
executablePath: await BrowserUtil.getFirefoxExecutablePath(),
|
||||
headless: true,
|
||||
firefoxUserPrefs: BrowserUtil.firefoxStabilityPrefs,
|
||||
};
|
||||
|
||||
if (proxyOptions) {
|
||||
launchOptions["proxy"] = proxyOptions;
|
||||
}
|
||||
|
||||
return firefox.launch(launchOptions);
|
||||
}
|
||||
|
||||
throw new Error("Invalid Browser Type.");
|
||||
}
|
||||
|
||||
async function launchBrowserWithRetry(
|
||||
browserType: BrowserType,
|
||||
proxy?: WorkerConfig["proxy"],
|
||||
): Promise<Browser> {
|
||||
let lastError: Error | undefined;
|
||||
|
||||
for (
|
||||
let attempt: number = 1;
|
||||
attempt <= MAX_BROWSER_LAUNCH_RETRIES;
|
||||
attempt++
|
||||
) {
|
||||
try {
|
||||
return await launchBrowserOnly(browserType, proxy);
|
||||
} catch (err: unknown) {
|
||||
lastError = err as Error;
|
||||
|
||||
if (attempt < MAX_BROWSER_LAUNCH_RETRIES) {
|
||||
await new Promise((resolve: (value: void) => void) => {
|
||||
setTimeout(resolve, BROWSER_LAUNCH_RETRY_DELAY_MS);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Failed to launch browser after ${MAX_BROWSER_LAUNCH_RETRIES} attempts. ` +
|
||||
`This is usually caused by insufficient memory in the container. ` +
|
||||
`Last error: ${lastError?.message || String(lastError)}`,
|
||||
);
|
||||
}
|
||||
|
||||
let executionsSinceLastLaunch: number = 0;
|
||||
|
||||
async function ensureBrowser(config: WorkerConfig): Promise<Browser> {
|
||||
const configProxyServer: string | null = config.proxy?.server || null;
|
||||
|
||||
// If we have a browser of the right type, same proxy, and it's still connected, reuse it
|
||||
if (
|
||||
currentBrowser &&
|
||||
currentBrowserType === config.browserType &&
|
||||
currentProxyServer === configProxyServer &&
|
||||
currentBrowser.isConnected()
|
||||
) {
|
||||
/*
|
||||
* Active health check: verify the browser can actually create pages,
|
||||
* not just that the WebSocket connection is alive. This catches zombie
|
||||
* browsers where the process is alive but internally broken.
|
||||
*/
|
||||
let isHealthy: boolean = true;
|
||||
|
||||
try {
|
||||
const healthContext: BrowserContext = await currentBrowser.newContext();
|
||||
const healthPage: Page = await healthContext.newPage();
|
||||
await healthPage.close();
|
||||
await healthContext.close();
|
||||
} catch {
|
||||
isHealthy = false;
|
||||
|
||||
if (process.send) {
|
||||
try {
|
||||
process.send({
|
||||
type: "log",
|
||||
message: `[SyntheticMonitorWorker] Warm browser failed health check, will relaunch`,
|
||||
});
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (currentBrowser.isConnected()) {
|
||||
await currentBrowser.close();
|
||||
}
|
||||
} catch {
|
||||
// ignore cleanup errors
|
||||
}
|
||||
currentBrowser = null;
|
||||
currentBrowserType = null;
|
||||
currentProxyServer = null;
|
||||
}
|
||||
|
||||
if (isHealthy && currentBrowser) {
|
||||
executionsSinceLastLaunch++;
|
||||
if (process.send) {
|
||||
try {
|
||||
process.send({
|
||||
type: "log",
|
||||
message: `[SyntheticMonitorWorker] Reusing warm ${config.browserType} browser (execution #${executionsSinceLastLaunch} since launch)`,
|
||||
});
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
return currentBrowser;
|
||||
}
|
||||
}
|
||||
|
||||
// Close existing browser if type/proxy changed or browser crashed
|
||||
const reason: string = !currentBrowser
|
||||
? "no browser"
|
||||
: currentBrowserType !== config.browserType
|
||||
? `type change (${currentBrowserType} → ${config.browserType})`
|
||||
: currentProxyServer !== configProxyServer
|
||||
? "proxy change"
|
||||
: "browser disconnected";
|
||||
|
||||
if (process.send) {
|
||||
try {
|
||||
process.send({
|
||||
type: "log",
|
||||
message: `[SyntheticMonitorWorker] Launching new browser — reason: ${reason}`,
|
||||
});
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
if (currentBrowser) {
|
||||
try {
|
||||
if (currentBrowser.isConnected()) {
|
||||
await currentBrowser.close();
|
||||
}
|
||||
} catch {
|
||||
// ignore cleanup errors
|
||||
}
|
||||
currentBrowser = null;
|
||||
currentBrowserType = null;
|
||||
currentProxyServer = null;
|
||||
}
|
||||
|
||||
executionsSinceLastLaunch = 1;
|
||||
|
||||
// Launch new browser
|
||||
currentBrowser = await launchBrowserWithRetry(
|
||||
config.browserType,
|
||||
config.proxy,
|
||||
);
|
||||
currentBrowserType = config.browserType;
|
||||
currentProxyServer = configProxyServer;
|
||||
|
||||
// Notify parent of browser type for affinity matching
|
||||
sendMessage({
|
||||
type: "ready",
|
||||
browserType: currentBrowserType,
|
||||
});
|
||||
|
||||
return currentBrowser;
|
||||
}
|
||||
|
||||
const MAX_CONTEXT_CREATE_RETRIES: number = 3;
|
||||
const CONTEXT_CREATE_RETRY_DELAY_MS: number = 1000;
|
||||
|
||||
async function createContextAndPage(
|
||||
config: WorkerConfig,
|
||||
): Promise<{ browser: Browser; context: BrowserContext; page: Page }> {
|
||||
const viewport: { height: number; width: number } =
|
||||
BrowserUtil.getViewportHeightAndWidth({
|
||||
screenSizeType: config.screenSizeType,
|
||||
});
|
||||
|
||||
let lastError: Error | undefined;
|
||||
|
||||
for (
|
||||
let attempt: number = 1;
|
||||
attempt <= MAX_CONTEXT_CREATE_RETRIES;
|
||||
attempt++
|
||||
) {
|
||||
const browserStartTime: [number, number] = process.hrtime();
|
||||
const browser: Browser = await ensureBrowser(config);
|
||||
const browserElapsed: [number, number] = process.hrtime(browserStartTime);
|
||||
const browserMs: number = Math.ceil(
|
||||
(browserElapsed[0] * 1000000000 + browserElapsed[1]) / 1000000,
|
||||
);
|
||||
|
||||
try {
|
||||
const context: BrowserContext = await browser.newContext({
|
||||
viewport: {
|
||||
width: viewport.width,
|
||||
height: viewport.height,
|
||||
},
|
||||
});
|
||||
|
||||
const page: Page = await context.newPage();
|
||||
|
||||
if (process.send) {
|
||||
try {
|
||||
process.send({
|
||||
type: "log",
|
||||
message: `[SyntheticMonitorWorker] Context+page created (attempt ${attempt}, ensureBrowser took ${browserMs}ms)`,
|
||||
});
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
return { browser, context, page };
|
||||
} catch (err: unknown) {
|
||||
lastError = err as Error;
|
||||
|
||||
if (process.send) {
|
||||
try {
|
||||
process.send({
|
||||
type: "log",
|
||||
message: `[SyntheticMonitorWorker] Context/page creation failed on attempt ${attempt}/${MAX_CONTEXT_CREATE_RETRIES}: ${(err as Error)?.message}. ensureBrowser took ${browserMs}ms`,
|
||||
});
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
// Browser died between launch and context/page creation — close and force relaunch
|
||||
if (currentBrowser) {
|
||||
try {
|
||||
if (currentBrowser.isConnected()) {
|
||||
await currentBrowser.close();
|
||||
}
|
||||
} catch {
|
||||
// ignore cleanup errors
|
||||
}
|
||||
}
|
||||
currentBrowser = null;
|
||||
currentBrowserType = null;
|
||||
currentProxyServer = null;
|
||||
|
||||
if (attempt < MAX_CONTEXT_CREATE_RETRIES) {
|
||||
await new Promise((resolve: (value: void) => void) => {
|
||||
setTimeout(resolve, CONTEXT_CREATE_RETRY_DELAY_MS);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Failed to create browser context/page after ${MAX_CONTEXT_CREATE_RETRIES} attempts. ` +
|
||||
`The browser may be crashing on startup due to insufficient memory or container restrictions. ` +
|
||||
`Last error: ${lastError?.message || String(lastError)}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function runExecution(config: WorkerConfig): Promise<WorkerResult> {
|
||||
const workerResult: WorkerResult = {
|
||||
logMessages: [],
|
||||
scriptError: undefined,
|
||||
result: undefined,
|
||||
screenshots: {},
|
||||
executionTimeInMS: 0,
|
||||
};
|
||||
|
||||
let context: BrowserContext | null = null;
|
||||
|
||||
try {
|
||||
const startTime: [number, number] = process.hrtime();
|
||||
|
||||
const session: {
|
||||
browser: Browser;
|
||||
context: BrowserContext;
|
||||
page: Page;
|
||||
} = await createContextAndPage(config);
|
||||
|
||||
const browser: Browser = session.browser;
|
||||
context = session.context;
|
||||
const page: Page = session.page;
|
||||
|
||||
// Track browser disconnection so we can give a clear error
|
||||
let browserDisconnected: boolean = false;
|
||||
const disconnectHandler: () => void = (): void => {
|
||||
browserDisconnected = true;
|
||||
};
|
||||
browser.on("disconnected", disconnectHandler);
|
||||
|
||||
// Set default timeouts so page operations don't hang indefinitely
|
||||
page.setDefaultTimeout(config.timeout);
|
||||
page.setDefaultNavigationTimeout(config.timeout);
|
||||
|
||||
const logMessages: string[] = [];
|
||||
|
||||
const sandbox: Context = {
|
||||
console: {
|
||||
log: (...args: unknown[]) => {
|
||||
logMessages.push(
|
||||
args
|
||||
.map((v: unknown) => {
|
||||
return typeof v === "object" ? JSON.stringify(v) : String(v);
|
||||
})
|
||||
.join(" "),
|
||||
);
|
||||
},
|
||||
},
|
||||
browser: browser,
|
||||
page: page,
|
||||
screenSizeType: config.screenSizeType,
|
||||
browserType: config.browserType,
|
||||
axios: axios,
|
||||
crypto: crypto,
|
||||
setTimeout: setTimeout,
|
||||
clearTimeout: clearTimeout,
|
||||
setInterval: setInterval,
|
||||
};
|
||||
|
||||
vm.createContext(sandbox);
|
||||
|
||||
const script: string = `(async()=>{
|
||||
${config.script}
|
||||
})()`;
|
||||
|
||||
let returnVal: unknown;
|
||||
|
||||
let scriptTimeoutTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
|
||||
const scriptTimeoutPromise: Promise<never> = new Promise<never>(
|
||||
(_: (value: never) => void, reject: (reason: Error) => void) => {
|
||||
scriptTimeoutTimer = setTimeout(() => {
|
||||
reject(
|
||||
new Error(
|
||||
`Synthetic monitor script timed out after ${config.timeout}ms. ` +
|
||||
`Consider optimizing your script or increasing the timeout.`,
|
||||
),
|
||||
);
|
||||
}, config.timeout);
|
||||
},
|
||||
);
|
||||
|
||||
try {
|
||||
returnVal = await Promise.race([
|
||||
vm.runInContext(script, sandbox, {
|
||||
timeout: config.timeout,
|
||||
}),
|
||||
scriptTimeoutPromise,
|
||||
]);
|
||||
} catch (scriptErr: unknown) {
|
||||
if (browserDisconnected) {
|
||||
throw new Error(
|
||||
"Browser crashed or was terminated during script execution. This is usually caused by high memory usage. Try simplifying the script or reducing the number of page navigations.",
|
||||
);
|
||||
}
|
||||
throw scriptErr;
|
||||
} finally {
|
||||
if (scriptTimeoutTimer) {
|
||||
clearTimeout(scriptTimeoutTimer);
|
||||
}
|
||||
browser.removeListener("disconnected", disconnectHandler);
|
||||
}
|
||||
|
||||
const endTime: [number, number] = process.hrtime(startTime);
|
||||
const executionTimeInMS: number = Math.ceil(
|
||||
(endTime[0] * 1000000000 + endTime[1]) / 1000000,
|
||||
);
|
||||
|
||||
workerResult.executionTimeInMS = executionTimeInMS;
|
||||
workerResult.logMessages = logMessages;
|
||||
|
||||
// Capture return value before closing context to extract screenshots
|
||||
const returnObj: Record<string, unknown> =
|
||||
returnVal && typeof returnVal === "object"
|
||||
? (returnVal as Record<string, unknown>)
|
||||
: {};
|
||||
|
||||
// Close context (NOT browser) to free per-execution memory
|
||||
if (context) {
|
||||
try {
|
||||
await context.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
context = null;
|
||||
}
|
||||
|
||||
/*
|
||||
* In --single-process mode, closing a context can destabilize the browser.
|
||||
* Proactively check health so the next execution doesn't waste time on a zombie.
|
||||
*/
|
||||
if (currentBrowser && !currentBrowser.isConnected()) {
|
||||
currentBrowser = null;
|
||||
currentBrowserType = null;
|
||||
currentProxyServer = null;
|
||||
}
|
||||
|
||||
// Convert screenshots from Buffer to base64
|
||||
if (returnObj["screenshots"]) {
|
||||
const screenshots: Record<string, unknown> = returnObj[
|
||||
"screenshots"
|
||||
] as Record<string, unknown>;
|
||||
|
||||
for (const screenshotName in screenshots) {
|
||||
if (!screenshots[screenshotName]) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!(screenshots[screenshotName] instanceof Buffer)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const screenshotBuffer: Buffer = screenshots[screenshotName] as Buffer;
|
||||
workerResult.screenshots[screenshotName] =
|
||||
screenshotBuffer.toString("base64");
|
||||
}
|
||||
}
|
||||
|
||||
workerResult.result = returnObj["data"];
|
||||
} catch (err: unknown) {
|
||||
workerResult.scriptError = (err as Error)?.message || String(err);
|
||||
} finally {
|
||||
// Close context if not already closed (error path) — leave browser warm
|
||||
if (context) {
|
||||
try {
|
||||
await context.close();
|
||||
} catch {
|
||||
// ignore cleanup errors
|
||||
}
|
||||
}
|
||||
|
||||
// Proactively detect zombie browser after context cleanup
|
||||
if (currentBrowser && !currentBrowser.isConnected()) {
|
||||
currentBrowser = null;
|
||||
currentBrowserType = null;
|
||||
currentProxyServer = null;
|
||||
}
|
||||
}
|
||||
|
||||
return workerResult;
|
||||
}
|
||||
|
||||
async function shutdownGracefully(): Promise<void> {
|
||||
if (currentBrowser) {
|
||||
try {
|
||||
// Close all contexts first
|
||||
const contexts: Array<BrowserContext> = currentBrowser.contexts();
|
||||
for (const ctx of contexts) {
|
||||
try {
|
||||
await ctx.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
if (currentBrowser.isConnected()) {
|
||||
await currentBrowser.close();
|
||||
}
|
||||
} catch {
|
||||
// ignore cleanup errors
|
||||
}
|
||||
currentBrowser = null;
|
||||
currentBrowserType = null;
|
||||
currentProxyServer = null;
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
function sendMessage(msg: ReadyMessage | ResultMessage | ErrorMessage): void {
|
||||
try {
|
||||
if (process.send) {
|
||||
process.send(msg);
|
||||
}
|
||||
} catch {
|
||||
// IPC channel closed — can't send. Worker will be cleaned up by pool timeout or exit handler.
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Safety timeout for process.send() callback in legacy mode.
|
||||
*/
|
||||
const IPC_FLUSH_TIMEOUT_MS: number = 10000;
|
||||
|
||||
function handleLegacyMessage(config: WorkerConfig): void {
|
||||
/*
|
||||
* Legacy one-shot mode: receive a plain WorkerConfig (no `type` field),
|
||||
* run once, send result, exit. This maintains backward compatibility.
|
||||
*/
|
||||
const safetyMarginMs: number = 15000;
|
||||
const globalSafetyTimer: ReturnType<typeof setTimeout> = setTimeout(() => {
|
||||
const errorResult: WorkerResult = {
|
||||
logMessages: [],
|
||||
scriptError:
|
||||
"Synthetic monitor worker safety timeout reached. " +
|
||||
"The script or browser cleanup took too long. " +
|
||||
"Consider simplifying the script or increasing the timeout.",
|
||||
result: undefined,
|
||||
screenshots: {},
|
||||
executionTimeInMS: 0,
|
||||
};
|
||||
|
||||
if (process.send) {
|
||||
process.send(errorResult, () => {
|
||||
process.exit(1);
|
||||
});
|
||||
setTimeout(() => {
|
||||
process.exit(1);
|
||||
}, 5000);
|
||||
} else {
|
||||
process.exit(1);
|
||||
}
|
||||
}, config.timeout + safetyMarginMs);
|
||||
|
||||
if (globalSafetyTimer.unref) {
|
||||
globalSafetyTimer.unref();
|
||||
}
|
||||
|
||||
runExecution(config)
|
||||
.then((result: WorkerResult) => {
|
||||
clearTimeout(globalSafetyTimer);
|
||||
// In legacy mode, close browser before exit since we won't reuse it
|
||||
const cleanup: Promise<void> = (async (): Promise<void> => {
|
||||
if (currentBrowser) {
|
||||
try {
|
||||
if (currentBrowser.isConnected()) {
|
||||
await currentBrowser.close();
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
currentBrowser = null;
|
||||
}
|
||||
})();
|
||||
|
||||
cleanup
|
||||
.then(() => {
|
||||
if (process.send) {
|
||||
const fallbackTimer: ReturnType<typeof setTimeout> = setTimeout(
|
||||
() => {
|
||||
process.exit(0);
|
||||
},
|
||||
IPC_FLUSH_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
process.send(result, () => {
|
||||
clearTimeout(fallbackTimer);
|
||||
process.exit(0);
|
||||
});
|
||||
} else {
|
||||
process.exit(0);
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
process.exit(1);
|
||||
});
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
clearTimeout(globalSafetyTimer);
|
||||
|
||||
const errorResult: WorkerResult = {
|
||||
logMessages: [],
|
||||
scriptError: (err as Error)?.message || String(err),
|
||||
result: undefined,
|
||||
screenshots: {},
|
||||
executionTimeInMS: 0,
|
||||
};
|
||||
|
||||
if (process.send) {
|
||||
const fallbackTimer: ReturnType<typeof setTimeout> = setTimeout(() => {
|
||||
process.exit(1);
|
||||
}, IPC_FLUSH_TIMEOUT_MS);
|
||||
|
||||
process.send(errorResult, () => {
|
||||
clearTimeout(fallbackTimer);
|
||||
process.exit(1);
|
||||
});
|
||||
} else {
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Entry point: receive messages via IPC
|
||||
process.on(
|
||||
"message",
|
||||
(msg: ExecuteMessage | ShutdownMessage | WorkerConfig) => {
|
||||
// Distinguish new protocol (has `type` field) from legacy (plain WorkerConfig)
|
||||
if ("type" in msg && typeof msg.type === "string") {
|
||||
if (msg.type === "execute") {
|
||||
const executeMsg: ExecuteMessage = msg as ExecuteMessage;
|
||||
|
||||
/*
|
||||
* Per-execution safety timer: if runExecution hangs (browser stuck, VM stuck),
|
||||
* send an error back before the pool's timeout SIGKILL-s us with no message.
|
||||
*/
|
||||
const safetyMarginMs: number = 15000;
|
||||
const executionSafetyTimer: ReturnType<typeof setTimeout> = setTimeout(
|
||||
() => {
|
||||
sendMessage({
|
||||
type: "error",
|
||||
id: executeMsg.id,
|
||||
error:
|
||||
"Synthetic monitor worker safety timeout reached. " +
|
||||
"The script or browser cleanup took too long.",
|
||||
});
|
||||
/*
|
||||
* Exit so the pool doesn't reuse this worker while runExecution
|
||||
* is still in progress (would cause two concurrent executions
|
||||
* sharing the same browser).
|
||||
*/
|
||||
setTimeout(() => {
|
||||
process.exit(1);
|
||||
}, 5000); // give IPC 5s to flush the error message
|
||||
},
|
||||
executeMsg.config.timeout + safetyMarginMs,
|
||||
);
|
||||
|
||||
if (executionSafetyTimer.unref) {
|
||||
executionSafetyTimer.unref();
|
||||
}
|
||||
|
||||
runExecution(executeMsg.config)
|
||||
.then((result: WorkerResult) => {
|
||||
clearTimeout(executionSafetyTimer);
|
||||
sendMessage({
|
||||
type: "result",
|
||||
id: executeMsg.id,
|
||||
data: result,
|
||||
});
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
clearTimeout(executionSafetyTimer);
|
||||
sendMessage({
|
||||
type: "error",
|
||||
id: executeMsg.id,
|
||||
error: (err as Error)?.message || String(err),
|
||||
});
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.type === "shutdown") {
|
||||
shutdownGracefully().catch(() => {
|
||||
process.exit(1);
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Legacy protocol: plain WorkerConfig
|
||||
handleLegacyMessage(msg as WorkerConfig);
|
||||
},
|
||||
);
|
||||
Reference in New Issue
Block a user