From 34737fbba4b58d2cacd811c92ca46027f721634f Mon Sep 17 00:00:00 2001 From: Nawaz Dhandala Date: Mon, 27 Oct 2025 16:26:46 +0000 Subject: [PATCH] feat(telemetry): account for Exceptions in usage billing and add avg exception row size - Update TelemetryUsageBilling description to include Exceptions. - Add AverageExceptionRowSizeInBytes env/config (env example, docker-compose, Helm values & schema). - Use ExceptionInstanceService in TelemetryUsageBillingService to include exception row counts when estimating bytes for Traces. - Add helper to read average exception row size and adjust billing calculations. --- .../DatabaseModels/TelemetryUsageBilling.ts | 2 +- Common/Server/EnvironmentConfig.ts | 3 + .../Services/TelemetryUsageBillingService.ts | 79 +++++++++++++++++-- .../Public/oneuptime/templates/_helpers.tpl | 2 + HelmChart/Public/oneuptime/values.schema.json | 4 + HelmChart/Public/oneuptime/values.yaml | 1 + config.example.env | 1 + docker-compose.base.yml | 1 + 8 files changed, 84 insertions(+), 9 deletions(-) diff --git a/Common/Models/DatabaseModels/TelemetryUsageBilling.ts b/Common/Models/DatabaseModels/TelemetryUsageBilling.ts index edd10ed9f5..890d7f3491 100644 --- a/Common/Models/DatabaseModels/TelemetryUsageBilling.ts +++ b/Common/Models/DatabaseModels/TelemetryUsageBilling.ts @@ -39,7 +39,7 @@ export const DEFAULT_RETENTION_IN_DAYS: number = 15; pluralName: "Telemetry Usage Billings", icon: IconProp.Billing, tableDescription: - "Stores historical usage billing data for your telemetry data like Logs, Metrics, and Traces.", + "Stores historical usage billing data for your telemetry data like Logs, Metrics, Traces, and Exceptions.", }) @Entity({ name: "TelemetryUsageBilling", diff --git a/Common/Server/EnvironmentConfig.ts b/Common/Server/EnvironmentConfig.ts index 75bb9d08e5..a8dbce614e 100644 --- a/Common/Server/EnvironmentConfig.ts +++ b/Common/Server/EnvironmentConfig.ts @@ -380,6 +380,9 @@ export const AverageMetricRowSizeInBytes: number = parsePositiveNumberFromEnv( 1024, ); +export const AverageExceptionRowSizeInBytes: number = + parsePositiveNumberFromEnv("AVERAGE_EXCEPTION_ROW_SIZE_IN_BYTES", 1024); + export const SlackAppClientId: string | null = process.env["SLACK_APP_CLIENT_ID"] || null; export const SlackAppClientSecret: string | null = diff --git a/Common/Server/Services/TelemetryUsageBillingService.ts b/Common/Server/Services/TelemetryUsageBillingService.ts index af61e74742..d4e00afacc 100644 --- a/Common/Server/Services/TelemetryUsageBillingService.ts +++ b/Common/Server/Services/TelemetryUsageBillingService.ts @@ -15,6 +15,7 @@ import TelemetryServiceService from "./TelemetryServiceService"; import SpanService from "./SpanService"; import LogService from "./LogService"; import MetricService from "./MetricService"; +import ExceptionInstanceService from "./ExceptionInstanceService"; import AnalyticsQueryHelper from "../Types/AnalyticsDatabase/QueryHelper"; import DiskSize from "../../Types/DiskSize"; import logger from "../Utils/Logger"; @@ -24,6 +25,7 @@ import { AverageSpanRowSizeInBytes, AverageLogRowSizeInBytes, AverageMetricRowSizeInBytes, + AverageExceptionRowSizeInBytes, IsBillingEnabled, } from "../EnvironmentConfig"; import CaptureSpan from "../Utils/Telemetry/CaptureSpan"; @@ -76,8 +78,21 @@ export class Service extends DatabaseService { const averageRowSizeInBytes: number = this.getAverageRowSizeForProduct( data.productType, ); + const averageExceptionRowSizeInBytes: number = + this.getAverageExceptionRowSize(); - if (averageRowSizeInBytes <= 0) { + if ( + data.productType !== ProductType.Traces && + averageRowSizeInBytes <= 0 + ) { + return; + } + + if ( + data.productType === ProductType.Traces && + averageRowSizeInBytes <= 0 && + averageExceptionRowSizeInBytes <= 0 + ) { return; } @@ -129,11 +144,11 @@ export class Service extends DatabaseService { continue; } - let rowCount: number = 0; + let estimatedBytes: number = 0; try { if (data.productType === ProductType.Traces) { - const count: PositiveNumber = await SpanService.countBy({ + const spanCount: PositiveNumber = await SpanService.countBy({ query: { projectId: data.projectId, serviceId: telemetryService.id, @@ -146,7 +161,30 @@ export class Service extends DatabaseService { }, }); - rowCount = count.toNumber(); + const exceptionCount: PositiveNumber = + await ExceptionInstanceService.countBy({ + query: { + projectId: data.projectId, + serviceId: telemetryService.id, + time: AnalyticsQueryHelper.inBetween(startOfDay, endOfDay), + }, + skip: 0, + limit: LIMIT_INFINITY, + props: { + isRoot: true, + }, + }); + + const totalSpanCount: number = spanCount.toNumber(); + const totalExceptionCount: number = exceptionCount.toNumber(); + + if (totalSpanCount <= 0 && totalExceptionCount <= 0) { + continue; + } + + estimatedBytes = + totalSpanCount * averageRowSizeInBytes + + totalExceptionCount * averageExceptionRowSizeInBytes; } else if (data.productType === ProductType.Logs) { const count: PositiveNumber = await LogService.countBy({ query: { @@ -161,7 +199,13 @@ export class Service extends DatabaseService { }, }); - rowCount = count.toNumber(); + const totalRowCount: number = count.toNumber(); + + if (totalRowCount <= 0) { + continue; + } + + estimatedBytes = totalRowCount * averageRowSizeInBytes; } else if (data.productType === ProductType.Metrics) { const count: PositiveNumber = await MetricService.countBy({ query: { @@ -176,7 +220,13 @@ export class Service extends DatabaseService { }, }); - rowCount = count.toNumber(); + const totalRowCount: number = count.toNumber(); + + if (totalRowCount <= 0) { + continue; + } + + estimatedBytes = totalRowCount * averageRowSizeInBytes; } } catch (error) { logger.error( @@ -186,11 +236,10 @@ export class Service extends DatabaseService { continue; } - if (rowCount <= 0) { + if (estimatedBytes <= 0) { continue; } - const estimatedBytes: number = rowCount * averageRowSizeInBytes; const estimatedGigabytes: number = DiskSize.byteSizeToGB(estimatedBytes); if (!Number.isFinite(estimatedGigabytes) || estimatedGigabytes <= 0) { @@ -344,6 +393,20 @@ export class Service extends DatabaseService { return value; } + + private getAverageExceptionRowSize(): number { + const fallbackSize: number = 1024; + + if (!Number.isFinite(AverageExceptionRowSizeInBytes)) { + return fallbackSize; + } + + if (AverageExceptionRowSizeInBytes <= 0) { + return fallbackSize; + } + + return AverageExceptionRowSizeInBytes; + } } export default new Service(); diff --git a/HelmChart/Public/oneuptime/templates/_helpers.tpl b/HelmChart/Public/oneuptime/templates/_helpers.tpl index 2652789a22..7c99a86a65 100644 --- a/HelmChart/Public/oneuptime/templates/_helpers.tpl +++ b/HelmChart/Public/oneuptime/templates/_helpers.tpl @@ -493,6 +493,8 @@ Usage: - name: AVERAGE_METRIC_ROW_SIZE_IN_BYTES value: {{ $.Values.billing.telemetry.averageMetricRowSizeInBytes | quote }} +- name: AVERAGE_EXCEPTION_ROW_SIZE_IN_BYTES + value: {{ $.Values.billing.telemetry.averageExceptionRowSizeInBytes | quote }} {{- end }} diff --git a/HelmChart/Public/oneuptime/values.schema.json b/HelmChart/Public/oneuptime/values.schema.json index 0dd03b31d8..7b7b7659fa 100644 --- a/HelmChart/Public/oneuptime/values.schema.json +++ b/HelmChart/Public/oneuptime/values.schema.json @@ -611,6 +611,10 @@ "averageMetricRowSizeInBytes": { "type": "integer", "minimum": 1 + }, + "averageExceptionRowSizeInBytes": { + "type": "integer", + "minimum": 1 } }, "additionalProperties": false diff --git a/HelmChart/Public/oneuptime/values.yaml b/HelmChart/Public/oneuptime/values.yaml index 11b0cc0b01..24d475d07f 100644 --- a/HelmChart/Public/oneuptime/values.yaml +++ b/HelmChart/Public/oneuptime/values.yaml @@ -234,6 +234,7 @@ billing: averageSpanRowSizeInBytes: 1024 averageLogRowSizeInBytes: 1024 averageMetricRowSizeInBytes: 1024 + averageExceptionRowSizeInBytes: 1024 subscriptionPlan: basic: diff --git a/config.example.env b/config.example.env index 62fb0a1814..f69cb45dcb 100644 --- a/config.example.env +++ b/config.example.env @@ -208,6 +208,7 @@ BILLING_PRIVATE_KEY= AVERAGE_SPAN_ROW_SIZE_IN_BYTES=1024 AVERAGE_LOG_ROW_SIZE_IN_BYTES=1024 AVERAGE_METRIC_ROW_SIZE_IN_BYTES=1024 +AVERAGE_EXCEPTION_ROW_SIZE_IN_BYTES=1024 # Use this when you want to disable incident creation. DISABLE_AUTOMATIC_INCIDENT_CREATION=false diff --git a/docker-compose.base.yml b/docker-compose.base.yml index a9efca3d53..2b53e21428 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -122,6 +122,7 @@ x-common-server-variables: &common-server-variables AVERAGE_SPAN_ROW_SIZE_IN_BYTES: ${AVERAGE_SPAN_ROW_SIZE_IN_BYTES} AVERAGE_LOG_ROW_SIZE_IN_BYTES: ${AVERAGE_LOG_ROW_SIZE_IN_BYTES} AVERAGE_METRIC_ROW_SIZE_IN_BYTES: ${AVERAGE_METRIC_ROW_SIZE_IN_BYTES} + AVERAGE_EXCEPTION_ROW_SIZE_IN_BYTES: ${AVERAGE_EXCEPTION_ROW_SIZE_IN_BYTES} WORKFLOW_SCRIPT_TIMEOUT_IN_MS: ${WORKFLOW_SCRIPT_TIMEOUT_IN_MS} WORKFLOW_TIMEOUT_IN_MS: ${WORKFLOW_TIMEOUT_IN_MS}