diff --git a/Common/Server/EnvironmentConfig.ts b/Common/Server/EnvironmentConfig.ts index 8ab721a79e..68e0fef5fe 100644 --- a/Common/Server/EnvironmentConfig.ts +++ b/Common/Server/EnvironmentConfig.ts @@ -23,6 +23,25 @@ export const getAllEnvVars: () => JSONObject = (): JSONObject => { return process.env; }; +const parsePositiveNumberFromEnv = ( + envKey: string, + fallback: number, +): number => { + const rawValue: string | undefined = process.env[envKey]; + + if (!rawValue) { + return fallback; + } + + const parsedValue: number = parseFloat(rawValue); + + if (!Number.isFinite(parsedValue) || parsedValue <= 0) { + return fallback; + } + + return parsedValue; +}; + export const IsBillingEnabled: boolean = BillingConfig.IsBillingEnabled; export const BillingPublicKey: string = BillingConfig.BillingPublicKey; export const BillingPrivateKey: string = BillingConfig.BillingPrivateKey; @@ -346,6 +365,15 @@ export const DocsClientUrl: URL = new URL( export const DisableTelemetry: boolean = process.env["DISABLE_TELEMETRY"] === "true"; +export const AverageSpanRowSizeInBytes: number = + parsePositiveNumberFromEnv("AVERAGE_SPAN_ROW_SIZE_IN_BYTES", 1024); + +export const AverageLogRowSizeInBytes: number = + parsePositiveNumberFromEnv("AVERAGE_LOG_ROW_SIZE_IN_BYTES", 1024); + +export const AverageMetricRowSizeInBytes: number = + parsePositiveNumberFromEnv("AVERAGE_METRIC_ROW_SIZE_IN_BYTES", 1024); + export const SlackAppClientId: string | null = process.env["SLACK_APP_CLIENT_ID"] || null; export const SlackAppClientSecret: string | null = diff --git a/Common/Server/Services/OpenTelemetryIngestService.ts b/Common/Server/Services/OpenTelemetryIngestService.ts index 8bc7c8a8e3..b9605abe2e 100644 --- a/Common/Server/Services/OpenTelemetryIngestService.ts +++ b/Common/Server/Services/OpenTelemetryIngestService.ts @@ -4,11 +4,6 @@ import ObjectID from "../../Types/ObjectID"; import Metric, { AggregationTemporality, } from "../../Models/AnalyticsModels/Metric"; -import Dictionary from "../../Types/Dictionary"; -import ProductType from "../../Types/MeteredPlan/ProductType"; -import { IsBillingEnabled } from "../../Server/EnvironmentConfig"; -import TelemetryUsageBillingService from "../../Server/Services/TelemetryUsageBillingService"; -import logger from "../../Server/Utils/Logger"; import TelemetryService from "../../Models/DatabaseModels/TelemetryService"; import TelemetryServiceService from "../../Server/Services/TelemetryServiceService"; import { DEFAULT_RETENTION_IN_DAYS } from "../../Models/DatabaseModels/TelemetryUsageBilling"; @@ -20,10 +15,9 @@ export enum OtelAggregationTemporality { Delta = "AGGREGATION_TEMPORALITY_DELTA", } -export interface TelemetryServiceDataIngested { +export interface TelemetryServiceMetadata { serviceName: string; serviceId: ObjectID; - dataIngestedInGB: number; dataRententionInDays: number; } @@ -80,38 +74,6 @@ export default class OTelIngestService { service.retainTelemetryDataForDays || DEFAULT_RETENTION_IN_DAYS, }; } - - @CaptureSpan() - public static async recordDataIngestedUsgaeBilling(data: { - services: Dictionary; - projectId: ObjectID; - productType: ProductType; - }): Promise { - if (!IsBillingEnabled) { - return; - } - - for (const serviceName in data.services) { - const serviceData: TelemetryServiceDataIngested | undefined = - data.services[serviceName]; - - if (!serviceData) { - continue; - } - - TelemetryUsageBillingService.updateUsageBilling({ - projectId: data.projectId, - productType: data.productType, - dataIngestedInGB: serviceData.dataIngestedInGB || 0, - telemetryServiceId: serviceData.serviceId, - retentionInDays: serviceData.dataRententionInDays, - }).catch((err: Error) => { - logger.error("Failed to update usage billing for OTel"); - logger.error(err); - }); - } - } - @CaptureSpan() public static getMetricFromDatapoint(data: { dbMetric: Metric; diff --git a/OpenTelemetryIngest/Services/OtelLogsIngestService.ts b/OpenTelemetryIngest/Services/OtelLogsIngestService.ts index 581d3f1120..c2d5143f92 100644 --- a/OpenTelemetryIngest/Services/OtelLogsIngestService.ts +++ b/OpenTelemetryIngest/Services/OtelLogsIngestService.ts @@ -1,11 +1,9 @@ import { TelemetryRequest } from "Common/Server/Middleware/TelemetryIngest"; import OTelIngestService, { - TelemetryServiceDataIngested, + TelemetryServiceMetadata, } from "Common/Server/Services/OpenTelemetryIngestService"; import OneUptimeDate from "Common/Types/Date"; import BadRequestException from "Common/Types/Exception/BadRequestException"; -import JSONFunctions from "Common/Types/JSONFunctions"; -import ProductType from "Common/Types/MeteredPlan/ProductType"; import Text from "Common/Types/Text"; import LogService from "Common/Server/Services/LogService"; import { @@ -97,7 +95,7 @@ export default class OtelLogsIngestService extends OtelIngestBaseService { } const dbLogs: Array = []; - const serviceDictionary: Dictionary = {}; + const serviceDictionary: Dictionary = {}; let totalLogsProcessed: number = 0; let resourceLogCounter: number = 0; @@ -127,7 +125,6 @@ export default class OtelLogsIngestService extends OtelIngestBaseService { serviceName: serviceName, serviceId: service.serviceId, dataRententionInDays: service.dataRententionInDays, - dataIngestedInGB: 0, }; } @@ -146,10 +143,6 @@ export default class OtelLogsIngestService extends OtelIngestBaseService { prefixKeysWithString: "resource", }), }; - - const sizeInGb: number = JSONFunctions.getSizeOfJSONinGB(resourceLog); - serviceDictionary[serviceName]!.dataIngestedInGB += sizeInGb; - const scopeLogs: JSONArray = resourceLog["scopeLogs"] as JSONArray; if (!scopeLogs || !Array.isArray(scopeLogs)) { @@ -326,12 +319,6 @@ export default class OtelLogsIngestService extends OtelIngestBaseService { return; } - await OTelIngestService.recordDataIngestedUsgaeBilling({ - services: serviceDictionary, - projectId: (req as TelemetryRequest).projectId, - productType: ProductType.Logs, - }); - logger.debug( `Successfully processed ${totalLogsProcessed} logs for project: ${(req as TelemetryRequest).projectId}`, ); diff --git a/OpenTelemetryIngest/Services/OtelMetricsIngestService.ts b/OpenTelemetryIngest/Services/OtelMetricsIngestService.ts index a98385a4ca..a7bb39ae97 100644 --- a/OpenTelemetryIngest/Services/OtelMetricsIngestService.ts +++ b/OpenTelemetryIngest/Services/OtelMetricsIngestService.ts @@ -1,11 +1,9 @@ import { TelemetryRequest } from "Common/Server/Middleware/TelemetryIngest"; import OTelIngestService, { OtelAggregationTemporality, - TelemetryServiceDataIngested, + TelemetryServiceMetadata, } from "Common/Server/Services/OpenTelemetryIngestService"; import BadRequestException from "Common/Types/Exception/BadRequestException"; -import JSONFunctions from "Common/Types/JSONFunctions"; -import ProductType from "Common/Types/MeteredPlan/ProductType"; import { ExpressRequest, ExpressResponse, @@ -104,7 +102,7 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService { } const dbMetrics: Array = []; - const serviceDictionary: Dictionary = {}; + const serviceDictionary: Dictionary = {}; const metricNameServiceNameMap: Dictionary = {}; let totalMetricsProcessed: number = 0; @@ -136,7 +134,6 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService { serviceName: serviceName, serviceId: service.serviceId, dataRententionInDays: service.dataRententionInDays, - dataIngestedInGB: 0, }; } @@ -155,11 +152,6 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService { prefixKeysWithString: "resource", }), }; - - const sizeInGb: number = - JSONFunctions.getSizeOfJSONinGB(resourceMetric); - serviceDictionary[serviceName]!.dataIngestedInGB += sizeInGb; - const scopeMetrics: JSONArray = resourceMetric[ "scopeMetrics" ] as JSONArray; @@ -372,12 +364,6 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService { logger.error(err); }); - await OTelIngestService.recordDataIngestedUsgaeBilling({ - services: serviceDictionary, - projectId: (req as TelemetryRequest).projectId, - productType: ProductType.Metrics, - }); - logger.debug( `Successfully processed ${totalMetricsProcessed} metrics for project: ${(req as TelemetryRequest).projectId}`, ); diff --git a/OpenTelemetryIngest/Services/OtelTracesIngestService.ts b/OpenTelemetryIngest/Services/OtelTracesIngestService.ts index 9fd88e4fdc..d631a61ba4 100644 --- a/OpenTelemetryIngest/Services/OtelTracesIngestService.ts +++ b/OpenTelemetryIngest/Services/OtelTracesIngestService.ts @@ -1,11 +1,9 @@ import { TelemetryRequest } from "Common/Server/Middleware/TelemetryIngest"; import OTelIngestService, { - TelemetryServiceDataIngested, + TelemetryServiceMetadata, } from "Common/Server/Services/OpenTelemetryIngestService"; import OneUptimeDate from "Common/Types/Date"; import BadRequestException from "Common/Types/Exception/BadRequestException"; -import JSONFunctions from "Common/Types/JSONFunctions"; -import ProductType from "Common/Types/MeteredPlan/ProductType"; import { ExpressRequest, ExpressResponse, @@ -136,7 +134,7 @@ export default class OtelTracesIngestService extends OtelIngestBaseService { const dbSpans: Array = []; const dbExceptions: Array = []; - const serviceDictionary: Dictionary = {}; + const serviceDictionary: Dictionary = {}; let totalSpansProcessed: number = 0; let resourceSpanCounter: number = 0; @@ -166,7 +164,6 @@ export default class OtelTracesIngestService extends OtelIngestBaseService { serviceName: serviceName, serviceId: service.serviceId, dataRententionInDays: service.dataRententionInDays, - dataIngestedInGB: 0, }; } @@ -186,10 +183,6 @@ export default class OtelTracesIngestService extends OtelIngestBaseService { }), }; - const sizeInGb: number = - JSONFunctions.getSizeOfJSONinGB(resourceSpan); - serviceDictionary[serviceName]!.dataIngestedInGB += sizeInGb; - const scopeSpans: JSONArray = resourceSpan["scopeSpans"] as JSONArray; if (!scopeSpans || !Array.isArray(scopeSpans)) { @@ -412,12 +405,6 @@ export default class OtelTracesIngestService extends OtelIngestBaseService { return; } - await OTelIngestService.recordDataIngestedUsgaeBilling({ - services: serviceDictionary, - projectId: (req as TelemetryRequest).projectId, - productType: ProductType.Traces, - }); - logger.debug( `Successfully processed ${totalSpansProcessed} spans for project: ${(req as TelemetryRequest).projectId}`, ); diff --git a/config.example.env b/config.example.env index 92c0e81138..62fb0a1814 100644 --- a/config.example.env +++ b/config.example.env @@ -204,6 +204,11 @@ BILLING_ENABLED=false BILLING_PUBLIC_KEY= BILLING_PRIVATE_KEY= +# Average telemetry row sizes in bytes used to estimate usage when reporting to the billing provider. +AVERAGE_SPAN_ROW_SIZE_IN_BYTES=1024 +AVERAGE_LOG_ROW_SIZE_IN_BYTES=1024 +AVERAGE_METRIC_ROW_SIZE_IN_BYTES=1024 + # Use this when you want to disable incident creation. DISABLE_AUTOMATIC_INCIDENT_CREATION=false diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 90aee88d0c..a9efca3d53 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -119,6 +119,10 @@ x-common-server-variables: &common-server-variables IS_SERVER: "true" + AVERAGE_SPAN_ROW_SIZE_IN_BYTES: ${AVERAGE_SPAN_ROW_SIZE_IN_BYTES} + AVERAGE_LOG_ROW_SIZE_IN_BYTES: ${AVERAGE_LOG_ROW_SIZE_IN_BYTES} + AVERAGE_METRIC_ROW_SIZE_IN_BYTES: ${AVERAGE_METRIC_ROW_SIZE_IN_BYTES} + WORKFLOW_SCRIPT_TIMEOUT_IN_MS: ${WORKFLOW_SCRIPT_TIMEOUT_IN_MS} WORKFLOW_TIMEOUT_IN_MS: ${WORKFLOW_TIMEOUT_IN_MS}