diff --git a/Common/Types/Date.ts b/Common/Types/Date.ts index aaa4109afa..c0a1de60a7 100644 --- a/Common/Types/Date.ts +++ b/Common/Types/Date.ts @@ -1453,4 +1453,9 @@ export default class OneUptimeDate { date = this.fromString(date); return moment(date).format("YYYY-MM-DD HH:mm:ss"); } + + public static toClickhouseDateTime(date: Date | string): string { + const parsedDate: Date = this.fromString(date); + return moment(parsedDate).utc().format("YYYY-MM-DD HH:mm:ss"); + } } diff --git a/OpenTelemetryIngest/Services/OtelLogsIngestService.ts b/OpenTelemetryIngest/Services/OtelLogsIngestService.ts index 8d3e32952d..12ade43099 100644 --- a/OpenTelemetryIngest/Services/OtelLogsIngestService.ts +++ b/OpenTelemetryIngest/Services/OtelLogsIngestService.ts @@ -200,9 +200,7 @@ export default class OtelLogsIngestService extends OtelIngestBaseService { let timeUnixNanoNumeric: number = OneUptimeDate.getCurrentDateAsUnixNano(); - let timeIsoString: string = OneUptimeDate.toString( - OneUptimeDate.getCurrentDate(), - ); + let timeDate: Date = OneUptimeDate.getCurrentDate(); if (log["timeUnixNano"]) { try { @@ -221,23 +219,19 @@ export default class OtelLogsIngestService extends OtelIngestBaseService { } timeUnixNanoNumeric = timeUnixNano; - timeIsoString = OneUptimeDate.toString( - OneUptimeDate.fromUnixNano(timeUnixNano), - ); + timeDate = OneUptimeDate.fromUnixNano(timeUnixNano); } catch (timeError) { logger.warn( `Error processing timestamp ${log["timeUnixNano"]}: ${timeError instanceof Error ? timeError.message : String(timeError)}, using current time`, ); - const currentTime: Date = OneUptimeDate.getCurrentDate(); timeUnixNanoNumeric = OneUptimeDate.getCurrentDateAsUnixNano(); - timeIsoString = OneUptimeDate.toString(currentTime); + timeDate = OneUptimeDate.getCurrentDate(); } } else { - const currentTime: Date = OneUptimeDate.getCurrentDate(); timeUnixNanoNumeric = OneUptimeDate.getCurrentDateAsUnixNano(); - timeIsoString = OneUptimeDate.toString(currentTime); + timeDate = OneUptimeDate.getCurrentDate(); } let logSeverityNumber: number = @@ -287,16 +281,18 @@ export default class OtelLogsIngestService extends OtelIngestBaseService { } const ingestionDate: Date = OneUptimeDate.getCurrentDate(); - const timestampIso: string = - OneUptimeDate.toString(ingestionDate); + const ingestionTimestamp: string = + OneUptimeDate.toClickhouseDateTime(ingestionDate); + const logTimestamp: string = + OneUptimeDate.toClickhouseDateTime(timeDate); const logRow: JSONObject = { _id: ObjectID.generate().toString(), - createdAt: timestampIso, - updatedAt: timestampIso, + createdAt: ingestionTimestamp, + updatedAt: ingestionTimestamp, projectId: projectId.toString(), serviceId: serviceId.toString(), - time: timeIsoString, + time: logTimestamp, timeUnixNano: Math.trunc(timeUnixNanoNumeric).toString(), severityNumber: logSeverityNumber, severityText: severityText, diff --git a/OpenTelemetryIngest/Services/OtelMetricsIngestService.ts b/OpenTelemetryIngest/Services/OtelMetricsIngestService.ts index 839d85a302..434800920d 100644 --- a/OpenTelemetryIngest/Services/OtelMetricsIngestService.ts +++ b/OpenTelemetryIngest/Services/OtelMetricsIngestService.ts @@ -30,6 +30,13 @@ import { OPEN_TELEMETRY_INGEST_METRIC_FLUSH_BATCH_SIZE } from "../Config"; import OneUptimeDate from "Common/Types/Date"; import MetricService from "Common/Server/Services/MetricService"; +type MetricTimestamp = { + nano: string; + iso: string; + db: string; + date: Date; +}; + export default class OtelMetricsIngestService extends OtelIngestBaseService { private static async flushMetricsBuffer( metrics: Array, @@ -389,9 +396,10 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService { isMonotonic?: boolean; }): JSONObject { const ingestionDate: Date = OneUptimeDate.getCurrentDate(); - const ingestionIso: string = OneUptimeDate.toString(ingestionDate); + const ingestionTimestamp: string = + OneUptimeDate.toClickhouseDateTime(ingestionDate); - const timeFields: { nano: string; iso: string } = this.safeParseUnixNano( + const timeFields: MetricTimestamp = this.safeParseUnixNano( data.datapoint["timeUnixNano"] as string | number | undefined, "metric datapoint timeUnixNano", ); @@ -400,7 +408,7 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService { "startTimeUnixNano" ] as string | number | undefined; - const startTimeFields: { nano: string; iso: string } | null = startTimeRaw + const startTimeFields: MetricTimestamp | null = startTimeRaw ? this.safeParseUnixNano( startTimeRaw, "metric datapoint startTimeUnixNano", @@ -464,13 +472,13 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService { const row: JSONObject = { _id: ObjectID.generate().toString(), - createdAt: ingestionIso, - updatedAt: ingestionIso, + createdAt: ingestionTimestamp, + updatedAt: ingestionTimestamp, projectId: data.projectId.toString(), serviceId: data.serviceId.toString(), serviceType: ServiceType.OpenTelemetry, name: data.metricName, - time: timeFields.iso, + time: timeFields.db, timeUnixNano: timeFields.nano, metricPointType: data.metricPointType, aggregationTemporality: this.mapAggregationTemporality( @@ -495,7 +503,7 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService { }; if (startTimeFields) { - row["startTime"] = startTimeFields.iso; + row["startTime"] = startTimeFields.db; row["startTimeUnixNano"] = startTimeFields.nano; } else { row["startTime"] = null; @@ -508,7 +516,7 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService { private static safeParseUnixNano( value: string | number | undefined, context: string, - ): { nano: string; iso: string } { + ): MetricTimestamp { let numericValue: number = OneUptimeDate.getCurrentDateAsUnixNano(); if (value !== undefined && value !== null) { @@ -533,13 +541,15 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService { } } - const dateIso: string = OneUptimeDate.toString( - OneUptimeDate.fromUnixNano(numericValue), - ); + const date: Date = OneUptimeDate.fromUnixNano(numericValue); + const iso: string = OneUptimeDate.toString(date); + const db: string = OneUptimeDate.toClickhouseDateTime(date); return { nano: Math.trunc(numericValue).toString(), - iso: dateIso, + iso: iso, + db: db, + date: date, }; } diff --git a/OpenTelemetryIngest/Services/OtelTracesIngestService.ts b/OpenTelemetryIngest/Services/OtelTracesIngestService.ts index d8535c6308..7d5c5dd0b6 100644 --- a/OpenTelemetryIngest/Services/OtelTracesIngestService.ts +++ b/OpenTelemetryIngest/Services/OtelTracesIngestService.ts @@ -617,16 +617,17 @@ export default class OtelTracesIngestService extends OtelIngestBaseService { links: Array; }): JSONObject { const ingestionDate: Date = OneUptimeDate.getCurrentDate(); - const ingestionIso: string = OneUptimeDate.toString(ingestionDate); + const ingestionTimestamp: string = + OneUptimeDate.toClickhouseDateTime(ingestionDate); return { _id: ObjectID.generate().toString(), - createdAt: ingestionIso, - updatedAt: ingestionIso, + createdAt: ingestionTimestamp, + updatedAt: ingestionTimestamp, projectId: data.projectId.toString(), serviceId: data.serviceId.toString(), - startTime: data.startTime.iso, - endTime: data.endTime.iso, + startTime: OneUptimeDate.toClickhouseDateTime(data.startTime.date), + endTime: OneUptimeDate.toClickhouseDateTime(data.endTime.date), startTimeUnixNano: data.startTime.nano, endTimeUnixNano: data.endTime.nano, durationUnixNano: data.durationUnixNano, @@ -647,15 +648,16 @@ export default class OtelTracesIngestService extends OtelIngestBaseService { private static buildExceptionRow(data: ExceptionEventPayload): JSONObject { const ingestionDate: Date = OneUptimeDate.getCurrentDate(); - const ingestionIso: string = OneUptimeDate.toString(ingestionDate); + const ingestionTimestamp: string = + OneUptimeDate.toClickhouseDateTime(ingestionDate); return { _id: ObjectID.generate().toString(), - createdAt: ingestionIso, - updatedAt: ingestionIso, + createdAt: ingestionTimestamp, + updatedAt: ingestionTimestamp, projectId: data.projectId.toString(), serviceId: data.serviceId.toString(), - time: data.time.iso, + time: OneUptimeDate.toClickhouseDateTime(data.time.date), timeUnixNano: data.time.nano, exceptionType: data.exceptionType || "", stackTrace: data.stackTrace || "",