github/oneuptime
1
1
Fork 0
mirror of https://github.com/OneUptime/oneuptime.git synced 2026-04-06 00:32:12 +02:00
Code Issues 193 Packages Projects Releases Wiki Activity

refactor(opentelemetry): format timestamps as ClickHouse UTC datetimes

Browse Source 
- add OneUptimeDate.toClickhouseDateTime to produce UTC "YYYY-MM-DD HH:mm:ss"
- use ClickHouse-formatted timestamps for createdAt/updatedAt and time fields in OtelLogsIngestService, OtelMetricsIngestService and OtelTracesIngestService
- extend metric timestamp parsing to include db/date (and propagate db for DB storage)
- switch intermediate handling to Date objects to avoid extra ISO-string conversions
This commit is contained in:
Nawaz Dhandala
2025-10-27 15:41:13 +00:00
parent 9d93d59f91
commit 2eacc90714
4 changed files with 49 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
Common/Types/Date.ts
View File

@@ -1453,4 +1453,9 @@ export default class OneUptimeDate {
date = this.fromString(date);
return moment(date).format("YYYY-MM-DD HH:mm:ss");
}
public static toClickhouseDateTime(date: Date | string): string {
const parsedDate: Date = this.fromString(date);
return moment(parsedDate).utc().format("YYYY-MM-DD HH:mm:ss");
}
}

26
OpenTelemetryIngest/Services/OtelLogsIngestService.ts
View File

@@ -200,9 +200,7 @@ export default class OtelLogsIngestService extends OtelIngestBaseService {
let timeUnixNanoNumeric: number =
OneUptimeDate.getCurrentDateAsUnixNano();
let timeIsoString: string = OneUptimeDate.toString(
OneUptimeDate.getCurrentDate(),
);
let timeDate: Date = OneUptimeDate.getCurrentDate();
if (log["timeUnixNano"]) {
try {
@@ -221,23 +219,19 @@ export default class OtelLogsIngestService extends OtelIngestBaseService {
}
timeUnixNanoNumeric = timeUnixNano;
timeIsoString = OneUptimeDate.toString(
OneUptimeDate.fromUnixNano(timeUnixNano),
);
timeDate = OneUptimeDate.fromUnixNano(timeUnixNano);
} catch (timeError) {
logger.warn(
`Error processing timestamp ${log["timeUnixNano"]}: ${timeError instanceof Error ? timeError.message : String(timeError)}, using current time`,
);
const currentTime: Date = OneUptimeDate.getCurrentDate();
timeUnixNanoNumeric =
OneUptimeDate.getCurrentDateAsUnixNano();
timeIsoString = OneUptimeDate.toString(currentTime);
timeDate = OneUptimeDate.getCurrentDate();
}
} else {
const currentTime: Date = OneUptimeDate.getCurrentDate();
timeUnixNanoNumeric =
OneUptimeDate.getCurrentDateAsUnixNano();
timeIsoString = OneUptimeDate.toString(currentTime);
timeDate = OneUptimeDate.getCurrentDate();
}
let logSeverityNumber: number =
@@ -287,16 +281,18 @@ export default class OtelLogsIngestService extends OtelIngestBaseService {
}
const ingestionDate: Date = OneUptimeDate.getCurrentDate();
const timestampIso: string =
OneUptimeDate.toString(ingestionDate);
const ingestionTimestamp: string =
OneUptimeDate.toClickhouseDateTime(ingestionDate);
const logTimestamp: string =
OneUptimeDate.toClickhouseDateTime(timeDate);
const logRow: JSONObject = {
_id: ObjectID.generate().toString(),
createdAt: timestampIso,
updatedAt: timestampIso,
createdAt: ingestionTimestamp,
updatedAt: ingestionTimestamp,
projectId: projectId.toString(),
serviceId: serviceId.toString(),
time: timeIsoString,
time: logTimestamp,
timeUnixNano: Math.trunc(timeUnixNanoNumeric).toString(),
severityNumber: logSeverityNumber,
severityText: severityText,

34
OpenTelemetryIngest/Services/OtelMetricsIngestService.ts
View File

@@ -30,6 +30,13 @@ import { OPEN_TELEMETRY_INGEST_METRIC_FLUSH_BATCH_SIZE } from "../Config";
import OneUptimeDate from "Common/Types/Date";
import MetricService from "Common/Server/Services/MetricService";
type MetricTimestamp = {
nano: string;
iso: string;
db: string;
date: Date;
};
export default class OtelMetricsIngestService extends OtelIngestBaseService {
private static async flushMetricsBuffer(
metrics: Array<JSONObject>,
@@ -389,9 +396,10 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService {
isMonotonic?: boolean;
}): JSONObject {
const ingestionDate: Date = OneUptimeDate.getCurrentDate();
const ingestionIso: string = OneUptimeDate.toString(ingestionDate);
const ingestionTimestamp: string =
OneUptimeDate.toClickhouseDateTime(ingestionDate);
const timeFields: { nano: string; iso: string } = this.safeParseUnixNano(
const timeFields: MetricTimestamp = this.safeParseUnixNano(
data.datapoint["timeUnixNano"] as string | number | undefined,
"metric datapoint timeUnixNano",
);
@@ -400,7 +408,7 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService {
"startTimeUnixNano"
] as string | number | undefined;
const startTimeFields: { nano: string; iso: string } | null = startTimeRaw
const startTimeFields: MetricTimestamp | null = startTimeRaw
? this.safeParseUnixNano(
startTimeRaw,
"metric datapoint startTimeUnixNano",
@@ -464,13 +472,13 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService {
const row: JSONObject = {
_id: ObjectID.generate().toString(),
createdAt: ingestionIso,
updatedAt: ingestionIso,
createdAt: ingestionTimestamp,
updatedAt: ingestionTimestamp,
projectId: data.projectId.toString(),
serviceId: data.serviceId.toString(),
serviceType: ServiceType.OpenTelemetry,
name: data.metricName,
time: timeFields.iso,
time: timeFields.db,
timeUnixNano: timeFields.nano,
metricPointType: data.metricPointType,
aggregationTemporality: this.mapAggregationTemporality(
@@ -495,7 +503,7 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService {
};
if (startTimeFields) {
row["startTime"] = startTimeFields.iso;
row["startTime"] = startTimeFields.db;
row["startTimeUnixNano"] = startTimeFields.nano;
} else {
row["startTime"] = null;
@@ -508,7 +516,7 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService {
private static safeParseUnixNano(
value: string | number | undefined,
context: string,
): { nano: string; iso: string } {
): MetricTimestamp {
let numericValue: number = OneUptimeDate.getCurrentDateAsUnixNano();
if (value !== undefined && value !== null) {
@@ -533,13 +541,15 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService {
}
}
const dateIso: string = OneUptimeDate.toString(
OneUptimeDate.fromUnixNano(numericValue),
);
const date: Date = OneUptimeDate.fromUnixNano(numericValue);
const iso: string = OneUptimeDate.toString(date);
const db: string = OneUptimeDate.toClickhouseDateTime(date);
return {
nano: Math.trunc(numericValue).toString(),
iso: dateIso,
iso: iso,
db: db,
date: date,
};
}

20
OpenTelemetryIngest/Services/OtelTracesIngestService.ts
View File

@@ -617,16 +617,17 @@ export default class OtelTracesIngestService extends OtelIngestBaseService {
links: Array<JSONObject>;
}): JSONObject {
const ingestionDate: Date = OneUptimeDate.getCurrentDate();
const ingestionIso: string = OneUptimeDate.toString(ingestionDate);
const ingestionTimestamp: string =
OneUptimeDate.toClickhouseDateTime(ingestionDate);
return {
_id: ObjectID.generate().toString(),
createdAt: ingestionIso,
updatedAt: ingestionIso,
createdAt: ingestionTimestamp,
updatedAt: ingestionTimestamp,
projectId: data.projectId.toString(),
serviceId: data.serviceId.toString(),
startTime: data.startTime.iso,
endTime: data.endTime.iso,
startTime: OneUptimeDate.toClickhouseDateTime(data.startTime.date),
endTime: OneUptimeDate.toClickhouseDateTime(data.endTime.date),
startTimeUnixNano: data.startTime.nano,
endTimeUnixNano: data.endTime.nano,
durationUnixNano: data.durationUnixNano,
@@ -647,15 +648,16 @@ export default class OtelTracesIngestService extends OtelIngestBaseService {
private static buildExceptionRow(data: ExceptionEventPayload): JSONObject {
const ingestionDate: Date = OneUptimeDate.getCurrentDate();
const ingestionIso: string = OneUptimeDate.toString(ingestionDate);
const ingestionTimestamp: string =
OneUptimeDate.toClickhouseDateTime(ingestionDate);
return {
_id: ObjectID.generate().toString(),
createdAt: ingestionIso,
updatedAt: ingestionIso,
createdAt: ingestionTimestamp,
updatedAt: ingestionTimestamp,
projectId: data.projectId.toString(),
serviceId: data.serviceId.toString(),
time: data.time.iso,
time: OneUptimeDate.toClickhouseDateTime(data.time.date),
timeUnixNano: data.time.nano,
exceptionType: data.exceptionType || "",
stackTrace: data.stackTrace || "",