github/oneuptime
1
1
Fork 0
mirror of https://github.com/OneUptime/oneuptime.git synced 2026-04-06 00:32:12 +02:00
Code Issues 193 Packages Projects Releases Wiki Activity

feat: update column types to MapStringString and BigNumber, and add projections in Log, Metric, and Span models

Browse Source
This commit is contained in:
Nawaz Dhandala
2026-03-16 12:08:30 +00:00
parent 046482a2a8
commit 8f9e5a46fa
6 changed files with 99 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Common/Models/AnalyticsModels/ExceptionInstance.ts
View File

@@ -422,7 +422,7 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
description: "Attributes",
required: true,
defaultValue: {},
type: TableColumnType.JSON,
type: TableColumnType.MapStringString,
accessControl: {
read: [
Permission.ProjectOwner,

10
Common/Models/AnalyticsModels/Log.ts
View File

@@ -165,7 +165,7 @@ export default class Log extends AnalyticsBaseModel {
description: "Attributes",
required: true,
defaultValue: {},
type: TableColumnType.JSON,
type: TableColumnType.MapStringString,
accessControl: {
read: [
Permission.ProjectOwner,
@@ -429,7 +429,13 @@ export default class Log extends AnalyticsBaseModel {
flagsColumn,
retentionDateColumn,
],
projections: [],
projections: [
{
name: "proj_severity_histogram",
query:
"SELECT projectId, severityText, toStartOfInterval(time, INTERVAL 1 MINUTE) AS minute, count() AS cnt ORDER BY (projectId, minute, severityText)",
},
],
sortKeys: ["projectId", "time", "serviceId"],
primaryKeys: ["projectId", "time", "serviceId"],
partitionKey: "sipHash64(projectId) % 16",

18
Common/Models/AnalyticsModels/Metric.ts
View File

@@ -169,6 +169,12 @@ export default class Metric extends AnalyticsBaseModel {
description: "Metric Point Type of this Metric",
required: false,
type: TableColumnType.Text,
skipIndex: {
name: "idx_metric_point_type",
type: SkipIndexType.Set,
params: [5],
granularity: 4,
},
accessControl: {
read: [
Permission.ProjectOwner,
@@ -286,7 +292,7 @@ export default class Metric extends AnalyticsBaseModel {
title: "Attributes",
description: "Attributes",
required: true,
type: TableColumnType.JSON,
type: TableColumnType.MapStringString,
defaultValue: {},
accessControl: {
read: [
@@ -357,7 +363,7 @@ export default class Metric extends AnalyticsBaseModel {
title: "Count",
description: "Count",
required: false,
type: TableColumnType.Number,
type: TableColumnType.BigNumber,
accessControl: {
read: [
Permission.ProjectOwner,
@@ -473,7 +479,7 @@ export default class Metric extends AnalyticsBaseModel {
description: "Bucket Counts",
required: true,
defaultValue: [],
type: TableColumnType.ArrayNumber,
type: TableColumnType.ArrayBigNumber,
accessControl: {
read: [
Permission.ProjectOwner,
@@ -498,7 +504,7 @@ export default class Metric extends AnalyticsBaseModel {
description: "Explicit Bonds",
required: true,
defaultValue: [],
type: TableColumnType.ArrayNumber,
type: TableColumnType.ArrayBigNumber,
accessControl: {
read: [
Permission.ProjectOwner,
@@ -583,8 +589,8 @@ export default class Metric extends AnalyticsBaseModel {
retentionDateColumn,
],
projections: [],
sortKeys: ["projectId", "time", "serviceId"],
primaryKeys: ["projectId", "time", "serviceId"],
sortKeys: ["projectId", "name", "serviceId", "time"],
primaryKeys: ["projectId", "name", "serviceId", "time"],
partitionKey: "sipHash64(projectId) % 16",
ttlExpression: "retentionDate DELETE",
});

16
Common/Models/AnalyticsModels/Span.ts
View File

@@ -330,8 +330,7 @@ export default class Span extends AnalyticsBaseModel {
description: "Attributes",
required: true,
defaultValue: {},
type: TableColumnType.JSON,
codec: { codec: "ZSTD", level: 3 },
type: TableColumnType.MapStringString,
accessControl: {
read: [
Permission.ProjectOwner,
@@ -629,7 +628,18 @@ export default class Span extends AnalyticsBaseModel {
hasExceptionColumn,
retentionDateColumn,
],
projections: [],
projections: [
{
name: "proj_agg_by_service",
query:
"SELECT projectId, serviceId, toStartOfMinute(startTime) AS minute, count() AS cnt, avg(durationUnixNano) AS avg_duration, quantile(0.99)(durationUnixNano) AS p99_duration ORDER BY (projectId, serviceId, minute)",
},
{
name: "proj_trace_by_id",
query:
"SELECT projectId, traceId, startTime, serviceId, spanId, parentSpanId, name, durationUnixNano, statusCode, hasException ORDER BY (projectId, traceId, startTime)",
},
],
sortKeys: ["projectId", "startTime", "serviceId", "traceId"],
primaryKeys: ["projectId", "startTime", "serviceId", "traceId"],
partitionKey: "sipHash64(projectId) % 16",

63
Common/Server/Utils/AnalyticsDatabase/StatementGenerator.ts
View File

@@ -251,6 +251,34 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
value = `CAST(${this.escapeStringLiteral(value.toString())} AS Int128)`;
}
if (column.type === TableColumnType.BigNumber) {
if (typeof value === "string") {
value = parseInt(value);
}
}
if (column.type === TableColumnType.ArrayBigNumber) {
value = `[${(value as Array<number>)
.map((v: number) => {
if (v && typeof v !== "number") {
v = parseFloat(v);
return isNaN(v) ? "NULL" : v;
}
return v;
})
.join(", ")}]`;
}
if (column.type === TableColumnType.MapStringString) {
const mapObj: Record<string, string> = value as Record<string, string>;
const entries: Array<string> = Object.entries(mapObj).map(
([k, v]: [string, string]) => {
return `${this.escapeStringLiteral(k)}, ${this.escapeStringLiteral(v)}`;
},
);
value = `map(${entries.join(", ")})`;
}
return value;
}
@@ -387,6 +415,25 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
} else {
whereStatement.append(SQL`AND ${key} IS NULL`);
}
} else if (
tableColumn.type === TableColumnType.MapStringString &&
typeof value === "object"
) {
const mapValue: Record<string, string> = value as Record<string, string>;
for (const mapKey in mapValue) {
if (mapValue[mapKey] === undefined) {
continue;
}
whereStatement.append(
SQL`AND ${key}[${{
value: mapKey,
type: TableColumnType.Text,
}}] = ${{
value: mapValue[mapKey] as string,
type: TableColumnType.Text,
}}`,
);
}
} else if (
(tableColumn.type === TableColumnType.JSON ||
tableColumn.type === TableColumnType.JSONArray) &&
@@ -640,6 +687,15 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
);
}
// Append projections after indexes
if (this.model.projections && this.model.projections.length > 0) {
for (const projection of this.model.projections) {
columns.append(
`, PROJECTION ${projection.name} (${projection.query})`,
);
}
}
return columns;
}
@@ -649,7 +705,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
return {
String: TableColumnType.Text,
Int32: TableColumnType.Number,
Int64: TableColumnType.LongNumber,
Int64: TableColumnType.BigNumber,
Int128: TableColumnType.LongNumber,
Float32: TableColumnType.Decimal,
Float64: TableColumnType.Decimal,
@@ -657,6 +713,8 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
"DateTime64(9)": TableColumnType.DateTime64,
"Array(String)": TableColumnType.ArrayText,
"Array(Int32)": TableColumnType.ArrayNumber,
"Array(Int64)": TableColumnType.ArrayBigNumber,
"Map(String, String)": TableColumnType.MapStringString,
JSON: TableColumnType.JSON, //JSONArray is also JSON
Bool: TableColumnType.Boolean,
}[clickhouseType];
@@ -676,8 +734,11 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
[TableColumnType.JSON]: SQL`String`, // we use JSON as a string because ClickHouse has really good JSON support for string types
[TableColumnType.JSONArray]: SQL`String`, // we use JSON as a string because ClickHouse has really good JSON support for string types
[TableColumnType.ArrayNumber]: SQL`Array(Int32)`,
[TableColumnType.ArrayBigNumber]: SQL`Array(Int64)`,
[TableColumnType.ArrayText]: SQL`Array(String)`,
[TableColumnType.LongNumber]: SQL`Int128`,
[TableColumnType.BigNumber]: SQL`Int64`,
[TableColumnType.MapStringString]: SQL`Map(String, String)`,
}[type];
if (!statement) {

3
Common/Types/AnalyticsDatabase/TableColumnType.ts
View File

@@ -10,9 +10,12 @@ enum ColumnType {
ArrayNumber = "Array of Numbers",
ArrayText = "Array of Text",
LongNumber = "Long Number",
BigNumber = "Big Number",
DateTime64 = "DateTime64",
IP = "IP",
Port = "Port",
MapStringString = "Map(String, String)",
ArrayBigNumber = "Array of Big Numbers",
}
export default ColumnType;