diff --git a/Common/Models/AnalyticsModels/ExceptionInstance.ts b/Common/Models/AnalyticsModels/ExceptionInstance.ts index 028ced01a0..ccb83f3cd7 100644 --- a/Common/Models/AnalyticsModels/ExceptionInstance.ts +++ b/Common/Models/AnalyticsModels/ExceptionInstance.ts @@ -422,7 +422,7 @@ export default class ExceptionInstance extends AnalyticsBaseModel { description: "Attributes", required: true, defaultValue: {}, - type: TableColumnType.JSON, + type: TableColumnType.MapStringString, accessControl: { read: [ Permission.ProjectOwner, diff --git a/Common/Models/AnalyticsModels/Log.ts b/Common/Models/AnalyticsModels/Log.ts index 7f345c451b..63e81680dc 100644 --- a/Common/Models/AnalyticsModels/Log.ts +++ b/Common/Models/AnalyticsModels/Log.ts @@ -165,7 +165,7 @@ export default class Log extends AnalyticsBaseModel { description: "Attributes", required: true, defaultValue: {}, - type: TableColumnType.JSON, + type: TableColumnType.MapStringString, accessControl: { read: [ Permission.ProjectOwner, @@ -429,7 +429,13 @@ export default class Log extends AnalyticsBaseModel { flagsColumn, retentionDateColumn, ], - projections: [], + projections: [ + { + name: "proj_severity_histogram", + query: + "SELECT projectId, severityText, toStartOfInterval(time, INTERVAL 1 MINUTE) AS minute, count() AS cnt ORDER BY (projectId, minute, severityText)", + }, + ], sortKeys: ["projectId", "time", "serviceId"], primaryKeys: ["projectId", "time", "serviceId"], partitionKey: "sipHash64(projectId) % 16", diff --git a/Common/Models/AnalyticsModels/Metric.ts b/Common/Models/AnalyticsModels/Metric.ts index 71f47e81e2..462104d02a 100644 --- a/Common/Models/AnalyticsModels/Metric.ts +++ b/Common/Models/AnalyticsModels/Metric.ts @@ -169,6 +169,12 @@ export default class Metric extends AnalyticsBaseModel { description: "Metric Point Type of this Metric", required: false, type: TableColumnType.Text, + skipIndex: { + name: "idx_metric_point_type", + type: SkipIndexType.Set, + params: [5], + granularity: 4, + }, accessControl: { read: [ Permission.ProjectOwner, @@ -286,7 +292,7 @@ export default class Metric extends AnalyticsBaseModel { title: "Attributes", description: "Attributes", required: true, - type: TableColumnType.JSON, + type: TableColumnType.MapStringString, defaultValue: {}, accessControl: { read: [ @@ -357,7 +363,7 @@ export default class Metric extends AnalyticsBaseModel { title: "Count", description: "Count", required: false, - type: TableColumnType.Number, + type: TableColumnType.BigNumber, accessControl: { read: [ Permission.ProjectOwner, @@ -473,7 +479,7 @@ export default class Metric extends AnalyticsBaseModel { description: "Bucket Counts", required: true, defaultValue: [], - type: TableColumnType.ArrayNumber, + type: TableColumnType.ArrayBigNumber, accessControl: { read: [ Permission.ProjectOwner, @@ -498,7 +504,7 @@ export default class Metric extends AnalyticsBaseModel { description: "Explicit Bonds", required: true, defaultValue: [], - type: TableColumnType.ArrayNumber, + type: TableColumnType.ArrayBigNumber, accessControl: { read: [ Permission.ProjectOwner, @@ -583,8 +589,8 @@ export default class Metric extends AnalyticsBaseModel { retentionDateColumn, ], projections: [], - sortKeys: ["projectId", "time", "serviceId"], - primaryKeys: ["projectId", "time", "serviceId"], + sortKeys: ["projectId", "name", "serviceId", "time"], + primaryKeys: ["projectId", "name", "serviceId", "time"], partitionKey: "sipHash64(projectId) % 16", ttlExpression: "retentionDate DELETE", }); diff --git a/Common/Models/AnalyticsModels/Span.ts b/Common/Models/AnalyticsModels/Span.ts index e6d5db4ef2..b644967c0f 100644 --- a/Common/Models/AnalyticsModels/Span.ts +++ b/Common/Models/AnalyticsModels/Span.ts @@ -330,8 +330,7 @@ export default class Span extends AnalyticsBaseModel { description: "Attributes", required: true, defaultValue: {}, - type: TableColumnType.JSON, - codec: { codec: "ZSTD", level: 3 }, + type: TableColumnType.MapStringString, accessControl: { read: [ Permission.ProjectOwner, @@ -629,7 +628,18 @@ export default class Span extends AnalyticsBaseModel { hasExceptionColumn, retentionDateColumn, ], - projections: [], + projections: [ + { + name: "proj_agg_by_service", + query: + "SELECT projectId, serviceId, toStartOfMinute(startTime) AS minute, count() AS cnt, avg(durationUnixNano) AS avg_duration, quantile(0.99)(durationUnixNano) AS p99_duration ORDER BY (projectId, serviceId, minute)", + }, + { + name: "proj_trace_by_id", + query: + "SELECT projectId, traceId, startTime, serviceId, spanId, parentSpanId, name, durationUnixNano, statusCode, hasException ORDER BY (projectId, traceId, startTime)", + }, + ], sortKeys: ["projectId", "startTime", "serviceId", "traceId"], primaryKeys: ["projectId", "startTime", "serviceId", "traceId"], partitionKey: "sipHash64(projectId) % 16", diff --git a/Common/Server/Utils/AnalyticsDatabase/StatementGenerator.ts b/Common/Server/Utils/AnalyticsDatabase/StatementGenerator.ts index 8bac3b3ef1..694a9dc87b 100644 --- a/Common/Server/Utils/AnalyticsDatabase/StatementGenerator.ts +++ b/Common/Server/Utils/AnalyticsDatabase/StatementGenerator.ts @@ -251,6 +251,34 @@ export default class StatementGenerator { value = `CAST(${this.escapeStringLiteral(value.toString())} AS Int128)`; } + if (column.type === TableColumnType.BigNumber) { + if (typeof value === "string") { + value = parseInt(value); + } + } + + if (column.type === TableColumnType.ArrayBigNumber) { + value = `[${(value as Array) + .map((v: number) => { + if (v && typeof v !== "number") { + v = parseFloat(v); + return isNaN(v) ? "NULL" : v; + } + return v; + }) + .join(", ")}]`; + } + + if (column.type === TableColumnType.MapStringString) { + const mapObj: Record = value as Record; + const entries: Array = Object.entries(mapObj).map( + ([k, v]: [string, string]) => { + return `${this.escapeStringLiteral(k)}, ${this.escapeStringLiteral(v)}`; + }, + ); + value = `map(${entries.join(", ")})`; + } + return value; } @@ -387,6 +415,25 @@ export default class StatementGenerator { } else { whereStatement.append(SQL`AND ${key} IS NULL`); } + } else if ( + tableColumn.type === TableColumnType.MapStringString && + typeof value === "object" + ) { + const mapValue: Record = value as Record; + for (const mapKey in mapValue) { + if (mapValue[mapKey] === undefined) { + continue; + } + whereStatement.append( + SQL`AND ${key}[${{ + value: mapKey, + type: TableColumnType.Text, + }}] = ${{ + value: mapValue[mapKey] as string, + type: TableColumnType.Text, + }}`, + ); + } } else if ( (tableColumn.type === TableColumnType.JSON || tableColumn.type === TableColumnType.JSONArray) && @@ -640,6 +687,15 @@ export default class StatementGenerator { ); } + // Append projections after indexes + if (this.model.projections && this.model.projections.length > 0) { + for (const projection of this.model.projections) { + columns.append( + `, PROJECTION ${projection.name} (${projection.query})`, + ); + } + } + return columns; } @@ -649,7 +705,7 @@ export default class StatementGenerator { return { String: TableColumnType.Text, Int32: TableColumnType.Number, - Int64: TableColumnType.LongNumber, + Int64: TableColumnType.BigNumber, Int128: TableColumnType.LongNumber, Float32: TableColumnType.Decimal, Float64: TableColumnType.Decimal, @@ -657,6 +713,8 @@ export default class StatementGenerator { "DateTime64(9)": TableColumnType.DateTime64, "Array(String)": TableColumnType.ArrayText, "Array(Int32)": TableColumnType.ArrayNumber, + "Array(Int64)": TableColumnType.ArrayBigNumber, + "Map(String, String)": TableColumnType.MapStringString, JSON: TableColumnType.JSON, //JSONArray is also JSON Bool: TableColumnType.Boolean, }[clickhouseType]; @@ -676,8 +734,11 @@ export default class StatementGenerator { [TableColumnType.JSON]: SQL`String`, // we use JSON as a string because ClickHouse has really good JSON support for string types [TableColumnType.JSONArray]: SQL`String`, // we use JSON as a string because ClickHouse has really good JSON support for string types [TableColumnType.ArrayNumber]: SQL`Array(Int32)`, + [TableColumnType.ArrayBigNumber]: SQL`Array(Int64)`, [TableColumnType.ArrayText]: SQL`Array(String)`, [TableColumnType.LongNumber]: SQL`Int128`, + [TableColumnType.BigNumber]: SQL`Int64`, + [TableColumnType.MapStringString]: SQL`Map(String, String)`, }[type]; if (!statement) { diff --git a/Common/Types/AnalyticsDatabase/TableColumnType.ts b/Common/Types/AnalyticsDatabase/TableColumnType.ts index 7120408bf4..ae6e7220b7 100644 --- a/Common/Types/AnalyticsDatabase/TableColumnType.ts +++ b/Common/Types/AnalyticsDatabase/TableColumnType.ts @@ -10,9 +10,12 @@ enum ColumnType { ArrayNumber = "Array of Numbers", ArrayText = "Array of Text", LongNumber = "Long Number", + BigNumber = "Big Number", DateTime64 = "DateTime64", IP = "IP", Port = "Port", + MapStringString = "Map(String, String)", + ArrayBigNumber = "Array of Big Numbers", } export default ColumnType;