diff --git a/App/FeatureSet/BaseAPI/Index.ts b/App/FeatureSet/BaseAPI/Index.ts index ec2b29a235..fccb233285 100644 --- a/App/FeatureSet/BaseAPI/Index.ts +++ b/App/FeatureSet/BaseAPI/Index.ts @@ -141,9 +141,6 @@ import LogService, { LogService as LogServiceType, } from "Common/Server/Services/LogService"; -import TelemetryAttributeService, { - TelemetryAttributeService as TelemetryAttributeServiceType, -} from "Common/Server/Services/TelemetryAttributeService"; import CopilotActionTypePriorityService, { Service as CopilotActionTypePriorityServiceType, } from "Common/Server/Services/CopilotActionTypePriorityService"; @@ -489,7 +486,6 @@ import WorkflowVariable from "Common/Models/DatabaseModels/WorkflowVariable"; import ProbeOwnerTeam from "Common/Models/DatabaseModels/ProbeOwnerTeam"; import ProbeOwnerUser from "Common/Models/DatabaseModels/ProbeOwnerUser"; import ServiceCatalogDependency from "Common/Models/DatabaseModels/ServiceCatalogDependency"; -import TelemetryAttribute from "Common/Models/AnalyticsModels/TelemetryAttribute"; import ExceptionInstance from "Common/Models/AnalyticsModels/ExceptionInstance"; import TelemetyException from "Common/Models/DatabaseModels/TelemetryException"; import CopilotActionTypePriority from "Common/Models/DatabaseModels/CopilotActionTypePriority"; @@ -613,14 +609,6 @@ const BaseAPIFeatureSet: FeatureSet = { const APP_NAME: string = "api"; - app.use( - `/${APP_NAME.toLocaleLowerCase()}`, - new BaseAnalyticsAPI( - TelemetryAttribute, - TelemetryAttributeService, - ).getRouter(), - ); - app.use(`/${APP_NAME.toLocaleLowerCase()}`, OpenAPI.getRouter()); app.use( diff --git a/Common/Models/AnalyticsModels/Index.ts b/Common/Models/AnalyticsModels/Index.ts index 28513ec7f4..622d36b5a2 100644 --- a/Common/Models/AnalyticsModels/Index.ts +++ b/Common/Models/AnalyticsModels/Index.ts @@ -2,7 +2,6 @@ import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel"; import Log from "./Log"; import Metric from "./Metric"; import Span from "./Span"; -import TelemetryAttribute from "./TelemetryAttribute"; import ExceptionInstance from "./ExceptionInstance"; import MonitorLog from "./MonitorLog"; @@ -10,7 +9,6 @@ const AnalyticsModels: Array<{ new (): AnalyticsBaseModel }> = [ Log, Span, Metric, - TelemetryAttribute, ExceptionInstance, MonitorLog, ]; diff --git a/Common/Server/Services/Index.ts b/Common/Server/Services/Index.ts index 82463a2415..388ef1bdbb 100644 --- a/Common/Server/Services/Index.ts +++ b/Common/Server/Services/Index.ts @@ -135,7 +135,6 @@ import WorkflowVariablesService from "./WorkflowVariableService"; import AnalyticsBaseModel from "../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel"; import CopilotPullRequestService from "./CopilotPullRequestService"; import ServiceCatalogDependencyService from "./ServiceCatalogDependencyService"; -import TelemetryAttributeService from "./TelemetryAttributeService"; import TelemetryExceptionService from "./TelemetryExceptionService"; import ExceptionInstanceService from "./ExceptionInstanceService"; import CopilotActionTypePriorityService from "./CopilotActionTypePriorityService"; @@ -356,7 +355,6 @@ export const AnalyticsServices: Array< LogService, SpanService, MetricService, - TelemetryAttributeService, ExceptionInstanceService, MonitorLogService, ]; diff --git a/Common/Server/Services/TelemetryAttributeService.ts b/Common/Server/Services/TelemetryAttributeService.ts index c0a37fef98..801692e936 100644 --- a/Common/Server/Services/TelemetryAttributeService.ts +++ b/Common/Server/Services/TelemetryAttributeService.ts @@ -1,13 +1,46 @@ +import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement"; import TelemetryType from "../../Types/Telemetry/TelemetryType"; -import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase"; -import AnalyticsDatabaseService from "./AnalyticsDatabaseService"; -import TelemetryAttribute from "../../Models/AnalyticsModels/TelemetryAttribute"; +import LogDatabaseService from "./LogService"; +import MetricDatabaseService from "./MetricService"; +import SpanDatabaseService from "./SpanService"; +import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType"; +import { JSONObject } from "../../Types/JSON"; import ObjectID from "../../Types/ObjectID"; import CaptureSpan from "../Utils/Telemetry/CaptureSpan"; +import type AnalyticsDatabaseService from "./AnalyticsDatabaseService"; -export class TelemetryAttributeService extends AnalyticsDatabaseService { - public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) { - super({ modelType: TelemetryAttribute, database: clickhouseDatabase }); +type TelemetrySource = { + service: AnalyticsDatabaseService; + tableName: string; + attributesColumn: string; +}; + +export class TelemetryAttributeService { + private static readonly ATTRIBUTES_LIMIT: number = 5000; + + private getTelemetrySource(telemetryType: TelemetryType): TelemetrySource | null { + switch (telemetryType) { + case TelemetryType.Log: + return { + service: LogDatabaseService, + tableName: LogDatabaseService.model.tableName, + attributesColumn: "attributes", + }; + case TelemetryType.Metric: + return { + service: MetricDatabaseService, + tableName: MetricDatabaseService.model.tableName, + attributesColumn: "attributes", + }; + case TelemetryType.Trace: + return { + service: SpanDatabaseService, + tableName: SpanDatabaseService.model.tableName, + attributesColumn: "attributes", + }; + default: + return null; + } } @CaptureSpan() @@ -15,58 +48,49 @@ export class TelemetryAttributeService extends AnalyticsDatabaseService { - const telemetryAttribute: TelemetryAttribute | null = await this.findOneBy({ - query: { - projectId: data.projectId, - telemetryType: data.telemetryType, - }, - select: { - attributes: true, - }, - props: { - isRoot: true, - }, - }); + const source: TelemetrySource | null = this.getTelemetrySource( + data.telemetryType, + ); - return telemetryAttribute && - telemetryAttribute.attributes && - telemetryAttribute - ? telemetryAttribute.attributes - : []; + if (!source) { + return []; + } + + const { service, tableName, attributesColumn } = source; + + const statement: Statement = SQL` + SELECT DISTINCT arrayJoin(JSONExtractKeys(${attributesColumn})) AS attribute + FROM ${tableName} + WHERE projectId = ${{ + type: TableColumnType.ObjectID, + value: data.projectId, + }} + AND ${attributesColumn} IS NOT NULL + AND ${attributesColumn} != '' + ORDER BY attribute ASC + LIMIT ${{ + type: TableColumnType.Number, + value: TelemetryAttributeService.ATTRIBUTES_LIMIT, + }} + `; + + const dbResult = await service.executeQuery(statement); + const response = await dbResult.json<{ + data?: Array; + }>(); + + const rows: Array = response.data || []; + + const attributeKeys: Array = rows + .map((row: JSONObject) => { + const attribute: unknown = row["attribute"]; + return typeof attribute === "string" ? attribute : null; + }) + .filter((attribute): attribute is string => Boolean(attribute)); + + return Array.from(new Set(attributeKeys)); } - @CaptureSpan() - public async refreshAttributes(data: { - projectId: ObjectID; - telemetryType: TelemetryType; - attributes: string[]; - }): Promise { - const { projectId, telemetryType, attributes } = data; - - // delete existing attributes - await this.deleteBy({ - query: { - projectId, - telemetryType, - }, - props: { - isRoot: true, - }, - }); - - const telemetryAttribute: TelemetryAttribute = new TelemetryAttribute(); - - telemetryAttribute.projectId = projectId; - telemetryAttribute.telemetryType = telemetryType; - telemetryAttribute.attributes = attributes; - - await this.create({ - data: telemetryAttribute, - props: { - isRoot: true, - }, - }); - } } export default new TelemetryAttributeService(); diff --git a/Common/Server/Utils/Telemetry/Telemetry.ts b/Common/Server/Utils/Telemetry/Telemetry.ts index c8e64b7fba..2907006424 100644 --- a/Common/Server/Utils/Telemetry/Telemetry.ts +++ b/Common/Server/Utils/Telemetry/Telemetry.ts @@ -175,12 +175,6 @@ export default class TelemetryUtil { cacheKey, mergedKeys, ); - - await TelemetryAttributeService.refreshAttributes({ - projectId: data.projectId, - telemetryType: data.telemetryType, - attributes: mergedKeys, - }); } } diff --git a/Common/Types/Permission.ts b/Common/Types/Permission.ts index 96ac5536f0..5a2709f8a2 100644 --- a/Common/Types/Permission.ts +++ b/Common/Types/Permission.ts @@ -80,9 +80,6 @@ enum Permission { EditTelemetryServiceMetrics = "EditTelemetryServiceMetrics", ReadTelemetryServiceMetrics = "ReadTelemetryServiceMetrics", - // Telemetry Attributes - DeleteTelemetryAttributes = "DeleteTelemetryAttributes", - // Billing Permissions (Owner Permission) ManageProjectBilling = "ManageProjectBilling", diff --git a/Docs/Content/mcp/resources.md b/Docs/Content/mcp/resources.md index 4171155081..61d6ee4a2c 100644 --- a/Docs/Content/mcp/resources.md +++ b/Docs/Content/mcp/resources.md @@ -153,7 +153,6 @@ The OneUptime MCP Server provides access to 126 different resource types across ### Telemetry Services - **TelemetryService**: Manage telemetry data sources - **TelemetryIngestionKey**: Keys for telemetry data ingestion -- **TelemetryAttribute**: Manage telemetry attributes - **Log**: Access and manage log data - **Span**: Distributed tracing spans - **Metric**: Application and infrastructure metrics diff --git a/MCP/README.md b/MCP/README.md index 6a3f19e04d..a0aceaf59e 100644 --- a/MCP/README.md +++ b/MCP/README.md @@ -32,7 +32,7 @@ The MCP server automatically generates tools for each OneUptime model with the f The server automatically generates tools for all OneUptime models including: **Database Models**: Incident, Alert, Monitor, Project, User, Team, StatusPage, and 100+ more -**Analytics Models**: Log, Metric, Span, TelemetryAttribute, ExceptionInstance, MonitorLog +**Analytics Models**: Log, Metric, Span, ExceptionInstance, MonitorLog ## Configuration diff --git a/Worker/DataMigrations/AddAttributesColumnToTelemetryAttribute.ts b/Worker/DataMigrations/AddAttributesColumnToTelemetryAttribute.ts index dbbe7a39b2..91bf4a48d9 100644 --- a/Worker/DataMigrations/AddAttributesColumnToTelemetryAttribute.ts +++ b/Worker/DataMigrations/AddAttributesColumnToTelemetryAttribute.ts @@ -1,8 +1,4 @@ import DataMigrationBase from "./DataMigrationBase"; -import AnalyticsTableColumn from "Common/Types/AnalyticsDatabase/TableColumn"; -import TableColumnType from "Common/Types/AnalyticsDatabase/TableColumnType"; -import TelemetryAttributeService from "Common/Server/Services/TelemetryAttributeService"; -import TelemetryAttribute from "Common/Models/AnalyticsModels/TelemetryAttribute"; export default class AddAttributesColumnToTelemetryAttribute extends DataMigrationBase { public constructor() { @@ -10,24 +6,8 @@ export default class AddAttributesColumnToTelemetryAttribute extends DataMigrati } public override async migrate(): Promise { - const column: AnalyticsTableColumn | undefined = - new TelemetryAttribute().tableColumns.find( - (column: AnalyticsTableColumn) => { - return column.key === "attributes"; - }, - ); - - if (!column) { - return; - } - - const columnType: TableColumnType | null = - await TelemetryAttributeService.getColumnTypeInDatabase(column); - - if (!columnType) { - await TelemetryAttributeService.dropColumnInDatabase("attribute"); - await TelemetryAttributeService.addColumnInDatabase(column); - } + // Telemetry attributes table has been deprecated; nothing to migrate. + return; } public override async rollback(): Promise { diff --git a/Worker/DataMigrations/DeleteAllTelemetryAttributes.ts b/Worker/DataMigrations/DeleteAllTelemetryAttributes.ts index 0f689a9a6b..dd83bed22d 100644 --- a/Worker/DataMigrations/DeleteAllTelemetryAttributes.ts +++ b/Worker/DataMigrations/DeleteAllTelemetryAttributes.ts @@ -1,5 +1,4 @@ import DataMigrationBase from "./DataMigrationBase"; -import TelemetryAttributeService from "Common/Server/Services/TelemetryAttributeService"; export default class DeleteAllTelemetryAttributes extends DataMigrationBase { public constructor() { @@ -7,12 +6,8 @@ export default class DeleteAllTelemetryAttributes extends DataMigrationBase { } public override async migrate(): Promise { - await TelemetryAttributeService.deleteBy({ - query: {}, - props: { - isRoot: true, - }, - }); + // Telemetry attributes now reside directly within telemetry data tables. + return; } public override async rollback(): Promise {