diff --git a/App/FeatureSet/BaseAPI/Index.ts b/App/FeatureSet/BaseAPI/Index.ts index d1741ff0f8..6b41ce6578 100644 --- a/App/FeatureSet/BaseAPI/Index.ts +++ b/App/FeatureSet/BaseAPI/Index.ts @@ -546,7 +546,7 @@ import MonitorFeedService, { Service as MonitorFeedServiceType, } from "Common/Server/Services/MonitorFeedService"; -// MetricType. +// MetricType. import MetricTypeService, { Service as MetricTypeServiceType, } from "Common/Server/Services/MetricTypeService"; diff --git a/Common/Models/DatabaseModels/Index.ts b/Common/Models/DatabaseModels/Index.ts index ea40416d0a..14c0272516 100644 --- a/Common/Models/DatabaseModels/Index.ts +++ b/Common/Models/DatabaseModels/Index.ts @@ -363,7 +363,7 @@ const AllModelTypes: Array<{ MonitorFeed, - MetricType + MetricType, ]; const modelTypeMap: { [key: string]: { new (): BaseModel } } = {}; diff --git a/Common/Models/DatabaseModels/MetricType.ts b/Common/Models/DatabaseModels/MetricType.ts index aab18c64d9..0c1d3ec7ce 100644 --- a/Common/Models/DatabaseModels/MetricType.ts +++ b/Common/Models/DatabaseModels/MetricType.ts @@ -105,7 +105,7 @@ export default class MetricType extends BaseModel { nullable: true, onDelete: "CASCADE", orphanedRowAction: "nullify", - } + }, ) @JoinColumn({ name: "projectId" }) public project?: Project = undefined; @@ -141,7 +141,7 @@ export default class MetricType extends BaseModel { () => { return TelemetryService; }, - { eager: false } + { eager: false }, ) @JoinTable({ name: "MetricTypeTelemetryService", @@ -249,7 +249,7 @@ export default class MetricType extends BaseModel { nullable: true, onDelete: "SET NULL", orphanedRowAction: "nullify", - } + }, ) @JoinColumn({ name: "createdByUserId" }) public createdByUser?: User = undefined; @@ -308,7 +308,7 @@ export default class MetricType extends BaseModel { nullable: true, onDelete: "SET NULL", orphanedRowAction: "nullify", - } + }, ) @JoinColumn({ name: "deletedByUserId" }) public deletedByUser?: User = undefined; diff --git a/Common/Server/Infrastructure/Postgres/SchemaMigrations/1743518485566-MigrationName.ts b/Common/Server/Infrastructure/Postgres/SchemaMigrations/1743518485566-MigrationName.ts index 8f78033dc1..ee11a0726b 100644 --- a/Common/Server/Infrastructure/Postgres/SchemaMigrations/1743518485566-MigrationName.ts +++ b/Common/Server/Infrastructure/Postgres/SchemaMigrations/1743518485566-MigrationName.ts @@ -1,32 +1,67 @@ import { MigrationInterface, QueryRunner } from "typeorm"; export class MigrationName1743518485566 implements MigrationInterface { - public name = 'MigrationName1743518485566' + public name = "MigrationName1743518485566"; - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`CREATE TABLE "MetricType" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "projectId" uuid NOT NULL, "name" character varying(100) NOT NULL, "createdByUserId" uuid, "deletedByUserId" uuid, CONSTRAINT "PK_3b19440ac8f314d9c775e026af5" PRIMARY KEY ("_id"))`); - await queryRunner.query(`CREATE INDEX "IDX_d25bfc3fab2ebac8e977d88593" ON "MetricType" ("projectId") `); - await queryRunner.query(`CREATE TABLE "MetricTypeTelemetryService" ("metricTypeId" uuid NOT NULL, "telemetryServiceId" uuid NOT NULL, CONSTRAINT "PK_ff3bdfa86c187345b15bf2d94e5" PRIMARY KEY ("metricTypeId", "telemetryServiceId"))`); - await queryRunner.query(`CREATE INDEX "IDX_2e26ea92e9cb5693040fd0a65b" ON "MetricTypeTelemetryService" ("metricTypeId") `); - await queryRunner.query(`CREATE INDEX "IDX_f5ca58781b68c634e61ce25868" ON "MetricTypeTelemetryService" ("telemetryServiceId") `); - await queryRunner.query(`ALTER TABLE "MetricType" ADD CONSTRAINT "FK_d25bfc3fab2ebac8e977d88593a" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`); - await queryRunner.query(`ALTER TABLE "MetricType" ADD CONSTRAINT "FK_0662070948eed6110c5e108e77f" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`); - await queryRunner.query(`ALTER TABLE "MetricType" ADD CONSTRAINT "FK_154d3b5c6f725d30753ef209666" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`); - await queryRunner.query(`ALTER TABLE "MetricTypeTelemetryService" ADD CONSTRAINT "FK_2e26ea92e9cb5693040fd0a65bb" FOREIGN KEY ("metricTypeId") REFERENCES "MetricType"("_id") ON DELETE CASCADE ON UPDATE CASCADE`); - await queryRunner.query(`ALTER TABLE "MetricTypeTelemetryService" ADD CONSTRAINT "FK_f5ca58781b68c634e61ce25868b" FOREIGN KEY ("telemetryServiceId") REFERENCES "TelemetryService"("_id") ON DELETE CASCADE ON UPDATE CASCADE`); - } - - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`ALTER TABLE "MetricTypeTelemetryService" DROP CONSTRAINT "FK_f5ca58781b68c634e61ce25868b"`); - await queryRunner.query(`ALTER TABLE "MetricTypeTelemetryService" DROP CONSTRAINT "FK_2e26ea92e9cb5693040fd0a65bb"`); - await queryRunner.query(`ALTER TABLE "MetricType" DROP CONSTRAINT "FK_154d3b5c6f725d30753ef209666"`); - await queryRunner.query(`ALTER TABLE "MetricType" DROP CONSTRAINT "FK_0662070948eed6110c5e108e77f"`); - await queryRunner.query(`ALTER TABLE "MetricType" DROP CONSTRAINT "FK_d25bfc3fab2ebac8e977d88593a"`); - await queryRunner.query(`DROP INDEX "public"."IDX_f5ca58781b68c634e61ce25868"`); - await queryRunner.query(`DROP INDEX "public"."IDX_2e26ea92e9cb5693040fd0a65b"`); - await queryRunner.query(`DROP TABLE "MetricTypeTelemetryService"`); - await queryRunner.query(`DROP INDEX "public"."IDX_d25bfc3fab2ebac8e977d88593"`); - await queryRunner.query(`DROP TABLE "MetricType"`); - } + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "MetricType" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "projectId" uuid NOT NULL, "name" character varying(100) NOT NULL, "createdByUserId" uuid, "deletedByUserId" uuid, CONSTRAINT "PK_3b19440ac8f314d9c775e026af5" PRIMARY KEY ("_id"))`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_d25bfc3fab2ebac8e977d88593" ON "MetricType" ("projectId") `, + ); + await queryRunner.query( + `CREATE TABLE "MetricTypeTelemetryService" ("metricTypeId" uuid NOT NULL, "telemetryServiceId" uuid NOT NULL, CONSTRAINT "PK_ff3bdfa86c187345b15bf2d94e5" PRIMARY KEY ("metricTypeId", "telemetryServiceId"))`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_2e26ea92e9cb5693040fd0a65b" ON "MetricTypeTelemetryService" ("metricTypeId") `, + ); + await queryRunner.query( + `CREATE INDEX "IDX_f5ca58781b68c634e61ce25868" ON "MetricTypeTelemetryService" ("telemetryServiceId") `, + ); + await queryRunner.query( + `ALTER TABLE "MetricType" ADD CONSTRAINT "FK_d25bfc3fab2ebac8e977d88593a" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "MetricType" ADD CONSTRAINT "FK_0662070948eed6110c5e108e77f" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "MetricType" ADD CONSTRAINT "FK_154d3b5c6f725d30753ef209666" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "MetricTypeTelemetryService" ADD CONSTRAINT "FK_2e26ea92e9cb5693040fd0a65bb" FOREIGN KEY ("metricTypeId") REFERENCES "MetricType"("_id") ON DELETE CASCADE ON UPDATE CASCADE`, + ); + await queryRunner.query( + `ALTER TABLE "MetricTypeTelemetryService" ADD CONSTRAINT "FK_f5ca58781b68c634e61ce25868b" FOREIGN KEY ("telemetryServiceId") REFERENCES "TelemetryService"("_id") ON DELETE CASCADE ON UPDATE CASCADE`, + ); + } + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "MetricTypeTelemetryService" DROP CONSTRAINT "FK_f5ca58781b68c634e61ce25868b"`, + ); + await queryRunner.query( + `ALTER TABLE "MetricTypeTelemetryService" DROP CONSTRAINT "FK_2e26ea92e9cb5693040fd0a65bb"`, + ); + await queryRunner.query( + `ALTER TABLE "MetricType" DROP CONSTRAINT "FK_154d3b5c6f725d30753ef209666"`, + ); + await queryRunner.query( + `ALTER TABLE "MetricType" DROP CONSTRAINT "FK_0662070948eed6110c5e108e77f"`, + ); + await queryRunner.query( + `ALTER TABLE "MetricType" DROP CONSTRAINT "FK_d25bfc3fab2ebac8e977d88593a"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_f5ca58781b68c634e61ce25868"`, + ); + await queryRunner.query( + `DROP INDEX "public"."IDX_2e26ea92e9cb5693040fd0a65b"`, + ); + await queryRunner.query(`DROP TABLE "MetricTypeTelemetryService"`); + await queryRunner.query( + `DROP INDEX "public"."IDX_d25bfc3fab2ebac8e977d88593"`, + ); + await queryRunner.query(`DROP TABLE "MetricType"`); + } } diff --git a/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts b/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts index 54fc976833..c2fa951649 100644 --- a/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +++ b/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts @@ -251,5 +251,5 @@ export default [ MigrationName1743005293206, MigrationName1743006662678, MigrationName1743186793413, - MigrationName1743518485566 + MigrationName1743518485566, ]; diff --git a/Common/Server/Services/MetricTypeService.ts b/Common/Server/Services/MetricTypeService.ts index e76ed650de..8c0fabe227 100644 --- a/Common/Server/Services/MetricTypeService.ts +++ b/Common/Server/Services/MetricTypeService.ts @@ -1,7 +1,6 @@ import DatabaseService from "./DatabaseService"; import Model from "Common/Models/DatabaseModels/MetricType"; - export class Service extends DatabaseService { public constructor() { super(Model); diff --git a/Common/Server/Utils/Telemetry/Telemetry.ts b/Common/Server/Utils/Telemetry/Telemetry.ts index f68844b499..8c32519090 100644 --- a/Common/Server/Utils/Telemetry/Telemetry.ts +++ b/Common/Server/Utils/Telemetry/Telemetry.ts @@ -6,10 +6,104 @@ import GlobalCache from "../../Infrastructure/GlobalCache"; import TelemetryAttributeService from "../../Services/TelemetryAttributeService"; import CaptureSpan from "./CaptureSpan"; import logger from "../Logger"; +import MetricType from "../../../Models/DatabaseModels/MetricType"; +import MetricTypeService from "../../Services/MetricTypeService"; +import TelemetryService from "../../../Models/DatabaseModels/TelemetryService"; export type AttributeType = string | number | boolean | null; export default class TelemetryUtil { + @CaptureSpan() + public static async indexMetricNameServiceNameMap(data: { + projectId: ObjectID; + metricNameServiceNameMap: Dictionary>; + }): Promise { + for (const metricName of Object.keys(data.metricNameServiceNameMap)) { + // fetch metric + const metricType: MetricType | null = await MetricTypeService.findOneBy({ + query: { + projectId: data.projectId, + name: metricName, + }, + select: { + telemetryServices: true, + }, + props: { + isRoot: true, + }, + }); + + if (metricType) { + if (!metricType.telemetryServices) { + metricType.telemetryServices = []; + } + + const telemetryServiceIds: Array = + metricType.telemetryServices!.map((service: TelemetryService) => { + return service.id!; + }); + + // check if telemetry services are same as the ones in the map + const telemetryServicesInMap: Array = + data.metricNameServiceNameMap[metricName] || []; + + let isSame: boolean = true; + + for (const telemetryServiceId of telemetryServicesInMap) { + if ( + !telemetryServiceIds.filter((serviceId: ObjectID) => { + return serviceId.toString() === telemetryServiceId.toString(); + }).length + ) { + isSame = false; + // add the service id to the list + const telemetryService: TelemetryService = new TelemetryService(); + telemetryService.id = telemetryServiceId; + metricType.telemetryServices!.push(telemetryService); + } + } + + // if its not the same then update the metric type + + if (!isSame) { + // update metric type + await MetricTypeService.updateOneById({ + _id: metricType.id!, + data: { + telemetryServices: metricType.telemetryServices || [], + }, + props: { + isRoot: true, + }, + } as any); + } + } else { + // create metric type + const metricType: MetricType = new MetricType(); + metricType.name = metricName; + metricType.projectId = data.projectId; + metricType.telemetryServices = []; + + const telemetryServiceIds: Array = + data.metricNameServiceNameMap[metricName] || []; + + for (const telemetryServiceId of telemetryServiceIds) { + const telemetryService: TelemetryService = new TelemetryService(); + telemetryService.id = telemetryServiceId; + metricType.telemetryServices!.push(telemetryService); + } + + // save metric type + await MetricTypeService.create({ + data: metricType, + props: { + isRoot: true, + }, + }); + } + } + } + @CaptureSpan() public static async indexAttributes(data: { attributes: Array; diff --git a/Dashboard/src/Components/Metrics/MetricsTable.tsx b/Dashboard/src/Components/Metrics/MetricsTable.tsx index 48592db4e3..d2c320069e 100644 --- a/Dashboard/src/Components/Metrics/MetricsTable.tsx +++ b/Dashboard/src/Components/Metrics/MetricsTable.tsx @@ -1,29 +1,16 @@ import ProjectUtil from "Common/UI/Utils/Project"; import SortOrder from "Common/Types/BaseDatabase/SortOrder"; import ObjectID from "Common/Types/ObjectID"; -import AnalyticsModelTable from "Common/UI/Components/ModelTable/AnalyticsModelTable"; import FieldType from "Common/UI/Components/Types/FieldType"; import Navigation from "Common/UI/Utils/Navigation"; -import Metric from "Common/Models/AnalyticsModels/Metric"; import RouteMap, { RouteUtil } from "../../Utils/RouteMap"; import PageMap from "../../Utils/PageMap"; import Route from "Common/Types/API/Route"; import URL from "Common/Types/API/URL"; -import React, { - Fragment, - FunctionComponent, - ReactElement, - useEffect, -} from "react"; -import { PromiseVoidFunction } from "Common/Types/FunctionTypes"; -import HTTPResponse from "Common/Types/API/HTTPResponse"; -import { JSONObject } from "Common/Types/JSON"; -import HTTPErrorResponse from "Common/Types/API/HTTPErrorResponse"; -import API from "Common/Utils/API"; -import { APP_API_URL } from "Common/UI/Config"; -import ModelAPI from "Common/UI/Utils/ModelAPI/ModelAPI"; -import PageLoader from "Common/UI/Components/Loader/PageLoader"; -import ErrorMessage from "Common/UI/Components/ErrorMessage/ErrorMessage"; +import React, { Fragment, FunctionComponent, ReactElement } from "react"; +import ModelTable from "Common/UI/Components/ModelTable/ModelTable"; +import MetricType from "Common/Models/DatabaseModels/MetricType"; +import Includes from "Common/Types/BaseDatabase/Includes"; export interface ComponentProps { telemetryServiceId?: ObjectID | undefined; @@ -33,61 +20,10 @@ export interface ComponentProps { const MetricsTable: FunctionComponent = ( props: ComponentProps, ): ReactElement => { - const [attributes, setAttributes] = React.useState>([]); - - const [isPageLoading, setIsPageLoading] = React.useState(true); - const [pageError, setPageError] = React.useState(""); - - const loadAttributes: PromiseVoidFunction = async (): Promise => { - try { - setIsPageLoading(true); - - const attributeRepsonse: HTTPResponse | HTTPErrorResponse = - await API.post( - URL.fromString(APP_API_URL.toString()).addRoute( - "/telemetry/metrics/get-attributes", - ), - {}, - { - ...ModelAPI.getCommonHeaders(), - }, - ); - - if (attributeRepsonse instanceof HTTPErrorResponse) { - throw attributeRepsonse; - } else { - const attributes: Array = attributeRepsonse.data[ - "attributes" - ] as Array; - setAttributes(attributes); - } - - setIsPageLoading(false); - setPageError(""); - } catch (err) { - setIsPageLoading(false); - setPageError(API.getFriendlyErrorMessage(err as Error)); - } - }; - - useEffect(() => { - loadAttributes().catch((err: Error) => { - setPageError(API.getFriendlyErrorMessage(err as Error)); - }); - }, []); - - if (isPageLoading) { - return ; - } - - if (pageError) { - return ; - } - return ( - - modelType={Metric} + + modelType={MetricType} id="metrics-table" isDeleteable={false} isEditable={false} @@ -103,10 +39,7 @@ const MetricsTable: FunctionComponent = ( description: "Metrics are the individual data points that make up a service. They are the building blocks of a service and represent the work done by a single service.", }} - groupBy={{ - name: true, - }} - onViewPage={async (item: Metric) => { + onViewPage={async (item: MetricType) => { if (!props.telemetryServiceId || !props.telemetryServiceName) { const route: Route = RouteUtil.populateRouteParams( RouteMap[PageMap.TELEMETRY_METRIC_VIEW]!, @@ -140,8 +73,8 @@ const MetricsTable: FunctionComponent = ( }} query={{ projectId: ProjectUtil.getCurrentProjectId()!, - serviceId: props.telemetryServiceId - ? props.telemetryServiceId + telemetryServices: props.telemetryServiceId + ? new Includes([props.telemetryServiceId]) : undefined, }} showViewIdButton={false} @@ -156,14 +89,6 @@ const MetricsTable: FunctionComponent = ( title: "Name", type: FieldType.Text, }, - { - field: { - attributes: true, - }, - type: FieldType.JSON, - title: "Attributes", - jsonKeys: attributes, - }, ]} columns={[ { diff --git a/OpenTelemetryIngest/Services/OtelIngest.ts b/OpenTelemetryIngest/Services/OtelIngest.ts index 4afb56f2c7..2114476ffa 100644 --- a/OpenTelemetryIngest/Services/OtelIngest.ts +++ b/OpenTelemetryIngest/Services/OtelIngest.ts @@ -275,6 +275,10 @@ export default class OtelIngestService { const attributeKeySet: Set = new Set(); const serviceDictionary: Dictionary = {}; + // Metric name to serviceId map + // example: "cpu.usage" -> [serviceId1, serviceId2] + const metricNameServiceNameMap: Dictionary> = {}; + for (const resourceMetric of resourceMetrics) { const serviceName: string = this.getServiceNameFromAttributes( ((resourceMetric["resource"] as JSONObject)?.[ @@ -331,10 +335,32 @@ export default class OtelIngestService { dbMetric.projectId = (req as TelemetryRequest).projectId; dbMetric.serviceId = serviceDictionary[serviceName]!.serviceId!; dbMetric.serviceType = ServiceType.OpenTelemetry; - dbMetric.name = metric["name"] as string; + dbMetric.name = (metric["name"] || "").toString().toLowerCase(); dbMetric.description = metric["description"] as string; dbMetric.unit = metric["unit"] as string; + if (dbMetric.name) { + // add this to metricNameServiceNameMap + if (!metricNameServiceNameMap[dbMetric.name]) { + metricNameServiceNameMap[dbMetric.name] = []; + } + + if ( + metricNameServiceNameMap[dbMetric.name]!.filter( + (serviceId: ObjectID) => { + return ( + serviceId.toString() === + serviceDictionary[serviceName]!.serviceId!.toString() + ); + }, + ).length > 0 + ) { + metricNameServiceNameMap[dbMetric.name]!.push( + dbMetric.serviceId, + ); + } + } + const attributesObject: Dictionary< AttributeType | Array > = { @@ -427,6 +453,10 @@ export default class OtelIngestService { projectId: (req as TelemetryRequest).projectId, telemetryType: TelemetryType.Metric, }), + TelemetryUtil.indexMetricNameServiceNameMap({ + metricNameServiceNameMap: metricNameServiceNameMap, + projectId: (req as TelemetryRequest).projectId, + }), OTelIngestService.recordDataIngestedUsgaeBilling({ services: serviceDictionary, projectId: (req as TelemetryRequest).projectId,