feat: implement metric name to service ID mapping in telemetry utility

2026-04-06 00:32:12 +02:00 · 2025-04-01 16:11:40 +01:00
parent 302282a2cb
commit 6ea5ff82c2
9 changed files with 202 additions and 119 deletions
--- a/App/FeatureSet/BaseAPI/Index.ts
+++ b/App/FeatureSet/BaseAPI/Index.ts
@@ -546,7 +546,7 @@ import MonitorFeedService, {
  Service as MonitorFeedServiceType,
 } from "Common/Server/Services/MonitorFeedService";

-// MetricType. 
+// MetricType.
 import MetricTypeService, {
  Service as MetricTypeServiceType,
 } from "Common/Server/Services/MetricTypeService";
--- a/Common/Models/DatabaseModels/Index.ts
+++ b/Common/Models/DatabaseModels/Index.ts
@@ -363,7 +363,7 @@ const AllModelTypes: Array<{

  MonitorFeed,

-  MetricType
+  MetricType,
 ];

 const modelTypeMap: { [key: string]: { new (): BaseModel } } = {};
--- a/Common/Models/DatabaseModels/MetricType.ts
+++ b/Common/Models/DatabaseModels/MetricType.ts
@@ -105,7 +105,7 @@ export default class MetricType extends BaseModel {
      nullable: true,
      onDelete: "CASCADE",
      orphanedRowAction: "nullify",
-    }
+    },
  )
  @JoinColumn({ name: "projectId" })
  public project?: Project = undefined;
@@ -141,7 +141,7 @@ export default class MetricType extends BaseModel {
    () => {
      return TelemetryService;
    },
-    { eager: false }
+    { eager: false },
  )
  @JoinTable({
    name: "MetricTypeTelemetryService",
@@ -249,7 +249,7 @@ export default class MetricType extends BaseModel {
      nullable: true,
      onDelete: "SET NULL",
      orphanedRowAction: "nullify",
-    }
+    },
  )
  @JoinColumn({ name: "createdByUserId" })
  public createdByUser?: User = undefined;
@@ -308,7 +308,7 @@ export default class MetricType extends BaseModel {
      nullable: true,
      onDelete: "SET NULL",
      orphanedRowAction: "nullify",
-    }
+    },
  )
  @JoinColumn({ name: "deletedByUserId" })
  public deletedByUser?: User = undefined;
--- a/Common/Server/Infrastructure/Postgres/SchemaMigrations/1743518485566-MigrationName.ts
+++ b/Common/Server/Infrastructure/Postgres/SchemaMigrations/1743518485566-MigrationName.ts
@@ -1,32 +1,67 @@
 import { MigrationInterface, QueryRunner } from "typeorm";

 export class MigrationName1743518485566 implements MigrationInterface {
-    public name = 'MigrationName1743518485566'
+  public name = "MigrationName1743518485566";

-    public async up(queryRunner: QueryRunner): Promise<void> {
-        await queryRunner.query(`CREATE TABLE "MetricType" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "projectId" uuid NOT NULL, "name" character varying(100) NOT NULL, "createdByUserId" uuid, "deletedByUserId" uuid, CONSTRAINT "PK_3b19440ac8f314d9c775e026af5" PRIMARY KEY ("_id"))`);
-        await queryRunner.query(`CREATE INDEX "IDX_d25bfc3fab2ebac8e977d88593" ON "MetricType" ("projectId") `);
-        await queryRunner.query(`CREATE TABLE "MetricTypeTelemetryService" ("metricTypeId" uuid NOT NULL, "telemetryServiceId" uuid NOT NULL, CONSTRAINT "PK_ff3bdfa86c187345b15bf2d94e5" PRIMARY KEY ("metricTypeId", "telemetryServiceId"))`);
-        await queryRunner.query(`CREATE INDEX "IDX_2e26ea92e9cb5693040fd0a65b" ON "MetricTypeTelemetryService" ("metricTypeId") `);
-        await queryRunner.query(`CREATE INDEX "IDX_f5ca58781b68c634e61ce25868" ON "MetricTypeTelemetryService" ("telemetryServiceId") `);
-        await queryRunner.query(`ALTER TABLE "MetricType" ADD CONSTRAINT "FK_d25bfc3fab2ebac8e977d88593a" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`);
-        await queryRunner.query(`ALTER TABLE "MetricType" ADD CONSTRAINT "FK_0662070948eed6110c5e108e77f" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`);
-        await queryRunner.query(`ALTER TABLE "MetricType" ADD CONSTRAINT "FK_154d3b5c6f725d30753ef209666" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`);
-        await queryRunner.query(`ALTER TABLE "MetricTypeTelemetryService" ADD CONSTRAINT "FK_2e26ea92e9cb5693040fd0a65bb" FOREIGN KEY ("metricTypeId") REFERENCES "MetricType"("_id") ON DELETE CASCADE ON UPDATE CASCADE`);
-        await queryRunner.query(`ALTER TABLE "MetricTypeTelemetryService" ADD CONSTRAINT "FK_f5ca58781b68c634e61ce25868b" FOREIGN KEY ("telemetryServiceId") REFERENCES "TelemetryService"("_id") ON DELETE CASCADE ON UPDATE CASCADE`);
-    }
-
-    public async down(queryRunner: QueryRunner): Promise<void> {
-        await queryRunner.query(`ALTER TABLE "MetricTypeTelemetryService" DROP CONSTRAINT "FK_f5ca58781b68c634e61ce25868b"`);
-        await queryRunner.query(`ALTER TABLE "MetricTypeTelemetryService" DROP CONSTRAINT "FK_2e26ea92e9cb5693040fd0a65bb"`);
-        await queryRunner.query(`ALTER TABLE "MetricType" DROP CONSTRAINT "FK_154d3b5c6f725d30753ef209666"`);
-        await queryRunner.query(`ALTER TABLE "MetricType" DROP CONSTRAINT "FK_0662070948eed6110c5e108e77f"`);
-        await queryRunner.query(`ALTER TABLE "MetricType" DROP CONSTRAINT "FK_d25bfc3fab2ebac8e977d88593a"`);
-        await queryRunner.query(`DROP INDEX "public"."IDX_f5ca58781b68c634e61ce25868"`);
-        await queryRunner.query(`DROP INDEX "public"."IDX_2e26ea92e9cb5693040fd0a65b"`);
-        await queryRunner.query(`DROP TABLE "MetricTypeTelemetryService"`);
-        await queryRunner.query(`DROP INDEX "public"."IDX_d25bfc3fab2ebac8e977d88593"`);
-        await queryRunner.query(`DROP TABLE "MetricType"`);
-    }
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "MetricType" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "projectId" uuid NOT NULL, "name" character varying(100) NOT NULL, "createdByUserId" uuid, "deletedByUserId" uuid, CONSTRAINT "PK_3b19440ac8f314d9c775e026af5" PRIMARY KEY ("_id"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_d25bfc3fab2ebac8e977d88593" ON "MetricType" ("projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE TABLE "MetricTypeTelemetryService" ("metricTypeId" uuid NOT NULL, "telemetryServiceId" uuid NOT NULL, CONSTRAINT "PK_ff3bdfa86c187345b15bf2d94e5" PRIMARY KEY ("metricTypeId", "telemetryServiceId"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_2e26ea92e9cb5693040fd0a65b" ON "MetricTypeTelemetryService" ("metricTypeId") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_f5ca58781b68c634e61ce25868" ON "MetricTypeTelemetryService" ("telemetryServiceId") `,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "MetricType" ADD CONSTRAINT "FK_d25bfc3fab2ebac8e977d88593a" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "MetricType" ADD CONSTRAINT "FK_0662070948eed6110c5e108e77f" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "MetricType" ADD CONSTRAINT "FK_154d3b5c6f725d30753ef209666" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "MetricTypeTelemetryService" ADD CONSTRAINT "FK_2e26ea92e9cb5693040fd0a65bb" FOREIGN KEY ("metricTypeId") REFERENCES "MetricType"("_id") ON DELETE CASCADE ON UPDATE CASCADE`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "MetricTypeTelemetryService" ADD CONSTRAINT "FK_f5ca58781b68c634e61ce25868b" FOREIGN KEY ("telemetryServiceId") REFERENCES "TelemetryService"("_id") ON DELETE CASCADE ON UPDATE CASCADE`,
+    );
+  }

+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "MetricTypeTelemetryService" DROP CONSTRAINT "FK_f5ca58781b68c634e61ce25868b"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "MetricTypeTelemetryService" DROP CONSTRAINT "FK_2e26ea92e9cb5693040fd0a65bb"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "MetricType" DROP CONSTRAINT "FK_154d3b5c6f725d30753ef209666"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "MetricType" DROP CONSTRAINT "FK_0662070948eed6110c5e108e77f"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "MetricType" DROP CONSTRAINT "FK_d25bfc3fab2ebac8e977d88593a"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_f5ca58781b68c634e61ce25868"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_2e26ea92e9cb5693040fd0a65b"`,
+    );
+    await queryRunner.query(`DROP TABLE "MetricTypeTelemetryService"`);
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_d25bfc3fab2ebac8e977d88593"`,
+    );
+    await queryRunner.query(`DROP TABLE "MetricType"`);
+  }
 }
--- a/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts
+++ b/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts
@@ -251,5 +251,5 @@ export default [
  MigrationName1743005293206,
  MigrationName1743006662678,
  MigrationName1743186793413,
-  MigrationName1743518485566
+  MigrationName1743518485566,
 ];
--- a/Common/Server/Services/MetricTypeService.ts
+++ b/Common/Server/Services/MetricTypeService.ts
@@ -1,7 +1,6 @@
 import DatabaseService from "./DatabaseService";
 import Model from "Common/Models/DatabaseModels/MetricType";

-
 export class Service extends DatabaseService<Model> {
  public constructor() {
    super(Model);
--- a/Common/Server/Utils/Telemetry/Telemetry.ts
+++ b/Common/Server/Utils/Telemetry/Telemetry.ts
@@ -6,10 +6,104 @@ import GlobalCache from "../../Infrastructure/GlobalCache";
 import TelemetryAttributeService from "../../Services/TelemetryAttributeService";
 import CaptureSpan from "./CaptureSpan";
 import logger from "../Logger";
+import MetricType from "../../../Models/DatabaseModels/MetricType";
+import MetricTypeService from "../../Services/MetricTypeService";
+import TelemetryService from "../../../Models/DatabaseModels/TelemetryService";

 export type AttributeType = string | number | boolean | null;

 export default class TelemetryUtil {
+  @CaptureSpan()
+  public static async indexMetricNameServiceNameMap(data: {
+    projectId: ObjectID;
+    metricNameServiceNameMap: Dictionary<Array<ObjectID>>;
+  }): Promise<void> {
+    for (const metricName of Object.keys(data.metricNameServiceNameMap)) {
+      // fetch metric
+      const metricType: MetricType | null = await MetricTypeService.findOneBy({
+        query: {
+          projectId: data.projectId,
+          name: metricName,
+        },
+        select: {
+          telemetryServices: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (metricType) {
+        if (!metricType.telemetryServices) {
+          metricType.telemetryServices = [];
+        }
+
+        const telemetryServiceIds: Array<ObjectID> =
+          metricType.telemetryServices!.map((service: TelemetryService) => {
+            return service.id!;
+          });
+
+        // check if telemetry services are same as the ones in the map
+        const telemetryServicesInMap: Array<ObjectID> =
+          data.metricNameServiceNameMap[metricName] || [];
+
+        let isSame: boolean = true;
+
+        for (const telemetryServiceId of telemetryServicesInMap) {
+          if (
+            !telemetryServiceIds.filter((serviceId: ObjectID) => {
+              return serviceId.toString() === telemetryServiceId.toString();
+            }).length
+          ) {
+            isSame = false;
+            // add the service id to the list
+            const telemetryService: TelemetryService = new TelemetryService();
+            telemetryService.id = telemetryServiceId;
+            metricType.telemetryServices!.push(telemetryService);
+          }
+        }
+
+        // if its not the same then update the metric type
+
+        if (!isSame) {
+          // update metric type
+          await MetricTypeService.updateOneById({
+            _id: metricType.id!,
+            data: {
+              telemetryServices: metricType.telemetryServices || [],
+            },
+            props: {
+              isRoot: true,
+            },
+          } as any);
+        }
+      } else {
+        // create metric type
+        const metricType: MetricType = new MetricType();
+        metricType.name = metricName;
+        metricType.projectId = data.projectId;
+        metricType.telemetryServices = [];
+
+        const telemetryServiceIds: Array<ObjectID> =
+          data.metricNameServiceNameMap[metricName] || [];
+
+        for (const telemetryServiceId of telemetryServiceIds) {
+          const telemetryService: TelemetryService = new TelemetryService();
+          telemetryService.id = telemetryServiceId;
+          metricType.telemetryServices!.push(telemetryService);
+        }
+
+        // save metric type
+        await MetricTypeService.create({
+          data: metricType,
+          props: {
+            isRoot: true,
+          },
+        });
+      }
+    }
+  }
+
  @CaptureSpan()
  public static async indexAttributes(data: {
    attributes: Array<string>;
--- a/Dashboard/src/Components/Metrics/MetricsTable.tsx
+++ b/Dashboard/src/Components/Metrics/MetricsTable.tsx
@@ -1,29 +1,16 @@
 import ProjectUtil from "Common/UI/Utils/Project";
 import SortOrder from "Common/Types/BaseDatabase/SortOrder";
 import ObjectID from "Common/Types/ObjectID";
-import AnalyticsModelTable from "Common/UI/Components/ModelTable/AnalyticsModelTable";
 import FieldType from "Common/UI/Components/Types/FieldType";
 import Navigation from "Common/UI/Utils/Navigation";
-import Metric from "Common/Models/AnalyticsModels/Metric";
 import RouteMap, { RouteUtil } from "../../Utils/RouteMap";
 import PageMap from "../../Utils/PageMap";
 import Route from "Common/Types/API/Route";
 import URL from "Common/Types/API/URL";
-import React, {
-  Fragment,
-  FunctionComponent,
-  ReactElement,
-  useEffect,
-} from "react";
-import { PromiseVoidFunction } from "Common/Types/FunctionTypes";
-import HTTPResponse from "Common/Types/API/HTTPResponse";
-import { JSONObject } from "Common/Types/JSON";
-import HTTPErrorResponse from "Common/Types/API/HTTPErrorResponse";
-import API from "Common/Utils/API";
-import { APP_API_URL } from "Common/UI/Config";
-import ModelAPI from "Common/UI/Utils/ModelAPI/ModelAPI";
-import PageLoader from "Common/UI/Components/Loader/PageLoader";
-import ErrorMessage from "Common/UI/Components/ErrorMessage/ErrorMessage";
+import React, { Fragment, FunctionComponent, ReactElement } from "react";
+import ModelTable from "Common/UI/Components/ModelTable/ModelTable";
+import MetricType from "Common/Models/DatabaseModels/MetricType";
+import Includes from "Common/Types/BaseDatabase/Includes";

 export interface ComponentProps {
  telemetryServiceId?: ObjectID | undefined;
@@ -33,61 +20,10 @@ export interface ComponentProps {
 const MetricsTable: FunctionComponent<ComponentProps> = (
  props: ComponentProps,
 ): ReactElement => {
-  const [attributes, setAttributes] = React.useState<Array<string>>([]);
-
-  const [isPageLoading, setIsPageLoading] = React.useState<boolean>(true);
-  const [pageError, setPageError] = React.useState<string>("");
-
-  const loadAttributes: PromiseVoidFunction = async (): Promise<void> => {
-    try {
-      setIsPageLoading(true);
-
-      const attributeRepsonse: HTTPResponse<JSONObject> | HTTPErrorResponse =
-        await API.post(
-          URL.fromString(APP_API_URL.toString()).addRoute(
-            "/telemetry/metrics/get-attributes",
-          ),
-          {},
-          {
-            ...ModelAPI.getCommonHeaders(),
-          },
-        );
-
-      if (attributeRepsonse instanceof HTTPErrorResponse) {
-        throw attributeRepsonse;
-      } else {
-        const attributes: Array<string> = attributeRepsonse.data[
-          "attributes"
-        ] as Array<string>;
-        setAttributes(attributes);
-      }
-
-      setIsPageLoading(false);
-      setPageError("");
-    } catch (err) {
-      setIsPageLoading(false);
-      setPageError(API.getFriendlyErrorMessage(err as Error));
-    }
-  };
-
-  useEffect(() => {
-    loadAttributes().catch((err: Error) => {
-      setPageError(API.getFriendlyErrorMessage(err as Error));
-    });
-  }, []);
-
-  if (isPageLoading) {
-    return <PageLoader isVisible={true} />;
-  }
-
-  if (pageError) {
-    return <ErrorMessage message={pageError} />;
-  }
-
  return (
    <Fragment>
-      <AnalyticsModelTable<Metric>
-        modelType={Metric}
+      <ModelTable<MetricType>
+        modelType={MetricType}
        id="metrics-table"
        isDeleteable={false}
        isEditable={false}
@@ -103,10 +39,7 @@ const MetricsTable: FunctionComponent<ComponentProps> = (
          description:
            "Metrics are the individual data points that make up a service. They are the building blocks of a service and represent the work done by a single service.",
        }}
-        groupBy={{
-          name: true,
-        }}
-        onViewPage={async (item: Metric) => {
+        onViewPage={async (item: MetricType) => {
          if (!props.telemetryServiceId || !props.telemetryServiceName) {
            const route: Route = RouteUtil.populateRouteParams(
              RouteMap[PageMap.TELEMETRY_METRIC_VIEW]!,
@@ -140,8 +73,8 @@ const MetricsTable: FunctionComponent<ComponentProps> = (
        }}
        query={{
          projectId: ProjectUtil.getCurrentProjectId()!,
-          serviceId: props.telemetryServiceId
-            ? props.telemetryServiceId
+          telemetryServices: props.telemetryServiceId
+            ? new Includes([props.telemetryServiceId])
            : undefined,
        }}
        showViewIdButton={false}
@@ -156,14 +89,6 @@ const MetricsTable: FunctionComponent<ComponentProps> = (
            title: "Name",
            type: FieldType.Text,
          },
-          {
-            field: {
-              attributes: true,
-            },
-            type: FieldType.JSON,
-            title: "Attributes",
-            jsonKeys: attributes,
-          },
        ]}
        columns={[
          {
--- a/OpenTelemetryIngest/Services/OtelIngest.ts
+++ b/OpenTelemetryIngest/Services/OtelIngest.ts
@@ -275,6 +275,10 @@ export default class OtelIngestService {
      const attributeKeySet: Set<string> = new Set<string>();
      const serviceDictionary: Dictionary<TelemetryServiceDataIngested> = {};

+      // Metric name to serviceId map
+      // example: "cpu.usage" -> [serviceId1, serviceId2]
+      const metricNameServiceNameMap: Dictionary<Array<ObjectID>> = {};
+
      for (const resourceMetric of resourceMetrics) {
        const serviceName: string = this.getServiceNameFromAttributes(
          ((resourceMetric["resource"] as JSONObject)?.[
@@ -331,10 +335,32 @@ export default class OtelIngestService {
            dbMetric.projectId = (req as TelemetryRequest).projectId;
            dbMetric.serviceId = serviceDictionary[serviceName]!.serviceId!;
            dbMetric.serviceType = ServiceType.OpenTelemetry;
-            dbMetric.name = metric["name"] as string;
+            dbMetric.name = (metric["name"] || "").toString().toLowerCase();
            dbMetric.description = metric["description"] as string;
            dbMetric.unit = metric["unit"] as string;

+            if (dbMetric.name) {
+              // add this to metricNameServiceNameMap
+              if (!metricNameServiceNameMap[dbMetric.name]) {
+                metricNameServiceNameMap[dbMetric.name] = [];
+              }
+
+              if (
+                metricNameServiceNameMap[dbMetric.name]!.filter(
+                  (serviceId: ObjectID) => {
+                    return (
+                      serviceId.toString() ===
+                      serviceDictionary[serviceName]!.serviceId!.toString()
+                    );
+                  },
+                ).length > 0
+              ) {
+                metricNameServiceNameMap[dbMetric.name]!.push(
+                  dbMetric.serviceId,
+                );
+              }
+            }
+
            const attributesObject: Dictionary<
              AttributeType | Array<AttributeType>
            > = {
@@ -427,6 +453,10 @@ export default class OtelIngestService {
          projectId: (req as TelemetryRequest).projectId,
          telemetryType: TelemetryType.Metric,
        }),
+        TelemetryUtil.indexMetricNameServiceNameMap({
+          metricNameServiceNameMap: metricNameServiceNameMap,
+          projectId: (req as TelemetryRequest).projectId,
+        }),
        OTelIngestService.recordDataIngestedUsgaeBilling({
          services: serviceDictionary,
          projectId: (req as TelemetryRequest).projectId,