From 3c8dc1eee1e85f3098038fe803a48a6080bb25ad Mon Sep 17 00:00:00 2001 From: Nawaz Dhandala Date: Fri, 27 Mar 2026 11:11:33 +0000 Subject: [PATCH] feat: Add ProfileTable component for displaying profiling data - Implemented ProfileTable component to visualize profiles with advanced filtering options. - Integrated telemetry services and attributes loading for dynamic filtering. - Added error handling and loading states for improved user experience. feat: Create ProfileUtil for stack frame parsing and formatting - Introduced ProfileUtil class with methods for frame type color coding and stack frame parsing. - Added utility functions for formatting profile values based on units. docs: Add documentation for telemetry profiles integration - Created comprehensive guide on sending continuous profiling data to OneUptime. - Included supported profile types, setup instructions, and instrumentation examples. feat: Implement ProfileAggregationService for flamegraph and function list retrieval - Developed ProfileAggregationService to aggregate profile samples and generate flamegraphs. - Added functionality to retrieve top functions based on various metrics. feat: Define MonitorStepProfileMonitor interface for profile monitoring - Created MonitorStepProfileMonitor interface and utility for building queries based on monitoring parameters. test: Add example OTLP profiles payload for testing - Included example JSON payload for OTLP profiles to assist in testing and integration. --- .../Dashboard/Toolbar/DashboardToolbar.tsx | 82 ++-- .../Components/Profiles/ProfileFlamegraph.tsx | 376 ++++++++++++++++ .../Profiles/ProfileFunctionList.tsx | 287 ++++++++++++ .../src/Components/Profiles/ProfileTable.tsx | 342 ++++++++++++++ .../Dashboard/src/Pages/Profiles/Index.tsx | 9 +- .../src/Pages/Profiles/View/Index.tsx | 27 +- .../Dashboard/src/Utils/ProfileUtil.ts | 148 +++++++ .../Docs/Content/telemetry/profiles.md | 184 ++++++++ Common/Server/API/TelemetryAPI.ts | 166 +++++++ Common/Server/EnvironmentConfig.ts | 8 + Common/Server/Services/IncidentService.ts | 13 +- .../Services/ProfileAggregationService.ts | 417 ++++++++++++++++++ .../Services/TelemetryUsageBillingService.ts | 81 +++- .../Billing/MeteredPlan/AllMeteredPlans.ts | 9 + .../Monitor/MonitorStepProfileMonitor.ts | 98 ++++ .../Public/oneuptime/templates/_helpers.tpl | 6 + HelmChart/Public/oneuptime/values.schema.json | 8 + HelmChart/Public/oneuptime/values.yaml | 2 + .../Docs/examples/otlp-profiles-payload.json | 281 ++++++++++++ 19 files changed, 2482 insertions(+), 62 deletions(-) create mode 100644 App/FeatureSet/Dashboard/src/Components/Profiles/ProfileFlamegraph.tsx create mode 100644 App/FeatureSet/Dashboard/src/Components/Profiles/ProfileFunctionList.tsx create mode 100644 App/FeatureSet/Dashboard/src/Components/Profiles/ProfileTable.tsx create mode 100644 App/FeatureSet/Dashboard/src/Utils/ProfileUtil.ts create mode 100644 App/FeatureSet/Docs/Content/telemetry/profiles.md create mode 100644 Common/Server/Services/ProfileAggregationService.ts create mode 100644 Common/Types/Monitor/MonitorStepProfileMonitor.ts create mode 100644 Telemetry/Docs/examples/otlp-profiles-payload.json diff --git a/App/FeatureSet/Dashboard/src/Components/Dashboard/Toolbar/DashboardToolbar.tsx b/App/FeatureSet/Dashboard/src/Components/Dashboard/Toolbar/DashboardToolbar.tsx index 92e515f648..ee4c71ee45 100644 --- a/App/FeatureSet/Dashboard/src/Components/Dashboard/Toolbar/DashboardToolbar.tsx +++ b/App/FeatureSet/Dashboard/src/Components/Dashboard/Toolbar/DashboardToolbar.tsx @@ -56,34 +56,61 @@ const DashboardToolbar: FunctionComponent = ( return (
- {/* Top row: Dashboard name + action buttons */} -
-
-

+ {/* Single row: Dashboard name + time range + variables + action buttons */} +
+
+

{props.dashboardName}

{isEditMode && ( - - + + Editing )} {hasComponents && !isEditMode && ( - + {props.dashboardViewConfig.components.length} widget {props.dashboardViewConfig.components.length !== 1 ? "s" : ""} )} + + {/* Time range + variables inline (only when components exist and not in edit mode) */} + {hasComponents && !isEditMode && ( + <> +
+ { + props.onStartAndEndDateChange(startAndEndDate); + }} + /> + + {/* Template variables */} + {props.variables && + props.variables.length > 0 && + props.onVariableValueChange && ( + <> +
+ + + )} + + )} + {/* Refreshing indicator */} {props.isRefreshing && props.autoRefreshInterval !== AutoRefreshInterval.OFF && ( - + Refreshing @@ -91,7 +118,7 @@ const DashboardToolbar: FunctionComponent = (
{!isSaving && ( -
+
{isEditMode ? ( <> @@ -157,7 +184,7 @@ const DashboardToolbar: FunctionComponent = ( /> -
+
- {/* Bottom row: Time range + variables (only when components exist and not in edit mode) */} - {hasComponents && !isEditMode && ( -
-
- { - props.onStartAndEndDateChange(startAndEndDate); - }} - /> -
- - {/* Template variables */} - {props.variables && - props.variables.length > 0 && - props.onVariableValueChange && ( - <> -
-
- -
- - )} -
- )} - {showCancelModal ? ( ; +} + +interface TooltipData { + name: string; + fileName: string; + selfValue: number; + totalValue: number; + x: number; + y: number; +} + +const ProfileFlamegraph: FunctionComponent = ( + props: ProfileFlamegraphProps, +): ReactElement => { + const [samples, setSamples] = useState>([]); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(""); + const [zoomStack, setZoomStack] = useState>([]); + const [tooltip, setTooltip] = useState(null); + + const loadSamples: () => Promise = async (): Promise => { + try { + setIsLoading(true); + setError(""); + + const result: ListResult = + await AnalyticsModelAPI.getList({ + modelType: ProfileSample, + query: { + projectId: ProjectUtil.getCurrentProjectId()!, + profileId: props.profileId, + }, + select: { + stacktrace: true, + frameTypes: true, + value: true, + profileType: true, + }, + limit: 10000, + skip: 0, + sort: { + value: SortOrder.Descending, + }, + }); + + setSamples(result.data || []); + } catch (err) { + setError(API.getFriendlyMessage(err)); + } finally { + setIsLoading(false); + } + }; + + useEffect(() => { + void loadSamples(); + }, [props.profileId]); + + const rootNode: FlamegraphNode = useMemo(() => { + const root: FlamegraphNode = { + name: "root", + fileName: "", + lineNumber: 0, + frameType: "", + selfValue: 0, + totalValue: 0, + children: new Map(), + }; + + for (const sample of samples) { + const stacktrace: Array = sample.stacktrace || []; + const frameTypes: Array = sample.frameTypes || []; + const value: number = sample.value || 0; + + let currentNode: FlamegraphNode = root; + root.totalValue += value; + + // Walk from root to leaf (stacktrace is ordered root-to-leaf) + for (let i: number = 0; i < stacktrace.length; i++) { + const frame: string = stacktrace[i]!; + const frameType: string = + i < frameTypes.length ? frameTypes[i]! : "unknown"; + + let child: FlamegraphNode | undefined = + currentNode.children.get(frame); + + if (!child) { + const parsed: ParsedStackFrame = ProfileUtil.parseStackFrame(frame); + child = { + name: parsed.functionName, + fileName: parsed.fileName, + lineNumber: parsed.lineNumber, + frameType, + selfValue: 0, + totalValue: 0, + children: new Map(), + }; + currentNode.children.set(frame, child); + } + + child.totalValue += value; + + // Last frame in the stack is the leaf -- add self value + if (i === stacktrace.length - 1) { + child.selfValue += value; + } + + currentNode = child; + } + } + + return root; + }, [samples]); + + const activeRoot: FlamegraphNode = useMemo(() => { + if (zoomStack.length > 0) { + return zoomStack[zoomStack.length - 1]!; + } + return rootNode; + }, [rootNode, zoomStack]); + + const handleClickNode: (node: FlamegraphNode) => void = useCallback( + (node: FlamegraphNode): void => { + if (node.children.size > 0) { + setZoomStack((prev: Array) => { + return [...prev, node]; + }); + } + }, + [], + ); + + const handleZoomOut: () => void = useCallback((): void => { + setZoomStack((prev: Array) => { + return prev.slice(0, prev.length - 1); + }); + }, []); + + const handleResetZoom: () => void = useCallback((): void => { + setZoomStack([]); + }, []); + + const handleMouseEnter: ( + node: FlamegraphNode, + event: React.MouseEvent, + ) => void = useCallback( + (node: FlamegraphNode, event: React.MouseEvent): void => { + setTooltip({ + name: node.name, + fileName: node.fileName, + selfValue: node.selfValue, + totalValue: node.totalValue, + x: event.clientX, + y: event.clientY, + }); + }, + [], + ); + + const handleMouseLeave: () => void = useCallback((): void => { + setTooltip(null); + }, []); + + if (isLoading) { + return ; + } + + if (error) { + return ( + { + void loadSamples(); + }} + /> + ); + } + + if (samples.length === 0) { + return ( +
+ No profile samples found for this profile. +
+ ); + } + + const renderNode: ( + node: FlamegraphNode, + parentTotal: number, + depth: number, + offsetFraction: number, + widthFraction: number, + ) => ReactElement | null = ( + node: FlamegraphNode, + parentTotal: number, + depth: number, + offsetFraction: number, + widthFraction: number, + ): ReactElement | null => { + if (widthFraction < 0.005) { + return null; + } + + const bgColor: string = ProfileUtil.getFrameTypeColor(node.frameType); + const percentage: number = + parentTotal > 0 ? (node.totalValue / parentTotal) * 100 : 0; + + const children: Array = Array.from( + node.children.values(), + ).sort((a: FlamegraphNode, b: FlamegraphNode) => { + return b.totalValue - a.totalValue; + }); + + let childOffset: number = 0; + + return ( + +
{ + handleClickNode(node); + }} + onMouseEnter={(e: React.MouseEvent) => { + handleMouseEnter(node, e); + }} + onMouseLeave={handleMouseLeave} + title={`${node.name} (${percentage.toFixed(1)}%)`} + > + {widthFraction > 0.03 ? node.name : ""} +
+ {children.map((child: FlamegraphNode) => { + const childWidth: number = + node.totalValue > 0 + ? (child.totalValue / node.totalValue) * widthFraction + : 0; + const currentOffset: number = offsetFraction + childOffset; + childOffset += childWidth; + + return renderNode( + child, + node.totalValue, + depth + 1, + currentOffset, + childWidth, + ); + })} + + ); + }; + + const getMaxDepth: (node: FlamegraphNode, depth: number) => number = ( + node: FlamegraphNode, + depth: number, + ): number => { + let max: number = depth; + for (const child of node.children.values()) { + const childDepth: number = getMaxDepth(child, depth + 1); + if (childDepth > max) { + max = childDepth; + } + } + return max; + }; + + const maxDepth: number = getMaxDepth(activeRoot, 0); + const height: number = (maxDepth + 1) * 26 + 10; + + return ( +
+ {zoomStack.length > 0 && ( +
+ + + + Zoomed into: {activeRoot.name} + +
+ )} + +
+ Frame Types: + {[ + "kernel", + "native", + "jvm", + "cpython", + "go", + "v8js", + "unknown", + ].map((type: string) => { + return ( + + + {type} + + ); + })} +
+ +
+ {renderNode(activeRoot, activeRoot.totalValue, 0, 0, 1)} +
+ + {tooltip && ( +
+
{tooltip.name}
+ {tooltip.fileName && ( +
{tooltip.fileName}
+ )} +
+ Self: {tooltip.selfValue.toLocaleString()} +
+
Total: {tooltip.totalValue.toLocaleString()}
+
+ )} +
+ ); +}; + +export default ProfileFlamegraph; diff --git a/App/FeatureSet/Dashboard/src/Components/Profiles/ProfileFunctionList.tsx b/App/FeatureSet/Dashboard/src/Components/Profiles/ProfileFunctionList.tsx new file mode 100644 index 0000000000..13cf27050b --- /dev/null +++ b/App/FeatureSet/Dashboard/src/Components/Profiles/ProfileFunctionList.tsx @@ -0,0 +1,287 @@ +import React, { + FunctionComponent, + ReactElement, + useCallback, + useEffect, + useMemo, + useState, +} from "react"; +import ProfileSample from "Common/Models/AnalyticsModels/ProfileSample"; +import AnalyticsModelAPI, { + ListResult, +} from "Common/UI/Utils/AnalyticsModelAPI/AnalyticsModelAPI"; +import ProjectUtil from "Common/UI/Utils/Project"; +import API from "Common/UI/Utils/API/API"; +import PageLoader from "Common/UI/Components/Loader/PageLoader"; +import ErrorMessage from "Common/UI/Components/ErrorMessage/ErrorMessage"; +import ProfileUtil, { ParsedStackFrame } from "../../Utils/ProfileUtil"; +import SortOrder from "Common/Types/BaseDatabase/SortOrder"; + +export interface ProfileFunctionListProps { + profileId: string; +} + +interface FunctionRow { + functionName: string; + fileName: string; + selfValue: number; + totalValue: number; + sampleCount: number; +} + +type SortField = + | "functionName" + | "fileName" + | "selfValue" + | "totalValue" + | "sampleCount"; + +const ProfileFunctionList: FunctionComponent = ( + props: ProfileFunctionListProps, +): ReactElement => { + const [samples, setSamples] = useState>([]); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(""); + const [sortField, setSortField] = useState("selfValue"); + const [sortDirection, setSortDirection] = useState<"asc" | "desc">("desc"); + + const loadSamples: () => Promise = async (): Promise => { + try { + setIsLoading(true); + setError(""); + + const result: ListResult = + await AnalyticsModelAPI.getList({ + modelType: ProfileSample, + query: { + projectId: ProjectUtil.getCurrentProjectId()!, + profileId: props.profileId, + }, + select: { + stacktrace: true, + frameTypes: true, + value: true, + profileType: true, + }, + limit: 10000, + skip: 0, + sort: { + value: SortOrder.Descending, + }, + }); + + setSamples(result.data || []); + } catch (err) { + setError(API.getFriendlyMessage(err)); + } finally { + setIsLoading(false); + } + }; + + useEffect(() => { + void loadSamples(); + }, [props.profileId]); + + const functionRows: Array = useMemo(() => { + const functionMap: Map< + string, + { + functionName: string; + fileName: string; + selfValue: number; + totalValue: number; + sampleCount: number; + } + > = new Map(); + + for (const sample of samples) { + const stacktrace: Array = sample.stacktrace || []; + const value: number = sample.value || 0; + + const seenInThisSample: Set = new Set(); + + for (let i: number = 0; i < stacktrace.length; i++) { + const frame: string = stacktrace[i]!; + const parsed: ParsedStackFrame = ProfileUtil.parseStackFrame(frame); + const key: string = `${parsed.functionName}@${parsed.fileName}`; + + let entry: FunctionRow | undefined = functionMap.get(key); + + if (!entry) { + entry = { + functionName: parsed.functionName, + fileName: parsed.fileName, + selfValue: 0, + totalValue: 0, + sampleCount: 0, + }; + functionMap.set(key, entry); + } + + // Only add total value once per sample (avoid double-counting recursive calls) + if (!seenInThisSample.has(key)) { + entry.totalValue += value; + entry.sampleCount += 1; + seenInThisSample.add(key); + } + + // Self value is only for the leaf frame + if (i === stacktrace.length - 1) { + entry.selfValue += value; + } + } + } + + return Array.from(functionMap.values()); + }, [samples]); + + const sortedRows: Array = useMemo(() => { + const rows: Array = [...functionRows]; + + rows.sort((a: FunctionRow, b: FunctionRow) => { + let aVal: string | number = a[sortField]; + let bVal: string | number = b[sortField]; + + if (typeof aVal === "string") { + aVal = aVal.toLowerCase(); + bVal = (bVal as string).toLowerCase(); + } + + if (aVal < bVal) { + return sortDirection === "asc" ? -1 : 1; + } + if (aVal > bVal) { + return sortDirection === "asc" ? 1 : -1; + } + return 0; + }); + + return rows; + }, [functionRows, sortField, sortDirection]); + + const handleSort: (field: SortField) => void = useCallback( + (field: SortField): void => { + if (field === sortField) { + setSortDirection((prev: "asc" | "desc") => { + return prev === "asc" ? "desc" : "asc"; + }); + } else { + setSortField(field); + setSortDirection("desc"); + } + }, + [sortField], + ); + + const getSortIndicator: (field: SortField) => string = useCallback( + (field: SortField): string => { + if (field !== sortField) { + return ""; + } + return sortDirection === "asc" ? " \u2191" : " \u2193"; + }, + [sortField, sortDirection], + ); + + if (isLoading) { + return ; + } + + if (error) { + return ( + { + void loadSamples(); + }} + /> + ); + } + + if (samples.length === 0) { + return ( +
+ No profile samples found for this profile. +
+ ); + } + + return ( +
+ + + + + + + + + + + + {sortedRows.map((row: FunctionRow, index: number) => { + return ( + + + + + + + + ); + })} + +
{ + handleSort("functionName"); + }} + > + Function{getSortIndicator("functionName")} + { + handleSort("fileName"); + }} + > + File{getSortIndicator("fileName")} + { + handleSort("selfValue"); + }} + > + Self Value{getSortIndicator("selfValue")} + { + handleSort("totalValue"); + }} + > + Total Value{getSortIndicator("totalValue")} + { + handleSort("sampleCount"); + }} + > + Samples{getSortIndicator("sampleCount")} +
+ {row.functionName} + + {row.fileName || "-"} + + {row.selfValue.toLocaleString()} + + {row.totalValue.toLocaleString()} + + {row.sampleCount.toLocaleString()} +
+
+ ); +}; + +export default ProfileFunctionList; diff --git a/App/FeatureSet/Dashboard/src/Components/Profiles/ProfileTable.tsx b/App/FeatureSet/Dashboard/src/Components/Profiles/ProfileTable.tsx new file mode 100644 index 0000000000..2c6dfa82f2 --- /dev/null +++ b/App/FeatureSet/Dashboard/src/Components/Profiles/ProfileTable.tsx @@ -0,0 +1,342 @@ +import ProjectUtil from "Common/UI/Utils/Project"; +import SortOrder from "Common/Types/BaseDatabase/SortOrder"; +import ObjectID from "Common/Types/ObjectID"; +import AnalyticsModelTable from "Common/UI/Components/ModelTable/AnalyticsModelTable"; +import FieldType from "Common/UI/Components/Types/FieldType"; +import Profile from "Common/Models/AnalyticsModels/Profile"; +import React, { + Fragment, + FunctionComponent, + ReactElement, + useEffect, + useState, +} from "react"; +import RouteMap, { RouteUtil } from "../../Utils/RouteMap"; +import PageMap from "../../Utils/PageMap"; +import { PromiseVoidFunction } from "Common/Types/FunctionTypes"; +import HTTPResponse from "Common/Types/API/HTTPResponse"; +import { JSONObject } from "Common/Types/JSON"; +import HTTPErrorResponse from "Common/Types/API/HTTPErrorResponse"; +import API from "Common/Utils/API"; +import { APP_API_URL } from "Common/UI/Config"; +import URL from "Common/Types/API/URL"; +import ModelAPI from "Common/UI/Utils/ModelAPI/ModelAPI"; +import PageLoader from "Common/UI/Components/Loader/PageLoader"; +import ErrorMessage from "Common/UI/Components/ErrorMessage/ErrorMessage"; +import Query from "Common/Types/BaseDatabase/Query"; +import ListResult from "Common/Types/BaseDatabase/ListResult"; +import Service from "Common/Models/DatabaseModels/Service"; +import { LIMIT_PER_PROJECT } from "Common/Types/Database/LimitMax"; +import ServiceElement from "../Service/ServiceElement"; + +export interface ComponentProps { + modelId?: ObjectID | undefined; + profileQuery?: Query | undefined; + isMinimalTable?: boolean | undefined; + noItemsMessage?: string | undefined; +} + +const ProfileTable: FunctionComponent = ( + props: ComponentProps, +): ReactElement => { + const modelId: ObjectID | undefined = props.modelId; + + const [attributes, setAttributes] = React.useState>([]); + const [attributesLoaded, setAttributesLoaded] = + React.useState(false); + const [attributesLoading, setAttributesLoading] = + React.useState(false); + const [attributesError, setAttributesError] = React.useState(""); + + const [isPageLoading, setIsPageLoading] = React.useState(true); + const [pageError, setPageError] = React.useState(""); + + const [telemetryServices, setServices] = React.useState>([]); + + const [areAdvancedFiltersVisible, setAreAdvancedFiltersVisible] = + useState(false); + + const query: Query = React.useMemo(() => { + const baseQuery: Query = { + ...(props.profileQuery || {}), + }; + + const projectId: ObjectID | null = ProjectUtil.getCurrentProjectId(); + + if (projectId) { + baseQuery.projectId = projectId; + } + + if (modelId) { + baseQuery.serviceId = modelId; + } + + return baseQuery; + }, [props.profileQuery, modelId]); + + const loadServices: PromiseVoidFunction = async (): Promise => { + try { + setIsPageLoading(true); + setPageError(""); + + const telemetryServicesResponse: ListResult = + await ModelAPI.getList({ + modelType: Service, + query: { + projectId: ProjectUtil.getCurrentProjectId()!, + }, + select: { + serviceColor: true, + name: true, + }, + limit: LIMIT_PER_PROJECT, + skip: 0, + sort: { + name: SortOrder.Ascending, + }, + }); + + setServices(telemetryServicesResponse.data || []); + } catch (err) { + setPageError(API.getFriendlyErrorMessage(err as Error)); + } finally { + setIsPageLoading(false); + } + }; + + const loadAttributes: PromiseVoidFunction = async (): Promise => { + if (attributesLoading || attributesLoaded) { + return; + } + + try { + setAttributesLoading(true); + setAttributesError(""); + + const attributeResponse: HTTPResponse | HTTPErrorResponse = + await API.post({ + url: URL.fromString(APP_API_URL.toString()).addRoute( + "/telemetry/profiles/get-attributes", + ), + data: {}, + headers: { + ...ModelAPI.getCommonHeaders(), + }, + }); + + if (attributeResponse instanceof HTTPErrorResponse) { + throw attributeResponse; + } + + const fetchedAttributes: Array = (attributeResponse.data[ + "attributes" + ] || []) as Array; + setAttributes(fetchedAttributes); + setAttributesLoaded(true); + } catch (err) { + setAttributes([]); + setAttributesLoaded(false); + setAttributesError(API.getFriendlyErrorMessage(err as Error)); + } finally { + setAttributesLoading(false); + } + }; + + useEffect(() => { + loadServices().catch((err: Error) => { + setPageError(API.getFriendlyErrorMessage(err as Error)); + }); + }, []); + + const handleAdvancedFiltersToggle: (show: boolean) => void = ( + show: boolean, + ): void => { + setAreAdvancedFiltersVisible(show); + + if (show && !attributesLoaded && !attributesLoading) { + void loadAttributes(); + } + }; + + if (isPageLoading) { + return ; + } + + return ( + + {pageError && ( +
+ { + void loadServices(); + }} + /> +
+ )} + + {areAdvancedFiltersVisible && attributesError && ( +
+ { + setAttributesLoaded(false); + void loadAttributes(); + }} + /> +
+ )} + +
+ + userPreferencesKey="profile-table" + disablePagination={props.isMinimalTable} + modelType={Profile} + id="profiles-table" + isDeleteable={false} + isEditable={false} + isCreateable={false} + singularName="Profile" + pluralName="Profiles" + name="Profiles" + isViewable={true} + cardProps={ + props.isMinimalTable + ? undefined + : { + title: "Profiles", + description: + "Continuous profiling data from your services. Profiles help you understand CPU, memory, and allocation hotspots in your applications.", + } + } + query={query} + showViewIdButton={true} + noItemsMessage={ + props.noItemsMessage + ? props.noItemsMessage + : "No profiles found." + } + showRefreshButton={true} + sortBy="startTime" + sortOrder={SortOrder.Descending} + onViewPage={(profile: Profile) => { + return Promise.resolve( + RouteUtil.populateRouteParams( + RouteMap[PageMap.PROFILE_VIEW]!, + { + modelId: profile.profileId!, + }, + ), + ); + }} + filters={[ + { + field: { + serviceId: true, + }, + type: FieldType.MultiSelectDropdown, + filterDropdownOptions: telemetryServices.map( + (service: Service) => { + return { + label: service.name!, + value: service.id!.toString(), + }; + }, + ), + title: "Service", + }, + { + field: { + profileType: true, + }, + type: FieldType.Text, + title: "Profile Type", + }, + { + field: { + traceId: true, + }, + type: FieldType.Text, + title: "Trace ID", + }, + { + field: { + startTime: true, + }, + type: FieldType.DateTime, + title: "Start Time", + }, + { + field: { + attributes: true, + }, + type: FieldType.JSON, + title: "Attributes", + jsonKeys: attributes, + isAdvancedFilter: true, + }, + ]} + onAdvancedFiltersToggle={handleAdvancedFiltersToggle} + columns={[ + { + field: { + profileId: true, + }, + title: "Profile ID", + type: FieldType.Text, + }, + { + field: { + profileType: true, + }, + title: "Profile Type", + type: FieldType.Text, + }, + { + field: { + serviceId: true, + }, + title: "Service", + type: FieldType.Element, + getElement: (profile: Profile): ReactElement => { + const telemetryService: Service | undefined = + telemetryServices.find((service: Service) => { + return ( + service.id?.toString() === + profile.serviceId?.toString() + ); + }); + + if (!telemetryService) { + return

Unknown

; + } + + return ( + + + + ); + }, + }, + { + field: { + sampleCount: true, + }, + title: "Samples", + type: FieldType.Number, + }, + { + field: { + startTime: true, + }, + title: "Start Time", + type: FieldType.DateTime, + }, + ]} + /> +
+ + ); +}; + +export default ProfileTable; diff --git a/App/FeatureSet/Dashboard/src/Pages/Profiles/Index.tsx b/App/FeatureSet/Dashboard/src/Pages/Profiles/Index.tsx index fab4bd9027..51a5c9a70d 100644 --- a/App/FeatureSet/Dashboard/src/Pages/Profiles/Index.tsx +++ b/App/FeatureSet/Dashboard/src/Pages/Profiles/Index.tsx @@ -12,7 +12,7 @@ import ModelAPI from "Common/UI/Utils/ModelAPI/ModelAPI"; import API from "Common/UI/Utils/API/API"; import PageLoader from "Common/UI/Components/Loader/PageLoader"; import { PromiseVoidFunction } from "Common/Types/FunctionTypes"; -import InfoCard from "Common/UI/Components/InfoCard/InfoCard"; +import ProfileTable from "../../Components/Profiles/ProfileTable"; const ProfilesPage: FunctionComponent = ( props: PageComponentProps, @@ -62,12 +62,7 @@ const ProfilesPage: FunctionComponent = ( return ; } - return ( - - ); + return ; }; export default ProfilesPage; diff --git a/App/FeatureSet/Dashboard/src/Pages/Profiles/View/Index.tsx b/App/FeatureSet/Dashboard/src/Pages/Profiles/View/Index.tsx index 5af5411755..071f5172a3 100644 --- a/App/FeatureSet/Dashboard/src/Pages/Profiles/View/Index.tsx +++ b/App/FeatureSet/Dashboard/src/Pages/Profiles/View/Index.tsx @@ -1,18 +1,35 @@ import PageComponentProps from "../../PageComponentProps"; import Navigation from "Common/UI/Utils/Navigation"; import React, { FunctionComponent, ReactElement } from "react"; -import InfoCard from "Common/UI/Components/InfoCard/InfoCard"; +import Tabs from "Common/UI/Components/Tabs/Tabs"; +import { Tab } from "Common/UI/Components/Tabs/Tab"; +import ProfileFlamegraph from "../../../Components/Profiles/ProfileFlamegraph"; +import ProfileFunctionList from "../../../Components/Profiles/ProfileFunctionList"; const ProfileViewPage: FunctionComponent< PageComponentProps > = (): ReactElement => { const profileId: string = Navigation.getLastParamAsString(0); + const tabs: Array = [ + { + name: "Flamegraph", + children: , + }, + { + name: "Function List", + children: , + }, + ]; + + const handleTabChange: (tab: Tab) => void = (_tab: Tab): void => { + // Tab content is rendered by the Tabs component via children + }; + return ( - +
+ +
); }; diff --git a/App/FeatureSet/Dashboard/src/Utils/ProfileUtil.ts b/App/FeatureSet/Dashboard/src/Utils/ProfileUtil.ts new file mode 100644 index 0000000000..a98fb7dd18 --- /dev/null +++ b/App/FeatureSet/Dashboard/src/Utils/ProfileUtil.ts @@ -0,0 +1,148 @@ +export interface ParsedStackFrame { + functionName: string; + fileName: string; + lineNumber: number; +} + +export default class ProfileUtil { + public static getFrameTypeColor(frameType: string): string { + const type: string = frameType.toLowerCase(); + + switch (type) { + case "kernel": + return "bg-red-500"; + case "native": + return "bg-orange-500"; + case "jvm": + return "bg-green-500"; + case "cpython": + return "bg-blue-500"; + case "go": + return "bg-cyan-500"; + case "v8js": + return "bg-yellow-500"; + default: + return "bg-gray-400"; + } + } + + public static getFrameTypeTextColor(frameType: string): string { + const type: string = frameType.toLowerCase(); + + switch (type) { + case "kernel": + return "text-red-700"; + case "native": + return "text-orange-700"; + case "jvm": + return "text-green-700"; + case "cpython": + return "text-blue-700"; + case "go": + return "text-cyan-700"; + case "v8js": + return "text-yellow-700"; + default: + return "text-gray-600"; + } + } + + public static getFrameTypeBgLight(frameType: string): string { + const type: string = frameType.toLowerCase(); + + switch (type) { + case "kernel": + return "bg-red-400"; + case "native": + return "bg-orange-400"; + case "jvm": + return "bg-green-400"; + case "cpython": + return "bg-blue-400"; + case "go": + return "bg-cyan-400"; + case "v8js": + return "bg-yellow-400"; + default: + return "bg-gray-300"; + } + } + + public static parseStackFrame(frame: string): ParsedStackFrame { + // Format: "function@file:line" + const atIndex: number = frame.indexOf("@"); + + if (atIndex === -1) { + return { + functionName: frame, + fileName: "", + lineNumber: 0, + }; + } + + const functionName: string = frame.substring(0, atIndex); + const rest: string = frame.substring(atIndex + 1); + + const lastColonIndex: number = rest.lastIndexOf(":"); + + if (lastColonIndex === -1) { + return { + functionName, + fileName: rest, + lineNumber: 0, + }; + } + + const fileName: string = rest.substring(0, lastColonIndex); + const lineStr: string = rest.substring(lastColonIndex + 1); + const lineNumber: number = parseInt(lineStr, 10); + + return { + functionName, + fileName, + lineNumber: isNaN(lineNumber) ? 0 : lineNumber, + }; + } + + public static formatProfileValue(value: number, unit: string): string { + const lowerUnit: string = unit.toLowerCase(); + + if (lowerUnit === "nanoseconds" || lowerUnit === "ns") { + if (value >= 1_000_000_000) { + return `${(value / 1_000_000_000).toFixed(2)}s`; + } + if (value >= 1_000_000) { + return `${(value / 1_000_000).toFixed(2)}ms`; + } + if (value >= 1_000) { + return `${(value / 1_000).toFixed(2)}us`; + } + return `${value}ns`; + } + + if (lowerUnit === "bytes" || lowerUnit === "byte") { + if (value >= 1_073_741_824) { + return `${(value / 1_073_741_824).toFixed(2)} GB`; + } + if (value >= 1_048_576) { + return `${(value / 1_048_576).toFixed(2)} MB`; + } + if (value >= 1_024) { + return `${(value / 1_024).toFixed(2)} KB`; + } + return `${value} B`; + } + + if (lowerUnit === "count" || lowerUnit === "samples") { + if (value >= 1_000_000) { + return `${(value / 1_000_000).toFixed(2)}M`; + } + if (value >= 1_000) { + return `${(value / 1_000).toFixed(2)}K`; + } + return `${value}`; + } + + return `${value} ${unit}`; + } +} diff --git a/App/FeatureSet/Docs/Content/telemetry/profiles.md b/App/FeatureSet/Docs/Content/telemetry/profiles.md new file mode 100644 index 0000000000..a6b3948e84 --- /dev/null +++ b/App/FeatureSet/Docs/Content/telemetry/profiles.md @@ -0,0 +1,184 @@ +# Send Continuous Profiling Data to OneUptime + +## Overview + +Continuous profiling is the fourth pillar of observability alongside logs, metrics, and traces. Profiles capture how your application spends CPU time, allocates memory, and uses system resources at the function level. OneUptime ingests profiling data via the OpenTelemetry Protocol (OTLP) and stores it alongside your other telemetry signals for unified analysis. + +With profiling data in OneUptime, you can identify hot functions consuming CPU, detect memory leaks, find contention bottlenecks, and correlate performance issues with specific traces and spans. + +## Supported Profile Types + +OneUptime supports the following profile types: + +| Profile Type | Description | Unit | +| --- | --- | --- | +| cpu | CPU time spent executing code | nanoseconds | +| wall | Wall-clock time (includes waiting/sleeping) | nanoseconds | +| alloc_objects | Number of heap allocations | count | +| alloc_space | Bytes of heap memory allocated | bytes | +| goroutine | Number of active goroutines (Go) | count | +| contention | Time spent waiting on locks/mutexes | nanoseconds | + +## Getting Started + +### Step 1 - Create a Telemetry Ingestion Token + +After you sign up to OneUptime and create a project, click on "More" in the Navigation bar and click on "Project Settings". + +On the Telemetry Ingestion Key page, click on "Create Ingestion Key" to create a token. + +![Create Service](/docs/static/images/TelemetryIngestionKeys.png) + +Once you created a token, click on "View" to view the token. + +![View Service](/docs/static/images/TelemetryIngestionKeyView.png) + +### Step 2 - Configure Your Profiler + +OneUptime accepts profiling data over both gRPC and HTTP using the OTLP profiles protocol. + +| Protocol | Endpoint | +| --- | --- | +| gRPC | `your-oneuptime-host:4317` (OTLP standard gRPC port) | +| HTTP | `https://your-oneuptime-host/otlp/v1/profiles` | + +**Environment Variables** + +Set the following environment variables to point your profiler at OneUptime: + +```bash +export OTEL_EXPORTER_OTLP_HEADERS=x-oneuptime-token=YOUR_ONEUPTIME_SERVICE_TOKEN +export OTEL_EXPORTER_OTLP_ENDPOINT=https://oneuptime.com/otlp +export OTEL_SERVICE_NAME=my-service +``` + +**Self Hosted OneUptime** + +If you are self-hosting OneUptime, replace the endpoint with your own host (e.g., `http(s)://YOUR-ONEUPTIME-HOST/otlp`). For gRPC, connect directly to port 4317 on your OneUptime host. + +## Instrumentation Guide + +### Using Grafana Alloy (eBPF-based profiling) + +Grafana Alloy (formerly Grafana Agent) can collect CPU profiles from all processes on a Linux host using eBPF, with zero code changes required. Configure it to export via OTLP to OneUptime. + +Example Alloy configuration: + +```hcl +pyroscope.ebpf "default" { + forward_to = [pyroscope.write.oneuptime.receiver] + targets = discovery.process.all.targets +} + +pyroscope.write "oneuptime" { + endpoint { + url = "https://oneuptime.com/otlp/v1/profiles" + headers = { + "x-oneuptime-token" = "YOUR_ONEUPTIME_SERVICE_TOKEN", + } + } +} +``` + +### Using async-profiler (Java) + +For Java applications, use [async-profiler](https://github.com/async-profiler/async-profiler) with the OpenTelemetry Java agent to send profiling data via OTLP. + +```bash +# Start your Java application with the OpenTelemetry Java agent +java -javaagent:opentelemetry-javaagent.jar \ + -Dotel.exporter.otlp.endpoint=https://oneuptime.com/otlp \ + -Dotel.exporter.otlp.headers=x-oneuptime-token=YOUR_ONEUPTIME_SERVICE_TOKEN \ + -Dotel.service.name=my-java-service \ + -jar my-app.jar +``` + +### Using Go pprof with OTLP Export + +For Go applications, you can use the standard `net/http/pprof` package alongside an OTLP exporter. Configure continuous profiling by periodically collecting pprof data and forwarding it to OneUptime. + +```go +import ( + "runtime/pprof" + "bytes" + "time" +) + +// Collect a 30-second CPU profile and export periodically +func collectProfile() { + var buf bytes.Buffer + pprof.StartCPUProfile(&buf) + time.Sleep(30 * time.Second) + pprof.StopCPUProfile() + // Convert pprof output to OTLP format and send to OneUptime +} +``` + +Alternatively, use the OpenTelemetry Collector with a profiling receiver that scrapes your Go application's `/debug/pprof` endpoint and exports via OTLP. + +### Using py-spy (Python) + +For Python applications, [py-spy](https://github.com/benfred/py-spy) can capture CPU profiles without code changes. Use the OpenTelemetry Collector to receive and forward profile data. + +```bash +# Capture profiles and send to a local OTLP collector +py-spy record --format speedscope --pid $PID -o profile.json +``` + +For continuous profiling, run py-spy alongside your application and configure the OpenTelemetry Collector to ingest and forward the profiles to OneUptime. + +## Using the OpenTelemetry Collector + +You can use the OpenTelemetry Collector as a proxy to receive profiles from your applications and forward them to OneUptime. + +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +exporters: + otlphttp: + endpoint: "https://oneuptime.com/otlp" + encoding: json + headers: + "Content-Type": "application/json" + "x-oneuptime-token": "YOUR_ONEUPTIME_SERVICE_TOKEN" + +service: + pipelines: + profiles: + receivers: [otlp] + exporters: [otlphttp] +``` + +## Features + +### Flamegraph Visualization + +OneUptime renders profile data as interactive flamegraphs. Each bar represents a function in the call stack, and its width is proportional to the time or resources consumed. You can click on any function to zoom in and see its callers and callees. + +### Function List + +View a sortable table of all functions captured in a profile, ranked by self time, total time, or allocation count. This helps you quickly identify the most expensive functions in your application. + +### Trace Correlation + +Profiles in OneUptime can be correlated with distributed traces. When a profile includes trace and span IDs (via the OTLP link table), you can navigate directly from a slow trace span to the corresponding CPU or memory profile to understand exactly what code was executing. + +### Filtering by Profile Type + +Filter profiles by type (cpu, wall, alloc_objects, alloc_space, goroutine, contention) to focus on the specific resource dimension you are investigating. + +## Data Retention + +Profile data retention is configured per telemetry service in your OneUptime project settings. The default retention period is 15 days. Data is automatically deleted after the retention period expires. + +To change the retention period for a service, navigate to **Telemetry > Services > [Your Service] > Settings** and update the data retention value. + +## Need Help? + +Please contact support@oneuptime.com if you need any help setting up profiling with OneUptime. diff --git a/Common/Server/API/TelemetryAPI.ts b/Common/Server/API/TelemetryAPI.ts index e9805211fe..8bc1c4f768 100644 --- a/Common/Server/API/TelemetryAPI.ts +++ b/Common/Server/API/TelemetryAPI.ts @@ -23,6 +23,12 @@ import LogAggregationService, { AnalyticsTopItem, AnalyticsTableRow, } from "../Services/LogAggregationService"; +import ProfileAggregationService, { + FlamegraphRequest, + FunctionListRequest, + FunctionListItem, + ProfileFlamegraphNode, +} from "../Services/ProfileAggregationService"; import ObjectID from "../../Types/ObjectID"; import OneUptimeDate from "../../Types/Date"; import { JSONObject } from "../../Types/JSON"; @@ -710,4 +716,164 @@ function computeDefaultBucketSize(startTime: Date, endTime: Date): number { return 1440; } +// --- Profile Get Attributes Endpoint --- + +router.post( + "/telemetry/profiles/get-attributes", + UserMiddleware.getUserMiddleware, + async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => { + return getAttributes(req, res, next, TelemetryType.Profile); + }, +); + +// --- Profile Flamegraph Endpoint --- + +router.post( + "/telemetry/profiles/flamegraph", + UserMiddleware.getUserMiddleware, + async ( + req: ExpressRequest, + res: ExpressResponse, + next: NextFunction, + ): Promise => { + try { + const databaseProps: DatabaseCommonInteractionProps = + await CommonAPI.getDatabaseCommonInteractionProps(req); + + if (!databaseProps?.tenantId) { + return Response.sendErrorResponse( + req, + res, + new BadDataException("Invalid Project ID"), + ); + } + + const body: JSONObject = req.body as JSONObject; + + const profileId: string | undefined = body["profileId"] + ? (body["profileId"] as string) + : undefined; + + const startTime: Date | undefined = body["startTime"] + ? OneUptimeDate.fromString(body["startTime"] as string) + : undefined; + + const endTime: Date | undefined = body["endTime"] + ? OneUptimeDate.fromString(body["endTime"] as string) + : undefined; + + const serviceIds: Array | undefined = body["serviceIds"] + ? (body["serviceIds"] as Array).map((id: string) => { + return new ObjectID(id); + }) + : undefined; + + const profileType: string | undefined = body["profileType"] + ? (body["profileType"] as string) + : undefined; + + if (!profileId && !startTime) { + return Response.sendErrorResponse( + req, + res, + new BadDataException( + "Either profileId or startTime must be provided", + ), + ); + } + + const request: FlamegraphRequest = { + projectId: databaseProps.tenantId, + profileId, + startTime, + endTime, + serviceIds, + profileType, + }; + + const flamegraph: ProfileFlamegraphNode = + await ProfileAggregationService.getFlamegraph(request); + + return Response.sendJsonObjectResponse(req, res, { + flamegraph: flamegraph as unknown as JSONObject, + }); + } catch (err: unknown) { + next(err); + } + }, +); + +// --- Profile Function List Endpoint --- + +router.post( + "/telemetry/profiles/function-list", + UserMiddleware.getUserMiddleware, + async ( + req: ExpressRequest, + res: ExpressResponse, + next: NextFunction, + ): Promise => { + try { + const databaseProps: DatabaseCommonInteractionProps = + await CommonAPI.getDatabaseCommonInteractionProps(req); + + if (!databaseProps?.tenantId) { + return Response.sendErrorResponse( + req, + res, + new BadDataException("Invalid Project ID"), + ); + } + + const body: JSONObject = req.body as JSONObject; + + const startTime: Date = body["startTime"] + ? OneUptimeDate.fromString(body["startTime"] as string) + : OneUptimeDate.addRemoveHours(OneUptimeDate.getCurrentDate(), -1); + + const endTime: Date = body["endTime"] + ? OneUptimeDate.fromString(body["endTime"] as string) + : OneUptimeDate.getCurrentDate(); + + const serviceIds: Array | undefined = body["serviceIds"] + ? (body["serviceIds"] as Array).map((id: string) => { + return new ObjectID(id); + }) + : undefined; + + const profileType: string | undefined = body["profileType"] + ? (body["profileType"] as string) + : undefined; + + const limit: number | undefined = body["limit"] + ? (body["limit"] as number) + : undefined; + + const sortBy: "selfValue" | "totalValue" | "sampleCount" | undefined = + body["sortBy"] + ? (body["sortBy"] as "selfValue" | "totalValue" | "sampleCount") + : undefined; + + const request: FunctionListRequest = { + projectId: databaseProps.tenantId, + startTime, + endTime, + serviceIds, + profileType, + limit, + sortBy, + }; + + const functions: Array = + await ProfileAggregationService.getFunctionList(request); + + return Response.sendJsonObjectResponse(req, res, { + functions: functions as unknown as JSONObject, + }); + } catch (err: unknown) { + next(err); + } + }, +); + export default router; diff --git a/Common/Server/EnvironmentConfig.ts b/Common/Server/EnvironmentConfig.ts index c1a6a3808b..8416964b5f 100644 --- a/Common/Server/EnvironmentConfig.ts +++ b/Common/Server/EnvironmentConfig.ts @@ -427,6 +427,14 @@ export const AverageMetricRowSizeInBytes: number = parsePositiveNumberFromEnv( export const AverageExceptionRowSizeInBytes: number = parsePositiveNumberFromEnv("AVERAGE_EXCEPTION_ROW_SIZE_IN_BYTES", 1024); +export const AverageProfileRowSizeInBytes: number = parsePositiveNumberFromEnv( + "AVERAGE_PROFILE_ROW_SIZE_IN_BYTES", + 1024, +); + +export const AverageProfileSampleRowSizeInBytes: number = + parsePositiveNumberFromEnv("AVERAGE_PROFILE_SAMPLE_ROW_SIZE_IN_BYTES", 512); + export const SlackAppClientId: string | null = process.env["SLACK_APP_CLIENT_ID"] || null; export const SlackAppClientSecret: string | null = diff --git a/Common/Server/Services/IncidentService.ts b/Common/Server/Services/IncidentService.ts index ccb8c1b8a0..0bf5ceb653 100644 --- a/Common/Server/Services/IncidentService.ts +++ b/Common/Server/Services/IncidentService.ts @@ -1350,19 +1350,22 @@ ${incident.remediationNotes || "No remediation notes provided."} ] as Date; // find the resolved state timeline to calculate time from resolution to postmortem + const resolvedStateId: ObjectID = + await IncidentStateTimelineService.getResolvedStateIdForProject( + projectId, + ); + const resolvedTimeline: IncidentStateTimeline | null = await IncidentStateTimelineService.findOneBy({ query: { incidentId: incidentId, + incidentStateId: resolvedStateId, }, select: { startsAt: true, - incidentState: { - isResolvedState: true, - }, }, sort: { - startsAt: SortOrder.Ascending, + startsAt: SortOrder.Descending, }, props: { isRoot: true, @@ -1370,7 +1373,7 @@ ${incident.remediationNotes || "No remediation notes provided."} }); // only emit if the incident has been resolved - if (resolvedTimeline?.incidentState?.isResolvedState && resolvedTimeline.startsAt) { + if (resolvedTimeline && resolvedTimeline.startsAt) { const postmortemMetric: Metric = new Metric(); postmortemMetric.projectId = projectId; postmortemMetric.serviceId = incidentId; diff --git a/Common/Server/Services/ProfileAggregationService.ts b/Common/Server/Services/ProfileAggregationService.ts new file mode 100644 index 0000000000..63f82b83f9 --- /dev/null +++ b/Common/Server/Services/ProfileAggregationService.ts @@ -0,0 +1,417 @@ +import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement"; +import ProfileSampleDatabaseService from "./ProfileSampleService"; +import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType"; +import { JSONObject } from "../../Types/JSON"; +import ObjectID from "../../Types/ObjectID"; +import Includes from "../../Types/BaseDatabase/Includes"; +import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName"; +import CaptureSpan from "../Utils/Telemetry/CaptureSpan"; +import { DbJSONResponse, Results } from "./AnalyticsDatabaseService"; + +// --- Interfaces --- + +export interface ProfileFlamegraphNode { + functionName: string; + fileName: string; + lineNumber: number; + selfValue: number; + totalValue: number; + children: ProfileFlamegraphNode[]; + frameType: string; +} + +export interface FlamegraphRequest { + projectId: ObjectID; + profileId?: string; + startTime?: Date; + endTime?: Date; + serviceIds?: Array; + profileType?: string; +} + +export interface FunctionListItem { + functionName: string; + fileName: string; + selfValue: number; + totalValue: number; + sampleCount: number; + frameType: string; +} + +export interface FunctionListRequest { + projectId: ObjectID; + startTime: Date; + endTime: Date; + serviceIds?: Array; + profileType?: string; + limit?: number; + sortBy?: "selfValue" | "totalValue" | "sampleCount"; +} + +interface ParsedFrame { + functionName: string; + fileName: string; + lineNumber: number; +} + +// --- Service --- + +export class ProfileAggregationService { + private static readonly TABLE_NAME: string = + AnalyticsTableName.ProfileSample; + private static readonly DEFAULT_FUNCTION_LIST_LIMIT: number = 50; + private static readonly MAX_SAMPLE_FETCH: number = 50000; + + /** + * Build a flamegraph tree from ProfileSample records. + * + * Each sample has a `stacktrace` array where each element follows the + * format "functionName@fileName:lineNumber". The array is ordered + * bottom-up (index 0 = root, last index = leaf). + * + * We aggregate samples that share common stack prefixes into a tree of + * ProfileFlamegraphNode objects. + */ + @CaptureSpan() + public static async getFlamegraph( + request: FlamegraphRequest, + ): Promise { + const statement: Statement = + ProfileAggregationService.buildFlamegraphQuery(request); + + const dbResult: Results = + await ProfileSampleDatabaseService.executeQuery(statement); + const response: DbJSONResponse = await dbResult.json<{ + data?: Array; + }>(); + + const rows: Array = response.data || []; + + // Build the tree from samples + const root: ProfileFlamegraphNode = { + functionName: "(root)", + fileName: "", + lineNumber: 0, + selfValue: 0, + totalValue: 0, + children: [], + frameType: "", + }; + + for (const row of rows) { + const stacktrace: Array = (row["stacktrace"] as Array) || []; + const frameTypes: Array = (row["frameTypes"] as Array) || []; + const value: number = Number(row["value"] || 0); + + if (stacktrace.length === 0) { + continue; + } + + // Walk down the tree, creating nodes as needed + let currentNode: ProfileFlamegraphNode = root; + currentNode.totalValue += value; + + for (let i: number = 0; i < stacktrace.length; i++) { + const frame: ParsedFrame = ProfileAggregationService.parseFrame( + stacktrace[i]!, + ); + const frameType: string = frameTypes[i] || ""; + + // Find or create child + let childNode: ProfileFlamegraphNode | undefined = + currentNode.children.find( + (child: ProfileFlamegraphNode): boolean => { + return ( + child.functionName === frame.functionName && + child.fileName === frame.fileName && + child.lineNumber === frame.lineNumber + ); + }, + ); + + if (!childNode) { + childNode = { + functionName: frame.functionName, + fileName: frame.fileName, + lineNumber: frame.lineNumber, + selfValue: 0, + totalValue: 0, + children: [], + frameType: frameType, + }; + currentNode.children.push(childNode); + } + + childNode.totalValue += value; + + // If this is the leaf frame, add to selfValue + if (i === stacktrace.length - 1) { + childNode.selfValue += value; + } + + currentNode = childNode; + } + } + + return root; + } + + /** + * Return the top functions aggregated across samples, sorted by the + * requested metric (selfValue, totalValue, or sampleCount). + */ + @CaptureSpan() + public static async getFunctionList( + request: FunctionListRequest, + ): Promise> { + const statement: Statement = + ProfileAggregationService.buildFunctionListQuery(request); + + const dbResult: Results = + await ProfileSampleDatabaseService.executeQuery(statement); + const response: DbJSONResponse = await dbResult.json<{ + data?: Array; + }>(); + + const rows: Array = response.data || []; + + // Aggregate per-function stats in-memory from the raw samples + const functionMap: Map< + string, + { + functionName: string; + fileName: string; + selfValue: number; + totalValue: number; + sampleCount: number; + frameType: string; + } + > = new Map(); + + for (const row of rows) { + const stacktrace: Array = (row["stacktrace"] as Array) || []; + const frameTypes: Array = (row["frameTypes"] as Array) || []; + const value: number = Number(row["value"] || 0); + + if (stacktrace.length === 0) { + continue; + } + + const seenInThisSample: Set = new Set(); + + for (let i: number = 0; i < stacktrace.length; i++) { + const frame: ParsedFrame = ProfileAggregationService.parseFrame( + stacktrace[i]!, + ); + const frameType: string = frameTypes[i] || ""; + const key: string = `${frame.functionName}@${frame.fileName}:${frame.lineNumber}`; + const isLeaf: boolean = i === stacktrace.length - 1; + + let entry = functionMap.get(key); + + if (!entry) { + entry = { + functionName: frame.functionName, + fileName: frame.fileName, + selfValue: 0, + totalValue: 0, + sampleCount: 0, + frameType: frameType, + }; + functionMap.set(key, entry); + } + + // totalValue: count the value once per unique function per sample + if (!seenInThisSample.has(key)) { + entry.totalValue += value; + entry.sampleCount += 1; + seenInThisSample.add(key); + } + + // selfValue: only the leaf frame + if (isLeaf) { + entry.selfValue += value; + } + } + } + + // Sort + const sortBy: string = request.sortBy || "selfValue"; + const items: Array = Array.from(functionMap.values()); + + items.sort((a, b) => { + if (sortBy === "totalValue") { + return b.totalValue - a.totalValue; + } + + if (sortBy === "sampleCount") { + return b.sampleCount - a.sampleCount; + } + + return b.selfValue - a.selfValue; + }); + + const limit: number = + request.limit ?? ProfileAggregationService.DEFAULT_FUNCTION_LIST_LIMIT; + + return items.slice(0, limit); + } + + // --- Query builders --- + + private static buildFlamegraphQuery(request: FlamegraphRequest): Statement { + const statement: Statement = SQL` + SELECT + stacktrace, + frameTypes, + value + FROM ${ProfileAggregationService.TABLE_NAME} + WHERE projectId = ${{ + type: TableColumnType.ObjectID, + value: request.projectId, + }} + `; + + if (request.profileId) { + statement.append( + SQL` AND profileId = ${{ + type: TableColumnType.Text, + value: request.profileId, + }}`, + ); + } + + if (request.startTime) { + statement.append( + SQL` AND time >= ${{ + type: TableColumnType.Date, + value: request.startTime, + }}`, + ); + } + + if (request.endTime) { + statement.append( + SQL` AND time <= ${{ + type: TableColumnType.Date, + value: request.endTime, + }}`, + ); + } + + ProfileAggregationService.appendCommonFilters(statement, request); + + statement.append( + SQL` LIMIT ${{ + type: TableColumnType.Number, + value: ProfileAggregationService.MAX_SAMPLE_FETCH, + }}`, + ); + + return statement; + } + + private static buildFunctionListQuery( + request: FunctionListRequest, + ): Statement { + const statement: Statement = SQL` + SELECT + stacktrace, + frameTypes, + value + FROM ${ProfileAggregationService.TABLE_NAME} + WHERE projectId = ${{ + type: TableColumnType.ObjectID, + value: request.projectId, + }} + AND time >= ${{ + type: TableColumnType.Date, + value: request.startTime, + }} + AND time <= ${{ + type: TableColumnType.Date, + value: request.endTime, + }} + `; + + ProfileAggregationService.appendCommonFilters(statement, request); + + statement.append( + SQL` LIMIT ${{ + type: TableColumnType.Number, + value: ProfileAggregationService.MAX_SAMPLE_FETCH, + }}`, + ); + + return statement; + } + + private static appendCommonFilters( + statement: Statement, + request: Pick, + ): void { + if (request.serviceIds && request.serviceIds.length > 0) { + statement.append( + SQL` AND serviceId IN (${{ + type: TableColumnType.ObjectID, + value: new Includes( + request.serviceIds.map((id: ObjectID) => { + return id.toString(); + }), + ), + }})`, + ); + } + + if (request.profileType) { + statement.append( + SQL` AND profileType = ${{ + type: TableColumnType.Text, + value: request.profileType, + }}`, + ); + } + } + + /** + * Parse a frame string in the format "functionName@fileName:lineNumber". + * Falls back gracefully if the format is unexpected. + */ + private static parseFrame(frame: string): ParsedFrame { + // Expected format: "functionName@fileName:lineNumber" + const atIndex: number = frame.indexOf("@"); + + if (atIndex === -1) { + return { + functionName: frame, + fileName: "", + lineNumber: 0, + }; + } + + const functionName: string = frame.substring(0, atIndex); + const rest: string = frame.substring(atIndex + 1); + + const lastColonIndex: number = rest.lastIndexOf(":"); + + if (lastColonIndex === -1) { + return { + functionName, + fileName: rest, + lineNumber: 0, + }; + } + + const fileName: string = rest.substring(0, lastColonIndex); + const lineNumberStr: string = rest.substring(lastColonIndex + 1); + const lineNumber: number = parseInt(lineNumberStr, 10) || 0; + + return { + functionName, + fileName, + lineNumber, + }; + } +} + +export default ProfileAggregationService; diff --git a/Common/Server/Services/TelemetryUsageBillingService.ts b/Common/Server/Services/TelemetryUsageBillingService.ts index 36a8007e23..e3f0bd982e 100644 --- a/Common/Server/Services/TelemetryUsageBillingService.ts +++ b/Common/Server/Services/TelemetryUsageBillingService.ts @@ -16,6 +16,8 @@ import SpanService from "./SpanService"; import LogService from "./LogService"; import MetricService from "./MetricService"; import ExceptionInstanceService from "./ExceptionInstanceService"; +import ProfileService from "./ProfileService"; +import ProfileSampleService from "./ProfileSampleService"; import AnalyticsQueryHelper from "../Types/AnalyticsDatabase/QueryHelper"; import DiskSize from "../../Types/DiskSize"; import logger from "../Utils/Logger"; @@ -26,6 +28,8 @@ import { AverageLogRowSizeInBytes, AverageMetricRowSizeInBytes, AverageExceptionRowSizeInBytes, + AverageProfileRowSizeInBytes, + AverageProfileSampleRowSizeInBytes, IsBillingEnabled, } from "../EnvironmentConfig"; import CaptureSpan from "../Utils/Telemetry/CaptureSpan"; @@ -81,7 +85,14 @@ export class Service extends DatabaseService { const averageExceptionRowSizeInBytes: number = this.getAverageExceptionRowSize(); - if (data.productType !== ProductType.Traces && averageRowSizeInBytes <= 0) { + const averageProfileSampleRowSizeInBytes: number = + this.getAverageProfileSampleRowSize(); + + if ( + data.productType !== ProductType.Traces && + data.productType !== ProductType.Profiles && + averageRowSizeInBytes <= 0 + ) { return; } @@ -93,6 +104,14 @@ export class Service extends DatabaseService { return; } + if ( + data.productType === ProductType.Profiles && + averageRowSizeInBytes <= 0 && + averageProfileSampleRowSizeInBytes <= 0 + ) { + return; + } + const usageDayString: string = OneUptimeDate.getDateString(usageDate); const startOfDay: Date = OneUptimeDate.getStartOfDay(usageDate); const endOfDay: Date = OneUptimeDate.getEndOfDay(usageDate); @@ -223,6 +242,45 @@ export class Service extends DatabaseService { } estimatedBytes = totalRowCount * averageRowSizeInBytes; + } else if (data.productType === ProductType.Profiles) { + const profileCount: PositiveNumber = await ProfileService.countBy({ + query: { + projectId: data.projectId, + serviceId: service.id, + startTime: AnalyticsQueryHelper.inBetween(startOfDay, endOfDay), + }, + skip: 0, + limit: LIMIT_INFINITY, + props: { + isRoot: true, + }, + }); + + const profileSampleCount: PositiveNumber = + await ProfileSampleService.countBy({ + query: { + projectId: data.projectId, + serviceId: service.id, + time: AnalyticsQueryHelper.inBetween(startOfDay, endOfDay), + }, + skip: 0, + limit: LIMIT_INFINITY, + props: { + isRoot: true, + }, + }); + + const totalProfileCount: number = profileCount.toNumber(); + const totalProfileSampleCount: number = + profileSampleCount.toNumber(); + + if (totalProfileCount <= 0 && totalProfileSampleCount <= 0) { + continue; + } + + estimatedBytes = + totalProfileCount * averageRowSizeInBytes + + totalProfileSampleCount * averageProfileSampleRowSizeInBytes; } } catch (error) { logger.error( @@ -268,7 +326,8 @@ export class Service extends DatabaseService { if ( data.productType !== ProductType.Traces && data.productType !== ProductType.Metrics && - data.productType !== ProductType.Logs + data.productType !== ProductType.Logs && + data.productType !== ProductType.Profiles ) { throw new BadDataException( "This product type is not a telemetry product type.", @@ -370,7 +429,8 @@ export class Service extends DatabaseService { if ( productType !== ProductType.Traces && productType !== ProductType.Logs && - productType !== ProductType.Metrics + productType !== ProductType.Metrics && + productType !== ProductType.Profiles ) { return fallbackSize; } @@ -380,6 +440,7 @@ export class Service extends DatabaseService { [ProductType.Traces]: AverageSpanRowSizeInBytes, [ProductType.Logs]: AverageLogRowSizeInBytes, [ProductType.Metrics]: AverageMetricRowSizeInBytes, + [ProductType.Profiles]: AverageProfileRowSizeInBytes, }[productType] ?? fallbackSize; if (!Number.isFinite(value) || value <= 0) { @@ -402,6 +463,20 @@ export class Service extends DatabaseService { return AverageExceptionRowSizeInBytes; } + + private getAverageProfileSampleRowSize(): number { + const fallbackSize: number = 512; + + if (!Number.isFinite(AverageProfileSampleRowSizeInBytes)) { + return fallbackSize; + } + + if (AverageProfileSampleRowSizeInBytes <= 0) { + return fallbackSize; + } + + return AverageProfileSampleRowSizeInBytes; + } } export default new Service(); diff --git a/Common/Server/Types/Billing/MeteredPlan/AllMeteredPlans.ts b/Common/Server/Types/Billing/MeteredPlan/AllMeteredPlans.ts index b48d090487..60abcee131 100644 --- a/Common/Server/Types/Billing/MeteredPlan/AllMeteredPlans.ts +++ b/Common/Server/Types/Billing/MeteredPlan/AllMeteredPlans.ts @@ -26,11 +26,18 @@ export const TracesDataIngestMetredPlan: TelemetryMeteredPlanType = unitCostInUSD: 0.1 / 15, // 0.10 per 15 days per GB }); +export const ProfilesDataIngestMeteredPlan: TelemetryMeteredPlanType = + new TelemetryMeteredPlanType({ + productType: ProductType.Profiles, + unitCostInUSD: 0.1 / 15, // 0.10 per 15 days per GB + }); + const AllMeteredPlans: Array = [ ActiveMonitoringMeteredPlan, LogDataIngestMeteredPlan, MetricsDataIngestMeteredPlan, TracesDataIngestMetredPlan, + ProfilesDataIngestMeteredPlan, ]; export class MeteredPlanUtil { @@ -44,6 +51,8 @@ export class MeteredPlanUtil { return MetricsDataIngestMeteredPlan; } else if (productType === ProductType.Traces) { return TracesDataIngestMetredPlan; + } else if (productType === ProductType.Profiles) { + return ProfilesDataIngestMeteredPlan; } else if (productType === ProductType.ActiveMonitoring) { return ActiveMonitoringMeteredPlan; } diff --git a/Common/Types/Monitor/MonitorStepProfileMonitor.ts b/Common/Types/Monitor/MonitorStepProfileMonitor.ts new file mode 100644 index 0000000000..bfe3e87e4b --- /dev/null +++ b/Common/Types/Monitor/MonitorStepProfileMonitor.ts @@ -0,0 +1,98 @@ +import Profile from "../../Models/AnalyticsModels/Profile"; +import InBetween from "../BaseDatabase/InBetween"; +import Includes from "../BaseDatabase/Includes"; +import Query from "../BaseDatabase/Query"; +import Search from "../BaseDatabase/Search"; +import OneUptimeDate from "../Date"; +import Dictionary from "../Dictionary"; +import { JSONObject } from "../JSON"; +import ObjectID from "../ObjectID"; + +export default interface MonitorStepProfileMonitor { + attributes: Dictionary; + profileTypes: Array; + telemetryServiceIds: Array; + lastXSecondsOfProfiles: number; + profileType: string; +} + +export class MonitorStepProfileMonitorUtil { + public static toQuery( + monitorStepProfileMonitor: MonitorStepProfileMonitor, + ): Query { + const query: Query = {}; + + if ( + monitorStepProfileMonitor.telemetryServiceIds && + monitorStepProfileMonitor.telemetryServiceIds.length > 0 + ) { + query.serviceId = new Includes( + monitorStepProfileMonitor.telemetryServiceIds, + ); + } + + if ( + monitorStepProfileMonitor.attributes && + Object.keys(monitorStepProfileMonitor.attributes).length > 0 + ) { + query.attributes = monitorStepProfileMonitor.attributes; + } + + if ( + monitorStepProfileMonitor.profileTypes && + monitorStepProfileMonitor.profileTypes.length > 0 + ) { + query.profileType = new Includes( + monitorStepProfileMonitor.profileTypes, + ); + } + + if (monitorStepProfileMonitor.profileType) { + query.profileType = new Search(monitorStepProfileMonitor.profileType); + } + + if (monitorStepProfileMonitor.lastXSecondsOfProfiles) { + const endDate: Date = OneUptimeDate.getCurrentDate(); + const startDate: Date = OneUptimeDate.addRemoveSeconds( + endDate, + monitorStepProfileMonitor.lastXSecondsOfProfiles * -1, + ); + query.startTime = new InBetween(startDate, endDate); + } + + return query; + } + + public static getDefault(): MonitorStepProfileMonitor { + return { + attributes: {}, + profileType: "", + profileTypes: [], + telemetryServiceIds: [], + lastXSecondsOfProfiles: 60, + }; + } + + public static fromJSON(json: JSONObject): MonitorStepProfileMonitor { + return { + attributes: + (json["attributes"] as Dictionary) || {}, + profileType: json["profileType"] as string, + profileTypes: json["profileTypes"] as Array, + telemetryServiceIds: ObjectID.fromJSONArray( + json["telemetryServiceIds"] as Array, + ), + lastXSecondsOfProfiles: json["lastXSecondsOfProfiles"] as number, + }; + } + + public static toJSON(monitor: MonitorStepProfileMonitor): JSONObject { + return { + attributes: monitor.attributes, + profileType: monitor.profileType, + profileTypes: monitor.profileTypes, + telemetryServiceIds: ObjectID.toJSONArray(monitor.telemetryServiceIds), + lastXSecondsOfProfiles: monitor.lastXSecondsOfProfiles, + }; + } +} diff --git a/HelmChart/Public/oneuptime/templates/_helpers.tpl b/HelmChart/Public/oneuptime/templates/_helpers.tpl index dfc77cffa7..281713f994 100644 --- a/HelmChart/Public/oneuptime/templates/_helpers.tpl +++ b/HelmChart/Public/oneuptime/templates/_helpers.tpl @@ -544,6 +544,12 @@ Usage: - name: AVERAGE_EXCEPTION_ROW_SIZE_IN_BYTES value: {{ $.Values.billing.telemetry.averageExceptionRowSizeInBytes | quote }} +- name: AVERAGE_PROFILE_ROW_SIZE_IN_BYTES + value: {{ $.Values.billing.telemetry.averageProfileRowSizeInBytes | quote }} + +- name: AVERAGE_PROFILE_SAMPLE_ROW_SIZE_IN_BYTES + value: {{ $.Values.billing.telemetry.averageProfileSampleRowSizeInBytes | quote }} + {{- include "oneuptime.env.oneuptimeSecret" . }} {{- end }} diff --git a/HelmChart/Public/oneuptime/values.schema.json b/HelmChart/Public/oneuptime/values.schema.json index b926442558..5f729c8417 100644 --- a/HelmChart/Public/oneuptime/values.schema.json +++ b/HelmChart/Public/oneuptime/values.schema.json @@ -672,6 +672,14 @@ "averageExceptionRowSizeInBytes": { "type": "integer", "minimum": 1 + }, + "averageProfileRowSizeInBytes": { + "type": "integer", + "minimum": 1 + }, + "averageProfileSampleRowSizeInBytes": { + "type": "integer", + "minimum": 1 } }, "additionalProperties": false diff --git a/HelmChart/Public/oneuptime/values.yaml b/HelmChart/Public/oneuptime/values.yaml index f59332aae9..8f9f6404ef 100644 --- a/HelmChart/Public/oneuptime/values.yaml +++ b/HelmChart/Public/oneuptime/values.yaml @@ -275,6 +275,8 @@ billing: averageLogRowSizeInBytes: 1024 averageMetricRowSizeInBytes: 1024 averageExceptionRowSizeInBytes: 1024 + averageProfileRowSizeInBytes: 1024 + averageProfileSampleRowSizeInBytes: 512 subscriptionPlan: basic: diff --git a/Telemetry/Docs/examples/otlp-profiles-payload.json b/Telemetry/Docs/examples/otlp-profiles-payload.json new file mode 100644 index 0000000000..a64a1420ce --- /dev/null +++ b/Telemetry/Docs/examples/otlp-profiles-payload.json @@ -0,0 +1,281 @@ +{ + "resourceProfiles": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "my-go-service" + } + }, + { + "key": "host.name", + "value": { + "stringValue": "prod-server-01" + } + }, + { + "key": "process.runtime.name", + "value": { + "stringValue": "go" + } + }, + { + "key": "process.runtime.version", + "value": { + "stringValue": "go1.22.0" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "go" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.28.0" + } + } + ] + }, + "scopeProfiles": [ + { + "scope": { + "name": "otel-profiling-go", + "version": "0.5.0" + }, + "profiles": [ + { + "profileId": "qg7PaWLjuqLhWlwvlHRU9A==", + "startTimeUnixNano": "1700000000000000000", + "endTimeUnixNano": "1700000030000000000", + "attributes": [ + { + "key": "profiling.data.type", + "value": { + "stringValue": "cpu" + } + } + ], + "originalPayloadFormat": "pprofext", + "profile": { + "stringTable": [ + "", + "cpu", + "nanoseconds", + "samples", + "count", + "main.handleRequest", + "/app/main.go", + "net/http.(*conn).serve", + "/usr/local/go/src/net/http/server.go", + "runtime.goexit", + "/usr/local/go/src/runtime/asm_amd64.s", + "main.processData", + "/app/processor.go", + "encoding/json.Marshal", + "/usr/local/go/src/encoding/json/encode.go", + "runtime.mallocgc", + "/usr/local/go/src/runtime/malloc.go", + "main.queryDatabase", + "/app/db.go", + "database/sql.(*DB).QueryContext", + "/usr/local/go/src/database/sql/sql.go" + ], + "sampleType": [ + { + "type": 1, + "unit": 2 + }, + { + "type": 3, + "unit": 4 + } + ], + "periodType": { + "type": 1, + "unit": 2 + }, + "period": 10000000, + "functionTable": [ + { + "name": 5, + "filename": 6 + }, + { + "name": 7, + "filename": 8 + }, + { + "name": 9, + "filename": 10 + }, + { + "name": 11, + "filename": 12 + }, + { + "name": 13, + "filename": 14 + }, + { + "name": 15, + "filename": 16 + }, + { + "name": 17, + "filename": 18 + }, + { + "name": 19, + "filename": 20 + } + ], + "locationTable": [ + { + "line": [ + { + "functionIndex": 0, + "line": 42 + } + ] + }, + { + "line": [ + { + "functionIndex": 1, + "line": 1960 + } + ] + }, + { + "line": [ + { + "functionIndex": 2, + "line": 1700 + } + ] + }, + { + "line": [ + { + "functionIndex": 3, + "line": 88 + } + ] + }, + { + "line": [ + { + "functionIndex": 4, + "line": 160 + } + ] + }, + { + "line": [ + { + "functionIndex": 5, + "line": 905 + } + ] + }, + { + "line": [ + { + "functionIndex": 6, + "line": 55 + } + ] + }, + { + "line": [ + { + "functionIndex": 7, + "line": 1612 + } + ] + } + ], + "stackTable": [ + { + "locationIndices": [0, 1, 2] + }, + { + "locationIndices": [3, 4, 0, 1, 2] + }, + { + "locationIndices": [5, 3, 0, 1, 2] + }, + { + "locationIndices": [6, 7, 0, 1, 2] + } + ], + "linkTable": [ + { + "traceId": "qg7PaWLjuqLhWlwvlHRU9A==", + "spanId": "r+N4WZXXfP4=" + } + ], + "attributeTable": [ + { + "key": "profile.frame.type", + "value": { + "stringValue": "go" + } + }, + { + "key": "thread.name", + "value": { + "stringValue": "main" + } + } + ], + "sample": [ + { + "stackIndex": 0, + "value": [50000000, 5], + "timestampsUnixNano": ["1700000005000000000"], + "linkIndex": 0, + "attributeIndices": [0, 1] + }, + { + "stackIndex": 1, + "value": [120000000, 12], + "timestampsUnixNano": ["1700000010000000000"], + "linkIndex": 0, + "attributeIndices": [0] + }, + { + "stackIndex": 2, + "value": [30000000, 3], + "timestampsUnixNano": ["1700000015000000000"], + "linkIndex": 0, + "attributeIndices": [0] + }, + { + "stackIndex": 3, + "value": [80000000, 8], + "timestampsUnixNano": ["1700000020000000000"], + "linkIndex": 0, + "attributeIndices": [0] + } + ] + } + } + ] + } + ], + "schemaUrl": "https://opentelemetry.io/schemas/1.21.0" + } + ] +}