mirror of
https://github.com/OneUptime/oneuptime.git
synced 2026-04-06 00:32:12 +02:00
feat: add LogPipelineProcessor model and related services
- Implemented LogPipelineProcessor model with necessary fields and access controls. - Created LogDropFilterAction enum for defining actions on log drop filters. - Introduced LogPipelineProcessorType enum and configuration interfaces for various processor types. - Developed LogExport utility for exporting logs in CSV and JSON formats. - Added LogDropFilterService for managing drop filters with caching. - Implemented LogPipelineService for loading and processing log pipelines with processors. - Created LogFilterEvaluator for evaluating filter queries against log entries.
This commit is contained in:
@@ -1277,6 +1277,7 @@ const DashboardLogsViewer: FunctionComponent<ComponentProps> = (
|
||||
viewMode={viewMode}
|
||||
onViewModeChange={setViewMode}
|
||||
analyticsServiceIds={serviceIdStrings}
|
||||
projectId={ProjectUtil.getCurrentProjectId() || undefined}
|
||||
analyticsAppliedFacetFilters={appliedFacetFilters}
|
||||
onUpdateCurrentSavedView={async () => {
|
||||
if (!selectedSavedView?.id) {
|
||||
|
||||
169
App/FeatureSet/Dashboard/src/Pages/Settings/LogDropFilters.tsx
Normal file
169
App/FeatureSet/Dashboard/src/Pages/Settings/LogDropFilters.tsx
Normal file
@@ -0,0 +1,169 @@
|
||||
import PageComponentProps from "../PageComponentProps";
|
||||
import FormFieldSchemaType from "Common/UI/Components/Forms/Types/FormFieldSchemaType";
|
||||
import ModelTable from "Common/UI/Components/ModelTable/ModelTable";
|
||||
import FieldType from "Common/UI/Components/Types/FieldType";
|
||||
import LogDropFilter from "Common/Models/DatabaseModels/LogDropFilter";
|
||||
import ProjectUtil from "Common/UI/Utils/Project";
|
||||
import Navigation from "Common/UI/Utils/Navigation";
|
||||
import React, { Fragment, FunctionComponent, ReactElement } from "react";
|
||||
|
||||
const LogDropFilters: FunctionComponent<
|
||||
PageComponentProps
|
||||
> = (): ReactElement => {
|
||||
return (
|
||||
<Fragment>
|
||||
<ModelTable<LogDropFilter>
|
||||
modelType={LogDropFilter}
|
||||
query={{
|
||||
projectId: ProjectUtil.getCurrentProjectId()!,
|
||||
}}
|
||||
id="log-drop-filters-table"
|
||||
name="Settings > Log Drop Filters"
|
||||
userPreferencesKey="log-drop-filters-table"
|
||||
isDeleteable={true}
|
||||
isEditable={true}
|
||||
isCreateable={true}
|
||||
cardProps={{
|
||||
title: "Log Drop Filters",
|
||||
description:
|
||||
"Drop filters let you discard or sample logs before they are stored. Matching logs are dropped or sampled at the configured percentage. Filters run in sort order before pipeline processing.",
|
||||
}}
|
||||
noItemsMessage={"No drop filters found."}
|
||||
viewPageRoute={Navigation.getCurrentRoute()}
|
||||
formFields={[
|
||||
{
|
||||
field: {
|
||||
name: true,
|
||||
},
|
||||
title: "Name",
|
||||
fieldType: FormFieldSchemaType.Text,
|
||||
required: true,
|
||||
placeholder: "e.g. Drop Debug Logs",
|
||||
validation: {
|
||||
minLength: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
field: {
|
||||
description: true,
|
||||
},
|
||||
title: "Description",
|
||||
fieldType: FormFieldSchemaType.LongText,
|
||||
required: false,
|
||||
placeholder: "Describe what this filter does.",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
filterQuery: true,
|
||||
},
|
||||
title: "Filter Query",
|
||||
fieldType: FormFieldSchemaType.LongText,
|
||||
required: true,
|
||||
placeholder:
|
||||
"e.g. severityText = 'DEBUG' OR body LIKE '%health%'",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
action: true,
|
||||
},
|
||||
title: "Action",
|
||||
fieldType: FormFieldSchemaType.Text,
|
||||
required: true,
|
||||
placeholder: "drop or sample",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
samplePercentage: true,
|
||||
},
|
||||
title: "Sample Percentage",
|
||||
fieldType: FormFieldSchemaType.Number,
|
||||
required: false,
|
||||
placeholder: "50 (keep this % of matching logs)",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
isEnabled: true,
|
||||
},
|
||||
title: "Enabled",
|
||||
fieldType: FormFieldSchemaType.Toggle,
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
sortOrder: true,
|
||||
},
|
||||
title: "Sort Order",
|
||||
fieldType: FormFieldSchemaType.Number,
|
||||
required: false,
|
||||
placeholder: "0",
|
||||
},
|
||||
]}
|
||||
showRefreshButton={true}
|
||||
showViewIdButton={true}
|
||||
filters={[
|
||||
{
|
||||
field: {
|
||||
name: true,
|
||||
},
|
||||
type: FieldType.Text,
|
||||
title: "Name",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
action: true,
|
||||
},
|
||||
type: FieldType.Text,
|
||||
title: "Action",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
isEnabled: true,
|
||||
},
|
||||
type: FieldType.Boolean,
|
||||
title: "Enabled",
|
||||
},
|
||||
]}
|
||||
columns={[
|
||||
{
|
||||
field: {
|
||||
name: true,
|
||||
},
|
||||
title: "Name",
|
||||
type: FieldType.Text,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
action: true,
|
||||
},
|
||||
title: "Action",
|
||||
type: FieldType.Text,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
samplePercentage: true,
|
||||
},
|
||||
title: "Sample %",
|
||||
type: FieldType.Number,
|
||||
noValueMessage: "-",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
isEnabled: true,
|
||||
},
|
||||
title: "Enabled",
|
||||
type: FieldType.Boolean,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
sortOrder: true,
|
||||
},
|
||||
title: "Sort Order",
|
||||
type: FieldType.Number,
|
||||
},
|
||||
]}
|
||||
/>
|
||||
</Fragment>
|
||||
);
|
||||
};
|
||||
|
||||
export default LogDropFilters;
|
||||
250
App/FeatureSet/Dashboard/src/Pages/Settings/LogPipelineView.tsx
Normal file
250
App/FeatureSet/Dashboard/src/Pages/Settings/LogPipelineView.tsx
Normal file
@@ -0,0 +1,250 @@
|
||||
import PageMap from "../../Utils/PageMap";
|
||||
import RouteMap from "../../Utils/RouteMap";
|
||||
import PageComponentProps from "../PageComponentProps";
|
||||
import Route from "Common/Types/API/Route";
|
||||
import ObjectID from "Common/Types/ObjectID";
|
||||
import FormFieldSchemaType from "Common/UI/Components/Forms/Types/FormFieldSchemaType";
|
||||
import ModelDelete from "Common/UI/Components/ModelDelete/ModelDelete";
|
||||
import CardModelDetail from "Common/UI/Components/ModelDetail/CardModelDetail";
|
||||
import ModelTable from "Common/UI/Components/ModelTable/ModelTable";
|
||||
import FieldType from "Common/UI/Components/Types/FieldType";
|
||||
import Navigation from "Common/UI/Utils/Navigation";
|
||||
import LogPipeline from "Common/Models/DatabaseModels/LogPipeline";
|
||||
import LogPipelineProcessor from "Common/Models/DatabaseModels/LogPipelineProcessor";
|
||||
import React, { Fragment, FunctionComponent, ReactElement } from "react";
|
||||
|
||||
const LogPipelineView: FunctionComponent<PageComponentProps> = (
|
||||
_props: PageComponentProps,
|
||||
): ReactElement => {
|
||||
const modelId: ObjectID = Navigation.getLastParamAsObjectID();
|
||||
|
||||
return (
|
||||
<Fragment>
|
||||
<CardModelDetail<LogPipeline>
|
||||
name="Log Pipeline Details"
|
||||
cardProps={{
|
||||
title: "Log Pipeline Details",
|
||||
description: "Details for this log pipeline.",
|
||||
}}
|
||||
isEditable={true}
|
||||
formFields={[
|
||||
{
|
||||
field: {
|
||||
name: true,
|
||||
},
|
||||
title: "Name",
|
||||
fieldType: FormFieldSchemaType.Text,
|
||||
required: true,
|
||||
placeholder: "Pipeline Name",
|
||||
validation: {
|
||||
minLength: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
field: {
|
||||
description: true,
|
||||
},
|
||||
title: "Description",
|
||||
fieldType: FormFieldSchemaType.LongText,
|
||||
required: false,
|
||||
placeholder: "Pipeline Description",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
filterQuery: true,
|
||||
},
|
||||
title: "Filter Query",
|
||||
fieldType: FormFieldSchemaType.LongText,
|
||||
required: false,
|
||||
placeholder:
|
||||
"e.g. severityText = 'ERROR' AND attributes.service = 'api'",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
isEnabled: true,
|
||||
},
|
||||
title: "Enabled",
|
||||
fieldType: FormFieldSchemaType.Toggle,
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
sortOrder: true,
|
||||
},
|
||||
title: "Sort Order",
|
||||
fieldType: FormFieldSchemaType.Number,
|
||||
required: false,
|
||||
},
|
||||
]}
|
||||
modelDetailProps={{
|
||||
modelType: LogPipeline,
|
||||
id: "model-detail-log-pipeline",
|
||||
fields: [
|
||||
{
|
||||
field: {
|
||||
name: true,
|
||||
},
|
||||
title: "Name",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
description: true,
|
||||
},
|
||||
title: "Description",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
filterQuery: true,
|
||||
},
|
||||
title: "Filter Query",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
isEnabled: true,
|
||||
},
|
||||
title: "Enabled",
|
||||
fieldType: FieldType.Boolean,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
sortOrder: true,
|
||||
},
|
||||
title: "Sort Order",
|
||||
fieldType: FieldType.Number,
|
||||
},
|
||||
],
|
||||
modelId: modelId,
|
||||
}}
|
||||
/>
|
||||
|
||||
<ModelTable<LogPipelineProcessor>
|
||||
modelType={LogPipelineProcessor}
|
||||
query={{
|
||||
logPipelineId: modelId,
|
||||
}}
|
||||
id="log-pipeline-processors-table"
|
||||
name="Log Pipeline > Processors"
|
||||
userPreferencesKey="log-pipeline-processors-table"
|
||||
isDeleteable={true}
|
||||
isEditable={true}
|
||||
isCreateable={true}
|
||||
cardProps={{
|
||||
title: "Processors",
|
||||
description:
|
||||
"Processors transform logs matched by this pipeline. They run in sort order.",
|
||||
}}
|
||||
noItemsMessage={"No processors configured for this pipeline."}
|
||||
formFields={[
|
||||
{
|
||||
field: {
|
||||
name: true,
|
||||
},
|
||||
title: "Name",
|
||||
fieldType: FormFieldSchemaType.Text,
|
||||
required: true,
|
||||
placeholder: "e.g. Remap severity",
|
||||
validation: {
|
||||
minLength: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
field: {
|
||||
processorType: true,
|
||||
},
|
||||
title: "Processor Type",
|
||||
fieldType: FormFieldSchemaType.Text,
|
||||
required: true,
|
||||
placeholder: "e.g. AttributeRemapper",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
configuration: true,
|
||||
},
|
||||
title: "Configuration (JSON)",
|
||||
fieldType: FormFieldSchemaType.JSON,
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
isEnabled: true,
|
||||
},
|
||||
title: "Enabled",
|
||||
fieldType: FormFieldSchemaType.Toggle,
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
sortOrder: true,
|
||||
},
|
||||
title: "Sort Order",
|
||||
fieldType: FormFieldSchemaType.Number,
|
||||
required: false,
|
||||
placeholder: "0",
|
||||
},
|
||||
]}
|
||||
showRefreshButton={true}
|
||||
createInitialValues={{
|
||||
logPipelineId: modelId,
|
||||
}}
|
||||
filters={[
|
||||
{
|
||||
field: {
|
||||
name: true,
|
||||
},
|
||||
type: FieldType.Text,
|
||||
title: "Name",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
isEnabled: true,
|
||||
},
|
||||
type: FieldType.Boolean,
|
||||
title: "Enabled",
|
||||
},
|
||||
]}
|
||||
columns={[
|
||||
{
|
||||
field: {
|
||||
name: true,
|
||||
},
|
||||
title: "Name",
|
||||
type: FieldType.Text,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
processorType: true,
|
||||
},
|
||||
title: "Type",
|
||||
type: FieldType.Text,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
isEnabled: true,
|
||||
},
|
||||
title: "Enabled",
|
||||
type: FieldType.Boolean,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
sortOrder: true,
|
||||
},
|
||||
title: "Sort Order",
|
||||
type: FieldType.Number,
|
||||
},
|
||||
]}
|
||||
/>
|
||||
|
||||
<ModelDelete
|
||||
modelType={LogPipeline}
|
||||
modelId={modelId}
|
||||
onDeleteSuccess={() => {
|
||||
Navigation.navigate(
|
||||
RouteMap[PageMap.SETTINGS_LOG_PIPELINES] as Route,
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</Fragment>
|
||||
);
|
||||
};
|
||||
|
||||
export default LogPipelineView;
|
||||
139
App/FeatureSet/Dashboard/src/Pages/Settings/LogPipelines.tsx
Normal file
139
App/FeatureSet/Dashboard/src/Pages/Settings/LogPipelines.tsx
Normal file
@@ -0,0 +1,139 @@
|
||||
import PageMap from "../../Utils/PageMap";
|
||||
import RouteMap, { RouteUtil } from "../../Utils/RouteMap";
|
||||
import PageComponentProps from "../PageComponentProps";
|
||||
import FormFieldSchemaType from "Common/UI/Components/Forms/Types/FormFieldSchemaType";
|
||||
import ModelTable from "Common/UI/Components/ModelTable/ModelTable";
|
||||
import FieldType from "Common/UI/Components/Types/FieldType";
|
||||
import LogPipeline from "Common/Models/DatabaseModels/LogPipeline";
|
||||
import ProjectUtil from "Common/UI/Utils/Project";
|
||||
import Route from "Common/Types/API/Route";
|
||||
import React, { Fragment, FunctionComponent, ReactElement } from "react";
|
||||
|
||||
const LogPipelines: FunctionComponent<PageComponentProps> = (): ReactElement => {
|
||||
return (
|
||||
<Fragment>
|
||||
<ModelTable<LogPipeline>
|
||||
modelType={LogPipeline}
|
||||
query={{
|
||||
projectId: ProjectUtil.getCurrentProjectId()!,
|
||||
}}
|
||||
id="log-pipelines-table"
|
||||
name="Settings > Log Pipelines"
|
||||
userPreferencesKey="log-pipelines-table"
|
||||
isDeleteable={true}
|
||||
isEditable={true}
|
||||
isCreateable={true}
|
||||
cardProps={{
|
||||
title: "Log Pipelines",
|
||||
description:
|
||||
"Configure server-side log processing pipelines that transform logs at ingest time. Pipelines run in sort order and apply processors to matching logs.",
|
||||
}}
|
||||
noItemsMessage={"No log pipelines found."}
|
||||
viewPageRoute={RouteUtil.populateRouteParams(
|
||||
RouteMap[PageMap.SETTINGS_LOG_PIPELINE_VIEW] as Route,
|
||||
)}
|
||||
formFields={[
|
||||
{
|
||||
field: {
|
||||
name: true,
|
||||
},
|
||||
title: "Name",
|
||||
fieldType: FormFieldSchemaType.Text,
|
||||
required: true,
|
||||
placeholder: "e.g. Parse Nginx Logs",
|
||||
validation: {
|
||||
minLength: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
field: {
|
||||
description: true,
|
||||
},
|
||||
title: "Description",
|
||||
fieldType: FormFieldSchemaType.LongText,
|
||||
required: false,
|
||||
placeholder: "Describe what this pipeline does.",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
filterQuery: true,
|
||||
},
|
||||
title: "Filter Query",
|
||||
fieldType: FormFieldSchemaType.LongText,
|
||||
required: false,
|
||||
placeholder:
|
||||
"e.g. severityText = 'ERROR' AND attributes.service = 'api'",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
isEnabled: true,
|
||||
},
|
||||
title: "Enabled",
|
||||
fieldType: FormFieldSchemaType.Toggle,
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
sortOrder: true,
|
||||
},
|
||||
title: "Sort Order",
|
||||
fieldType: FormFieldSchemaType.Number,
|
||||
required: false,
|
||||
placeholder: "0",
|
||||
},
|
||||
]}
|
||||
showRefreshButton={true}
|
||||
showViewIdButton={true}
|
||||
filters={[
|
||||
{
|
||||
field: {
|
||||
name: true,
|
||||
},
|
||||
type: FieldType.Text,
|
||||
title: "Name",
|
||||
},
|
||||
{
|
||||
field: {
|
||||
isEnabled: true,
|
||||
},
|
||||
type: FieldType.Boolean,
|
||||
title: "Enabled",
|
||||
},
|
||||
]}
|
||||
columns={[
|
||||
{
|
||||
field: {
|
||||
name: true,
|
||||
},
|
||||
title: "Name",
|
||||
type: FieldType.Text,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
description: true,
|
||||
},
|
||||
noValueMessage: "-",
|
||||
title: "Description",
|
||||
type: FieldType.LongText,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
isEnabled: true,
|
||||
},
|
||||
title: "Enabled",
|
||||
type: FieldType.Boolean,
|
||||
},
|
||||
{
|
||||
field: {
|
||||
sortOrder: true,
|
||||
},
|
||||
title: "Sort Order",
|
||||
type: FieldType.Number,
|
||||
},
|
||||
]}
|
||||
/>
|
||||
</Fragment>
|
||||
);
|
||||
};
|
||||
|
||||
export default LogPipelines;
|
||||
@@ -68,6 +68,24 @@ const DashboardSideMenu: () => JSX.Element = (): ReactElement => {
|
||||
},
|
||||
icon: IconProp.Terminal,
|
||||
},
|
||||
{
|
||||
link: {
|
||||
title: "Log Pipelines",
|
||||
to: RouteUtil.populateRouteParams(
|
||||
RouteMap[PageMap.SETTINGS_LOG_PIPELINES] as Route,
|
||||
),
|
||||
},
|
||||
icon: IconProp.Logs,
|
||||
},
|
||||
{
|
||||
link: {
|
||||
title: "Log Drop Filters",
|
||||
to: RouteUtil.populateRouteParams(
|
||||
RouteMap[PageMap.SETTINGS_LOG_DROP_FILTERS] as Route,
|
||||
),
|
||||
},
|
||||
icon: IconProp.Filter,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -59,6 +59,10 @@ import SettingsUsageHistory from "../Pages/Settings/UsageHistory";
|
||||
|
||||
import SettingsSlackIntegration from "../Pages/Settings/SlackIntegration";
|
||||
|
||||
import SettingsLogPipelines from "../Pages/Settings/LogPipelines";
|
||||
import SettingsLogPipelineView from "../Pages/Settings/LogPipelineView";
|
||||
import SettingsLogDropFilters from "../Pages/Settings/LogDropFilters";
|
||||
|
||||
const SettingsRoutes: FunctionComponent<ComponentProps> = (
|
||||
props: ComponentProps,
|
||||
): ReactElement => {
|
||||
@@ -375,6 +379,49 @@ const SettingsRoutes: FunctionComponent<ComponentProps> = (
|
||||
/>
|
||||
}
|
||||
/>
|
||||
|
||||
<PageRoute
|
||||
path={RouteUtil.getLastPathForKey(
|
||||
PageMap.SETTINGS_LOG_PIPELINES,
|
||||
)}
|
||||
element={
|
||||
<SettingsLogPipelines
|
||||
{...props}
|
||||
pageRoute={
|
||||
RouteMap[PageMap.SETTINGS_LOG_PIPELINES] as Route
|
||||
}
|
||||
/>
|
||||
}
|
||||
/>
|
||||
|
||||
<PageRoute
|
||||
path={RouteUtil.getLastPathForKey(
|
||||
PageMap.SETTINGS_LOG_PIPELINE_VIEW,
|
||||
2,
|
||||
)}
|
||||
element={
|
||||
<SettingsLogPipelineView
|
||||
{...props}
|
||||
pageRoute={
|
||||
RouteMap[PageMap.SETTINGS_LOG_PIPELINE_VIEW] as Route
|
||||
}
|
||||
/>
|
||||
}
|
||||
/>
|
||||
|
||||
<PageRoute
|
||||
path={RouteUtil.getLastPathForKey(
|
||||
PageMap.SETTINGS_LOG_DROP_FILTERS,
|
||||
)}
|
||||
element={
|
||||
<SettingsLogDropFilters
|
||||
{...props}
|
||||
pageRoute={
|
||||
RouteMap[PageMap.SETTINGS_LOG_DROP_FILTERS] as Route
|
||||
}
|
||||
/>
|
||||
}
|
||||
/>
|
||||
</PageRoute>
|
||||
</Routes>
|
||||
);
|
||||
|
||||
@@ -342,6 +342,10 @@ enum PageMap {
|
||||
SETTINGS_TELEMETRY_INGESTION_KEYS = "SETTINGS_TELEMETRY_INGESTION_KEYS",
|
||||
SETTINGS_TELEMETRY_INGESTION_KEY_VIEW = "SETTINGS_TELEMETRY_INGESTION_KEY_VIEW",
|
||||
|
||||
SETTINGS_LOG_PIPELINES = "SETTINGS_LOG_PIPELINES",
|
||||
SETTINGS_LOG_PIPELINE_VIEW = "SETTINGS_LOG_PIPELINE_VIEW",
|
||||
SETTINGS_LOG_DROP_FILTERS = "SETTINGS_LOG_DROP_FILTERS",
|
||||
|
||||
// API Keys.
|
||||
SETTINGS_APIKEYS = "SETTINGS_APIKEYS",
|
||||
SETTINGS_APIKEY_VIEW = "SETTINGS_APIKEY_VIEW",
|
||||
|
||||
@@ -304,6 +304,9 @@ export const SettingsRoutePath: Dictionary<string> = {
|
||||
[PageMap.SETTINGS_APIKEY_VIEW]: `api-keys/${RouteParams.ModelID}`,
|
||||
[PageMap.SETTINGS_TELEMETRY_INGESTION_KEYS]: `telemetry-ingestion-keys`,
|
||||
[PageMap.SETTINGS_TELEMETRY_INGESTION_KEY_VIEW]: `telemetry-ingestion-keys/${RouteParams.ModelID}`,
|
||||
[PageMap.SETTINGS_LOG_PIPELINES]: `log-pipelines`,
|
||||
[PageMap.SETTINGS_LOG_PIPELINE_VIEW]: `log-pipelines/${RouteParams.ModelID}`,
|
||||
[PageMap.SETTINGS_LOG_DROP_FILTERS]: `log-drop-filters`,
|
||||
[PageMap.SETTINGS_SLACK_INTEGRATION]: "slack-integration",
|
||||
[PageMap.SETTINGS_MICROSOFT_TEAMS_INTEGRATION]: "microsoft-teams-integration",
|
||||
|
||||
@@ -2124,6 +2127,24 @@ const RouteMap: Dictionary<Route> = {
|
||||
}`,
|
||||
),
|
||||
|
||||
[PageMap.SETTINGS_LOG_PIPELINES]: new Route(
|
||||
`/dashboard/${RouteParams.ProjectID}/settings/${
|
||||
SettingsRoutePath[PageMap.SETTINGS_LOG_PIPELINES]
|
||||
}`,
|
||||
),
|
||||
|
||||
[PageMap.SETTINGS_LOG_PIPELINE_VIEW]: new Route(
|
||||
`/dashboard/${RouteParams.ProjectID}/settings/${
|
||||
SettingsRoutePath[PageMap.SETTINGS_LOG_PIPELINE_VIEW]
|
||||
}`,
|
||||
),
|
||||
|
||||
[PageMap.SETTINGS_LOG_DROP_FILTERS]: new Route(
|
||||
`/dashboard/${RouteParams.ProjectID}/settings/${
|
||||
SettingsRoutePath[PageMap.SETTINGS_LOG_DROP_FILTERS]
|
||||
}`,
|
||||
),
|
||||
|
||||
[PageMap.SETTINGS_APIKEY_VIEW]: new Route(
|
||||
`/dashboard/${RouteParams.ProjectID}/settings/${
|
||||
SettingsRoutePath[PageMap.SETTINGS_APIKEY_VIEW]
|
||||
|
||||
@@ -37,6 +37,9 @@ import IncidentTemplateOwnerUser from "./IncidentTemplateOwnerUser";
|
||||
//Labels.
|
||||
import Label from "./Label";
|
||||
import LogSavedView from "./LogSavedView";
|
||||
import LogPipeline from "./LogPipeline";
|
||||
import LogPipelineProcessor from "./LogPipelineProcessor";
|
||||
import LogDropFilter from "./LogDropFilter";
|
||||
// Monitors
|
||||
import Monitor from "./Monitor";
|
||||
import MonitorCustomField from "./MonitorCustomField";
|
||||
@@ -252,6 +255,9 @@ const AllModelTypes: Array<{
|
||||
ApiKey,
|
||||
Label,
|
||||
LogSavedView,
|
||||
LogPipeline,
|
||||
LogPipelineProcessor,
|
||||
LogDropFilter,
|
||||
ApiKeyPermission,
|
||||
ProjectSmtpConfig,
|
||||
StatusPage,
|
||||
|
||||
480
Common/Models/DatabaseModels/LogDropFilter.ts
Normal file
480
Common/Models/DatabaseModels/LogDropFilter.ts
Normal file
@@ -0,0 +1,480 @@
|
||||
import Project from "./Project";
|
||||
import User from "./User";
|
||||
import BaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
|
||||
import Route from "../../Types/API/Route";
|
||||
import ColumnAccessControl from "../../Types/Database/AccessControl/ColumnAccessControl";
|
||||
import TableAccessControl from "../../Types/Database/AccessControl/TableAccessControl";
|
||||
import TableBillingAccessControl from "../../Types/Database/AccessControl/TableBillingAccessControl";
|
||||
import ColumnLength from "../../Types/Database/ColumnLength";
|
||||
import ColumnType from "../../Types/Database/ColumnType";
|
||||
import CrudApiEndpoint from "../../Types/Database/CrudApiEndpoint";
|
||||
import EnableDocumentation from "../../Types/Database/EnableDocumentation";
|
||||
import TableColumn from "../../Types/Database/TableColumn";
|
||||
import TableColumnType from "../../Types/Database/TableColumnType";
|
||||
import TableMetadata from "../../Types/Database/TableMetadata";
|
||||
import TenantColumn from "../../Types/Database/TenantColumn";
|
||||
import IconProp from "../../Types/Icon/IconProp";
|
||||
import ObjectID from "../../Types/ObjectID";
|
||||
import Permission from "../../Types/Permission";
|
||||
import { PlanType } from "../../Types/Billing/SubscriptionPlan";
|
||||
import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
|
||||
|
||||
@EnableDocumentation()
|
||||
@TableBillingAccessControl({
|
||||
create: PlanType.Free,
|
||||
read: PlanType.Free,
|
||||
update: PlanType.Free,
|
||||
delete: PlanType.Free,
|
||||
})
|
||||
@TenantColumn("projectId")
|
||||
@CrudApiEndpoint(new Route("/log-drop-filter"))
|
||||
@Entity({
|
||||
name: "LogDropFilter",
|
||||
})
|
||||
@TableMetadata({
|
||||
tableName: "LogDropFilter",
|
||||
singularName: "Log Drop Filter",
|
||||
pluralName: "Log Drop Filters",
|
||||
icon: IconProp.Filter,
|
||||
tableDescription:
|
||||
"Configure rules to drop or sample logs before storage to reduce volume and cost.",
|
||||
})
|
||||
@TableAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogDropFilter,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
delete: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.DeleteProjectLogDropFilter,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogDropFilter,
|
||||
],
|
||||
})
|
||||
export default class LogDropFilter extends BaseModel {
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogDropFilter,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
manyToOneRelationColumn: "projectId",
|
||||
type: TableColumnType.Entity,
|
||||
modelType: Project,
|
||||
title: "Project",
|
||||
description: "Relation to the project this log drop filter belongs to.",
|
||||
})
|
||||
@ManyToOne(
|
||||
() => {
|
||||
return Project;
|
||||
},
|
||||
{
|
||||
eager: false,
|
||||
nullable: true,
|
||||
onDelete: "CASCADE",
|
||||
orphanedRowAction: "nullify",
|
||||
},
|
||||
)
|
||||
@JoinColumn({ name: "projectId" })
|
||||
public project?: Project = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogDropFilter,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@Index()
|
||||
@TableColumn({
|
||||
type: TableColumnType.ObjectID,
|
||||
required: true,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Project ID",
|
||||
description: "ID of the project this log drop filter belongs to.",
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.ObjectID,
|
||||
nullable: false,
|
||||
transformer: ObjectID.getDatabaseTransformer(),
|
||||
})
|
||||
public projectId?: ObjectID = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogDropFilter,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogDropFilter,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
required: true,
|
||||
type: TableColumnType.Name,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Name",
|
||||
description: "Friendly name for this drop filter.",
|
||||
})
|
||||
@Column({
|
||||
nullable: false,
|
||||
type: ColumnType.Name,
|
||||
length: ColumnLength.Name,
|
||||
})
|
||||
public name?: string = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogDropFilter,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogDropFilter,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
required: false,
|
||||
type: TableColumnType.LongText,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Description",
|
||||
description: "Description of what this drop filter does.",
|
||||
})
|
||||
@Column({
|
||||
nullable: true,
|
||||
type: ColumnType.LongText,
|
||||
length: ColumnLength.LongText,
|
||||
})
|
||||
public description?: string = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogDropFilter,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogDropFilter,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
required: true,
|
||||
type: TableColumnType.LongText,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Filter Query",
|
||||
description:
|
||||
"Filter expression that identifies which logs to drop or sample.",
|
||||
})
|
||||
@Column({
|
||||
nullable: false,
|
||||
type: ColumnType.LongText,
|
||||
length: ColumnLength.LongText,
|
||||
})
|
||||
public filterQuery?: string = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogDropFilter,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogDropFilter,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
required: true,
|
||||
type: TableColumnType.ShortText,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Action",
|
||||
description:
|
||||
"What to do with matching logs: 'drop' to discard entirely, 'sample' to keep a percentage.",
|
||||
})
|
||||
@Column({
|
||||
nullable: false,
|
||||
type: ColumnType.ShortText,
|
||||
length: ColumnLength.ShortText,
|
||||
default: "drop",
|
||||
})
|
||||
public action?: string = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogDropFilter,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogDropFilter,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
title: "Sample Percentage",
|
||||
required: false,
|
||||
type: TableColumnType.Number,
|
||||
canReadOnRelationQuery: true,
|
||||
description:
|
||||
"When action is 'sample', the percentage of matching logs to keep (1-99).",
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.Number,
|
||||
nullable: true,
|
||||
})
|
||||
public samplePercentage?: number = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogDropFilter,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogDropFilter,
|
||||
],
|
||||
})
|
||||
@Index()
|
||||
@TableColumn({
|
||||
required: true,
|
||||
type: TableColumnType.Boolean,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Enabled",
|
||||
description: "Whether this drop filter is active.",
|
||||
defaultValue: true,
|
||||
})
|
||||
@Column({
|
||||
nullable: false,
|
||||
type: ColumnType.Boolean,
|
||||
default: true,
|
||||
})
|
||||
public isEnabled?: boolean = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogDropFilter,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogDropFilter,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
title: "Sort Order",
|
||||
required: true,
|
||||
type: TableColumnType.Number,
|
||||
canReadOnRelationQuery: true,
|
||||
description:
|
||||
"Determines the evaluation order of this filter relative to others.",
|
||||
defaultValue: 0,
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.Number,
|
||||
nullable: false,
|
||||
default: 0,
|
||||
})
|
||||
public sortOrder?: number = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
manyToOneRelationColumn: "createdByUserId",
|
||||
type: TableColumnType.Entity,
|
||||
modelType: User,
|
||||
title: "Created By User",
|
||||
description: "Relation to the user who created this log drop filter.",
|
||||
})
|
||||
@ManyToOne(
|
||||
() => {
|
||||
return User;
|
||||
},
|
||||
{
|
||||
eager: false,
|
||||
nullable: true,
|
||||
onDelete: "SET NULL",
|
||||
orphanedRowAction: "nullify",
|
||||
},
|
||||
)
|
||||
@JoinColumn({ name: "createdByUserId" })
|
||||
public createdByUser?: User = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
type: TableColumnType.ObjectID,
|
||||
title: "Created By User ID",
|
||||
description: "ID of the user who created this log drop filter.",
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.ObjectID,
|
||||
nullable: true,
|
||||
transformer: ObjectID.getDatabaseTransformer(),
|
||||
})
|
||||
public createdByUserId?: ObjectID = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
manyToOneRelationColumn: "deletedByUserId",
|
||||
type: TableColumnType.Entity,
|
||||
modelType: User,
|
||||
title: "Deleted By User",
|
||||
description: "Relation to the user who deleted this log drop filter.",
|
||||
})
|
||||
@ManyToOne(
|
||||
() => {
|
||||
return User;
|
||||
},
|
||||
{
|
||||
eager: false,
|
||||
nullable: true,
|
||||
onDelete: "SET NULL",
|
||||
orphanedRowAction: "nullify",
|
||||
},
|
||||
)
|
||||
@JoinColumn({ name: "deletedByUserId" })
|
||||
public deletedByUser?: User = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogDropFilter,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
type: TableColumnType.ObjectID,
|
||||
title: "Deleted By User ID",
|
||||
description: "ID of the user who deleted this log drop filter.",
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.ObjectID,
|
||||
nullable: true,
|
||||
transformer: ObjectID.getDatabaseTransformer(),
|
||||
})
|
||||
public deletedByUserId?: ObjectID = undefined;
|
||||
}
|
||||
412
Common/Models/DatabaseModels/LogPipeline.ts
Normal file
412
Common/Models/DatabaseModels/LogPipeline.ts
Normal file
@@ -0,0 +1,412 @@
|
||||
import Project from "./Project";
|
||||
import User from "./User";
|
||||
import BaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
|
||||
import Route from "../../Types/API/Route";
|
||||
import ColumnAccessControl from "../../Types/Database/AccessControl/ColumnAccessControl";
|
||||
import TableAccessControl from "../../Types/Database/AccessControl/TableAccessControl";
|
||||
import TableBillingAccessControl from "../../Types/Database/AccessControl/TableBillingAccessControl";
|
||||
import ColumnLength from "../../Types/Database/ColumnLength";
|
||||
import ColumnType from "../../Types/Database/ColumnType";
|
||||
import CrudApiEndpoint from "../../Types/Database/CrudApiEndpoint";
|
||||
import EnableDocumentation from "../../Types/Database/EnableDocumentation";
|
||||
import TableColumn from "../../Types/Database/TableColumn";
|
||||
import TableColumnType from "../../Types/Database/TableColumnType";
|
||||
import TableMetadata from "../../Types/Database/TableMetadata";
|
||||
import TenantColumn from "../../Types/Database/TenantColumn";
|
||||
import IconProp from "../../Types/Icon/IconProp";
|
||||
import ObjectID from "../../Types/ObjectID";
|
||||
import Permission from "../../Types/Permission";
|
||||
import { PlanType } from "../../Types/Billing/SubscriptionPlan";
|
||||
import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
|
||||
|
||||
@EnableDocumentation()
|
||||
@TableBillingAccessControl({
|
||||
create: PlanType.Free,
|
||||
read: PlanType.Free,
|
||||
update: PlanType.Free,
|
||||
delete: PlanType.Free,
|
||||
})
|
||||
@TenantColumn("projectId")
|
||||
@CrudApiEndpoint(new Route("/log-pipeline"))
|
||||
@Entity({
|
||||
name: "LogPipeline",
|
||||
})
|
||||
@TableMetadata({
|
||||
tableName: "LogPipeline",
|
||||
singularName: "Log Pipeline",
|
||||
pluralName: "Log Pipelines",
|
||||
icon: IconProp.Logs,
|
||||
tableDescription:
|
||||
"Configure server-side log processing pipelines that transform logs at ingest time.",
|
||||
})
|
||||
@TableAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipeline,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipeline,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
delete: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.DeleteProjectLogPipeline,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogPipeline,
|
||||
],
|
||||
})
|
||||
export default class LogPipeline extends BaseModel {
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipeline,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipeline,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
manyToOneRelationColumn: "projectId",
|
||||
type: TableColumnType.Entity,
|
||||
modelType: Project,
|
||||
title: "Project",
|
||||
description: "Relation to the project this log pipeline belongs to.",
|
||||
})
|
||||
@ManyToOne(
|
||||
() => {
|
||||
return Project;
|
||||
},
|
||||
{
|
||||
eager: false,
|
||||
nullable: true,
|
||||
onDelete: "CASCADE",
|
||||
orphanedRowAction: "nullify",
|
||||
},
|
||||
)
|
||||
@JoinColumn({ name: "projectId" })
|
||||
public project?: Project = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipeline,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipeline,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@Index()
|
||||
@TableColumn({
|
||||
type: TableColumnType.ObjectID,
|
||||
required: true,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Project ID",
|
||||
description: "ID of the project this log pipeline belongs to.",
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.ObjectID,
|
||||
nullable: false,
|
||||
transformer: ObjectID.getDatabaseTransformer(),
|
||||
})
|
||||
public projectId?: ObjectID = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipeline,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipeline,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogPipeline,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
required: true,
|
||||
type: TableColumnType.Name,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Name",
|
||||
description: "Friendly name for this log pipeline.",
|
||||
})
|
||||
@Column({
|
||||
nullable: false,
|
||||
type: ColumnType.Name,
|
||||
length: ColumnLength.Name,
|
||||
})
|
||||
public name?: string = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipeline,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipeline,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogPipeline,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
required: false,
|
||||
type: TableColumnType.LongText,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Description",
|
||||
description: "Description of what this log pipeline does.",
|
||||
})
|
||||
@Column({
|
||||
nullable: true,
|
||||
type: ColumnType.LongText,
|
||||
length: ColumnLength.LongText,
|
||||
})
|
||||
public description?: string = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipeline,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipeline,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogPipeline,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
required: false,
|
||||
type: TableColumnType.LongText,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Filter Query",
|
||||
description:
|
||||
"Filter expression that determines which logs this pipeline applies to.",
|
||||
})
|
||||
@Column({
|
||||
nullable: true,
|
||||
type: ColumnType.LongText,
|
||||
length: ColumnLength.LongText,
|
||||
})
|
||||
public filterQuery?: string = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipeline,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipeline,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogPipeline,
|
||||
],
|
||||
})
|
||||
@Index()
|
||||
@TableColumn({
|
||||
required: true,
|
||||
type: TableColumnType.Boolean,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Enabled",
|
||||
description: "Whether this log pipeline is active.",
|
||||
defaultValue: true,
|
||||
})
|
||||
@Column({
|
||||
nullable: false,
|
||||
type: ColumnType.Boolean,
|
||||
default: true,
|
||||
})
|
||||
public isEnabled?: boolean = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipeline,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipeline,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogPipeline,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
title: "Sort Order",
|
||||
required: true,
|
||||
type: TableColumnType.Number,
|
||||
canReadOnRelationQuery: true,
|
||||
description:
|
||||
"Determines the execution order of this pipeline relative to others.",
|
||||
defaultValue: 0,
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.Number,
|
||||
nullable: false,
|
||||
default: 0,
|
||||
})
|
||||
public sortOrder?: number = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipeline,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
manyToOneRelationColumn: "createdByUserId",
|
||||
type: TableColumnType.Entity,
|
||||
modelType: User,
|
||||
title: "Created By User",
|
||||
description: "Relation to the user who created this log pipeline.",
|
||||
})
|
||||
@ManyToOne(
|
||||
() => {
|
||||
return User;
|
||||
},
|
||||
{
|
||||
eager: false,
|
||||
nullable: true,
|
||||
onDelete: "SET NULL",
|
||||
orphanedRowAction: "nullify",
|
||||
},
|
||||
)
|
||||
@JoinColumn({ name: "createdByUserId" })
|
||||
public createdByUser?: User = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipeline,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
type: TableColumnType.ObjectID,
|
||||
title: "Created By User ID",
|
||||
description: "ID of the user who created this log pipeline.",
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.ObjectID,
|
||||
nullable: true,
|
||||
transformer: ObjectID.getDatabaseTransformer(),
|
||||
})
|
||||
public createdByUserId?: ObjectID = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipeline,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
manyToOneRelationColumn: "deletedByUserId",
|
||||
type: TableColumnType.Entity,
|
||||
modelType: User,
|
||||
title: "Deleted By User",
|
||||
description: "Relation to the user who deleted this log pipeline.",
|
||||
})
|
||||
@ManyToOne(
|
||||
() => {
|
||||
return User;
|
||||
},
|
||||
{
|
||||
eager: false,
|
||||
nullable: true,
|
||||
onDelete: "SET NULL",
|
||||
orphanedRowAction: "nullify",
|
||||
},
|
||||
)
|
||||
@JoinColumn({ name: "deletedByUserId" })
|
||||
public deletedByUser?: User = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipeline,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
type: TableColumnType.ObjectID,
|
||||
title: "Deleted By User ID",
|
||||
description: "ID of the user who deleted this log pipeline.",
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.ObjectID,
|
||||
nullable: true,
|
||||
transformer: ObjectID.getDatabaseTransformer(),
|
||||
})
|
||||
public deletedByUserId?: ObjectID = undefined;
|
||||
}
|
||||
434
Common/Models/DatabaseModels/LogPipelineProcessor.ts
Normal file
434
Common/Models/DatabaseModels/LogPipelineProcessor.ts
Normal file
@@ -0,0 +1,434 @@
|
||||
import Project from "./Project";
|
||||
import User from "./User";
|
||||
import LogPipeline from "./LogPipeline";
|
||||
import BaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
|
||||
import Route from "../../Types/API/Route";
|
||||
import ColumnAccessControl from "../../Types/Database/AccessControl/ColumnAccessControl";
|
||||
import TableAccessControl from "../../Types/Database/AccessControl/TableAccessControl";
|
||||
import TableBillingAccessControl from "../../Types/Database/AccessControl/TableBillingAccessControl";
|
||||
import ColumnLength from "../../Types/Database/ColumnLength";
|
||||
import ColumnType from "../../Types/Database/ColumnType";
|
||||
import CrudApiEndpoint from "../../Types/Database/CrudApiEndpoint";
|
||||
import EnableDocumentation from "../../Types/Database/EnableDocumentation";
|
||||
import TableColumn from "../../Types/Database/TableColumn";
|
||||
import TableColumnType from "../../Types/Database/TableColumnType";
|
||||
import TableMetadata from "../../Types/Database/TableMetadata";
|
||||
import TenantColumn from "../../Types/Database/TenantColumn";
|
||||
import IconProp from "../../Types/Icon/IconProp";
|
||||
import ObjectID from "../../Types/ObjectID";
|
||||
import Permission from "../../Types/Permission";
|
||||
import { PlanType } from "../../Types/Billing/SubscriptionPlan";
|
||||
import { JSONObject } from "../../Types/JSON";
|
||||
import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
|
||||
|
||||
@EnableDocumentation()
|
||||
@TableBillingAccessControl({
|
||||
create: PlanType.Free,
|
||||
read: PlanType.Free,
|
||||
update: PlanType.Free,
|
||||
delete: PlanType.Free,
|
||||
})
|
||||
@TenantColumn("projectId")
|
||||
@CrudApiEndpoint(new Route("/log-pipeline-processor"))
|
||||
@Entity({
|
||||
name: "LogPipelineProcessor",
|
||||
})
|
||||
@TableMetadata({
|
||||
tableName: "LogPipelineProcessor",
|
||||
singularName: "Log Pipeline Processor",
|
||||
pluralName: "Log Pipeline Processors",
|
||||
icon: IconProp.Settings,
|
||||
tableDescription:
|
||||
"Individual processors within a log pipeline that transform log data during ingestion.",
|
||||
})
|
||||
@TableAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipelineProcessor,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipelineProcessor,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
delete: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.DeleteProjectLogPipelineProcessor,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogPipelineProcessor,
|
||||
],
|
||||
})
|
||||
export default class LogPipelineProcessor extends BaseModel {
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipelineProcessor,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipelineProcessor,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
manyToOneRelationColumn: "projectId",
|
||||
type: TableColumnType.Entity,
|
||||
modelType: Project,
|
||||
title: "Project",
|
||||
description:
|
||||
"Relation to the project this log pipeline processor belongs to.",
|
||||
})
|
||||
@ManyToOne(
|
||||
() => {
|
||||
return Project;
|
||||
},
|
||||
{
|
||||
eager: false,
|
||||
nullable: true,
|
||||
onDelete: "CASCADE",
|
||||
orphanedRowAction: "nullify",
|
||||
},
|
||||
)
|
||||
@JoinColumn({ name: "projectId" })
|
||||
public project?: Project = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipelineProcessor,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipelineProcessor,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@Index()
|
||||
@TableColumn({
|
||||
type: TableColumnType.ObjectID,
|
||||
required: true,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Project ID",
|
||||
description:
|
||||
"ID of the project this log pipeline processor belongs to.",
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.ObjectID,
|
||||
nullable: false,
|
||||
transformer: ObjectID.getDatabaseTransformer(),
|
||||
})
|
||||
public projectId?: ObjectID = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipelineProcessor,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipelineProcessor,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
manyToOneRelationColumn: "logPipelineId",
|
||||
type: TableColumnType.Entity,
|
||||
modelType: LogPipeline,
|
||||
title: "Log Pipeline",
|
||||
description:
|
||||
"Relation to the log pipeline this processor belongs to.",
|
||||
})
|
||||
@ManyToOne(
|
||||
() => {
|
||||
return LogPipeline;
|
||||
},
|
||||
{
|
||||
eager: false,
|
||||
nullable: false,
|
||||
onDelete: "CASCADE",
|
||||
orphanedRowAction: "delete",
|
||||
},
|
||||
)
|
||||
@JoinColumn({ name: "logPipelineId" })
|
||||
public logPipeline?: LogPipeline = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipelineProcessor,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipelineProcessor,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@Index()
|
||||
@TableColumn({
|
||||
type: TableColumnType.ObjectID,
|
||||
required: true,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Log Pipeline ID",
|
||||
description:
|
||||
"ID of the log pipeline this processor belongs to.",
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.ObjectID,
|
||||
nullable: false,
|
||||
transformer: ObjectID.getDatabaseTransformer(),
|
||||
})
|
||||
public logPipelineId?: ObjectID = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipelineProcessor,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipelineProcessor,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogPipelineProcessor,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
required: true,
|
||||
type: TableColumnType.Name,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Name",
|
||||
description: "Friendly name for this processor.",
|
||||
})
|
||||
@Column({
|
||||
nullable: false,
|
||||
type: ColumnType.Name,
|
||||
length: ColumnLength.Name,
|
||||
})
|
||||
public name?: string = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipelineProcessor,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipelineProcessor,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogPipelineProcessor,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
required: true,
|
||||
type: TableColumnType.ShortText,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Processor Type",
|
||||
description:
|
||||
"The type of processor: GrokParser, AttributeRemapper, SeverityRemapper, or CategoryProcessor.",
|
||||
})
|
||||
@Column({
|
||||
nullable: false,
|
||||
type: ColumnType.ShortText,
|
||||
length: ColumnLength.ShortText,
|
||||
})
|
||||
public processorType?: string = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipelineProcessor,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipelineProcessor,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogPipelineProcessor,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
title: "Configuration",
|
||||
required: true,
|
||||
type: TableColumnType.JSON,
|
||||
canReadOnRelationQuery: true,
|
||||
description:
|
||||
"Processor-specific configuration as JSON (e.g., grok pattern, source/target fields, mapping rules).",
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.JSON,
|
||||
nullable: false,
|
||||
default: () => {
|
||||
return "'{}'";
|
||||
},
|
||||
})
|
||||
public configuration?: JSONObject = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipelineProcessor,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipelineProcessor,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogPipelineProcessor,
|
||||
],
|
||||
})
|
||||
@Index()
|
||||
@TableColumn({
|
||||
required: true,
|
||||
type: TableColumnType.Boolean,
|
||||
canReadOnRelationQuery: true,
|
||||
title: "Enabled",
|
||||
description: "Whether this processor is active.",
|
||||
defaultValue: true,
|
||||
})
|
||||
@Column({
|
||||
nullable: false,
|
||||
type: ColumnType.Boolean,
|
||||
default: true,
|
||||
})
|
||||
public isEnabled?: boolean = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.CreateProjectLogPipelineProcessor,
|
||||
],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipelineProcessor,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.EditProjectLogPipelineProcessor,
|
||||
],
|
||||
})
|
||||
@TableColumn({
|
||||
title: "Sort Order",
|
||||
required: true,
|
||||
type: TableColumnType.Number,
|
||||
canReadOnRelationQuery: true,
|
||||
description:
|
||||
"Determines the execution order of this processor within its pipeline.",
|
||||
defaultValue: 0,
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.Number,
|
||||
nullable: false,
|
||||
default: 0,
|
||||
})
|
||||
public sortOrder?: number = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipelineProcessor,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
manyToOneRelationColumn: "createdByUserId",
|
||||
type: TableColumnType.Entity,
|
||||
modelType: User,
|
||||
title: "Created By User",
|
||||
description:
|
||||
"Relation to the user who created this log pipeline processor.",
|
||||
})
|
||||
@ManyToOne(
|
||||
() => {
|
||||
return User;
|
||||
},
|
||||
{
|
||||
eager: false,
|
||||
nullable: true,
|
||||
onDelete: "SET NULL",
|
||||
orphanedRowAction: "nullify",
|
||||
},
|
||||
)
|
||||
@JoinColumn({ name: "createdByUserId" })
|
||||
public createdByUser?: User = undefined;
|
||||
|
||||
@ColumnAccessControl({
|
||||
create: [],
|
||||
read: [
|
||||
Permission.ProjectOwner,
|
||||
Permission.ProjectAdmin,
|
||||
Permission.ProjectMember,
|
||||
Permission.ReadProjectLogPipelineProcessor,
|
||||
Permission.ReadAllProjectResources,
|
||||
],
|
||||
update: [],
|
||||
})
|
||||
@TableColumn({
|
||||
type: TableColumnType.ObjectID,
|
||||
title: "Created By User ID",
|
||||
description:
|
||||
"ID of the user who created this log pipeline processor.",
|
||||
})
|
||||
@Column({
|
||||
type: ColumnType.ObjectID,
|
||||
nullable: true,
|
||||
transformer: ObjectID.getDatabaseTransformer(),
|
||||
})
|
||||
public createdByUserId?: ObjectID = undefined;
|
||||
}
|
||||
@@ -410,6 +410,272 @@ router.post(
|
||||
},
|
||||
);
|
||||
|
||||
// --- Log Export Endpoint ---
|
||||
|
||||
router.post(
|
||||
"/telemetry/logs/export",
|
||||
UserMiddleware.getUserMiddleware,
|
||||
async (
|
||||
req: ExpressRequest,
|
||||
res: ExpressResponse,
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const databaseProps: DatabaseCommonInteractionProps =
|
||||
await CommonAPI.getDatabaseCommonInteractionProps(req);
|
||||
|
||||
if (!databaseProps?.tenantId) {
|
||||
return Response.sendErrorResponse(
|
||||
req,
|
||||
res,
|
||||
new BadDataException("Invalid Project ID"),
|
||||
);
|
||||
}
|
||||
|
||||
const body: JSONObject = req.body as JSONObject;
|
||||
|
||||
const startTime: Date = body["startTime"]
|
||||
? OneUptimeDate.fromString(body["startTime"] as string)
|
||||
: OneUptimeDate.addRemoveHours(OneUptimeDate.getCurrentDate(), -1);
|
||||
|
||||
const endTime: Date = body["endTime"]
|
||||
? OneUptimeDate.fromString(body["endTime"] as string)
|
||||
: OneUptimeDate.getCurrentDate();
|
||||
|
||||
const limit: number = Math.min(
|
||||
(body["limit"] as number) || 10000,
|
||||
10000,
|
||||
);
|
||||
|
||||
const format: string = (body["format"] as string) || "json";
|
||||
|
||||
const serviceIds: Array<ObjectID> | undefined = body["serviceIds"]
|
||||
? (body["serviceIds"] as Array<string>).map((id: string) => {
|
||||
return new ObjectID(id);
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const severityTexts: Array<string> | undefined = body["severityTexts"]
|
||||
? (body["severityTexts"] as Array<string>)
|
||||
: undefined;
|
||||
|
||||
const bodySearchText: string | undefined = body["bodySearchText"]
|
||||
? (body["bodySearchText"] as string)
|
||||
: undefined;
|
||||
|
||||
const traceIds: Array<string> | undefined = body["traceIds"]
|
||||
? (body["traceIds"] as Array<string>)
|
||||
: undefined;
|
||||
|
||||
const spanIds: Array<string> | undefined = body["spanIds"]
|
||||
? (body["spanIds"] as Array<string>)
|
||||
: undefined;
|
||||
|
||||
const rows: Array<JSONObject> =
|
||||
await LogAggregationService.getExportLogs({
|
||||
projectId: databaseProps.tenantId,
|
||||
startTime,
|
||||
endTime,
|
||||
limit,
|
||||
serviceIds,
|
||||
severityTexts,
|
||||
bodySearchText,
|
||||
traceIds,
|
||||
spanIds,
|
||||
});
|
||||
|
||||
if (format === "csv") {
|
||||
const header: string =
|
||||
"time,serviceId,severityText,severityNumber,body,traceId,spanId,attributes";
|
||||
const csvRows: Array<string> = rows.map((row: JSONObject) => {
|
||||
const escapeCsv: (val: unknown) => string = (
|
||||
val: unknown,
|
||||
): string => {
|
||||
const str: string = val === null || val === undefined ? "" : String(val);
|
||||
if (
|
||||
str.includes(",") ||
|
||||
str.includes('"') ||
|
||||
str.includes("\n")
|
||||
) {
|
||||
return `"${str.replace(/"/g, '""')}"`;
|
||||
}
|
||||
return str;
|
||||
};
|
||||
|
||||
return [
|
||||
escapeCsv(row["time"]),
|
||||
escapeCsv(row["serviceId"]),
|
||||
escapeCsv(row["severityText"]),
|
||||
escapeCsv(row["severityNumber"]),
|
||||
escapeCsv(row["body"]),
|
||||
escapeCsv(row["traceId"]),
|
||||
escapeCsv(row["spanId"]),
|
||||
escapeCsv(JSON.stringify(row["attributes"] || {})),
|
||||
].join(",");
|
||||
});
|
||||
|
||||
const csv: string = [header, ...csvRows].join("\n");
|
||||
res.setHeader("Content-Type", "text/csv; charset=utf-8");
|
||||
res.setHeader(
|
||||
"Content-Disposition",
|
||||
"attachment; filename=logs-export.csv",
|
||||
);
|
||||
res.status(200).send(csv);
|
||||
return;
|
||||
}
|
||||
|
||||
// JSON format
|
||||
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
||||
res.setHeader(
|
||||
"Content-Disposition",
|
||||
"attachment; filename=logs-export.json",
|
||||
);
|
||||
res.status(200).send(JSON.stringify(rows, null, 2));
|
||||
} catch (err: unknown) {
|
||||
next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// --- Log Context Endpoint ---
|
||||
|
||||
router.post(
|
||||
"/telemetry/logs/context",
|
||||
UserMiddleware.getUserMiddleware,
|
||||
async (
|
||||
req: ExpressRequest,
|
||||
res: ExpressResponse,
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const databaseProps: DatabaseCommonInteractionProps =
|
||||
await CommonAPI.getDatabaseCommonInteractionProps(req);
|
||||
|
||||
if (!databaseProps?.tenantId) {
|
||||
return Response.sendErrorResponse(
|
||||
req,
|
||||
res,
|
||||
new BadDataException("Invalid Project ID"),
|
||||
);
|
||||
}
|
||||
|
||||
const body: JSONObject = req.body as JSONObject;
|
||||
|
||||
const logId: string | undefined = body["logId"] as string | undefined;
|
||||
const serviceId: string | undefined = body["serviceId"] as
|
||||
| string
|
||||
| undefined;
|
||||
const time: string | undefined = body["time"] as string | undefined;
|
||||
|
||||
if (!logId || !serviceId || !time) {
|
||||
return Response.sendErrorResponse(
|
||||
req,
|
||||
res,
|
||||
new BadDataException("logId, serviceId, and time are required"),
|
||||
);
|
||||
}
|
||||
|
||||
const count: number = (body["count"] as number) || 5;
|
||||
|
||||
const result: {
|
||||
before: Array<JSONObject>;
|
||||
after: Array<JSONObject>;
|
||||
} = await LogAggregationService.getLogContext({
|
||||
projectId: databaseProps.tenantId,
|
||||
serviceId: new ObjectID(serviceId),
|
||||
time: OneUptimeDate.fromString(time),
|
||||
logId,
|
||||
count,
|
||||
});
|
||||
|
||||
return Response.sendJsonObjectResponse(req, res, {
|
||||
before: result.before as unknown as JSONObject,
|
||||
after: result.after as unknown as JSONObject,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// --- Drop Filter Estimate Endpoint ---
|
||||
|
||||
router.post(
|
||||
"/telemetry/logs/drop-filter-estimate",
|
||||
UserMiddleware.getUserMiddleware,
|
||||
async (
|
||||
req: ExpressRequest,
|
||||
res: ExpressResponse,
|
||||
next: NextFunction,
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const databaseProps: DatabaseCommonInteractionProps =
|
||||
await CommonAPI.getDatabaseCommonInteractionProps(req);
|
||||
|
||||
if (!databaseProps?.tenantId) {
|
||||
return Response.sendErrorResponse(
|
||||
req,
|
||||
res,
|
||||
new BadDataException("Invalid Project ID"),
|
||||
);
|
||||
}
|
||||
|
||||
const body: JSONObject = req.body as JSONObject;
|
||||
|
||||
const filterQuery: string | undefined = body["filterQuery"] as
|
||||
| string
|
||||
| undefined;
|
||||
|
||||
if (!filterQuery) {
|
||||
return Response.sendErrorResponse(
|
||||
req,
|
||||
res,
|
||||
new BadDataException("filterQuery is required"),
|
||||
);
|
||||
}
|
||||
|
||||
const startTime: Date = body["startTime"]
|
||||
? OneUptimeDate.fromString(body["startTime"] as string)
|
||||
: OneUptimeDate.addRemoveHours(OneUptimeDate.getCurrentDate(), -24);
|
||||
|
||||
const endTime: Date = body["endTime"]
|
||||
? OneUptimeDate.fromString(body["endTime"] as string)
|
||||
: OneUptimeDate.getCurrentDate();
|
||||
|
||||
const serviceIds: Array<ObjectID> | undefined = body["serviceIds"]
|
||||
? (body["serviceIds"] as Array<string>).map((id: string) => {
|
||||
return new ObjectID(id);
|
||||
})
|
||||
: undefined;
|
||||
|
||||
const severityTexts: Array<string> | undefined = body["severityTexts"]
|
||||
? (body["severityTexts"] as Array<string>)
|
||||
: undefined;
|
||||
|
||||
const result: {
|
||||
totalLogs: number;
|
||||
matchingLogs: number;
|
||||
estimatedReductionPercent: number;
|
||||
} = await LogAggregationService.getDropFilterEstimate({
|
||||
projectId: databaseProps.tenantId,
|
||||
startTime,
|
||||
endTime,
|
||||
filterQuery,
|
||||
serviceIds,
|
||||
severityTexts,
|
||||
});
|
||||
|
||||
return Response.sendJsonObjectResponse(req, res, {
|
||||
totalLogs: result.totalLogs,
|
||||
matchingLogs: result.matchingLogs,
|
||||
estimatedReductionPercent: result.estimatedReductionPercent,
|
||||
} as unknown as JSONObject);
|
||||
} catch (err: unknown) {
|
||||
next(err);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
function computeDefaultBucketSize(startTime: Date, endTime: Date): number {
|
||||
|
||||
@@ -641,6 +641,243 @@ export class LogAggregationService {
|
||||
}
|
||||
}
|
||||
|
||||
@CaptureSpan()
|
||||
public static async getExportLogs(request: {
|
||||
projectId: ObjectID;
|
||||
startTime: Date;
|
||||
endTime: Date;
|
||||
limit: number;
|
||||
serviceIds?: Array<ObjectID> | undefined;
|
||||
severityTexts?: Array<string> | undefined;
|
||||
bodySearchText?: string | undefined;
|
||||
traceIds?: Array<string> | undefined;
|
||||
spanIds?: Array<string> | undefined;
|
||||
}): Promise<Array<JSONObject>> {
|
||||
const maxLimit: number = Math.min(request.limit || 10000, 10000);
|
||||
|
||||
const statement: Statement = SQL`
|
||||
SELECT
|
||||
time,
|
||||
serviceId,
|
||||
severityText,
|
||||
severityNumber,
|
||||
body,
|
||||
traceId,
|
||||
spanId,
|
||||
attributes
|
||||
FROM ${LogAggregationService.TABLE_NAME}
|
||||
WHERE projectId = ${{
|
||||
type: TableColumnType.ObjectID,
|
||||
value: request.projectId,
|
||||
}}
|
||||
AND time >= ${{
|
||||
type: TableColumnType.Date,
|
||||
value: request.startTime,
|
||||
}}
|
||||
AND time <= ${{
|
||||
type: TableColumnType.Date,
|
||||
value: request.endTime,
|
||||
}}
|
||||
`;
|
||||
|
||||
LogAggregationService.appendCommonFilters(statement, request);
|
||||
|
||||
statement.append(
|
||||
SQL` ORDER BY time DESC LIMIT ${{
|
||||
type: TableColumnType.Number,
|
||||
value: maxLimit,
|
||||
}}`,
|
||||
);
|
||||
|
||||
const dbResult: Results = await LogDatabaseService.executeQuery(statement);
|
||||
const response: DbJSONResponse = await dbResult.json<{
|
||||
data?: Array<JSONObject>;
|
||||
}>();
|
||||
|
||||
return response.data || [];
|
||||
}
|
||||
|
||||
@CaptureSpan()
|
||||
public static async getLogContext(request: {
|
||||
projectId: ObjectID;
|
||||
serviceId: ObjectID;
|
||||
time: Date;
|
||||
logId: string;
|
||||
count: number;
|
||||
}): Promise<{ before: Array<JSONObject>; after: Array<JSONObject> }> {
|
||||
const count: number = Math.min(request.count || 5, 20);
|
||||
|
||||
const beforeStatement: Statement = SQL`
|
||||
SELECT
|
||||
_id,
|
||||
time,
|
||||
timeUnixNano,
|
||||
serviceId,
|
||||
severityText,
|
||||
severityNumber,
|
||||
body,
|
||||
traceId,
|
||||
spanId,
|
||||
attributes
|
||||
FROM ${LogAggregationService.TABLE_NAME}
|
||||
WHERE projectId = ${{
|
||||
type: TableColumnType.ObjectID,
|
||||
value: request.projectId,
|
||||
}}
|
||||
AND serviceId = ${{
|
||||
type: TableColumnType.ObjectID,
|
||||
value: request.serviceId,
|
||||
}}
|
||||
AND time <= ${{
|
||||
type: TableColumnType.Date,
|
||||
value: request.time,
|
||||
}}
|
||||
AND _id != ${{
|
||||
type: TableColumnType.Text,
|
||||
value: request.logId,
|
||||
}}
|
||||
ORDER BY time DESC, timeUnixNano DESC
|
||||
LIMIT ${{
|
||||
type: TableColumnType.Number,
|
||||
value: count,
|
||||
}}
|
||||
`;
|
||||
|
||||
const afterStatement: Statement = SQL`
|
||||
SELECT
|
||||
_id,
|
||||
time,
|
||||
timeUnixNano,
|
||||
serviceId,
|
||||
severityText,
|
||||
severityNumber,
|
||||
body,
|
||||
traceId,
|
||||
spanId,
|
||||
attributes
|
||||
FROM ${LogAggregationService.TABLE_NAME}
|
||||
WHERE projectId = ${{
|
||||
type: TableColumnType.ObjectID,
|
||||
value: request.projectId,
|
||||
}}
|
||||
AND serviceId = ${{
|
||||
type: TableColumnType.ObjectID,
|
||||
value: request.serviceId,
|
||||
}}
|
||||
AND time >= ${{
|
||||
type: TableColumnType.Date,
|
||||
value: request.time,
|
||||
}}
|
||||
AND _id != ${{
|
||||
type: TableColumnType.Text,
|
||||
value: request.logId,
|
||||
}}
|
||||
ORDER BY time ASC, timeUnixNano ASC
|
||||
LIMIT ${{
|
||||
type: TableColumnType.Number,
|
||||
value: count,
|
||||
}}
|
||||
`;
|
||||
|
||||
const [beforeResult, afterResult] = await Promise.all([
|
||||
LogDatabaseService.executeQuery(beforeStatement),
|
||||
LogDatabaseService.executeQuery(afterStatement),
|
||||
]);
|
||||
|
||||
const beforeResponse: DbJSONResponse = await beforeResult.json<{
|
||||
data?: Array<JSONObject>;
|
||||
}>();
|
||||
const afterResponse: DbJSONResponse = await afterResult.json<{
|
||||
data?: Array<JSONObject>;
|
||||
}>();
|
||||
|
||||
const beforeRows: Array<JSONObject> = (beforeResponse.data || []).reverse();
|
||||
const afterRows: Array<JSONObject> = afterResponse.data || [];
|
||||
|
||||
return { before: beforeRows, after: afterRows };
|
||||
}
|
||||
|
||||
@CaptureSpan()
|
||||
public static async getDropFilterEstimate(request: {
|
||||
projectId: ObjectID;
|
||||
startTime: Date;
|
||||
endTime: Date;
|
||||
filterQuery: string;
|
||||
serviceIds?: Array<ObjectID> | undefined;
|
||||
severityTexts?: Array<string> | undefined;
|
||||
bodySearchText?: string | undefined;
|
||||
}): Promise<{
|
||||
totalLogs: number;
|
||||
matchingLogs: number;
|
||||
estimatedReductionPercent: number;
|
||||
}> {
|
||||
// Get total count
|
||||
const totalStatement: Statement = SQL`
|
||||
SELECT count() AS cnt
|
||||
FROM ${LogAggregationService.TABLE_NAME}
|
||||
WHERE projectId = ${{
|
||||
type: TableColumnType.ObjectID,
|
||||
value: request.projectId,
|
||||
}}
|
||||
AND time >= ${{
|
||||
type: TableColumnType.Date,
|
||||
value: request.startTime,
|
||||
}}
|
||||
AND time <= ${{
|
||||
type: TableColumnType.Date,
|
||||
value: request.endTime,
|
||||
}}
|
||||
`;
|
||||
|
||||
LogAggregationService.appendCommonFilters(totalStatement, request);
|
||||
|
||||
// Get matching count using the filter query as body search
|
||||
const matchStatement: Statement = SQL`
|
||||
SELECT count() AS cnt
|
||||
FROM ${LogAggregationService.TABLE_NAME}
|
||||
WHERE projectId = ${{
|
||||
type: TableColumnType.ObjectID,
|
||||
value: request.projectId,
|
||||
}}
|
||||
AND time >= ${{
|
||||
type: TableColumnType.Date,
|
||||
value: request.startTime,
|
||||
}}
|
||||
AND time <= ${{
|
||||
type: TableColumnType.Date,
|
||||
value: request.endTime,
|
||||
}}
|
||||
`;
|
||||
|
||||
LogAggregationService.appendCommonFilters(matchStatement, {
|
||||
...request,
|
||||
bodySearchText: request.filterQuery,
|
||||
});
|
||||
|
||||
const [totalResult, matchResult] = await Promise.all([
|
||||
LogDatabaseService.executeQuery(totalStatement),
|
||||
LogDatabaseService.executeQuery(matchStatement),
|
||||
]);
|
||||
|
||||
const totalResponse: DbJSONResponse = await totalResult.json<{
|
||||
data?: Array<JSONObject>;
|
||||
}>();
|
||||
const matchResponse: DbJSONResponse = await matchResult.json<{
|
||||
data?: Array<JSONObject>;
|
||||
}>();
|
||||
|
||||
const totalLogs: number = Number(
|
||||
(totalResponse.data || [])[0]?.["cnt"] || 0,
|
||||
);
|
||||
const matchingLogs: number = Number(
|
||||
(matchResponse.data || [])[0]?.["cnt"] || 0,
|
||||
);
|
||||
const estimatedReductionPercent: number =
|
||||
totalLogs > 0 ? Math.round((matchingLogs / totalLogs) * 100) : 0;
|
||||
|
||||
return { totalLogs, matchingLogs, estimatedReductionPercent };
|
||||
}
|
||||
|
||||
private static isTopLevelColumn(key: string): boolean {
|
||||
return LogAggregationService.TOP_LEVEL_COLUMNS.has(key);
|
||||
}
|
||||
|
||||
6
Common/Types/Log/LogDropFilterAction.ts
Normal file
6
Common/Types/Log/LogDropFilterAction.ts
Normal file
@@ -0,0 +1,6 @@
|
||||
enum LogDropFilterAction {
|
||||
Drop = "drop",
|
||||
Sample = "sample",
|
||||
}
|
||||
|
||||
export default LogDropFilterAction;
|
||||
44
Common/Types/Log/LogPipelineProcessorType.ts
Normal file
44
Common/Types/Log/LogPipelineProcessorType.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
enum LogPipelineProcessorType {
|
||||
GrokParser = "GrokParser",
|
||||
AttributeRemapper = "AttributeRemapper",
|
||||
SeverityRemapper = "SeverityRemapper",
|
||||
CategoryProcessor = "CategoryProcessor",
|
||||
}
|
||||
|
||||
export interface GrokParserConfig {
|
||||
source: string; // field to parse, e.g. "body"
|
||||
pattern: string; // grok pattern
|
||||
targetPrefix?: string; // prefix for extracted attributes
|
||||
}
|
||||
|
||||
export interface AttributeRemapperConfig {
|
||||
sourceKey: string; // source attribute key
|
||||
targetKey: string; // target attribute key
|
||||
preserveSource?: boolean; // keep original key (default false)
|
||||
overrideOnConflict?: boolean; // overwrite if target exists (default true)
|
||||
}
|
||||
|
||||
export interface SeverityRemapperConfig {
|
||||
sourceKey: string; // attribute key containing severity info
|
||||
mappings: Array<{
|
||||
matchValue: string; // value to match (case-insensitive)
|
||||
severityText: string; // mapped severity text
|
||||
severityNumber: number; // mapped severity number
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface CategoryProcessorConfig {
|
||||
targetKey: string; // attribute key to store the category
|
||||
categories: Array<{
|
||||
name: string; // category name/value
|
||||
filterQuery: string; // condition to match
|
||||
}>;
|
||||
}
|
||||
|
||||
export type LogPipelineProcessorConfig =
|
||||
| GrokParserConfig
|
||||
| AttributeRemapperConfig
|
||||
| SeverityRemapperConfig
|
||||
| CategoryProcessorConfig;
|
||||
|
||||
export default LogPipelineProcessorType;
|
||||
@@ -82,6 +82,24 @@ enum Permission {
|
||||
EditTelemetryServiceLog = "EditTelemetryServiceLog",
|
||||
ReadTelemetryServiceLog = "ReadTelemetryServiceLog",
|
||||
|
||||
// Log Pipelines
|
||||
CreateProjectLogPipeline = "CreateProjectLogPipeline",
|
||||
DeleteProjectLogPipeline = "DeleteProjectLogPipeline",
|
||||
EditProjectLogPipeline = "EditProjectLogPipeline",
|
||||
ReadProjectLogPipeline = "ReadProjectLogPipeline",
|
||||
|
||||
// Log Pipeline Processors
|
||||
CreateProjectLogPipelineProcessor = "CreateProjectLogPipelineProcessor",
|
||||
DeleteProjectLogPipelineProcessor = "DeleteProjectLogPipelineProcessor",
|
||||
EditProjectLogPipelineProcessor = "EditProjectLogPipelineProcessor",
|
||||
ReadProjectLogPipelineProcessor = "ReadProjectLogPipelineProcessor",
|
||||
|
||||
// Log Drop Filters
|
||||
CreateProjectLogDropFilter = "CreateProjectLogDropFilter",
|
||||
DeleteProjectLogDropFilter = "DeleteProjectLogDropFilter",
|
||||
EditProjectLogDropFilter = "EditProjectLogDropFilter",
|
||||
ReadProjectLogDropFilter = "ReadProjectLogDropFilter",
|
||||
|
||||
// Exceptions
|
||||
CreateTelemetryException = "CreateTelemetryException",
|
||||
DeleteTelemetryException = "DeleteTelemetryException",
|
||||
@@ -3981,6 +3999,120 @@ export class PermissionHelper {
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
|
||||
// Log Pipeline Permissions
|
||||
{
|
||||
permission: Permission.CreateProjectLogPipeline,
|
||||
title: "Create Log Pipeline",
|
||||
description:
|
||||
"This permission can create Log Pipelines in this project.",
|
||||
isAssignableToTenant: true,
|
||||
isAccessControlPermission: false,
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
{
|
||||
permission: Permission.DeleteProjectLogPipeline,
|
||||
title: "Delete Log Pipeline",
|
||||
description:
|
||||
"This permission can delete Log Pipelines of this project.",
|
||||
isAssignableToTenant: true,
|
||||
isAccessControlPermission: false,
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
{
|
||||
permission: Permission.EditProjectLogPipeline,
|
||||
title: "Edit Log Pipeline",
|
||||
description:
|
||||
"This permission can edit Log Pipelines of this project.",
|
||||
isAssignableToTenant: true,
|
||||
isAccessControlPermission: false,
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
{
|
||||
permission: Permission.ReadProjectLogPipeline,
|
||||
title: "Read Log Pipeline",
|
||||
description:
|
||||
"This permission can read Log Pipelines of this project.",
|
||||
isAssignableToTenant: true,
|
||||
isAccessControlPermission: false,
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
|
||||
// Log Pipeline Processor Permissions
|
||||
{
|
||||
permission: Permission.CreateProjectLogPipelineProcessor,
|
||||
title: "Create Log Pipeline Processor",
|
||||
description:
|
||||
"This permission can create Log Pipeline Processors in this project.",
|
||||
isAssignableToTenant: true,
|
||||
isAccessControlPermission: false,
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
{
|
||||
permission: Permission.DeleteProjectLogPipelineProcessor,
|
||||
title: "Delete Log Pipeline Processor",
|
||||
description:
|
||||
"This permission can delete Log Pipeline Processors of this project.",
|
||||
isAssignableToTenant: true,
|
||||
isAccessControlPermission: false,
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
{
|
||||
permission: Permission.EditProjectLogPipelineProcessor,
|
||||
title: "Edit Log Pipeline Processor",
|
||||
description:
|
||||
"This permission can edit Log Pipeline Processors of this project.",
|
||||
isAssignableToTenant: true,
|
||||
isAccessControlPermission: false,
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
{
|
||||
permission: Permission.ReadProjectLogPipelineProcessor,
|
||||
title: "Read Log Pipeline Processor",
|
||||
description:
|
||||
"This permission can read Log Pipeline Processors of this project.",
|
||||
isAssignableToTenant: true,
|
||||
isAccessControlPermission: false,
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
|
||||
// Log Drop Filter Permissions
|
||||
{
|
||||
permission: Permission.CreateProjectLogDropFilter,
|
||||
title: "Create Log Drop Filter",
|
||||
description:
|
||||
"This permission can create Log Drop Filters in this project.",
|
||||
isAssignableToTenant: true,
|
||||
isAccessControlPermission: false,
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
{
|
||||
permission: Permission.DeleteProjectLogDropFilter,
|
||||
title: "Delete Log Drop Filter",
|
||||
description:
|
||||
"This permission can delete Log Drop Filters of this project.",
|
||||
isAssignableToTenant: true,
|
||||
isAccessControlPermission: false,
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
{
|
||||
permission: Permission.EditProjectLogDropFilter,
|
||||
title: "Edit Log Drop Filter",
|
||||
description:
|
||||
"This permission can edit Log Drop Filters of this project.",
|
||||
isAssignableToTenant: true,
|
||||
isAccessControlPermission: false,
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
{
|
||||
permission: Permission.ReadProjectLogDropFilter,
|
||||
title: "Read Log Drop Filter",
|
||||
description:
|
||||
"This permission can read Log Drop Filters of this project.",
|
||||
isAssignableToTenant: true,
|
||||
isAccessControlPermission: false,
|
||||
group: PermissionGroup.Telemetry,
|
||||
},
|
||||
|
||||
{
|
||||
permission: Permission.CreateTelemetryException,
|
||||
title: "Create Telemetry Service Exception",
|
||||
|
||||
@@ -54,6 +54,8 @@ import LogsAnalyticsView from "./components/LogsAnalyticsView";
|
||||
import { queryStringToFilter } from "../../../Types/Log/LogQueryToFilter";
|
||||
import RangeStartAndEndDateTime from "../../../Types/Time/RangeStartAndEndDateTime";
|
||||
import TimeRange from "../../../Types/Time/TimeRange";
|
||||
import { exportLogs, LogExportFormat } from "../../Utils/LogExport";
|
||||
import ObjectID from "../../../Types/ObjectID";
|
||||
|
||||
export interface ComponentProps {
|
||||
logs: Array<Log>;
|
||||
@@ -64,6 +66,7 @@ export interface ComponentProps {
|
||||
noLogsMessage?: string | undefined;
|
||||
getTraceRoute?: (traceId: string, log: Log) => Route | URL | undefined;
|
||||
getSpanRoute?: (spanId: string, log: Log) => Route | URL | undefined;
|
||||
projectId?: ObjectID | undefined;
|
||||
totalCount?: number | undefined;
|
||||
page?: number | undefined;
|
||||
pageSize?: number | undefined;
|
||||
@@ -655,6 +658,12 @@ const LogsViewer: FunctionComponent<ComponentProps> = (
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
onExportCSV: () => {
|
||||
exportLogs(displayedLogs, LogExportFormat.CSV, selectedColumns);
|
||||
},
|
||||
onExportJSON: () => {
|
||||
exportLogs(displayedLogs, LogExportFormat.JSON, selectedColumns);
|
||||
},
|
||||
...(props.liveOptions ? { liveOptions: props.liveOptions } : {}),
|
||||
...(props.timeRange && props.onTimeRangeChange
|
||||
? {
|
||||
@@ -766,6 +775,7 @@ const LogsViewer: FunctionComponent<ComponentProps> = (
|
||||
getTraceRoute={props.getTraceRoute}
|
||||
getSpanRoute={props.getSpanRoute}
|
||||
variant="embedded"
|
||||
projectId={props.projectId}
|
||||
/>
|
||||
);
|
||||
}}
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
import React, { FunctionComponent, ReactElement, useMemo } from "react";
|
||||
import React, {
|
||||
FunctionComponent,
|
||||
ReactElement,
|
||||
useCallback,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useState,
|
||||
} from "react";
|
||||
import Log from "../../../../Models/AnalyticsModels/Log";
|
||||
import Service from "../../../../Models/DatabaseModels/Service";
|
||||
import Dictionary from "../../../../Types/Dictionary";
|
||||
@@ -11,6 +18,15 @@ import Link from "../../Link/Link";
|
||||
import OneUptimeDate from "../../../../Types/Date";
|
||||
import JSONFunctions from "../../../../Types/JSONFunctions";
|
||||
import SeverityBadge from "./SeverityBadge";
|
||||
import { JSONObject } from "../../../../Types/JSON";
|
||||
import API from "../../../Utils/API/API";
|
||||
import ModelAPI from "../../../Utils/ModelAPI/ModelAPI";
|
||||
import { APP_API_URL } from "../../../Config";
|
||||
import HTTPResponse from "../../../../Types/API/HTTPResponse";
|
||||
import HTTPErrorResponse from "../../../../Types/API/HTTPErrorResponse";
|
||||
import ObjectID from "../../../../Types/ObjectID";
|
||||
|
||||
type LogDetailTab = "details" | "context";
|
||||
|
||||
export interface LogDetailsPanelProps {
|
||||
log: Log;
|
||||
@@ -23,6 +39,8 @@ export interface LogDetailsPanelProps {
|
||||
| ((spanId: string, log: Log) => Route | URL | undefined)
|
||||
| undefined;
|
||||
variant?: "floating" | "embedded";
|
||||
projectId?: ObjectID | undefined;
|
||||
onLogSelect?: ((log: Log) => void) | undefined;
|
||||
}
|
||||
|
||||
interface PreparedBody {
|
||||
@@ -32,6 +50,14 @@ interface PreparedBody {
|
||||
raw: string;
|
||||
}
|
||||
|
||||
interface ContextLog {
|
||||
id: string;
|
||||
time: string;
|
||||
severity: string;
|
||||
body: string;
|
||||
serviceId: string;
|
||||
}
|
||||
|
||||
const prepareBody: (body: string | undefined) => PreparedBody = (
|
||||
body: string | undefined,
|
||||
): PreparedBody => {
|
||||
@@ -45,7 +71,7 @@ const prepareBody: (body: string | undefined) => PreparedBody = (
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed: any = JSON.parse(body);
|
||||
const parsed: unknown = JSON.parse(body);
|
||||
const pretty: string = JSON.stringify(parsed, null, 2);
|
||||
const compact: string = JSON.stringify(parsed);
|
||||
return {
|
||||
@@ -64,9 +90,26 @@ const prepareBody: (body: string | undefined) => PreparedBody = (
|
||||
}
|
||||
};
|
||||
|
||||
function parseContextRow(row: JSONObject): ContextLog {
|
||||
return {
|
||||
id: String(row["_id"] || ""),
|
||||
time: String(row["time"] || ""),
|
||||
severity: String(row["severityText"] || "Unspecified"),
|
||||
body: String(row["body"] || ""),
|
||||
serviceId: String(row["serviceId"] || ""),
|
||||
};
|
||||
}
|
||||
|
||||
const LogDetailsPanel: FunctionComponent<LogDetailsPanelProps> = (
|
||||
props: LogDetailsPanelProps,
|
||||
): ReactElement => {
|
||||
const [activeTab, setActiveTab] = useState<LogDetailTab>("details");
|
||||
const [contextBefore, setContextBefore] = useState<Array<ContextLog>>([]);
|
||||
const [contextAfter, setContextAfter] = useState<Array<ContextLog>>([]);
|
||||
const [contextLoading, setContextLoading] = useState<boolean>(false);
|
||||
const [contextError, setContextError] = useState<string>("");
|
||||
const [contextLoaded, setContextLoaded] = useState<boolean>(false);
|
||||
|
||||
const variant: "floating" | "embedded" = props.variant || "floating";
|
||||
const serviceId: string = props.log.serviceId?.toString() || "";
|
||||
const service: Service | undefined = props.serviceMap[serviceId];
|
||||
@@ -84,9 +127,8 @@ const LogDetailsPanel: FunctionComponent<LogDetailsPanelProps> = (
|
||||
}
|
||||
|
||||
try {
|
||||
const normalized: Record<string, unknown> = JSONFunctions.unflattenObject(
|
||||
props.log.attributes || {},
|
||||
);
|
||||
const normalized: Record<string, unknown> =
|
||||
JSONFunctions.unflattenObject(props.log.attributes || {});
|
||||
return JSON.stringify(normalized, null, 2);
|
||||
} catch {
|
||||
return null;
|
||||
@@ -137,6 +179,70 @@ const LogDetailsPanel: FunctionComponent<LogDetailsPanelProps> = (
|
||||
return undefined;
|
||||
}, [spanId, props, traceId]);
|
||||
|
||||
const loadContext: () => Promise<void> = useCallback(async (): Promise<void> => {
|
||||
if (!props.projectId || !serviceId || !props.log.time) {
|
||||
setContextError("Missing project or service information for context.");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
setContextLoading(true);
|
||||
setContextError("");
|
||||
|
||||
const response: HTTPResponse<JSONObject> | HTTPErrorResponse =
|
||||
await API.post({
|
||||
url: URL.fromString(APP_API_URL.toString()).addRoute(
|
||||
"/telemetry/logs/context",
|
||||
),
|
||||
data: {
|
||||
logId: props.log.getColumnValue("_id")?.toString() || "",
|
||||
serviceId: serviceId,
|
||||
time: props.log.time
|
||||
? OneUptimeDate.toString(props.log.time)
|
||||
: "",
|
||||
count: 5,
|
||||
},
|
||||
headers: {
|
||||
...ModelAPI.getCommonHeaders(),
|
||||
},
|
||||
});
|
||||
|
||||
if (response instanceof HTTPErrorResponse) {
|
||||
throw response;
|
||||
}
|
||||
|
||||
const before: Array<JSONObject> =
|
||||
(response.data["before"] as Array<JSONObject>) || [];
|
||||
const after: Array<JSONObject> =
|
||||
(response.data["after"] as Array<JSONObject>) || [];
|
||||
|
||||
setContextBefore(before.map(parseContextRow));
|
||||
setContextAfter(after.map(parseContextRow));
|
||||
setContextLoaded(true);
|
||||
} catch (err) {
|
||||
setContextError(
|
||||
`Failed to load log context. ${API.getFriendlyErrorMessage(err as Error)}`,
|
||||
);
|
||||
} finally {
|
||||
setContextLoading(false);
|
||||
}
|
||||
}, [props.projectId, serviceId, props.log]);
|
||||
|
||||
useEffect(() => {
|
||||
if (activeTab === "context" && !contextLoaded && !contextLoading) {
|
||||
void loadContext();
|
||||
}
|
||||
}, [activeTab, contextLoaded, contextLoading, loadContext]);
|
||||
|
||||
// Reset context when log changes
|
||||
useEffect(() => {
|
||||
setContextLoaded(false);
|
||||
setContextBefore([]);
|
||||
setContextAfter([]);
|
||||
setContextError("");
|
||||
setActiveTab("details");
|
||||
}, [props.log]);
|
||||
|
||||
const containerClassName: string =
|
||||
variant === "embedded"
|
||||
? "rounded-lg border border-gray-200 bg-white p-5 shadow-sm"
|
||||
@@ -149,6 +255,47 @@ const LogDetailsPanel: FunctionComponent<LogDetailsPanelProps> = (
|
||||
const smallBadgeClass: string =
|
||||
"inline-flex items-center gap-1 rounded-full border border-gray-200 bg-gray-50 px-2 py-1 text-[11px] font-mono uppercase tracking-wide text-gray-600";
|
||||
|
||||
const tabClass: (isActive: boolean) => string = (
|
||||
isActive: boolean,
|
||||
): string => {
|
||||
return `px-3 py-1.5 text-xs font-medium rounded-md transition-colors ${
|
||||
isActive
|
||||
? "bg-indigo-50 text-indigo-700 border border-indigo-200"
|
||||
: "text-gray-600 hover:text-gray-800 hover:bg-gray-50 border border-transparent"
|
||||
}`;
|
||||
};
|
||||
|
||||
const renderContextLogRow: (
|
||||
ctxLog: ContextLog,
|
||||
isCurrent: boolean,
|
||||
) => ReactElement = (
|
||||
ctxLog: ContextLog,
|
||||
isCurrent: boolean,
|
||||
): ReactElement => {
|
||||
const rowClass: string = isCurrent
|
||||
? "border-l-2 border-l-indigo-500 bg-indigo-50"
|
||||
: "border-l-2 border-l-transparent hover:bg-gray-50";
|
||||
|
||||
return (
|
||||
<div
|
||||
key={ctxLog.id || ctxLog.time}
|
||||
className={`flex items-start gap-3 px-3 py-2 ${rowClass}`}
|
||||
>
|
||||
<span className="flex-none whitespace-nowrap font-mono text-[11px] text-gray-400">
|
||||
{ctxLog.time
|
||||
? OneUptimeDate.getDateAsUserFriendlyFormattedString(
|
||||
new Date(ctxLog.time),
|
||||
)
|
||||
: "-"}
|
||||
</span>
|
||||
<SeverityBadge severity={ctxLog.severity} />
|
||||
<span className="min-w-0 flex-1 truncate font-mono text-xs text-gray-700">
|
||||
{ctxLog.body.slice(0, 200) || "-"}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className={containerClassName}>
|
||||
<div
|
||||
@@ -204,134 +351,204 @@ const LogDetailsPanel: FunctionComponent<LogDetailsPanelProps> = (
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="mt-4 space-y-5 text-sm text-gray-700">
|
||||
<section className="space-y-3">
|
||||
<header className="flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
|
||||
<span>Log Body</span>
|
||||
<CopyTextButton
|
||||
textToBeCopied={bodyDetails.raw}
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
iconOnly={false}
|
||||
title="Copy log body"
|
||||
/>
|
||||
</header>
|
||||
|
||||
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
|
||||
{bodyDetails.isJson ? (
|
||||
<pre className="max-h-80 overflow-auto whitespace-pre-wrap break-words font-mono text-[13px] leading-6 text-gray-800">
|
||||
{bodyDetails.pretty}
|
||||
</pre>
|
||||
) : (
|
||||
<p className="whitespace-pre-wrap break-words font-mono text-[13px] leading-6 text-gray-800">
|
||||
{bodyDetails.pretty || "-"}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{(traceId || spanId) && (
|
||||
<section className="grid gap-4 md:grid-cols-2">
|
||||
{traceId && (
|
||||
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
|
||||
<div className="mb-2 flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
|
||||
<span>Trace ID</span>
|
||||
<CopyTextButton
|
||||
textToBeCopied={traceId}
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
iconOnly={true}
|
||||
title="Copy trace id"
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
{traceRoute ? (
|
||||
<Link
|
||||
to={traceRoute}
|
||||
className="max-w-full truncate font-mono text-xs text-indigo-600 hover:text-indigo-500"
|
||||
title={`View trace ${traceId}`}
|
||||
>
|
||||
{traceId}
|
||||
</Link>
|
||||
) : (
|
||||
<span
|
||||
className="max-w-full truncate font-mono text-xs text-gray-700"
|
||||
title={traceId}
|
||||
>
|
||||
{traceId}
|
||||
</span>
|
||||
)}
|
||||
{traceRoute && (
|
||||
<Icon
|
||||
icon={IconProp.ExternalLink}
|
||||
className="h-4 w-4 flex-none text-indigo-400"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{spanId && (
|
||||
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
|
||||
<div className="mb-2 flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
|
||||
<span>Span ID</span>
|
||||
<CopyTextButton
|
||||
textToBeCopied={spanId}
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
iconOnly={true}
|
||||
title="Copy span id"
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
{spanRoute ? (
|
||||
<Link
|
||||
to={spanRoute}
|
||||
className="max-w-full truncate font-mono text-xs text-indigo-600 hover:text-indigo-500"
|
||||
title={`View span ${spanId}`}
|
||||
>
|
||||
{spanId}
|
||||
</Link>
|
||||
) : (
|
||||
<span
|
||||
className="max-w-full truncate font-mono text-xs text-gray-700"
|
||||
title={spanId}
|
||||
>
|
||||
{spanId}
|
||||
</span>
|
||||
)}
|
||||
{spanRoute && (
|
||||
<Icon
|
||||
icon={IconProp.ExternalLink}
|
||||
className="h-4 w-4 flex-none text-indigo-400"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
{/* Tab bar */}
|
||||
<div className="mt-3 flex items-center gap-1">
|
||||
<button
|
||||
type="button"
|
||||
className={tabClass(activeTab === "details")}
|
||||
onClick={() => {
|
||||
setActiveTab("details");
|
||||
}}
|
||||
>
|
||||
Details
|
||||
</button>
|
||||
{props.projectId && (
|
||||
<button
|
||||
type="button"
|
||||
className={tabClass(activeTab === "context")}
|
||||
onClick={() => {
|
||||
setActiveTab("context");
|
||||
}}
|
||||
>
|
||||
Context
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{prettyAttributes && (
|
||||
{/* Tab content */}
|
||||
{activeTab === "details" && (
|
||||
<div className="mt-4 space-y-5 text-sm text-gray-700">
|
||||
<section className="space-y-3">
|
||||
<header className="flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
|
||||
<span>Attributes</span>
|
||||
<span>Log Body</span>
|
||||
<CopyTextButton
|
||||
textToBeCopied={prettyAttributes}
|
||||
textToBeCopied={bodyDetails.raw}
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
iconOnly={false}
|
||||
title="Copy attributes"
|
||||
title="Copy log body"
|
||||
/>
|
||||
</header>
|
||||
|
||||
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
|
||||
<pre className="max-h-72 overflow-auto whitespace-pre-wrap break-words font-mono text-[13px] leading-6 text-gray-800">
|
||||
{prettyAttributes}
|
||||
</pre>
|
||||
{bodyDetails.isJson ? (
|
||||
<pre className="max-h-80 overflow-auto whitespace-pre-wrap break-words font-mono text-[13px] leading-6 text-gray-800">
|
||||
{bodyDetails.pretty}
|
||||
</pre>
|
||||
) : (
|
||||
<p className="whitespace-pre-wrap break-words font-mono text-[13px] leading-6 text-gray-800">
|
||||
{bodyDetails.pretty || "-"}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{(traceId || spanId) && (
|
||||
<section className="grid gap-4 md:grid-cols-2">
|
||||
{traceId && (
|
||||
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
|
||||
<div className="mb-2 flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
|
||||
<span>Trace ID</span>
|
||||
<CopyTextButton
|
||||
textToBeCopied={traceId}
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
iconOnly={true}
|
||||
title="Copy trace id"
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
{traceRoute ? (
|
||||
<Link
|
||||
to={traceRoute}
|
||||
className="max-w-full truncate font-mono text-xs text-indigo-600 hover:text-indigo-500"
|
||||
title={`View trace ${traceId}`}
|
||||
>
|
||||
{traceId}
|
||||
</Link>
|
||||
) : (
|
||||
<span
|
||||
className="max-w-full truncate font-mono text-xs text-gray-700"
|
||||
title={traceId}
|
||||
>
|
||||
{traceId}
|
||||
</span>
|
||||
)}
|
||||
{traceRoute && (
|
||||
<Icon
|
||||
icon={IconProp.ExternalLink}
|
||||
className="h-4 w-4 flex-none text-indigo-400"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{spanId && (
|
||||
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
|
||||
<div className="mb-2 flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
|
||||
<span>Span ID</span>
|
||||
<CopyTextButton
|
||||
textToBeCopied={spanId}
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
iconOnly={true}
|
||||
title="Copy span id"
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
{spanRoute ? (
|
||||
<Link
|
||||
to={spanRoute}
|
||||
className="max-w-full truncate font-mono text-xs text-indigo-600 hover:text-indigo-500"
|
||||
title={`View span ${spanId}`}
|
||||
>
|
||||
{spanId}
|
||||
</Link>
|
||||
) : (
|
||||
<span
|
||||
className="max-w-full truncate font-mono text-xs text-gray-700"
|
||||
title={spanId}
|
||||
>
|
||||
{spanId}
|
||||
</span>
|
||||
)}
|
||||
{spanRoute && (
|
||||
<Icon
|
||||
icon={IconProp.ExternalLink}
|
||||
className="h-4 w-4 flex-none text-indigo-400"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
)}
|
||||
|
||||
{prettyAttributes && (
|
||||
<section className="space-y-3">
|
||||
<header className="flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
|
||||
<span>Attributes</span>
|
||||
<CopyTextButton
|
||||
textToBeCopied={prettyAttributes}
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
iconOnly={false}
|
||||
title="Copy attributes"
|
||||
/>
|
||||
</header>
|
||||
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
|
||||
<pre className="max-h-72 overflow-auto whitespace-pre-wrap break-words font-mono text-[13px] leading-6 text-gray-800">
|
||||
{prettyAttributes}
|
||||
</pre>
|
||||
</div>
|
||||
</section>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{activeTab === "context" && (
|
||||
<div className="mt-4 text-sm text-gray-700">
|
||||
{contextLoading && (
|
||||
<div className="flex items-center justify-center py-8 text-xs text-gray-400">
|
||||
Loading surrounding logs...
|
||||
</div>
|
||||
)}
|
||||
{contextError && (
|
||||
<div className="rounded-md bg-red-50 p-3 text-xs text-red-600">
|
||||
{contextError}
|
||||
</div>
|
||||
)}
|
||||
{!contextLoading && !contextError && contextLoaded && (
|
||||
<div className="divide-y divide-gray-100 rounded-lg border border-gray-200">
|
||||
{contextBefore.length === 0 && contextAfter.length === 0 && (
|
||||
<div className="px-3 py-6 text-center text-xs text-gray-400">
|
||||
No surrounding logs found for this service.
|
||||
</div>
|
||||
)}
|
||||
{contextBefore.map((ctxLog: ContextLog) => {
|
||||
return renderContextLogRow(ctxLog, false);
|
||||
})}
|
||||
{renderContextLogRow(
|
||||
{
|
||||
id: props.log.getColumnValue("_id")?.toString() || "current",
|
||||
time: props.log.time
|
||||
? props.log.time.toString()
|
||||
: "",
|
||||
severity:
|
||||
props.log.severityText?.toString() || "Unspecified",
|
||||
body: props.log.body || "",
|
||||
serviceId: serviceId,
|
||||
},
|
||||
true,
|
||||
)}
|
||||
{contextAfter.map((ctxLog: ContextLog) => {
|
||||
return renderContextLogRow(ctxLog, false);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import React, { FunctionComponent, ReactElement } from "react";
|
||||
import React, { FunctionComponent, ReactElement, useRef, useState } from "react";
|
||||
import LiveLogsToggle from "./LiveLogsToggle";
|
||||
import LogTimeRangePicker from "./LogTimeRangePicker";
|
||||
import ColumnSelector from "./ColumnSelector";
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
LogsViewMode,
|
||||
} from "../types";
|
||||
import RangeStartAndEndDateTime from "../../../../Types/Time/RangeStartAndEndDateTime";
|
||||
import useComponentOutsideClick from "../../../Types/UseComponentOutsideClick";
|
||||
|
||||
export interface LogsViewerToolbarProps {
|
||||
resultCount: number;
|
||||
@@ -31,6 +32,8 @@ export interface LogsViewerToolbarProps {
|
||||
onSelectedColumnsChange?: ((columns: Array<string>) => void) | undefined;
|
||||
viewMode?: LogsViewMode | undefined;
|
||||
onViewModeChange?: ((mode: LogsViewMode) => void) | undefined;
|
||||
onExportCSV?: (() => void) | undefined;
|
||||
onExportJSON?: (() => void) | undefined;
|
||||
}
|
||||
|
||||
const LogsViewerToolbar: FunctionComponent<LogsViewerToolbarProps> = (
|
||||
@@ -41,6 +44,15 @@ const LogsViewerToolbar: FunctionComponent<LogsViewerToolbarProps> = (
|
||||
currentPage && totalPages && totalPages > 0,
|
||||
);
|
||||
|
||||
const exportDropdownRef: React.RefObject<HTMLDivElement | null> =
|
||||
useRef<HTMLDivElement | null>(null);
|
||||
const [isExportOpen, setIsExportOpen] = useState<boolean>(false);
|
||||
useComponentOutsideClick(exportDropdownRef, () => {
|
||||
setIsExportOpen(false);
|
||||
});
|
||||
|
||||
const showExport: boolean = Boolean(props.onExportCSV || props.onExportJSON);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={`flex items-center justify-between gap-3 ${props.className || ""}`}
|
||||
@@ -141,6 +153,74 @@ const LogsViewerToolbar: FunctionComponent<LogsViewerToolbarProps> = (
|
||||
/>
|
||||
)}
|
||||
|
||||
{showExport && (
|
||||
<div className="relative" ref={exportDropdownRef}>
|
||||
<button
|
||||
type="button"
|
||||
className="inline-flex items-center gap-1.5 rounded-md border border-gray-200 bg-white px-2.5 py-1.5 text-xs font-medium text-gray-700 shadow-sm transition-colors hover:border-gray-300 hover:bg-gray-50"
|
||||
onClick={() => {
|
||||
setIsExportOpen(!isExportOpen);
|
||||
}}
|
||||
>
|
||||
<svg
|
||||
className="h-3.5 w-3.5"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
strokeWidth={1.5}
|
||||
stroke="currentColor"
|
||||
>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="M3 16.5v2.25A2.25 2.25 0 0 0 5.25 21h13.5A2.25 2.25 0 0 0 21 18.75V16.5M16.5 12 12 16.5m0 0L7.5 12m4.5 4.5V3"
|
||||
/>
|
||||
</svg>
|
||||
Export
|
||||
<svg
|
||||
className="h-3 w-3"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
strokeWidth={2}
|
||||
stroke="currentColor"
|
||||
>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
d="m19.5 8.25-7.5 7.5-7.5-7.5"
|
||||
/>
|
||||
</svg>
|
||||
</button>
|
||||
{isExportOpen && (
|
||||
<div className="absolute right-0 z-20 mt-1 w-40 rounded-md border border-gray-200 bg-white py-1 shadow-lg">
|
||||
{props.onExportCSV && (
|
||||
<button
|
||||
type="button"
|
||||
className="flex w-full items-center gap-2 px-3 py-1.5 text-left text-xs text-gray-700 hover:bg-gray-50"
|
||||
onClick={() => {
|
||||
setIsExportOpen(false);
|
||||
props.onExportCSV!();
|
||||
}}
|
||||
>
|
||||
Export as CSV
|
||||
</button>
|
||||
)}
|
||||
{props.onExportJSON && (
|
||||
<button
|
||||
type="button"
|
||||
className="flex w-full items-center gap-2 px-3 py-1.5 text-left text-xs text-gray-700 hover:bg-gray-50"
|
||||
onClick={() => {
|
||||
setIsExportOpen(false);
|
||||
props.onExportJSON!();
|
||||
}}
|
||||
>
|
||||
Export as JSON
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{props.timeRange && props.onTimeRangeChange && (
|
||||
<LogTimeRangePicker
|
||||
value={props.timeRange}
|
||||
|
||||
160
Common/UI/Utils/LogExport.ts
Normal file
160
Common/UI/Utils/LogExport.ts
Normal file
@@ -0,0 +1,160 @@
|
||||
import Log from "../../Models/AnalyticsModels/Log";
|
||||
import { JSONObject } from "../../Types/JSON";
|
||||
import {
|
||||
isLogsAttributeColumnId,
|
||||
getLogsAttributeKeyFromColumnId,
|
||||
} from "../Components/LogsViewer/types";
|
||||
|
||||
export enum LogExportFormat {
|
||||
CSV = "csv",
|
||||
JSON = "json",
|
||||
}
|
||||
|
||||
function escapeCsvValue(value: string): string {
|
||||
if (
|
||||
value.includes(",") ||
|
||||
value.includes('"') ||
|
||||
value.includes("\n") ||
|
||||
value.includes("\r")
|
||||
) {
|
||||
return `"${value.replace(/"/g, '""')}"`;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function getLogFieldValue(log: Log, columnId: string): string {
|
||||
if (isLogsAttributeColumnId(columnId)) {
|
||||
const attrKey: string | null = getLogsAttributeKeyFromColumnId(columnId);
|
||||
if (attrKey && log.attributes) {
|
||||
const val: unknown = (log.attributes as Record<string, unknown>)[attrKey];
|
||||
if (val === undefined || val === null) {
|
||||
return "";
|
||||
}
|
||||
if (typeof val === "object") {
|
||||
return JSON.stringify(val);
|
||||
}
|
||||
return String(val);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
switch (columnId) {
|
||||
case "time":
|
||||
return log.time ? new Date(log.time).toISOString() : "";
|
||||
case "service":
|
||||
return log.serviceId?.toString() || "";
|
||||
case "severity":
|
||||
return log.severityText?.toString() || "";
|
||||
case "message":
|
||||
return log.body || "";
|
||||
case "traceId":
|
||||
return log.traceId || "";
|
||||
case "spanId":
|
||||
return log.spanId || "";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function getColumnLabel(columnId: string): string {
|
||||
if (isLogsAttributeColumnId(columnId)) {
|
||||
return getLogsAttributeKeyFromColumnId(columnId) || columnId;
|
||||
}
|
||||
|
||||
switch (columnId) {
|
||||
case "time":
|
||||
return "Time";
|
||||
case "service":
|
||||
return "Service ID";
|
||||
case "severity":
|
||||
return "Severity";
|
||||
case "message":
|
||||
return "Message";
|
||||
case "traceId":
|
||||
return "Trace ID";
|
||||
case "spanId":
|
||||
return "Span ID";
|
||||
default:
|
||||
return columnId;
|
||||
}
|
||||
}
|
||||
|
||||
export function exportLogsToCSV(
|
||||
logs: Array<Log>,
|
||||
columns: Array<string>,
|
||||
): string {
|
||||
const header: string = columns
|
||||
.map((col: string) => {
|
||||
return escapeCsvValue(getColumnLabel(col));
|
||||
})
|
||||
.join(",");
|
||||
|
||||
const rows: Array<string> = logs.map((log: Log) => {
|
||||
return columns
|
||||
.map((col: string) => {
|
||||
return escapeCsvValue(getLogFieldValue(log, col));
|
||||
})
|
||||
.join(",");
|
||||
});
|
||||
|
||||
return [header, ...rows].join("\n");
|
||||
}
|
||||
|
||||
export function exportLogsToJSON(logs: Array<Log>): string {
|
||||
const data: Array<JSONObject> = logs.map((log: Log) => {
|
||||
const obj: JSONObject = {};
|
||||
obj["time"] = log.time ? new Date(log.time).toISOString() : null;
|
||||
obj["serviceId"] = log.serviceId?.toString() || null;
|
||||
obj["severity"] = log.severityText?.toString() || null;
|
||||
obj["severityNumber"] = log.severityNumber || null;
|
||||
obj["body"] = log.body || null;
|
||||
obj["traceId"] = log.traceId || null;
|
||||
obj["spanId"] = log.spanId || null;
|
||||
obj["attributes"] = (log.attributes as JSONObject) || {};
|
||||
return obj;
|
||||
});
|
||||
|
||||
return JSON.stringify(data, null, 2);
|
||||
}
|
||||
|
||||
function downloadFile(
|
||||
content: string,
|
||||
filename: string,
|
||||
mimeType: string,
|
||||
): void {
|
||||
const blob: Blob = new Blob([content], { type: mimeType });
|
||||
const url: string = window.URL.createObjectURL(blob);
|
||||
const anchor: HTMLAnchorElement = document.createElement("a");
|
||||
anchor.href = url;
|
||||
anchor.download = filename;
|
||||
document.body.appendChild(anchor);
|
||||
anchor.click();
|
||||
document.body.removeChild(anchor);
|
||||
window.URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
function getTimestampFilename(format: LogExportFormat): string {
|
||||
const now: string = new Date()
|
||||
.toISOString()
|
||||
.replace(/[:.]/g, "-")
|
||||
.slice(0, 19);
|
||||
return `logs-${now}.${format}`;
|
||||
}
|
||||
|
||||
export function exportLogs(
|
||||
logs: Array<Log>,
|
||||
format: LogExportFormat,
|
||||
columns: Array<string>,
|
||||
): void {
|
||||
if (format === LogExportFormat.CSV) {
|
||||
const csv: string = exportLogsToCSV(logs, columns);
|
||||
downloadFile(csv, getTimestampFilename(format), "text/csv;charset=utf-8;");
|
||||
} else {
|
||||
const json: string = exportLogsToJSON(logs);
|
||||
downloadFile(
|
||||
json,
|
||||
getTimestampFilename(format),
|
||||
"application/json;charset=utf-8;",
|
||||
);
|
||||
}
|
||||
}
|
||||
89
Telemetry/Services/LogDropFilterService.ts
Normal file
89
Telemetry/Services/LogDropFilterService.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import LogDropFilter from "Common/Models/DatabaseModels/LogDropFilter";
|
||||
import DatabaseService from "Common/Server/Services/DatabaseService";
|
||||
import SortOrder from "Common/Types/BaseDatabase/SortOrder";
|
||||
import LIMIT_MAX from "Common/Types/Database/LimitMax";
|
||||
import ObjectID from "Common/Types/ObjectID";
|
||||
import { JSONObject } from "Common/Types/JSON";
|
||||
import LogDropFilterAction from "Common/Types/Log/LogDropFilterAction";
|
||||
import { evaluateFilter } from "../Utils/LogFilterEvaluator";
|
||||
|
||||
interface CacheEntry {
|
||||
filters: Array<LogDropFilter>;
|
||||
loadedAt: number;
|
||||
}
|
||||
|
||||
const CACHE_TTL_MS: number = 60 * 1000; // 60 seconds
|
||||
|
||||
const dropFilterCache: Map<string, CacheEntry> = new Map();
|
||||
|
||||
export class LogDropFilterService {
|
||||
public static async loadDropFilters(
|
||||
projectId: ObjectID,
|
||||
): Promise<Array<LogDropFilter>> {
|
||||
const cacheKey: string = projectId.toString();
|
||||
const cached: CacheEntry | undefined = dropFilterCache.get(cacheKey);
|
||||
|
||||
if (cached && Date.now() - cached.loadedAt < CACHE_TTL_MS) {
|
||||
return cached.filters;
|
||||
}
|
||||
|
||||
const service: DatabaseService<LogDropFilter> =
|
||||
new DatabaseService<LogDropFilter>(LogDropFilter);
|
||||
|
||||
const filters: Array<LogDropFilter> = await service.findBy({
|
||||
query: {
|
||||
projectId: projectId,
|
||||
isEnabled: true,
|
||||
},
|
||||
skip: 0,
|
||||
limit: LIMIT_MAX,
|
||||
sort: {
|
||||
sortOrder: SortOrder.Ascending,
|
||||
},
|
||||
select: {
|
||||
_id: true,
|
||||
name: true,
|
||||
filterQuery: true,
|
||||
action: true,
|
||||
samplePercentage: true,
|
||||
sortOrder: true,
|
||||
},
|
||||
props: {
|
||||
isRoot: true,
|
||||
},
|
||||
});
|
||||
|
||||
dropFilterCache.set(cacheKey, { filters, loadedAt: Date.now() });
|
||||
return filters;
|
||||
}
|
||||
|
||||
public static shouldDropLog(
|
||||
logRow: JSONObject,
|
||||
filters: Array<LogDropFilter>,
|
||||
): boolean {
|
||||
for (const filter of filters) {
|
||||
const filterQuery: string = (filter.filterQuery as string) || "";
|
||||
|
||||
if (!evaluateFilter(logRow, filterQuery)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Filter matches this log
|
||||
if (filter.action === LogDropFilterAction.Drop) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (filter.action === LogDropFilterAction.Sample) {
|
||||
const samplePercentage: number = filter.samplePercentage || 50;
|
||||
// Keep samplePercentage% of logs, drop the rest
|
||||
if (Math.random() * 100 >= samplePercentage) {
|
||||
return true; // drop this log
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export default LogDropFilterService;
|
||||
232
Telemetry/Services/LogPipelineService.ts
Normal file
232
Telemetry/Services/LogPipelineService.ts
Normal file
@@ -0,0 +1,232 @@
|
||||
import LogPipeline from "Common/Models/DatabaseModels/LogPipeline";
|
||||
import LogPipelineProcessor from "Common/Models/DatabaseModels/LogPipelineProcessor";
|
||||
import DatabaseService from "Common/Server/Services/DatabaseService";
|
||||
import SortOrder from "Common/Types/BaseDatabase/SortOrder";
|
||||
import LIMIT_MAX from "Common/Types/Database/LimitMax";
|
||||
import ObjectID from "Common/Types/ObjectID";
|
||||
import { JSONObject } from "Common/Types/JSON";
|
||||
import LogPipelineProcessorType, {
|
||||
AttributeRemapperConfig,
|
||||
SeverityRemapperConfig,
|
||||
CategoryProcessorConfig,
|
||||
} from "Common/Types/Log/LogPipelineProcessorType";
|
||||
import { evaluateFilter } from "../Utils/LogFilterEvaluator";
|
||||
import logger from "Common/Server/Utils/Logger";
|
||||
|
||||
export interface LoadedPipeline {
|
||||
pipeline: LogPipeline;
|
||||
processors: Array<LogPipelineProcessor>;
|
||||
}
|
||||
|
||||
interface CacheEntry {
|
||||
pipelines: Array<LoadedPipeline>;
|
||||
loadedAt: number;
|
||||
}
|
||||
|
||||
const CACHE_TTL_MS: number = 60 * 1000; // 60 seconds
|
||||
|
||||
const pipelineCache: Map<string, CacheEntry> = new Map();
|
||||
|
||||
export class LogPipelineService {
|
||||
public static async loadPipelines(
|
||||
projectId: ObjectID,
|
||||
): Promise<Array<LoadedPipeline>> {
|
||||
const cacheKey: string = projectId.toString();
|
||||
const cached: CacheEntry | undefined = pipelineCache.get(cacheKey);
|
||||
|
||||
if (cached && Date.now() - cached.loadedAt < CACHE_TTL_MS) {
|
||||
return cached.pipelines;
|
||||
}
|
||||
|
||||
const pipelineService: DatabaseService<LogPipeline> =
|
||||
new DatabaseService<LogPipeline>(LogPipeline);
|
||||
|
||||
const pipelines: Array<LogPipeline> = await pipelineService.findBy({
|
||||
query: {
|
||||
projectId: projectId,
|
||||
isEnabled: true,
|
||||
},
|
||||
skip: 0,
|
||||
limit: LIMIT_MAX,
|
||||
sort: {
|
||||
sortOrder: SortOrder.Ascending,
|
||||
},
|
||||
select: {
|
||||
_id: true,
|
||||
name: true,
|
||||
filterQuery: true,
|
||||
sortOrder: true,
|
||||
},
|
||||
props: {
|
||||
isRoot: true,
|
||||
},
|
||||
});
|
||||
|
||||
const processorService: DatabaseService<LogPipelineProcessor> =
|
||||
new DatabaseService<LogPipelineProcessor>(LogPipelineProcessor);
|
||||
|
||||
const loaded: Array<LoadedPipeline> = [];
|
||||
|
||||
for (const pipeline of pipelines) {
|
||||
const processors: Array<LogPipelineProcessor> =
|
||||
await processorService.findBy({
|
||||
query: {
|
||||
logPipelineId: pipeline._id,
|
||||
isEnabled: true,
|
||||
},
|
||||
skip: 0,
|
||||
limit: LIMIT_MAX,
|
||||
sort: {
|
||||
sortOrder: SortOrder.Ascending,
|
||||
},
|
||||
select: {
|
||||
_id: true,
|
||||
name: true,
|
||||
processorType: true,
|
||||
configuration: true,
|
||||
sortOrder: true,
|
||||
},
|
||||
props: {
|
||||
isRoot: true,
|
||||
},
|
||||
});
|
||||
|
||||
loaded.push({ pipeline, processors });
|
||||
}
|
||||
|
||||
pipelineCache.set(cacheKey, { pipelines: loaded, loadedAt: Date.now() });
|
||||
return loaded;
|
||||
}
|
||||
|
||||
public static processLog(
|
||||
logRow: JSONObject,
|
||||
pipelines: Array<LoadedPipeline>,
|
||||
): JSONObject {
|
||||
let result: JSONObject = { ...logRow };
|
||||
|
||||
for (const { pipeline, processors } of pipelines) {
|
||||
// Check if this pipeline's filter matches the log
|
||||
const filterQuery: string = (pipeline.filterQuery as string) || "";
|
||||
if (!evaluateFilter(result, filterQuery)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Apply each processor in order
|
||||
for (const processor of processors) {
|
||||
try {
|
||||
result = LogPipelineService.applyProcessor(result, processor);
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`Error applying processor "${processor.name}" in pipeline "${pipeline.name}": ${err}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static applyProcessor(
|
||||
logRow: JSONObject,
|
||||
processor: LogPipelineProcessor,
|
||||
): JSONObject {
|
||||
const config: JSONObject =
|
||||
(processor.configuration as JSONObject) || {};
|
||||
|
||||
switch (processor.processorType) {
|
||||
case LogPipelineProcessorType.AttributeRemapper:
|
||||
return LogPipelineService.applyAttributeRemapper(
|
||||
logRow,
|
||||
config as unknown as AttributeRemapperConfig,
|
||||
);
|
||||
case LogPipelineProcessorType.SeverityRemapper:
|
||||
return LogPipelineService.applySeverityRemapper(
|
||||
logRow,
|
||||
config as unknown as SeverityRemapperConfig,
|
||||
);
|
||||
case LogPipelineProcessorType.CategoryProcessor:
|
||||
return LogPipelineService.applyCategoryProcessor(
|
||||
logRow,
|
||||
config as unknown as CategoryProcessorConfig,
|
||||
);
|
||||
default:
|
||||
return logRow;
|
||||
}
|
||||
}
|
||||
|
||||
private static applyAttributeRemapper(
|
||||
logRow: JSONObject,
|
||||
config: AttributeRemapperConfig,
|
||||
): JSONObject {
|
||||
const attrs: Record<string, unknown> = {
|
||||
...((logRow["attributes"] as Record<string, unknown>) || {}),
|
||||
};
|
||||
|
||||
const sourceVal: unknown = attrs[config.sourceKey];
|
||||
if (sourceVal === undefined) {
|
||||
return logRow;
|
||||
}
|
||||
|
||||
const overrideOnConflict: boolean = config.overrideOnConflict !== false;
|
||||
if (!overrideOnConflict && attrs[config.targetKey] !== undefined) {
|
||||
return logRow;
|
||||
}
|
||||
|
||||
attrs[config.targetKey] = sourceVal;
|
||||
|
||||
if (!config.preserveSource) {
|
||||
delete attrs[config.sourceKey];
|
||||
}
|
||||
|
||||
// Update attributeKeys
|
||||
const attributeKeys: Array<string> = Object.keys(attrs);
|
||||
|
||||
return { ...logRow, attributes: attrs, attributeKeys };
|
||||
}
|
||||
|
||||
private static applySeverityRemapper(
|
||||
logRow: JSONObject,
|
||||
config: SeverityRemapperConfig,
|
||||
): JSONObject {
|
||||
const attrs: Record<string, unknown> =
|
||||
(logRow["attributes"] as Record<string, unknown>) || {};
|
||||
const sourceVal: unknown = attrs[config.sourceKey];
|
||||
if (sourceVal === undefined || sourceVal === null) {
|
||||
return logRow;
|
||||
}
|
||||
|
||||
const sourceStr: string = String(sourceVal).toLowerCase();
|
||||
|
||||
for (const mapping of config.mappings || []) {
|
||||
if (mapping.matchValue.toLowerCase() === sourceStr) {
|
||||
return {
|
||||
...logRow,
|
||||
severityText: mapping.severityText,
|
||||
severityNumber: mapping.severityNumber,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return logRow;
|
||||
}
|
||||
|
||||
private static applyCategoryProcessor(
|
||||
logRow: JSONObject,
|
||||
config: CategoryProcessorConfig,
|
||||
): JSONObject {
|
||||
for (const category of config.categories || []) {
|
||||
if (evaluateFilter(logRow, category.filterQuery)) {
|
||||
const attrs: Record<string, unknown> = {
|
||||
...((logRow["attributes"] as Record<string, unknown>) || {}),
|
||||
};
|
||||
attrs[config.targetKey] = category.name;
|
||||
const attributeKeys: Array<string> = Object.keys(attrs);
|
||||
return { ...logRow, attributes: attrs, attributeKeys };
|
||||
}
|
||||
}
|
||||
|
||||
return logRow;
|
||||
}
|
||||
}
|
||||
|
||||
export default LogPipelineService;
|
||||
@@ -24,6 +24,9 @@ import LogsQueueService from "./Queue/LogsQueueService";
|
||||
import OtelIngestBaseService from "./OtelIngestBaseService";
|
||||
import { TELEMETRY_LOG_FLUSH_BATCH_SIZE } from "../Config";
|
||||
import LogService from "Common/Server/Services/LogService";
|
||||
import LogPipelineService, { LoadedPipeline } from "./LogPipelineService";
|
||||
import LogDropFilterService from "./LogDropFilterService";
|
||||
import LogDropFilter from "Common/Models/DatabaseModels/LogDropFilter";
|
||||
|
||||
export default class OtelLogsIngestService extends OtelIngestBaseService {
|
||||
private static async flushLogsBuffer(
|
||||
@@ -92,6 +95,19 @@ export default class OtelLogsIngestService extends OtelIngestBaseService {
|
||||
const serviceDictionary: Dictionary<TelemetryServiceMetadata> = {};
|
||||
let totalLogsProcessed: number = 0;
|
||||
|
||||
// Load pipelines and drop filters once per batch
|
||||
const projectId: ObjectID = (req as TelemetryRequest).projectId;
|
||||
let loadedPipelines: Array<LoadedPipeline> = [];
|
||||
let loadedDropFilters: Array<LogDropFilter> = [];
|
||||
try {
|
||||
loadedPipelines = await LogPipelineService.loadPipelines(projectId);
|
||||
loadedDropFilters =
|
||||
await LogDropFilterService.loadDropFilters(projectId);
|
||||
} catch (loadError) {
|
||||
logger.error("Error loading pipelines/drop filters:");
|
||||
logger.error(loadError);
|
||||
}
|
||||
|
||||
let resourceLogCounter: number = 0;
|
||||
for (const resourceLog of resourceLogs) {
|
||||
try {
|
||||
@@ -316,7 +332,7 @@ export default class OtelLogsIngestService extends OtelIngestBaseService {
|
||||
serviceDictionary[serviceName]!.dataRententionInDays || 15,
|
||||
);
|
||||
|
||||
const logRow: JSONObject = {
|
||||
let logRow: JSONObject = {
|
||||
_id: ObjectID.generate().toString(),
|
||||
createdAt: ingestionTimestamp,
|
||||
updatedAt: ingestionTimestamp,
|
||||
@@ -339,6 +355,25 @@ export default class OtelLogsIngestService extends OtelIngestBaseService {
|
||||
OneUptimeDate.toClickhouseDateTime(retentionDate),
|
||||
};
|
||||
|
||||
// Drop filter check (before pipeline processing)
|
||||
if (
|
||||
loadedDropFilters.length > 0 &&
|
||||
LogDropFilterService.shouldDropLog(
|
||||
logRow,
|
||||
loadedDropFilters,
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Pipeline processing
|
||||
if (loadedPipelines.length > 0) {
|
||||
logRow = LogPipelineService.processLog(
|
||||
logRow,
|
||||
loadedPipelines,
|
||||
);
|
||||
}
|
||||
|
||||
dbLogs.push(logRow);
|
||||
totalLogsProcessed++;
|
||||
|
||||
|
||||
371
Telemetry/Utils/LogFilterEvaluator.ts
Normal file
371
Telemetry/Utils/LogFilterEvaluator.ts
Normal file
@@ -0,0 +1,371 @@
|
||||
import { JSONObject } from "Common/Types/JSON";
|
||||
|
||||
// Simple filter evaluator for log rows used by pipelines and drop filters.
|
||||
// Supports: =, !=, LIKE, IN, AND, OR, NOT, parentheses
|
||||
// Field paths: severityText, body, serviceId, attributes.<key>
|
||||
|
||||
interface Token {
|
||||
type:
|
||||
| "field"
|
||||
| "operator"
|
||||
| "value"
|
||||
| "and"
|
||||
| "or"
|
||||
| "not"
|
||||
| "lparen"
|
||||
| "rparen";
|
||||
value: string;
|
||||
}
|
||||
|
||||
function getFieldValue(logRow: JSONObject, fieldPath: string): string {
|
||||
if (fieldPath.startsWith("attributes.")) {
|
||||
const attrKey: string = fieldPath.slice("attributes.".length);
|
||||
const attrs: Record<string, unknown> =
|
||||
(logRow["attributes"] as Record<string, unknown>) || {};
|
||||
const val: unknown = attrs[attrKey];
|
||||
if (val === undefined || val === null) {
|
||||
return "";
|
||||
}
|
||||
if (typeof val === "object") {
|
||||
return JSON.stringify(val);
|
||||
}
|
||||
return String(val);
|
||||
}
|
||||
|
||||
const val: unknown = logRow[fieldPath];
|
||||
if (val === undefined || val === null) {
|
||||
return "";
|
||||
}
|
||||
return String(val);
|
||||
}
|
||||
|
||||
function tokenize(query: string): Array<Token> {
|
||||
const tokens: Array<Token> = [];
|
||||
let i: number = 0;
|
||||
const len: number = query.length;
|
||||
|
||||
while (i < len) {
|
||||
// Skip whitespace
|
||||
if (/\s/.test(query[i]!)) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Parentheses
|
||||
if (query[i] === "(") {
|
||||
tokens.push({ type: "lparen", value: "(" });
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (query[i] === ")") {
|
||||
tokens.push({ type: "rparen", value: ")" });
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check for keywords (AND, OR, NOT, LIKE, IN)
|
||||
const remaining: string = query.slice(i);
|
||||
const keywordMatch: RegExpMatchArray | null = remaining.match(
|
||||
/^(AND|OR|NOT|LIKE|IN|!=)\b/i,
|
||||
);
|
||||
if (keywordMatch) {
|
||||
const kw: string = keywordMatch[1]!.toUpperCase();
|
||||
if (kw === "AND") {
|
||||
tokens.push({ type: "and", value: "AND" });
|
||||
} else if (kw === "OR") {
|
||||
tokens.push({ type: "or", value: "OR" });
|
||||
} else if (kw === "NOT") {
|
||||
tokens.push({ type: "not", value: "NOT" });
|
||||
} else if (kw === "LIKE" || kw === "IN") {
|
||||
tokens.push({ type: "operator", value: kw });
|
||||
} else if (kw === "!=") {
|
||||
tokens.push({ type: "operator", value: "!=" });
|
||||
}
|
||||
i += keywordMatch[0]!.length;
|
||||
continue;
|
||||
}
|
||||
|
||||
// != operator (check before = to avoid conflict)
|
||||
if (query[i] === "!" && i + 1 < len && query[i + 1] === "=") {
|
||||
tokens.push({ type: "operator", value: "!=" });
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
// = operator
|
||||
if (query[i] === "=") {
|
||||
tokens.push({ type: "operator", value: "=" });
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Quoted string value
|
||||
if (query[i] === "'" || query[i] === '"') {
|
||||
const quote: string = query[i]!;
|
||||
i++;
|
||||
let val: string = "";
|
||||
while (i < len && query[i] !== quote) {
|
||||
if (query[i] === "\\" && i + 1 < len) {
|
||||
i++;
|
||||
val += query[i];
|
||||
} else {
|
||||
val += query[i];
|
||||
}
|
||||
i++;
|
||||
}
|
||||
i++; // skip closing quote
|
||||
tokens.push({ type: "value", value: val });
|
||||
continue;
|
||||
}
|
||||
|
||||
// Field name or unquoted value
|
||||
let word: string = "";
|
||||
while (i < len && !/[\s()=!]/.test(query[i]!)) {
|
||||
word += query[i];
|
||||
i++;
|
||||
}
|
||||
if (word.length > 0) {
|
||||
// Determine if this is a field or a value based on context
|
||||
const lastToken: Token | undefined = tokens[tokens.length - 1];
|
||||
if (lastToken && lastToken.type === "operator") {
|
||||
tokens.push({ type: "value", value: word });
|
||||
} else {
|
||||
tokens.push({ type: "field", value: word });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
interface FilterExpression {
|
||||
type: "comparison" | "and" | "or" | "not";
|
||||
}
|
||||
|
||||
interface ComparisonExpr extends FilterExpression {
|
||||
type: "comparison";
|
||||
field: string;
|
||||
operator: string;
|
||||
value: string | Array<string>;
|
||||
}
|
||||
|
||||
interface AndExpr extends FilterExpression {
|
||||
type: "and";
|
||||
left: FilterExpression;
|
||||
right: FilterExpression;
|
||||
}
|
||||
|
||||
interface OrExpr extends FilterExpression {
|
||||
type: "or";
|
||||
left: FilterExpression;
|
||||
right: FilterExpression;
|
||||
}
|
||||
|
||||
interface NotExpr extends FilterExpression {
|
||||
type: "not";
|
||||
expr: FilterExpression;
|
||||
}
|
||||
|
||||
class Parser {
|
||||
private tokens: Array<Token>;
|
||||
private pos: number;
|
||||
|
||||
public constructor(tokens: Array<Token>) {
|
||||
this.tokens = tokens;
|
||||
this.pos = 0;
|
||||
}
|
||||
|
||||
public parse(): FilterExpression {
|
||||
const expr: FilterExpression = this.parseOr();
|
||||
return expr;
|
||||
}
|
||||
|
||||
private parseOr(): FilterExpression {
|
||||
let left: FilterExpression = this.parseAnd();
|
||||
while (this.pos < this.tokens.length && this.tokens[this.pos]!.type === "or") {
|
||||
this.pos++;
|
||||
const right: FilterExpression = this.parseAnd();
|
||||
left = { type: "or", left, right } as OrExpr;
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
private parseAnd(): FilterExpression {
|
||||
let left: FilterExpression = this.parseUnary();
|
||||
while (this.pos < this.tokens.length && this.tokens[this.pos]!.type === "and") {
|
||||
this.pos++;
|
||||
const right: FilterExpression = this.parseUnary();
|
||||
left = { type: "and", left, right } as AndExpr;
|
||||
}
|
||||
return left;
|
||||
}
|
||||
|
||||
private parseUnary(): FilterExpression {
|
||||
if (
|
||||
this.pos < this.tokens.length &&
|
||||
this.tokens[this.pos]!.type === "not"
|
||||
) {
|
||||
this.pos++;
|
||||
const expr: FilterExpression = this.parseUnary();
|
||||
return { type: "not", expr } as NotExpr;
|
||||
}
|
||||
return this.parsePrimary();
|
||||
}
|
||||
|
||||
private parsePrimary(): FilterExpression {
|
||||
if (
|
||||
this.pos < this.tokens.length &&
|
||||
this.tokens[this.pos]!.type === "lparen"
|
||||
) {
|
||||
this.pos++; // skip (
|
||||
const expr: FilterExpression = this.parseOr();
|
||||
if (
|
||||
this.pos < this.tokens.length &&
|
||||
this.tokens[this.pos]!.type === "rparen"
|
||||
) {
|
||||
this.pos++; // skip )
|
||||
}
|
||||
return expr;
|
||||
}
|
||||
|
||||
// Comparison: field operator value
|
||||
const fieldToken: Token | undefined = this.tokens[this.pos];
|
||||
if (!fieldToken || fieldToken.type !== "field") {
|
||||
throw new Error(
|
||||
`Expected field at position ${this.pos}, got: ${fieldToken?.type || "end of input"}`,
|
||||
);
|
||||
}
|
||||
this.pos++;
|
||||
|
||||
const opToken: Token | undefined = this.tokens[this.pos];
|
||||
if (!opToken || opToken.type !== "operator") {
|
||||
throw new Error(
|
||||
`Expected operator at position ${this.pos}, got: ${opToken?.type || "end of input"}`,
|
||||
);
|
||||
}
|
||||
this.pos++;
|
||||
|
||||
if (opToken.value === "IN") {
|
||||
// Parse IN (val1, val2, val3)
|
||||
const values: Array<string> = this.parseInValues();
|
||||
return {
|
||||
type: "comparison",
|
||||
field: fieldToken.value,
|
||||
operator: "IN",
|
||||
value: values,
|
||||
} as ComparisonExpr;
|
||||
}
|
||||
|
||||
const valToken: Token | undefined = this.tokens[this.pos];
|
||||
if (!valToken || valToken.type !== "value") {
|
||||
throw new Error(
|
||||
`Expected value at position ${this.pos}, got: ${valToken?.type || "end of input"}`,
|
||||
);
|
||||
}
|
||||
this.pos++;
|
||||
|
||||
return {
|
||||
type: "comparison",
|
||||
field: fieldToken.value,
|
||||
operator: opToken.value,
|
||||
value: valToken.value,
|
||||
} as ComparisonExpr;
|
||||
}
|
||||
|
||||
private parseInValues(): Array<string> {
|
||||
const values: Array<string> = [];
|
||||
|
||||
// Expect (
|
||||
if (
|
||||
this.pos < this.tokens.length &&
|
||||
this.tokens[this.pos]!.type === "lparen"
|
||||
) {
|
||||
this.pos++;
|
||||
}
|
||||
|
||||
while (this.pos < this.tokens.length) {
|
||||
if (this.tokens[this.pos]!.type === "rparen") {
|
||||
this.pos++;
|
||||
break;
|
||||
}
|
||||
if (this.tokens[this.pos]!.type === "value") {
|
||||
values.push(this.tokens[this.pos]!.value);
|
||||
this.pos++;
|
||||
} else {
|
||||
this.pos++; // skip commas etc.
|
||||
}
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
}
|
||||
|
||||
function evaluateExpr(logRow: JSONObject, expr: FilterExpression): boolean {
|
||||
switch (expr.type) {
|
||||
case "comparison": {
|
||||
const comp: ComparisonExpr = expr as ComparisonExpr;
|
||||
const fieldVal: string = getFieldValue(logRow, comp.field);
|
||||
|
||||
switch (comp.operator) {
|
||||
case "=":
|
||||
return fieldVal === comp.value;
|
||||
case "!=":
|
||||
return fieldVal !== comp.value;
|
||||
case "LIKE": {
|
||||
// Convert SQL LIKE pattern to regex: % -> .*, _ -> .
|
||||
const pattern: string = String(comp.value)
|
||||
.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") // escape regex chars first
|
||||
.replace(/%/g, ".*")
|
||||
.replace(/_/g, ".");
|
||||
return new RegExp(`^${pattern}$`, "i").test(fieldVal);
|
||||
}
|
||||
case "IN": {
|
||||
const values: Array<string> = comp.value as Array<string>;
|
||||
return values.includes(fieldVal);
|
||||
}
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
case "and": {
|
||||
const andExpr: AndExpr = expr as AndExpr;
|
||||
return (
|
||||
evaluateExpr(logRow, andExpr.left) &&
|
||||
evaluateExpr(logRow, andExpr.right)
|
||||
);
|
||||
}
|
||||
case "or": {
|
||||
const orExpr: OrExpr = expr as OrExpr;
|
||||
return (
|
||||
evaluateExpr(logRow, orExpr.left) || evaluateExpr(logRow, orExpr.right)
|
||||
);
|
||||
}
|
||||
case "not": {
|
||||
const notExpr: NotExpr = expr as NotExpr;
|
||||
return !evaluateExpr(logRow, notExpr.expr);
|
||||
}
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function evaluateFilter(logRow: JSONObject, filterQuery: string): boolean {
|
||||
if (!filterQuery || filterQuery.trim().length === 0) {
|
||||
return true; // empty filter matches everything
|
||||
}
|
||||
|
||||
try {
|
||||
const tokens: Array<Token> = tokenize(filterQuery);
|
||||
if (tokens.length === 0) {
|
||||
return true;
|
||||
}
|
||||
const parser: Parser = new Parser(tokens);
|
||||
const expr: FilterExpression = parser.parse();
|
||||
return evaluateExpr(logRow, expr);
|
||||
} catch {
|
||||
// If filter can't be parsed, don't match (safe default)
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export default evaluateFilter;
|
||||
Reference in New Issue
Block a user