feat: add LogPipelineProcessor model and related services

- Implemented LogPipelineProcessor model with necessary fields and access controls.
- Created LogDropFilterAction enum for defining actions on log drop filters.
- Introduced LogPipelineProcessorType enum and configuration interfaces for various processor types.
- Developed LogExport utility for exporting logs in CSV and JSON formats.
- Added LogDropFilterService for managing drop filters with caching.
- Implemented LogPipelineService for loading and processing log pipelines with processors.
- Created LogFilterEvaluator for evaluating filter queries against log entries.
This commit is contained in:
Nawaz Dhandala
2026-03-13 11:50:07 +00:00
parent fd1ee0c248
commit 0e93929a3f
25 changed files with 3982 additions and 122 deletions

View File

@@ -1277,6 +1277,7 @@ const DashboardLogsViewer: FunctionComponent<ComponentProps> = (
viewMode={viewMode}
onViewModeChange={setViewMode}
analyticsServiceIds={serviceIdStrings}
projectId={ProjectUtil.getCurrentProjectId() || undefined}
analyticsAppliedFacetFilters={appliedFacetFilters}
onUpdateCurrentSavedView={async () => {
if (!selectedSavedView?.id) {

View File

@@ -0,0 +1,169 @@
import PageComponentProps from "../PageComponentProps";
import FormFieldSchemaType from "Common/UI/Components/Forms/Types/FormFieldSchemaType";
import ModelTable from "Common/UI/Components/ModelTable/ModelTable";
import FieldType from "Common/UI/Components/Types/FieldType";
import LogDropFilter from "Common/Models/DatabaseModels/LogDropFilter";
import ProjectUtil from "Common/UI/Utils/Project";
import Navigation from "Common/UI/Utils/Navigation";
import React, { Fragment, FunctionComponent, ReactElement } from "react";
const LogDropFilters: FunctionComponent<
PageComponentProps
> = (): ReactElement => {
return (
<Fragment>
<ModelTable<LogDropFilter>
modelType={LogDropFilter}
query={{
projectId: ProjectUtil.getCurrentProjectId()!,
}}
id="log-drop-filters-table"
name="Settings > Log Drop Filters"
userPreferencesKey="log-drop-filters-table"
isDeleteable={true}
isEditable={true}
isCreateable={true}
cardProps={{
title: "Log Drop Filters",
description:
"Drop filters let you discard or sample logs before they are stored. Matching logs are dropped or sampled at the configured percentage. Filters run in sort order before pipeline processing.",
}}
noItemsMessage={"No drop filters found."}
viewPageRoute={Navigation.getCurrentRoute()}
formFields={[
{
field: {
name: true,
},
title: "Name",
fieldType: FormFieldSchemaType.Text,
required: true,
placeholder: "e.g. Drop Debug Logs",
validation: {
minLength: 2,
},
},
{
field: {
description: true,
},
title: "Description",
fieldType: FormFieldSchemaType.LongText,
required: false,
placeholder: "Describe what this filter does.",
},
{
field: {
filterQuery: true,
},
title: "Filter Query",
fieldType: FormFieldSchemaType.LongText,
required: true,
placeholder:
"e.g. severityText = 'DEBUG' OR body LIKE '%health%'",
},
{
field: {
action: true,
},
title: "Action",
fieldType: FormFieldSchemaType.Text,
required: true,
placeholder: "drop or sample",
},
{
field: {
samplePercentage: true,
},
title: "Sample Percentage",
fieldType: FormFieldSchemaType.Number,
required: false,
placeholder: "50 (keep this % of matching logs)",
},
{
field: {
isEnabled: true,
},
title: "Enabled",
fieldType: FormFieldSchemaType.Toggle,
required: false,
},
{
field: {
sortOrder: true,
},
title: "Sort Order",
fieldType: FormFieldSchemaType.Number,
required: false,
placeholder: "0",
},
]}
showRefreshButton={true}
showViewIdButton={true}
filters={[
{
field: {
name: true,
},
type: FieldType.Text,
title: "Name",
},
{
field: {
action: true,
},
type: FieldType.Text,
title: "Action",
},
{
field: {
isEnabled: true,
},
type: FieldType.Boolean,
title: "Enabled",
},
]}
columns={[
{
field: {
name: true,
},
title: "Name",
type: FieldType.Text,
},
{
field: {
action: true,
},
title: "Action",
type: FieldType.Text,
},
{
field: {
samplePercentage: true,
},
title: "Sample %",
type: FieldType.Number,
noValueMessage: "-",
},
{
field: {
isEnabled: true,
},
title: "Enabled",
type: FieldType.Boolean,
},
{
field: {
sortOrder: true,
},
title: "Sort Order",
type: FieldType.Number,
},
]}
/>
</Fragment>
);
};
export default LogDropFilters;

View File

@@ -0,0 +1,250 @@
import PageMap from "../../Utils/PageMap";
import RouteMap from "../../Utils/RouteMap";
import PageComponentProps from "../PageComponentProps";
import Route from "Common/Types/API/Route";
import ObjectID from "Common/Types/ObjectID";
import FormFieldSchemaType from "Common/UI/Components/Forms/Types/FormFieldSchemaType";
import ModelDelete from "Common/UI/Components/ModelDelete/ModelDelete";
import CardModelDetail from "Common/UI/Components/ModelDetail/CardModelDetail";
import ModelTable from "Common/UI/Components/ModelTable/ModelTable";
import FieldType from "Common/UI/Components/Types/FieldType";
import Navigation from "Common/UI/Utils/Navigation";
import LogPipeline from "Common/Models/DatabaseModels/LogPipeline";
import LogPipelineProcessor from "Common/Models/DatabaseModels/LogPipelineProcessor";
import React, { Fragment, FunctionComponent, ReactElement } from "react";
const LogPipelineView: FunctionComponent<PageComponentProps> = (
_props: PageComponentProps,
): ReactElement => {
const modelId: ObjectID = Navigation.getLastParamAsObjectID();
return (
<Fragment>
<CardModelDetail<LogPipeline>
name="Log Pipeline Details"
cardProps={{
title: "Log Pipeline Details",
description: "Details for this log pipeline.",
}}
isEditable={true}
formFields={[
{
field: {
name: true,
},
title: "Name",
fieldType: FormFieldSchemaType.Text,
required: true,
placeholder: "Pipeline Name",
validation: {
minLength: 2,
},
},
{
field: {
description: true,
},
title: "Description",
fieldType: FormFieldSchemaType.LongText,
required: false,
placeholder: "Pipeline Description",
},
{
field: {
filterQuery: true,
},
title: "Filter Query",
fieldType: FormFieldSchemaType.LongText,
required: false,
placeholder:
"e.g. severityText = 'ERROR' AND attributes.service = 'api'",
},
{
field: {
isEnabled: true,
},
title: "Enabled",
fieldType: FormFieldSchemaType.Toggle,
required: false,
},
{
field: {
sortOrder: true,
},
title: "Sort Order",
fieldType: FormFieldSchemaType.Number,
required: false,
},
]}
modelDetailProps={{
modelType: LogPipeline,
id: "model-detail-log-pipeline",
fields: [
{
field: {
name: true,
},
title: "Name",
},
{
field: {
description: true,
},
title: "Description",
},
{
field: {
filterQuery: true,
},
title: "Filter Query",
},
{
field: {
isEnabled: true,
},
title: "Enabled",
fieldType: FieldType.Boolean,
},
{
field: {
sortOrder: true,
},
title: "Sort Order",
fieldType: FieldType.Number,
},
],
modelId: modelId,
}}
/>
<ModelTable<LogPipelineProcessor>
modelType={LogPipelineProcessor}
query={{
logPipelineId: modelId,
}}
id="log-pipeline-processors-table"
name="Log Pipeline > Processors"
userPreferencesKey="log-pipeline-processors-table"
isDeleteable={true}
isEditable={true}
isCreateable={true}
cardProps={{
title: "Processors",
description:
"Processors transform logs matched by this pipeline. They run in sort order.",
}}
noItemsMessage={"No processors configured for this pipeline."}
formFields={[
{
field: {
name: true,
},
title: "Name",
fieldType: FormFieldSchemaType.Text,
required: true,
placeholder: "e.g. Remap severity",
validation: {
minLength: 2,
},
},
{
field: {
processorType: true,
},
title: "Processor Type",
fieldType: FormFieldSchemaType.Text,
required: true,
placeholder: "e.g. AttributeRemapper",
},
{
field: {
configuration: true,
},
title: "Configuration (JSON)",
fieldType: FormFieldSchemaType.JSON,
required: true,
},
{
field: {
isEnabled: true,
},
title: "Enabled",
fieldType: FormFieldSchemaType.Toggle,
required: false,
},
{
field: {
sortOrder: true,
},
title: "Sort Order",
fieldType: FormFieldSchemaType.Number,
required: false,
placeholder: "0",
},
]}
showRefreshButton={true}
createInitialValues={{
logPipelineId: modelId,
}}
filters={[
{
field: {
name: true,
},
type: FieldType.Text,
title: "Name",
},
{
field: {
isEnabled: true,
},
type: FieldType.Boolean,
title: "Enabled",
},
]}
columns={[
{
field: {
name: true,
},
title: "Name",
type: FieldType.Text,
},
{
field: {
processorType: true,
},
title: "Type",
type: FieldType.Text,
},
{
field: {
isEnabled: true,
},
title: "Enabled",
type: FieldType.Boolean,
},
{
field: {
sortOrder: true,
},
title: "Sort Order",
type: FieldType.Number,
},
]}
/>
<ModelDelete
modelType={LogPipeline}
modelId={modelId}
onDeleteSuccess={() => {
Navigation.navigate(
RouteMap[PageMap.SETTINGS_LOG_PIPELINES] as Route,
);
}}
/>
</Fragment>
);
};
export default LogPipelineView;

View File

@@ -0,0 +1,139 @@
import PageMap from "../../Utils/PageMap";
import RouteMap, { RouteUtil } from "../../Utils/RouteMap";
import PageComponentProps from "../PageComponentProps";
import FormFieldSchemaType from "Common/UI/Components/Forms/Types/FormFieldSchemaType";
import ModelTable from "Common/UI/Components/ModelTable/ModelTable";
import FieldType from "Common/UI/Components/Types/FieldType";
import LogPipeline from "Common/Models/DatabaseModels/LogPipeline";
import ProjectUtil from "Common/UI/Utils/Project";
import Route from "Common/Types/API/Route";
import React, { Fragment, FunctionComponent, ReactElement } from "react";
const LogPipelines: FunctionComponent<PageComponentProps> = (): ReactElement => {
return (
<Fragment>
<ModelTable<LogPipeline>
modelType={LogPipeline}
query={{
projectId: ProjectUtil.getCurrentProjectId()!,
}}
id="log-pipelines-table"
name="Settings > Log Pipelines"
userPreferencesKey="log-pipelines-table"
isDeleteable={true}
isEditable={true}
isCreateable={true}
cardProps={{
title: "Log Pipelines",
description:
"Configure server-side log processing pipelines that transform logs at ingest time. Pipelines run in sort order and apply processors to matching logs.",
}}
noItemsMessage={"No log pipelines found."}
viewPageRoute={RouteUtil.populateRouteParams(
RouteMap[PageMap.SETTINGS_LOG_PIPELINE_VIEW] as Route,
)}
formFields={[
{
field: {
name: true,
},
title: "Name",
fieldType: FormFieldSchemaType.Text,
required: true,
placeholder: "e.g. Parse Nginx Logs",
validation: {
minLength: 2,
},
},
{
field: {
description: true,
},
title: "Description",
fieldType: FormFieldSchemaType.LongText,
required: false,
placeholder: "Describe what this pipeline does.",
},
{
field: {
filterQuery: true,
},
title: "Filter Query",
fieldType: FormFieldSchemaType.LongText,
required: false,
placeholder:
"e.g. severityText = 'ERROR' AND attributes.service = 'api'",
},
{
field: {
isEnabled: true,
},
title: "Enabled",
fieldType: FormFieldSchemaType.Toggle,
required: false,
},
{
field: {
sortOrder: true,
},
title: "Sort Order",
fieldType: FormFieldSchemaType.Number,
required: false,
placeholder: "0",
},
]}
showRefreshButton={true}
showViewIdButton={true}
filters={[
{
field: {
name: true,
},
type: FieldType.Text,
title: "Name",
},
{
field: {
isEnabled: true,
},
type: FieldType.Boolean,
title: "Enabled",
},
]}
columns={[
{
field: {
name: true,
},
title: "Name",
type: FieldType.Text,
},
{
field: {
description: true,
},
noValueMessage: "-",
title: "Description",
type: FieldType.LongText,
},
{
field: {
isEnabled: true,
},
title: "Enabled",
type: FieldType.Boolean,
},
{
field: {
sortOrder: true,
},
title: "Sort Order",
type: FieldType.Number,
},
]}
/>
</Fragment>
);
};
export default LogPipelines;

View File

@@ -68,6 +68,24 @@ const DashboardSideMenu: () => JSX.Element = (): ReactElement => {
},
icon: IconProp.Terminal,
},
{
link: {
title: "Log Pipelines",
to: RouteUtil.populateRouteParams(
RouteMap[PageMap.SETTINGS_LOG_PIPELINES] as Route,
),
},
icon: IconProp.Logs,
},
{
link: {
title: "Log Drop Filters",
to: RouteUtil.populateRouteParams(
RouteMap[PageMap.SETTINGS_LOG_DROP_FILTERS] as Route,
),
},
icon: IconProp.Filter,
},
],
},
{

View File

@@ -59,6 +59,10 @@ import SettingsUsageHistory from "../Pages/Settings/UsageHistory";
import SettingsSlackIntegration from "../Pages/Settings/SlackIntegration";
import SettingsLogPipelines from "../Pages/Settings/LogPipelines";
import SettingsLogPipelineView from "../Pages/Settings/LogPipelineView";
import SettingsLogDropFilters from "../Pages/Settings/LogDropFilters";
const SettingsRoutes: FunctionComponent<ComponentProps> = (
props: ComponentProps,
): ReactElement => {
@@ -375,6 +379,49 @@ const SettingsRoutes: FunctionComponent<ComponentProps> = (
/>
}
/>
<PageRoute
path={RouteUtil.getLastPathForKey(
PageMap.SETTINGS_LOG_PIPELINES,
)}
element={
<SettingsLogPipelines
{...props}
pageRoute={
RouteMap[PageMap.SETTINGS_LOG_PIPELINES] as Route
}
/>
}
/>
<PageRoute
path={RouteUtil.getLastPathForKey(
PageMap.SETTINGS_LOG_PIPELINE_VIEW,
2,
)}
element={
<SettingsLogPipelineView
{...props}
pageRoute={
RouteMap[PageMap.SETTINGS_LOG_PIPELINE_VIEW] as Route
}
/>
}
/>
<PageRoute
path={RouteUtil.getLastPathForKey(
PageMap.SETTINGS_LOG_DROP_FILTERS,
)}
element={
<SettingsLogDropFilters
{...props}
pageRoute={
RouteMap[PageMap.SETTINGS_LOG_DROP_FILTERS] as Route
}
/>
}
/>
</PageRoute>
</Routes>
);

View File

@@ -342,6 +342,10 @@ enum PageMap {
SETTINGS_TELEMETRY_INGESTION_KEYS = "SETTINGS_TELEMETRY_INGESTION_KEYS",
SETTINGS_TELEMETRY_INGESTION_KEY_VIEW = "SETTINGS_TELEMETRY_INGESTION_KEY_VIEW",
SETTINGS_LOG_PIPELINES = "SETTINGS_LOG_PIPELINES",
SETTINGS_LOG_PIPELINE_VIEW = "SETTINGS_LOG_PIPELINE_VIEW",
SETTINGS_LOG_DROP_FILTERS = "SETTINGS_LOG_DROP_FILTERS",
// API Keys.
SETTINGS_APIKEYS = "SETTINGS_APIKEYS",
SETTINGS_APIKEY_VIEW = "SETTINGS_APIKEY_VIEW",

View File

@@ -304,6 +304,9 @@ export const SettingsRoutePath: Dictionary<string> = {
[PageMap.SETTINGS_APIKEY_VIEW]: `api-keys/${RouteParams.ModelID}`,
[PageMap.SETTINGS_TELEMETRY_INGESTION_KEYS]: `telemetry-ingestion-keys`,
[PageMap.SETTINGS_TELEMETRY_INGESTION_KEY_VIEW]: `telemetry-ingestion-keys/${RouteParams.ModelID}`,
[PageMap.SETTINGS_LOG_PIPELINES]: `log-pipelines`,
[PageMap.SETTINGS_LOG_PIPELINE_VIEW]: `log-pipelines/${RouteParams.ModelID}`,
[PageMap.SETTINGS_LOG_DROP_FILTERS]: `log-drop-filters`,
[PageMap.SETTINGS_SLACK_INTEGRATION]: "slack-integration",
[PageMap.SETTINGS_MICROSOFT_TEAMS_INTEGRATION]: "microsoft-teams-integration",
@@ -2124,6 +2127,24 @@ const RouteMap: Dictionary<Route> = {
}`,
),
[PageMap.SETTINGS_LOG_PIPELINES]: new Route(
`/dashboard/${RouteParams.ProjectID}/settings/${
SettingsRoutePath[PageMap.SETTINGS_LOG_PIPELINES]
}`,
),
[PageMap.SETTINGS_LOG_PIPELINE_VIEW]: new Route(
`/dashboard/${RouteParams.ProjectID}/settings/${
SettingsRoutePath[PageMap.SETTINGS_LOG_PIPELINE_VIEW]
}`,
),
[PageMap.SETTINGS_LOG_DROP_FILTERS]: new Route(
`/dashboard/${RouteParams.ProjectID}/settings/${
SettingsRoutePath[PageMap.SETTINGS_LOG_DROP_FILTERS]
}`,
),
[PageMap.SETTINGS_APIKEY_VIEW]: new Route(
`/dashboard/${RouteParams.ProjectID}/settings/${
SettingsRoutePath[PageMap.SETTINGS_APIKEY_VIEW]

View File

@@ -37,6 +37,9 @@ import IncidentTemplateOwnerUser from "./IncidentTemplateOwnerUser";
//Labels.
import Label from "./Label";
import LogSavedView from "./LogSavedView";
import LogPipeline from "./LogPipeline";
import LogPipelineProcessor from "./LogPipelineProcessor";
import LogDropFilter from "./LogDropFilter";
// Monitors
import Monitor from "./Monitor";
import MonitorCustomField from "./MonitorCustomField";
@@ -252,6 +255,9 @@ const AllModelTypes: Array<{
ApiKey,
Label,
LogSavedView,
LogPipeline,
LogPipelineProcessor,
LogDropFilter,
ApiKeyPermission,
ProjectSmtpConfig,
StatusPage,

View File

@@ -0,0 +1,480 @@
import Project from "./Project";
import User from "./User";
import BaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
import Route from "../../Types/API/Route";
import ColumnAccessControl from "../../Types/Database/AccessControl/ColumnAccessControl";
import TableAccessControl from "../../Types/Database/AccessControl/TableAccessControl";
import TableBillingAccessControl from "../../Types/Database/AccessControl/TableBillingAccessControl";
import ColumnLength from "../../Types/Database/ColumnLength";
import ColumnType from "../../Types/Database/ColumnType";
import CrudApiEndpoint from "../../Types/Database/CrudApiEndpoint";
import EnableDocumentation from "../../Types/Database/EnableDocumentation";
import TableColumn from "../../Types/Database/TableColumn";
import TableColumnType from "../../Types/Database/TableColumnType";
import TableMetadata from "../../Types/Database/TableMetadata";
import TenantColumn from "../../Types/Database/TenantColumn";
import IconProp from "../../Types/Icon/IconProp";
import ObjectID from "../../Types/ObjectID";
import Permission from "../../Types/Permission";
import { PlanType } from "../../Types/Billing/SubscriptionPlan";
import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
@EnableDocumentation()
@TableBillingAccessControl({
create: PlanType.Free,
read: PlanType.Free,
update: PlanType.Free,
delete: PlanType.Free,
})
@TenantColumn("projectId")
@CrudApiEndpoint(new Route("/log-drop-filter"))
@Entity({
name: "LogDropFilter",
})
@TableMetadata({
tableName: "LogDropFilter",
singularName: "Log Drop Filter",
pluralName: "Log Drop Filters",
icon: IconProp.Filter,
tableDescription:
"Configure rules to drop or sample logs before storage to reduce volume and cost.",
})
@TableAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogDropFilter,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
delete: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.DeleteProjectLogDropFilter,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogDropFilter,
],
})
export default class LogDropFilter extends BaseModel {
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogDropFilter,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
manyToOneRelationColumn: "projectId",
type: TableColumnType.Entity,
modelType: Project,
title: "Project",
description: "Relation to the project this log drop filter belongs to.",
})
@ManyToOne(
() => {
return Project;
},
{
eager: false,
nullable: true,
onDelete: "CASCADE",
orphanedRowAction: "nullify",
},
)
@JoinColumn({ name: "projectId" })
public project?: Project = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogDropFilter,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [],
})
@Index()
@TableColumn({
type: TableColumnType.ObjectID,
required: true,
canReadOnRelationQuery: true,
title: "Project ID",
description: "ID of the project this log drop filter belongs to.",
})
@Column({
type: ColumnType.ObjectID,
nullable: false,
transformer: ObjectID.getDatabaseTransformer(),
})
public projectId?: ObjectID = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogDropFilter,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogDropFilter,
],
})
@TableColumn({
required: true,
type: TableColumnType.Name,
canReadOnRelationQuery: true,
title: "Name",
description: "Friendly name for this drop filter.",
})
@Column({
nullable: false,
type: ColumnType.Name,
length: ColumnLength.Name,
})
public name?: string = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogDropFilter,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogDropFilter,
],
})
@TableColumn({
required: false,
type: TableColumnType.LongText,
canReadOnRelationQuery: true,
title: "Description",
description: "Description of what this drop filter does.",
})
@Column({
nullable: true,
type: ColumnType.LongText,
length: ColumnLength.LongText,
})
public description?: string = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogDropFilter,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogDropFilter,
],
})
@TableColumn({
required: true,
type: TableColumnType.LongText,
canReadOnRelationQuery: true,
title: "Filter Query",
description:
"Filter expression that identifies which logs to drop or sample.",
})
@Column({
nullable: false,
type: ColumnType.LongText,
length: ColumnLength.LongText,
})
public filterQuery?: string = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogDropFilter,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogDropFilter,
],
})
@TableColumn({
required: true,
type: TableColumnType.ShortText,
canReadOnRelationQuery: true,
title: "Action",
description:
"What to do with matching logs: 'drop' to discard entirely, 'sample' to keep a percentage.",
})
@Column({
nullable: false,
type: ColumnType.ShortText,
length: ColumnLength.ShortText,
default: "drop",
})
public action?: string = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogDropFilter,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogDropFilter,
],
})
@TableColumn({
title: "Sample Percentage",
required: false,
type: TableColumnType.Number,
canReadOnRelationQuery: true,
description:
"When action is 'sample', the percentage of matching logs to keep (1-99).",
})
@Column({
type: ColumnType.Number,
nullable: true,
})
public samplePercentage?: number = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogDropFilter,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogDropFilter,
],
})
@Index()
@TableColumn({
required: true,
type: TableColumnType.Boolean,
canReadOnRelationQuery: true,
title: "Enabled",
description: "Whether this drop filter is active.",
defaultValue: true,
})
@Column({
nullable: false,
type: ColumnType.Boolean,
default: true,
})
public isEnabled?: boolean = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogDropFilter,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogDropFilter,
],
})
@TableColumn({
title: "Sort Order",
required: true,
type: TableColumnType.Number,
canReadOnRelationQuery: true,
description:
"Determines the evaluation order of this filter relative to others.",
defaultValue: 0,
})
@Column({
type: ColumnType.Number,
nullable: false,
default: 0,
})
public sortOrder?: number = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
manyToOneRelationColumn: "createdByUserId",
type: TableColumnType.Entity,
modelType: User,
title: "Created By User",
description: "Relation to the user who created this log drop filter.",
})
@ManyToOne(
() => {
return User;
},
{
eager: false,
nullable: true,
onDelete: "SET NULL",
orphanedRowAction: "nullify",
},
)
@JoinColumn({ name: "createdByUserId" })
public createdByUser?: User = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
type: TableColumnType.ObjectID,
title: "Created By User ID",
description: "ID of the user who created this log drop filter.",
})
@Column({
type: ColumnType.ObjectID,
nullable: true,
transformer: ObjectID.getDatabaseTransformer(),
})
public createdByUserId?: ObjectID = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
manyToOneRelationColumn: "deletedByUserId",
type: TableColumnType.Entity,
modelType: User,
title: "Deleted By User",
description: "Relation to the user who deleted this log drop filter.",
})
@ManyToOne(
() => {
return User;
},
{
eager: false,
nullable: true,
onDelete: "SET NULL",
orphanedRowAction: "nullify",
},
)
@JoinColumn({ name: "deletedByUserId" })
public deletedByUser?: User = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogDropFilter,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
type: TableColumnType.ObjectID,
title: "Deleted By User ID",
description: "ID of the user who deleted this log drop filter.",
})
@Column({
type: ColumnType.ObjectID,
nullable: true,
transformer: ObjectID.getDatabaseTransformer(),
})
public deletedByUserId?: ObjectID = undefined;
}

View File

@@ -0,0 +1,412 @@
import Project from "./Project";
import User from "./User";
import BaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
import Route from "../../Types/API/Route";
import ColumnAccessControl from "../../Types/Database/AccessControl/ColumnAccessControl";
import TableAccessControl from "../../Types/Database/AccessControl/TableAccessControl";
import TableBillingAccessControl from "../../Types/Database/AccessControl/TableBillingAccessControl";
import ColumnLength from "../../Types/Database/ColumnLength";
import ColumnType from "../../Types/Database/ColumnType";
import CrudApiEndpoint from "../../Types/Database/CrudApiEndpoint";
import EnableDocumentation from "../../Types/Database/EnableDocumentation";
import TableColumn from "../../Types/Database/TableColumn";
import TableColumnType from "../../Types/Database/TableColumnType";
import TableMetadata from "../../Types/Database/TableMetadata";
import TenantColumn from "../../Types/Database/TenantColumn";
import IconProp from "../../Types/Icon/IconProp";
import ObjectID from "../../Types/ObjectID";
import Permission from "../../Types/Permission";
import { PlanType } from "../../Types/Billing/SubscriptionPlan";
import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
@EnableDocumentation()
@TableBillingAccessControl({
create: PlanType.Free,
read: PlanType.Free,
update: PlanType.Free,
delete: PlanType.Free,
})
@TenantColumn("projectId")
@CrudApiEndpoint(new Route("/log-pipeline"))
@Entity({
name: "LogPipeline",
})
@TableMetadata({
tableName: "LogPipeline",
singularName: "Log Pipeline",
pluralName: "Log Pipelines",
icon: IconProp.Logs,
tableDescription:
"Configure server-side log processing pipelines that transform logs at ingest time.",
})
@TableAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
delete: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.DeleteProjectLogPipeline,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipeline,
],
})
export default class LogPipeline extends BaseModel {
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
manyToOneRelationColumn: "projectId",
type: TableColumnType.Entity,
modelType: Project,
title: "Project",
description: "Relation to the project this log pipeline belongs to.",
})
@ManyToOne(
() => {
return Project;
},
{
eager: false,
nullable: true,
onDelete: "CASCADE",
orphanedRowAction: "nullify",
},
)
@JoinColumn({ name: "projectId" })
public project?: Project = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [],
})
@Index()
@TableColumn({
type: TableColumnType.ObjectID,
required: true,
canReadOnRelationQuery: true,
title: "Project ID",
description: "ID of the project this log pipeline belongs to.",
})
@Column({
type: ColumnType.ObjectID,
nullable: false,
transformer: ObjectID.getDatabaseTransformer(),
})
public projectId?: ObjectID = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipeline,
],
})
@TableColumn({
required: true,
type: TableColumnType.Name,
canReadOnRelationQuery: true,
title: "Name",
description: "Friendly name for this log pipeline.",
})
@Column({
nullable: false,
type: ColumnType.Name,
length: ColumnLength.Name,
})
public name?: string = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipeline,
],
})
@TableColumn({
required: false,
type: TableColumnType.LongText,
canReadOnRelationQuery: true,
title: "Description",
description: "Description of what this log pipeline does.",
})
@Column({
nullable: true,
type: ColumnType.LongText,
length: ColumnLength.LongText,
})
public description?: string = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipeline,
],
})
@TableColumn({
required: false,
type: TableColumnType.LongText,
canReadOnRelationQuery: true,
title: "Filter Query",
description:
"Filter expression that determines which logs this pipeline applies to.",
})
@Column({
nullable: true,
type: ColumnType.LongText,
length: ColumnLength.LongText,
})
public filterQuery?: string = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipeline,
],
})
@Index()
@TableColumn({
required: true,
type: TableColumnType.Boolean,
canReadOnRelationQuery: true,
title: "Enabled",
description: "Whether this log pipeline is active.",
defaultValue: true,
})
@Column({
nullable: false,
type: ColumnType.Boolean,
default: true,
})
public isEnabled?: boolean = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipeline,
],
})
@TableColumn({
title: "Sort Order",
required: true,
type: TableColumnType.Number,
canReadOnRelationQuery: true,
description:
"Determines the execution order of this pipeline relative to others.",
defaultValue: 0,
})
@Column({
type: ColumnType.Number,
nullable: false,
default: 0,
})
public sortOrder?: number = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
manyToOneRelationColumn: "createdByUserId",
type: TableColumnType.Entity,
modelType: User,
title: "Created By User",
description: "Relation to the user who created this log pipeline.",
})
@ManyToOne(
() => {
return User;
},
{
eager: false,
nullable: true,
onDelete: "SET NULL",
orphanedRowAction: "nullify",
},
)
@JoinColumn({ name: "createdByUserId" })
public createdByUser?: User = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
type: TableColumnType.ObjectID,
title: "Created By User ID",
description: "ID of the user who created this log pipeline.",
})
@Column({
type: ColumnType.ObjectID,
nullable: true,
transformer: ObjectID.getDatabaseTransformer(),
})
public createdByUserId?: ObjectID = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
manyToOneRelationColumn: "deletedByUserId",
type: TableColumnType.Entity,
modelType: User,
title: "Deleted By User",
description: "Relation to the user who deleted this log pipeline.",
})
@ManyToOne(
() => {
return User;
},
{
eager: false,
nullable: true,
onDelete: "SET NULL",
orphanedRowAction: "nullify",
},
)
@JoinColumn({ name: "deletedByUserId" })
public deletedByUser?: User = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
type: TableColumnType.ObjectID,
title: "Deleted By User ID",
description: "ID of the user who deleted this log pipeline.",
})
@Column({
type: ColumnType.ObjectID,
nullable: true,
transformer: ObjectID.getDatabaseTransformer(),
})
public deletedByUserId?: ObjectID = undefined;
}

View File

@@ -0,0 +1,434 @@
import Project from "./Project";
import User from "./User";
import LogPipeline from "./LogPipeline";
import BaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
import Route from "../../Types/API/Route";
import ColumnAccessControl from "../../Types/Database/AccessControl/ColumnAccessControl";
import TableAccessControl from "../../Types/Database/AccessControl/TableAccessControl";
import TableBillingAccessControl from "../../Types/Database/AccessControl/TableBillingAccessControl";
import ColumnLength from "../../Types/Database/ColumnLength";
import ColumnType from "../../Types/Database/ColumnType";
import CrudApiEndpoint from "../../Types/Database/CrudApiEndpoint";
import EnableDocumentation from "../../Types/Database/EnableDocumentation";
import TableColumn from "../../Types/Database/TableColumn";
import TableColumnType from "../../Types/Database/TableColumnType";
import TableMetadata from "../../Types/Database/TableMetadata";
import TenantColumn from "../../Types/Database/TenantColumn";
import IconProp from "../../Types/Icon/IconProp";
import ObjectID from "../../Types/ObjectID";
import Permission from "../../Types/Permission";
import { PlanType } from "../../Types/Billing/SubscriptionPlan";
import { JSONObject } from "../../Types/JSON";
import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
@EnableDocumentation()
@TableBillingAccessControl({
create: PlanType.Free,
read: PlanType.Free,
update: PlanType.Free,
delete: PlanType.Free,
})
@TenantColumn("projectId")
@CrudApiEndpoint(new Route("/log-pipeline-processor"))
@Entity({
name: "LogPipelineProcessor",
})
@TableMetadata({
tableName: "LogPipelineProcessor",
singularName: "Log Pipeline Processor",
pluralName: "Log Pipeline Processors",
icon: IconProp.Settings,
tableDescription:
"Individual processors within a log pipeline that transform log data during ingestion.",
})
@TableAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipelineProcessor,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipelineProcessor,
Permission.ReadAllProjectResources,
],
delete: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.DeleteProjectLogPipelineProcessor,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipelineProcessor,
],
})
export default class LogPipelineProcessor extends BaseModel {
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipelineProcessor,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipelineProcessor,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
manyToOneRelationColumn: "projectId",
type: TableColumnType.Entity,
modelType: Project,
title: "Project",
description:
"Relation to the project this log pipeline processor belongs to.",
})
@ManyToOne(
() => {
return Project;
},
{
eager: false,
nullable: true,
onDelete: "CASCADE",
orphanedRowAction: "nullify",
},
)
@JoinColumn({ name: "projectId" })
public project?: Project = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipelineProcessor,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipelineProcessor,
Permission.ReadAllProjectResources,
],
update: [],
})
@Index()
@TableColumn({
type: TableColumnType.ObjectID,
required: true,
canReadOnRelationQuery: true,
title: "Project ID",
description:
"ID of the project this log pipeline processor belongs to.",
})
@Column({
type: ColumnType.ObjectID,
nullable: false,
transformer: ObjectID.getDatabaseTransformer(),
})
public projectId?: ObjectID = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipelineProcessor,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipelineProcessor,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
manyToOneRelationColumn: "logPipelineId",
type: TableColumnType.Entity,
modelType: LogPipeline,
title: "Log Pipeline",
description:
"Relation to the log pipeline this processor belongs to.",
})
@ManyToOne(
() => {
return LogPipeline;
},
{
eager: false,
nullable: false,
onDelete: "CASCADE",
orphanedRowAction: "delete",
},
)
@JoinColumn({ name: "logPipelineId" })
public logPipeline?: LogPipeline = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipelineProcessor,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipelineProcessor,
Permission.ReadAllProjectResources,
],
update: [],
})
@Index()
@TableColumn({
type: TableColumnType.ObjectID,
required: true,
canReadOnRelationQuery: true,
title: "Log Pipeline ID",
description:
"ID of the log pipeline this processor belongs to.",
})
@Column({
type: ColumnType.ObjectID,
nullable: false,
transformer: ObjectID.getDatabaseTransformer(),
})
public logPipelineId?: ObjectID = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipelineProcessor,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipelineProcessor,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipelineProcessor,
],
})
@TableColumn({
required: true,
type: TableColumnType.Name,
canReadOnRelationQuery: true,
title: "Name",
description: "Friendly name for this processor.",
})
@Column({
nullable: false,
type: ColumnType.Name,
length: ColumnLength.Name,
})
public name?: string = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipelineProcessor,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipelineProcessor,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipelineProcessor,
],
})
@TableColumn({
required: true,
type: TableColumnType.ShortText,
canReadOnRelationQuery: true,
title: "Processor Type",
description:
"The type of processor: GrokParser, AttributeRemapper, SeverityRemapper, or CategoryProcessor.",
})
@Column({
nullable: false,
type: ColumnType.ShortText,
length: ColumnLength.ShortText,
})
public processorType?: string = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipelineProcessor,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipelineProcessor,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipelineProcessor,
],
})
@TableColumn({
title: "Configuration",
required: true,
type: TableColumnType.JSON,
canReadOnRelationQuery: true,
description:
"Processor-specific configuration as JSON (e.g., grok pattern, source/target fields, mapping rules).",
})
@Column({
type: ColumnType.JSON,
nullable: false,
default: () => {
return "'{}'";
},
})
public configuration?: JSONObject = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipelineProcessor,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipelineProcessor,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipelineProcessor,
],
})
@Index()
@TableColumn({
required: true,
type: TableColumnType.Boolean,
canReadOnRelationQuery: true,
title: "Enabled",
description: "Whether this processor is active.",
defaultValue: true,
})
@Column({
nullable: false,
type: ColumnType.Boolean,
default: true,
})
public isEnabled?: boolean = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipelineProcessor,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipelineProcessor,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipelineProcessor,
],
})
@TableColumn({
title: "Sort Order",
required: true,
type: TableColumnType.Number,
canReadOnRelationQuery: true,
description:
"Determines the execution order of this processor within its pipeline.",
defaultValue: 0,
})
@Column({
type: ColumnType.Number,
nullable: false,
default: 0,
})
public sortOrder?: number = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipelineProcessor,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
manyToOneRelationColumn: "createdByUserId",
type: TableColumnType.Entity,
modelType: User,
title: "Created By User",
description:
"Relation to the user who created this log pipeline processor.",
})
@ManyToOne(
() => {
return User;
},
{
eager: false,
nullable: true,
onDelete: "SET NULL",
orphanedRowAction: "nullify",
},
)
@JoinColumn({ name: "createdByUserId" })
public createdByUser?: User = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipelineProcessor,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
type: TableColumnType.ObjectID,
title: "Created By User ID",
description:
"ID of the user who created this log pipeline processor.",
})
@Column({
type: ColumnType.ObjectID,
nullable: true,
transformer: ObjectID.getDatabaseTransformer(),
})
public createdByUserId?: ObjectID = undefined;
}

View File

@@ -410,6 +410,272 @@ router.post(
},
);
// --- Log Export Endpoint ---
router.post(
"/telemetry/logs/export",
UserMiddleware.getUserMiddleware,
async (
req: ExpressRequest,
res: ExpressResponse,
next: NextFunction,
): Promise<void> => {
try {
const databaseProps: DatabaseCommonInteractionProps =
await CommonAPI.getDatabaseCommonInteractionProps(req);
if (!databaseProps?.tenantId) {
return Response.sendErrorResponse(
req,
res,
new BadDataException("Invalid Project ID"),
);
}
const body: JSONObject = req.body as JSONObject;
const startTime: Date = body["startTime"]
? OneUptimeDate.fromString(body["startTime"] as string)
: OneUptimeDate.addRemoveHours(OneUptimeDate.getCurrentDate(), -1);
const endTime: Date = body["endTime"]
? OneUptimeDate.fromString(body["endTime"] as string)
: OneUptimeDate.getCurrentDate();
const limit: number = Math.min(
(body["limit"] as number) || 10000,
10000,
);
const format: string = (body["format"] as string) || "json";
const serviceIds: Array<ObjectID> | undefined = body["serviceIds"]
? (body["serviceIds"] as Array<string>).map((id: string) => {
return new ObjectID(id);
})
: undefined;
const severityTexts: Array<string> | undefined = body["severityTexts"]
? (body["severityTexts"] as Array<string>)
: undefined;
const bodySearchText: string | undefined = body["bodySearchText"]
? (body["bodySearchText"] as string)
: undefined;
const traceIds: Array<string> | undefined = body["traceIds"]
? (body["traceIds"] as Array<string>)
: undefined;
const spanIds: Array<string> | undefined = body["spanIds"]
? (body["spanIds"] as Array<string>)
: undefined;
const rows: Array<JSONObject> =
await LogAggregationService.getExportLogs({
projectId: databaseProps.tenantId,
startTime,
endTime,
limit,
serviceIds,
severityTexts,
bodySearchText,
traceIds,
spanIds,
});
if (format === "csv") {
const header: string =
"time,serviceId,severityText,severityNumber,body,traceId,spanId,attributes";
const csvRows: Array<string> = rows.map((row: JSONObject) => {
const escapeCsv: (val: unknown) => string = (
val: unknown,
): string => {
const str: string = val === null || val === undefined ? "" : String(val);
if (
str.includes(",") ||
str.includes('"') ||
str.includes("\n")
) {
return `"${str.replace(/"/g, '""')}"`;
}
return str;
};
return [
escapeCsv(row["time"]),
escapeCsv(row["serviceId"]),
escapeCsv(row["severityText"]),
escapeCsv(row["severityNumber"]),
escapeCsv(row["body"]),
escapeCsv(row["traceId"]),
escapeCsv(row["spanId"]),
escapeCsv(JSON.stringify(row["attributes"] || {})),
].join(",");
});
const csv: string = [header, ...csvRows].join("\n");
res.setHeader("Content-Type", "text/csv; charset=utf-8");
res.setHeader(
"Content-Disposition",
"attachment; filename=logs-export.csv",
);
res.status(200).send(csv);
return;
}
// JSON format
res.setHeader("Content-Type", "application/json; charset=utf-8");
res.setHeader(
"Content-Disposition",
"attachment; filename=logs-export.json",
);
res.status(200).send(JSON.stringify(rows, null, 2));
} catch (err: unknown) {
next(err);
}
},
);
// --- Log Context Endpoint ---
router.post(
"/telemetry/logs/context",
UserMiddleware.getUserMiddleware,
async (
req: ExpressRequest,
res: ExpressResponse,
next: NextFunction,
): Promise<void> => {
try {
const databaseProps: DatabaseCommonInteractionProps =
await CommonAPI.getDatabaseCommonInteractionProps(req);
if (!databaseProps?.tenantId) {
return Response.sendErrorResponse(
req,
res,
new BadDataException("Invalid Project ID"),
);
}
const body: JSONObject = req.body as JSONObject;
const logId: string | undefined = body["logId"] as string | undefined;
const serviceId: string | undefined = body["serviceId"] as
| string
| undefined;
const time: string | undefined = body["time"] as string | undefined;
if (!logId || !serviceId || !time) {
return Response.sendErrorResponse(
req,
res,
new BadDataException("logId, serviceId, and time are required"),
);
}
const count: number = (body["count"] as number) || 5;
const result: {
before: Array<JSONObject>;
after: Array<JSONObject>;
} = await LogAggregationService.getLogContext({
projectId: databaseProps.tenantId,
serviceId: new ObjectID(serviceId),
time: OneUptimeDate.fromString(time),
logId,
count,
});
return Response.sendJsonObjectResponse(req, res, {
before: result.before as unknown as JSONObject,
after: result.after as unknown as JSONObject,
});
} catch (err: unknown) {
next(err);
}
},
);
// --- Drop Filter Estimate Endpoint ---
router.post(
"/telemetry/logs/drop-filter-estimate",
UserMiddleware.getUserMiddleware,
async (
req: ExpressRequest,
res: ExpressResponse,
next: NextFunction,
): Promise<void> => {
try {
const databaseProps: DatabaseCommonInteractionProps =
await CommonAPI.getDatabaseCommonInteractionProps(req);
if (!databaseProps?.tenantId) {
return Response.sendErrorResponse(
req,
res,
new BadDataException("Invalid Project ID"),
);
}
const body: JSONObject = req.body as JSONObject;
const filterQuery: string | undefined = body["filterQuery"] as
| string
| undefined;
if (!filterQuery) {
return Response.sendErrorResponse(
req,
res,
new BadDataException("filterQuery is required"),
);
}
const startTime: Date = body["startTime"]
? OneUptimeDate.fromString(body["startTime"] as string)
: OneUptimeDate.addRemoveHours(OneUptimeDate.getCurrentDate(), -24);
const endTime: Date = body["endTime"]
? OneUptimeDate.fromString(body["endTime"] as string)
: OneUptimeDate.getCurrentDate();
const serviceIds: Array<ObjectID> | undefined = body["serviceIds"]
? (body["serviceIds"] as Array<string>).map((id: string) => {
return new ObjectID(id);
})
: undefined;
const severityTexts: Array<string> | undefined = body["severityTexts"]
? (body["severityTexts"] as Array<string>)
: undefined;
const result: {
totalLogs: number;
matchingLogs: number;
estimatedReductionPercent: number;
} = await LogAggregationService.getDropFilterEstimate({
projectId: databaseProps.tenantId,
startTime,
endTime,
filterQuery,
serviceIds,
severityTexts,
});
return Response.sendJsonObjectResponse(req, res, {
totalLogs: result.totalLogs,
matchingLogs: result.matchingLogs,
estimatedReductionPercent: result.estimatedReductionPercent,
} as unknown as JSONObject);
} catch (err: unknown) {
next(err);
}
},
);
// --- Helpers ---
function computeDefaultBucketSize(startTime: Date, endTime: Date): number {

View File

@@ -641,6 +641,243 @@ export class LogAggregationService {
}
}
@CaptureSpan()
public static async getExportLogs(request: {
projectId: ObjectID;
startTime: Date;
endTime: Date;
limit: number;
serviceIds?: Array<ObjectID> | undefined;
severityTexts?: Array<string> | undefined;
bodySearchText?: string | undefined;
traceIds?: Array<string> | undefined;
spanIds?: Array<string> | undefined;
}): Promise<Array<JSONObject>> {
const maxLimit: number = Math.min(request.limit || 10000, 10000);
const statement: Statement = SQL`
SELECT
time,
serviceId,
severityText,
severityNumber,
body,
traceId,
spanId,
attributes
FROM ${LogAggregationService.TABLE_NAME}
WHERE projectId = ${{
type: TableColumnType.ObjectID,
value: request.projectId,
}}
AND time >= ${{
type: TableColumnType.Date,
value: request.startTime,
}}
AND time <= ${{
type: TableColumnType.Date,
value: request.endTime,
}}
`;
LogAggregationService.appendCommonFilters(statement, request);
statement.append(
SQL` ORDER BY time DESC LIMIT ${{
type: TableColumnType.Number,
value: maxLimit,
}}`,
);
const dbResult: Results = await LogDatabaseService.executeQuery(statement);
const response: DbJSONResponse = await dbResult.json<{
data?: Array<JSONObject>;
}>();
return response.data || [];
}
@CaptureSpan()
public static async getLogContext(request: {
projectId: ObjectID;
serviceId: ObjectID;
time: Date;
logId: string;
count: number;
}): Promise<{ before: Array<JSONObject>; after: Array<JSONObject> }> {
const count: number = Math.min(request.count || 5, 20);
const beforeStatement: Statement = SQL`
SELECT
_id,
time,
timeUnixNano,
serviceId,
severityText,
severityNumber,
body,
traceId,
spanId,
attributes
FROM ${LogAggregationService.TABLE_NAME}
WHERE projectId = ${{
type: TableColumnType.ObjectID,
value: request.projectId,
}}
AND serviceId = ${{
type: TableColumnType.ObjectID,
value: request.serviceId,
}}
AND time <= ${{
type: TableColumnType.Date,
value: request.time,
}}
AND _id != ${{
type: TableColumnType.Text,
value: request.logId,
}}
ORDER BY time DESC, timeUnixNano DESC
LIMIT ${{
type: TableColumnType.Number,
value: count,
}}
`;
const afterStatement: Statement = SQL`
SELECT
_id,
time,
timeUnixNano,
serviceId,
severityText,
severityNumber,
body,
traceId,
spanId,
attributes
FROM ${LogAggregationService.TABLE_NAME}
WHERE projectId = ${{
type: TableColumnType.ObjectID,
value: request.projectId,
}}
AND serviceId = ${{
type: TableColumnType.ObjectID,
value: request.serviceId,
}}
AND time >= ${{
type: TableColumnType.Date,
value: request.time,
}}
AND _id != ${{
type: TableColumnType.Text,
value: request.logId,
}}
ORDER BY time ASC, timeUnixNano ASC
LIMIT ${{
type: TableColumnType.Number,
value: count,
}}
`;
const [beforeResult, afterResult] = await Promise.all([
LogDatabaseService.executeQuery(beforeStatement),
LogDatabaseService.executeQuery(afterStatement),
]);
const beforeResponse: DbJSONResponse = await beforeResult.json<{
data?: Array<JSONObject>;
}>();
const afterResponse: DbJSONResponse = await afterResult.json<{
data?: Array<JSONObject>;
}>();
const beforeRows: Array<JSONObject> = (beforeResponse.data || []).reverse();
const afterRows: Array<JSONObject> = afterResponse.data || [];
return { before: beforeRows, after: afterRows };
}
@CaptureSpan()
public static async getDropFilterEstimate(request: {
projectId: ObjectID;
startTime: Date;
endTime: Date;
filterQuery: string;
serviceIds?: Array<ObjectID> | undefined;
severityTexts?: Array<string> | undefined;
bodySearchText?: string | undefined;
}): Promise<{
totalLogs: number;
matchingLogs: number;
estimatedReductionPercent: number;
}> {
// Get total count
const totalStatement: Statement = SQL`
SELECT count() AS cnt
FROM ${LogAggregationService.TABLE_NAME}
WHERE projectId = ${{
type: TableColumnType.ObjectID,
value: request.projectId,
}}
AND time >= ${{
type: TableColumnType.Date,
value: request.startTime,
}}
AND time <= ${{
type: TableColumnType.Date,
value: request.endTime,
}}
`;
LogAggregationService.appendCommonFilters(totalStatement, request);
// Get matching count using the filter query as body search
const matchStatement: Statement = SQL`
SELECT count() AS cnt
FROM ${LogAggregationService.TABLE_NAME}
WHERE projectId = ${{
type: TableColumnType.ObjectID,
value: request.projectId,
}}
AND time >= ${{
type: TableColumnType.Date,
value: request.startTime,
}}
AND time <= ${{
type: TableColumnType.Date,
value: request.endTime,
}}
`;
LogAggregationService.appendCommonFilters(matchStatement, {
...request,
bodySearchText: request.filterQuery,
});
const [totalResult, matchResult] = await Promise.all([
LogDatabaseService.executeQuery(totalStatement),
LogDatabaseService.executeQuery(matchStatement),
]);
const totalResponse: DbJSONResponse = await totalResult.json<{
data?: Array<JSONObject>;
}>();
const matchResponse: DbJSONResponse = await matchResult.json<{
data?: Array<JSONObject>;
}>();
const totalLogs: number = Number(
(totalResponse.data || [])[0]?.["cnt"] || 0,
);
const matchingLogs: number = Number(
(matchResponse.data || [])[0]?.["cnt"] || 0,
);
const estimatedReductionPercent: number =
totalLogs > 0 ? Math.round((matchingLogs / totalLogs) * 100) : 0;
return { totalLogs, matchingLogs, estimatedReductionPercent };
}
private static isTopLevelColumn(key: string): boolean {
return LogAggregationService.TOP_LEVEL_COLUMNS.has(key);
}

View File

@@ -0,0 +1,6 @@
enum LogDropFilterAction {
Drop = "drop",
Sample = "sample",
}
export default LogDropFilterAction;

View File

@@ -0,0 +1,44 @@
enum LogPipelineProcessorType {
GrokParser = "GrokParser",
AttributeRemapper = "AttributeRemapper",
SeverityRemapper = "SeverityRemapper",
CategoryProcessor = "CategoryProcessor",
}
export interface GrokParserConfig {
source: string; // field to parse, e.g. "body"
pattern: string; // grok pattern
targetPrefix?: string; // prefix for extracted attributes
}
export interface AttributeRemapperConfig {
sourceKey: string; // source attribute key
targetKey: string; // target attribute key
preserveSource?: boolean; // keep original key (default false)
overrideOnConflict?: boolean; // overwrite if target exists (default true)
}
export interface SeverityRemapperConfig {
sourceKey: string; // attribute key containing severity info
mappings: Array<{
matchValue: string; // value to match (case-insensitive)
severityText: string; // mapped severity text
severityNumber: number; // mapped severity number
}>;
}
export interface CategoryProcessorConfig {
targetKey: string; // attribute key to store the category
categories: Array<{
name: string; // category name/value
filterQuery: string; // condition to match
}>;
}
export type LogPipelineProcessorConfig =
| GrokParserConfig
| AttributeRemapperConfig
| SeverityRemapperConfig
| CategoryProcessorConfig;
export default LogPipelineProcessorType;

View File

@@ -82,6 +82,24 @@ enum Permission {
EditTelemetryServiceLog = "EditTelemetryServiceLog",
ReadTelemetryServiceLog = "ReadTelemetryServiceLog",
// Log Pipelines
CreateProjectLogPipeline = "CreateProjectLogPipeline",
DeleteProjectLogPipeline = "DeleteProjectLogPipeline",
EditProjectLogPipeline = "EditProjectLogPipeline",
ReadProjectLogPipeline = "ReadProjectLogPipeline",
// Log Pipeline Processors
CreateProjectLogPipelineProcessor = "CreateProjectLogPipelineProcessor",
DeleteProjectLogPipelineProcessor = "DeleteProjectLogPipelineProcessor",
EditProjectLogPipelineProcessor = "EditProjectLogPipelineProcessor",
ReadProjectLogPipelineProcessor = "ReadProjectLogPipelineProcessor",
// Log Drop Filters
CreateProjectLogDropFilter = "CreateProjectLogDropFilter",
DeleteProjectLogDropFilter = "DeleteProjectLogDropFilter",
EditProjectLogDropFilter = "EditProjectLogDropFilter",
ReadProjectLogDropFilter = "ReadProjectLogDropFilter",
// Exceptions
CreateTelemetryException = "CreateTelemetryException",
DeleteTelemetryException = "DeleteTelemetryException",
@@ -3981,6 +3999,120 @@ export class PermissionHelper {
group: PermissionGroup.Telemetry,
},
// Log Pipeline Permissions
{
permission: Permission.CreateProjectLogPipeline,
title: "Create Log Pipeline",
description:
"This permission can create Log Pipelines in this project.",
isAssignableToTenant: true,
isAccessControlPermission: false,
group: PermissionGroup.Telemetry,
},
{
permission: Permission.DeleteProjectLogPipeline,
title: "Delete Log Pipeline",
description:
"This permission can delete Log Pipelines of this project.",
isAssignableToTenant: true,
isAccessControlPermission: false,
group: PermissionGroup.Telemetry,
},
{
permission: Permission.EditProjectLogPipeline,
title: "Edit Log Pipeline",
description:
"This permission can edit Log Pipelines of this project.",
isAssignableToTenant: true,
isAccessControlPermission: false,
group: PermissionGroup.Telemetry,
},
{
permission: Permission.ReadProjectLogPipeline,
title: "Read Log Pipeline",
description:
"This permission can read Log Pipelines of this project.",
isAssignableToTenant: true,
isAccessControlPermission: false,
group: PermissionGroup.Telemetry,
},
// Log Pipeline Processor Permissions
{
permission: Permission.CreateProjectLogPipelineProcessor,
title: "Create Log Pipeline Processor",
description:
"This permission can create Log Pipeline Processors in this project.",
isAssignableToTenant: true,
isAccessControlPermission: false,
group: PermissionGroup.Telemetry,
},
{
permission: Permission.DeleteProjectLogPipelineProcessor,
title: "Delete Log Pipeline Processor",
description:
"This permission can delete Log Pipeline Processors of this project.",
isAssignableToTenant: true,
isAccessControlPermission: false,
group: PermissionGroup.Telemetry,
},
{
permission: Permission.EditProjectLogPipelineProcessor,
title: "Edit Log Pipeline Processor",
description:
"This permission can edit Log Pipeline Processors of this project.",
isAssignableToTenant: true,
isAccessControlPermission: false,
group: PermissionGroup.Telemetry,
},
{
permission: Permission.ReadProjectLogPipelineProcessor,
title: "Read Log Pipeline Processor",
description:
"This permission can read Log Pipeline Processors of this project.",
isAssignableToTenant: true,
isAccessControlPermission: false,
group: PermissionGroup.Telemetry,
},
// Log Drop Filter Permissions
{
permission: Permission.CreateProjectLogDropFilter,
title: "Create Log Drop Filter",
description:
"This permission can create Log Drop Filters in this project.",
isAssignableToTenant: true,
isAccessControlPermission: false,
group: PermissionGroup.Telemetry,
},
{
permission: Permission.DeleteProjectLogDropFilter,
title: "Delete Log Drop Filter",
description:
"This permission can delete Log Drop Filters of this project.",
isAssignableToTenant: true,
isAccessControlPermission: false,
group: PermissionGroup.Telemetry,
},
{
permission: Permission.EditProjectLogDropFilter,
title: "Edit Log Drop Filter",
description:
"This permission can edit Log Drop Filters of this project.",
isAssignableToTenant: true,
isAccessControlPermission: false,
group: PermissionGroup.Telemetry,
},
{
permission: Permission.ReadProjectLogDropFilter,
title: "Read Log Drop Filter",
description:
"This permission can read Log Drop Filters of this project.",
isAssignableToTenant: true,
isAccessControlPermission: false,
group: PermissionGroup.Telemetry,
},
{
permission: Permission.CreateTelemetryException,
title: "Create Telemetry Service Exception",

View File

@@ -54,6 +54,8 @@ import LogsAnalyticsView from "./components/LogsAnalyticsView";
import { queryStringToFilter } from "../../../Types/Log/LogQueryToFilter";
import RangeStartAndEndDateTime from "../../../Types/Time/RangeStartAndEndDateTime";
import TimeRange from "../../../Types/Time/TimeRange";
import { exportLogs, LogExportFormat } from "../../Utils/LogExport";
import ObjectID from "../../../Types/ObjectID";
export interface ComponentProps {
logs: Array<Log>;
@@ -64,6 +66,7 @@ export interface ComponentProps {
noLogsMessage?: string | undefined;
getTraceRoute?: (traceId: string, log: Log) => Route | URL | undefined;
getSpanRoute?: (spanId: string, log: Log) => Route | URL | undefined;
projectId?: ObjectID | undefined;
totalCount?: number | undefined;
page?: number | undefined;
pageSize?: number | undefined;
@@ -655,6 +658,12 @@ const LogsViewer: FunctionComponent<ComponentProps> = (
},
}
: {}),
onExportCSV: () => {
exportLogs(displayedLogs, LogExportFormat.CSV, selectedColumns);
},
onExportJSON: () => {
exportLogs(displayedLogs, LogExportFormat.JSON, selectedColumns);
},
...(props.liveOptions ? { liveOptions: props.liveOptions } : {}),
...(props.timeRange && props.onTimeRangeChange
? {
@@ -766,6 +775,7 @@ const LogsViewer: FunctionComponent<ComponentProps> = (
getTraceRoute={props.getTraceRoute}
getSpanRoute={props.getSpanRoute}
variant="embedded"
projectId={props.projectId}
/>
);
}}

View File

@@ -1,4 +1,11 @@
import React, { FunctionComponent, ReactElement, useMemo } from "react";
import React, {
FunctionComponent,
ReactElement,
useCallback,
useEffect,
useMemo,
useState,
} from "react";
import Log from "../../../../Models/AnalyticsModels/Log";
import Service from "../../../../Models/DatabaseModels/Service";
import Dictionary from "../../../../Types/Dictionary";
@@ -11,6 +18,15 @@ import Link from "../../Link/Link";
import OneUptimeDate from "../../../../Types/Date";
import JSONFunctions from "../../../../Types/JSONFunctions";
import SeverityBadge from "./SeverityBadge";
import { JSONObject } from "../../../../Types/JSON";
import API from "../../../Utils/API/API";
import ModelAPI from "../../../Utils/ModelAPI/ModelAPI";
import { APP_API_URL } from "../../../Config";
import HTTPResponse from "../../../../Types/API/HTTPResponse";
import HTTPErrorResponse from "../../../../Types/API/HTTPErrorResponse";
import ObjectID from "../../../../Types/ObjectID";
type LogDetailTab = "details" | "context";
export interface LogDetailsPanelProps {
log: Log;
@@ -23,6 +39,8 @@ export interface LogDetailsPanelProps {
| ((spanId: string, log: Log) => Route | URL | undefined)
| undefined;
variant?: "floating" | "embedded";
projectId?: ObjectID | undefined;
onLogSelect?: ((log: Log) => void) | undefined;
}
interface PreparedBody {
@@ -32,6 +50,14 @@ interface PreparedBody {
raw: string;
}
interface ContextLog {
id: string;
time: string;
severity: string;
body: string;
serviceId: string;
}
const prepareBody: (body: string | undefined) => PreparedBody = (
body: string | undefined,
): PreparedBody => {
@@ -45,7 +71,7 @@ const prepareBody: (body: string | undefined) => PreparedBody = (
}
try {
const parsed: any = JSON.parse(body);
const parsed: unknown = JSON.parse(body);
const pretty: string = JSON.stringify(parsed, null, 2);
const compact: string = JSON.stringify(parsed);
return {
@@ -64,9 +90,26 @@ const prepareBody: (body: string | undefined) => PreparedBody = (
}
};
function parseContextRow(row: JSONObject): ContextLog {
return {
id: String(row["_id"] || ""),
time: String(row["time"] || ""),
severity: String(row["severityText"] || "Unspecified"),
body: String(row["body"] || ""),
serviceId: String(row["serviceId"] || ""),
};
}
const LogDetailsPanel: FunctionComponent<LogDetailsPanelProps> = (
props: LogDetailsPanelProps,
): ReactElement => {
const [activeTab, setActiveTab] = useState<LogDetailTab>("details");
const [contextBefore, setContextBefore] = useState<Array<ContextLog>>([]);
const [contextAfter, setContextAfter] = useState<Array<ContextLog>>([]);
const [contextLoading, setContextLoading] = useState<boolean>(false);
const [contextError, setContextError] = useState<string>("");
const [contextLoaded, setContextLoaded] = useState<boolean>(false);
const variant: "floating" | "embedded" = props.variant || "floating";
const serviceId: string = props.log.serviceId?.toString() || "";
const service: Service | undefined = props.serviceMap[serviceId];
@@ -84,9 +127,8 @@ const LogDetailsPanel: FunctionComponent<LogDetailsPanelProps> = (
}
try {
const normalized: Record<string, unknown> = JSONFunctions.unflattenObject(
props.log.attributes || {},
);
const normalized: Record<string, unknown> =
JSONFunctions.unflattenObject(props.log.attributes || {});
return JSON.stringify(normalized, null, 2);
} catch {
return null;
@@ -137,6 +179,70 @@ const LogDetailsPanel: FunctionComponent<LogDetailsPanelProps> = (
return undefined;
}, [spanId, props, traceId]);
const loadContext: () => Promise<void> = useCallback(async (): Promise<void> => {
if (!props.projectId || !serviceId || !props.log.time) {
setContextError("Missing project or service information for context.");
return;
}
try {
setContextLoading(true);
setContextError("");
const response: HTTPResponse<JSONObject> | HTTPErrorResponse =
await API.post({
url: URL.fromString(APP_API_URL.toString()).addRoute(
"/telemetry/logs/context",
),
data: {
logId: props.log.getColumnValue("_id")?.toString() || "",
serviceId: serviceId,
time: props.log.time
? OneUptimeDate.toString(props.log.time)
: "",
count: 5,
},
headers: {
...ModelAPI.getCommonHeaders(),
},
});
if (response instanceof HTTPErrorResponse) {
throw response;
}
const before: Array<JSONObject> =
(response.data["before"] as Array<JSONObject>) || [];
const after: Array<JSONObject> =
(response.data["after"] as Array<JSONObject>) || [];
setContextBefore(before.map(parseContextRow));
setContextAfter(after.map(parseContextRow));
setContextLoaded(true);
} catch (err) {
setContextError(
`Failed to load log context. ${API.getFriendlyErrorMessage(err as Error)}`,
);
} finally {
setContextLoading(false);
}
}, [props.projectId, serviceId, props.log]);
useEffect(() => {
if (activeTab === "context" && !contextLoaded && !contextLoading) {
void loadContext();
}
}, [activeTab, contextLoaded, contextLoading, loadContext]);
// Reset context when log changes
useEffect(() => {
setContextLoaded(false);
setContextBefore([]);
setContextAfter([]);
setContextError("");
setActiveTab("details");
}, [props.log]);
const containerClassName: string =
variant === "embedded"
? "rounded-lg border border-gray-200 bg-white p-5 shadow-sm"
@@ -149,6 +255,47 @@ const LogDetailsPanel: FunctionComponent<LogDetailsPanelProps> = (
const smallBadgeClass: string =
"inline-flex items-center gap-1 rounded-full border border-gray-200 bg-gray-50 px-2 py-1 text-[11px] font-mono uppercase tracking-wide text-gray-600";
const tabClass: (isActive: boolean) => string = (
isActive: boolean,
): string => {
return `px-3 py-1.5 text-xs font-medium rounded-md transition-colors ${
isActive
? "bg-indigo-50 text-indigo-700 border border-indigo-200"
: "text-gray-600 hover:text-gray-800 hover:bg-gray-50 border border-transparent"
}`;
};
const renderContextLogRow: (
ctxLog: ContextLog,
isCurrent: boolean,
) => ReactElement = (
ctxLog: ContextLog,
isCurrent: boolean,
): ReactElement => {
const rowClass: string = isCurrent
? "border-l-2 border-l-indigo-500 bg-indigo-50"
: "border-l-2 border-l-transparent hover:bg-gray-50";
return (
<div
key={ctxLog.id || ctxLog.time}
className={`flex items-start gap-3 px-3 py-2 ${rowClass}`}
>
<span className="flex-none whitespace-nowrap font-mono text-[11px] text-gray-400">
{ctxLog.time
? OneUptimeDate.getDateAsUserFriendlyFormattedString(
new Date(ctxLog.time),
)
: "-"}
</span>
<SeverityBadge severity={ctxLog.severity} />
<span className="min-w-0 flex-1 truncate font-mono text-xs text-gray-700">
{ctxLog.body.slice(0, 200) || "-"}
</span>
</div>
);
};
return (
<div className={containerClassName}>
<div
@@ -204,134 +351,204 @@ const LogDetailsPanel: FunctionComponent<LogDetailsPanelProps> = (
)}
</div>
<div className="mt-4 space-y-5 text-sm text-gray-700">
<section className="space-y-3">
<header className="flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
<span>Log Body</span>
<CopyTextButton
textToBeCopied={bodyDetails.raw}
size="xs"
variant="ghost"
iconOnly={false}
title="Copy log body"
/>
</header>
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
{bodyDetails.isJson ? (
<pre className="max-h-80 overflow-auto whitespace-pre-wrap break-words font-mono text-[13px] leading-6 text-gray-800">
{bodyDetails.pretty}
</pre>
) : (
<p className="whitespace-pre-wrap break-words font-mono text-[13px] leading-6 text-gray-800">
{bodyDetails.pretty || "-"}
</p>
)}
</div>
</section>
{(traceId || spanId) && (
<section className="grid gap-4 md:grid-cols-2">
{traceId && (
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
<div className="mb-2 flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
<span>Trace ID</span>
<CopyTextButton
textToBeCopied={traceId}
size="xs"
variant="ghost"
iconOnly={true}
title="Copy trace id"
/>
</div>
<div className="flex items-center justify-between gap-2">
{traceRoute ? (
<Link
to={traceRoute}
className="max-w-full truncate font-mono text-xs text-indigo-600 hover:text-indigo-500"
title={`View trace ${traceId}`}
>
{traceId}
</Link>
) : (
<span
className="max-w-full truncate font-mono text-xs text-gray-700"
title={traceId}
>
{traceId}
</span>
)}
{traceRoute && (
<Icon
icon={IconProp.ExternalLink}
className="h-4 w-4 flex-none text-indigo-400"
/>
)}
</div>
</div>
)}
{spanId && (
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
<div className="mb-2 flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
<span>Span ID</span>
<CopyTextButton
textToBeCopied={spanId}
size="xs"
variant="ghost"
iconOnly={true}
title="Copy span id"
/>
</div>
<div className="flex items-center justify-between gap-2">
{spanRoute ? (
<Link
to={spanRoute}
className="max-w-full truncate font-mono text-xs text-indigo-600 hover:text-indigo-500"
title={`View span ${spanId}`}
>
{spanId}
</Link>
) : (
<span
className="max-w-full truncate font-mono text-xs text-gray-700"
title={spanId}
>
{spanId}
</span>
)}
{spanRoute && (
<Icon
icon={IconProp.ExternalLink}
className="h-4 w-4 flex-none text-indigo-400"
/>
)}
</div>
</div>
)}
</section>
{/* Tab bar */}
<div className="mt-3 flex items-center gap-1">
<button
type="button"
className={tabClass(activeTab === "details")}
onClick={() => {
setActiveTab("details");
}}
>
Details
</button>
{props.projectId && (
<button
type="button"
className={tabClass(activeTab === "context")}
onClick={() => {
setActiveTab("context");
}}
>
Context
</button>
)}
</div>
{prettyAttributes && (
{/* Tab content */}
{activeTab === "details" && (
<div className="mt-4 space-y-5 text-sm text-gray-700">
<section className="space-y-3">
<header className="flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
<span>Attributes</span>
<span>Log Body</span>
<CopyTextButton
textToBeCopied={prettyAttributes}
textToBeCopied={bodyDetails.raw}
size="xs"
variant="ghost"
iconOnly={false}
title="Copy attributes"
title="Copy log body"
/>
</header>
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
<pre className="max-h-72 overflow-auto whitespace-pre-wrap break-words font-mono text-[13px] leading-6 text-gray-800">
{prettyAttributes}
</pre>
{bodyDetails.isJson ? (
<pre className="max-h-80 overflow-auto whitespace-pre-wrap break-words font-mono text-[13px] leading-6 text-gray-800">
{bodyDetails.pretty}
</pre>
) : (
<p className="whitespace-pre-wrap break-words font-mono text-[13px] leading-6 text-gray-800">
{bodyDetails.pretty || "-"}
</p>
)}
</div>
</section>
)}
</div>
{(traceId || spanId) && (
<section className="grid gap-4 md:grid-cols-2">
{traceId && (
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
<div className="mb-2 flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
<span>Trace ID</span>
<CopyTextButton
textToBeCopied={traceId}
size="xs"
variant="ghost"
iconOnly={true}
title="Copy trace id"
/>
</div>
<div className="flex items-center justify-between gap-2">
{traceRoute ? (
<Link
to={traceRoute}
className="max-w-full truncate font-mono text-xs text-indigo-600 hover:text-indigo-500"
title={`View trace ${traceId}`}
>
{traceId}
</Link>
) : (
<span
className="max-w-full truncate font-mono text-xs text-gray-700"
title={traceId}
>
{traceId}
</span>
)}
{traceRoute && (
<Icon
icon={IconProp.ExternalLink}
className="h-4 w-4 flex-none text-indigo-400"
/>
)}
</div>
</div>
)}
{spanId && (
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
<div className="mb-2 flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
<span>Span ID</span>
<CopyTextButton
textToBeCopied={spanId}
size="xs"
variant="ghost"
iconOnly={true}
title="Copy span id"
/>
</div>
<div className="flex items-center justify-between gap-2">
{spanRoute ? (
<Link
to={spanRoute}
className="max-w-full truncate font-mono text-xs text-indigo-600 hover:text-indigo-500"
title={`View span ${spanId}`}
>
{spanId}
</Link>
) : (
<span
className="max-w-full truncate font-mono text-xs text-gray-700"
title={spanId}
>
{spanId}
</span>
)}
{spanRoute && (
<Icon
icon={IconProp.ExternalLink}
className="h-4 w-4 flex-none text-indigo-400"
/>
)}
</div>
</div>
)}
</section>
)}
{prettyAttributes && (
<section className="space-y-3">
<header className="flex items-center justify-between text-[11px] uppercase tracking-wide text-gray-400">
<span>Attributes</span>
<CopyTextButton
textToBeCopied={prettyAttributes}
size="xs"
variant="ghost"
iconOnly={false}
title="Copy attributes"
/>
</header>
<div className={`rounded-lg border ${surfaceCardClass} p-4`}>
<pre className="max-h-72 overflow-auto whitespace-pre-wrap break-words font-mono text-[13px] leading-6 text-gray-800">
{prettyAttributes}
</pre>
</div>
</section>
)}
</div>
)}
{activeTab === "context" && (
<div className="mt-4 text-sm text-gray-700">
{contextLoading && (
<div className="flex items-center justify-center py-8 text-xs text-gray-400">
Loading surrounding logs...
</div>
)}
{contextError && (
<div className="rounded-md bg-red-50 p-3 text-xs text-red-600">
{contextError}
</div>
)}
{!contextLoading && !contextError && contextLoaded && (
<div className="divide-y divide-gray-100 rounded-lg border border-gray-200">
{contextBefore.length === 0 && contextAfter.length === 0 && (
<div className="px-3 py-6 text-center text-xs text-gray-400">
No surrounding logs found for this service.
</div>
)}
{contextBefore.map((ctxLog: ContextLog) => {
return renderContextLogRow(ctxLog, false);
})}
{renderContextLogRow(
{
id: props.log.getColumnValue("_id")?.toString() || "current",
time: props.log.time
? props.log.time.toString()
: "",
severity:
props.log.severityText?.toString() || "Unspecified",
body: props.log.body || "",
serviceId: serviceId,
},
true,
)}
{contextAfter.map((ctxLog: ContextLog) => {
return renderContextLogRow(ctxLog, false);
})}
</div>
)}
</div>
)}
</div>
);
};

View File

@@ -1,4 +1,4 @@
import React, { FunctionComponent, ReactElement } from "react";
import React, { FunctionComponent, ReactElement, useRef, useState } from "react";
import LiveLogsToggle from "./LiveLogsToggle";
import LogTimeRangePicker from "./LogTimeRangePicker";
import ColumnSelector from "./ColumnSelector";
@@ -10,6 +10,7 @@ import {
LogsViewMode,
} from "../types";
import RangeStartAndEndDateTime from "../../../../Types/Time/RangeStartAndEndDateTime";
import useComponentOutsideClick from "../../../Types/UseComponentOutsideClick";
export interface LogsViewerToolbarProps {
resultCount: number;
@@ -31,6 +32,8 @@ export interface LogsViewerToolbarProps {
onSelectedColumnsChange?: ((columns: Array<string>) => void) | undefined;
viewMode?: LogsViewMode | undefined;
onViewModeChange?: ((mode: LogsViewMode) => void) | undefined;
onExportCSV?: (() => void) | undefined;
onExportJSON?: (() => void) | undefined;
}
const LogsViewerToolbar: FunctionComponent<LogsViewerToolbarProps> = (
@@ -41,6 +44,15 @@ const LogsViewerToolbar: FunctionComponent<LogsViewerToolbarProps> = (
currentPage && totalPages && totalPages > 0,
);
const exportDropdownRef: React.RefObject<HTMLDivElement | null> =
useRef<HTMLDivElement | null>(null);
const [isExportOpen, setIsExportOpen] = useState<boolean>(false);
useComponentOutsideClick(exportDropdownRef, () => {
setIsExportOpen(false);
});
const showExport: boolean = Boolean(props.onExportCSV || props.onExportJSON);
return (
<div
className={`flex items-center justify-between gap-3 ${props.className || ""}`}
@@ -141,6 +153,74 @@ const LogsViewerToolbar: FunctionComponent<LogsViewerToolbarProps> = (
/>
)}
{showExport && (
<div className="relative" ref={exportDropdownRef}>
<button
type="button"
className="inline-flex items-center gap-1.5 rounded-md border border-gray-200 bg-white px-2.5 py-1.5 text-xs font-medium text-gray-700 shadow-sm transition-colors hover:border-gray-300 hover:bg-gray-50"
onClick={() => {
setIsExportOpen(!isExportOpen);
}}
>
<svg
className="h-3.5 w-3.5"
fill="none"
viewBox="0 0 24 24"
strokeWidth={1.5}
stroke="currentColor"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
d="M3 16.5v2.25A2.25 2.25 0 0 0 5.25 21h13.5A2.25 2.25 0 0 0 21 18.75V16.5M16.5 12 12 16.5m0 0L7.5 12m4.5 4.5V3"
/>
</svg>
Export
<svg
className="h-3 w-3"
fill="none"
viewBox="0 0 24 24"
strokeWidth={2}
stroke="currentColor"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
d="m19.5 8.25-7.5 7.5-7.5-7.5"
/>
</svg>
</button>
{isExportOpen && (
<div className="absolute right-0 z-20 mt-1 w-40 rounded-md border border-gray-200 bg-white py-1 shadow-lg">
{props.onExportCSV && (
<button
type="button"
className="flex w-full items-center gap-2 px-3 py-1.5 text-left text-xs text-gray-700 hover:bg-gray-50"
onClick={() => {
setIsExportOpen(false);
props.onExportCSV!();
}}
>
Export as CSV
</button>
)}
{props.onExportJSON && (
<button
type="button"
className="flex w-full items-center gap-2 px-3 py-1.5 text-left text-xs text-gray-700 hover:bg-gray-50"
onClick={() => {
setIsExportOpen(false);
props.onExportJSON!();
}}
>
Export as JSON
</button>
)}
</div>
)}
</div>
)}
{props.timeRange && props.onTimeRangeChange && (
<LogTimeRangePicker
value={props.timeRange}

View File

@@ -0,0 +1,160 @@
import Log from "../../Models/AnalyticsModels/Log";
import { JSONObject } from "../../Types/JSON";
import {
isLogsAttributeColumnId,
getLogsAttributeKeyFromColumnId,
} from "../Components/LogsViewer/types";
export enum LogExportFormat {
CSV = "csv",
JSON = "json",
}
function escapeCsvValue(value: string): string {
if (
value.includes(",") ||
value.includes('"') ||
value.includes("\n") ||
value.includes("\r")
) {
return `"${value.replace(/"/g, '""')}"`;
}
return value;
}
function getLogFieldValue(log: Log, columnId: string): string {
if (isLogsAttributeColumnId(columnId)) {
const attrKey: string | null = getLogsAttributeKeyFromColumnId(columnId);
if (attrKey && log.attributes) {
const val: unknown = (log.attributes as Record<string, unknown>)[attrKey];
if (val === undefined || val === null) {
return "";
}
if (typeof val === "object") {
return JSON.stringify(val);
}
return String(val);
}
return "";
}
switch (columnId) {
case "time":
return log.time ? new Date(log.time).toISOString() : "";
case "service":
return log.serviceId?.toString() || "";
case "severity":
return log.severityText?.toString() || "";
case "message":
return log.body || "";
case "traceId":
return log.traceId || "";
case "spanId":
return log.spanId || "";
default:
return "";
}
}
function getColumnLabel(columnId: string): string {
if (isLogsAttributeColumnId(columnId)) {
return getLogsAttributeKeyFromColumnId(columnId) || columnId;
}
switch (columnId) {
case "time":
return "Time";
case "service":
return "Service ID";
case "severity":
return "Severity";
case "message":
return "Message";
case "traceId":
return "Trace ID";
case "spanId":
return "Span ID";
default:
return columnId;
}
}
export function exportLogsToCSV(
logs: Array<Log>,
columns: Array<string>,
): string {
const header: string = columns
.map((col: string) => {
return escapeCsvValue(getColumnLabel(col));
})
.join(",");
const rows: Array<string> = logs.map((log: Log) => {
return columns
.map((col: string) => {
return escapeCsvValue(getLogFieldValue(log, col));
})
.join(",");
});
return [header, ...rows].join("\n");
}
export function exportLogsToJSON(logs: Array<Log>): string {
const data: Array<JSONObject> = logs.map((log: Log) => {
const obj: JSONObject = {};
obj["time"] = log.time ? new Date(log.time).toISOString() : null;
obj["serviceId"] = log.serviceId?.toString() || null;
obj["severity"] = log.severityText?.toString() || null;
obj["severityNumber"] = log.severityNumber || null;
obj["body"] = log.body || null;
obj["traceId"] = log.traceId || null;
obj["spanId"] = log.spanId || null;
obj["attributes"] = (log.attributes as JSONObject) || {};
return obj;
});
return JSON.stringify(data, null, 2);
}
function downloadFile(
content: string,
filename: string,
mimeType: string,
): void {
const blob: Blob = new Blob([content], { type: mimeType });
const url: string = window.URL.createObjectURL(blob);
const anchor: HTMLAnchorElement = document.createElement("a");
anchor.href = url;
anchor.download = filename;
document.body.appendChild(anchor);
anchor.click();
document.body.removeChild(anchor);
window.URL.revokeObjectURL(url);
}
function getTimestampFilename(format: LogExportFormat): string {
const now: string = new Date()
.toISOString()
.replace(/[:.]/g, "-")
.slice(0, 19);
return `logs-${now}.${format}`;
}
export function exportLogs(
logs: Array<Log>,
format: LogExportFormat,
columns: Array<string>,
): void {
if (format === LogExportFormat.CSV) {
const csv: string = exportLogsToCSV(logs, columns);
downloadFile(csv, getTimestampFilename(format), "text/csv;charset=utf-8;");
} else {
const json: string = exportLogsToJSON(logs);
downloadFile(
json,
getTimestampFilename(format),
"application/json;charset=utf-8;",
);
}
}

View File

@@ -0,0 +1,89 @@
import LogDropFilter from "Common/Models/DatabaseModels/LogDropFilter";
import DatabaseService from "Common/Server/Services/DatabaseService";
import SortOrder from "Common/Types/BaseDatabase/SortOrder";
import LIMIT_MAX from "Common/Types/Database/LimitMax";
import ObjectID from "Common/Types/ObjectID";
import { JSONObject } from "Common/Types/JSON";
import LogDropFilterAction from "Common/Types/Log/LogDropFilterAction";
import { evaluateFilter } from "../Utils/LogFilterEvaluator";
interface CacheEntry {
filters: Array<LogDropFilter>;
loadedAt: number;
}
const CACHE_TTL_MS: number = 60 * 1000; // 60 seconds
const dropFilterCache: Map<string, CacheEntry> = new Map();
export class LogDropFilterService {
public static async loadDropFilters(
projectId: ObjectID,
): Promise<Array<LogDropFilter>> {
const cacheKey: string = projectId.toString();
const cached: CacheEntry | undefined = dropFilterCache.get(cacheKey);
if (cached && Date.now() - cached.loadedAt < CACHE_TTL_MS) {
return cached.filters;
}
const service: DatabaseService<LogDropFilter> =
new DatabaseService<LogDropFilter>(LogDropFilter);
const filters: Array<LogDropFilter> = await service.findBy({
query: {
projectId: projectId,
isEnabled: true,
},
skip: 0,
limit: LIMIT_MAX,
sort: {
sortOrder: SortOrder.Ascending,
},
select: {
_id: true,
name: true,
filterQuery: true,
action: true,
samplePercentage: true,
sortOrder: true,
},
props: {
isRoot: true,
},
});
dropFilterCache.set(cacheKey, { filters, loadedAt: Date.now() });
return filters;
}
public static shouldDropLog(
logRow: JSONObject,
filters: Array<LogDropFilter>,
): boolean {
for (const filter of filters) {
const filterQuery: string = (filter.filterQuery as string) || "";
if (!evaluateFilter(logRow, filterQuery)) {
continue;
}
// Filter matches this log
if (filter.action === LogDropFilterAction.Drop) {
return true;
}
if (filter.action === LogDropFilterAction.Sample) {
const samplePercentage: number = filter.samplePercentage || 50;
// Keep samplePercentage% of logs, drop the rest
if (Math.random() * 100 >= samplePercentage) {
return true; // drop this log
}
}
}
return false;
}
}
export default LogDropFilterService;

View File

@@ -0,0 +1,232 @@
import LogPipeline from "Common/Models/DatabaseModels/LogPipeline";
import LogPipelineProcessor from "Common/Models/DatabaseModels/LogPipelineProcessor";
import DatabaseService from "Common/Server/Services/DatabaseService";
import SortOrder from "Common/Types/BaseDatabase/SortOrder";
import LIMIT_MAX from "Common/Types/Database/LimitMax";
import ObjectID from "Common/Types/ObjectID";
import { JSONObject } from "Common/Types/JSON";
import LogPipelineProcessorType, {
AttributeRemapperConfig,
SeverityRemapperConfig,
CategoryProcessorConfig,
} from "Common/Types/Log/LogPipelineProcessorType";
import { evaluateFilter } from "../Utils/LogFilterEvaluator";
import logger from "Common/Server/Utils/Logger";
export interface LoadedPipeline {
pipeline: LogPipeline;
processors: Array<LogPipelineProcessor>;
}
interface CacheEntry {
pipelines: Array<LoadedPipeline>;
loadedAt: number;
}
const CACHE_TTL_MS: number = 60 * 1000; // 60 seconds
const pipelineCache: Map<string, CacheEntry> = new Map();
export class LogPipelineService {
public static async loadPipelines(
projectId: ObjectID,
): Promise<Array<LoadedPipeline>> {
const cacheKey: string = projectId.toString();
const cached: CacheEntry | undefined = pipelineCache.get(cacheKey);
if (cached && Date.now() - cached.loadedAt < CACHE_TTL_MS) {
return cached.pipelines;
}
const pipelineService: DatabaseService<LogPipeline> =
new DatabaseService<LogPipeline>(LogPipeline);
const pipelines: Array<LogPipeline> = await pipelineService.findBy({
query: {
projectId: projectId,
isEnabled: true,
},
skip: 0,
limit: LIMIT_MAX,
sort: {
sortOrder: SortOrder.Ascending,
},
select: {
_id: true,
name: true,
filterQuery: true,
sortOrder: true,
},
props: {
isRoot: true,
},
});
const processorService: DatabaseService<LogPipelineProcessor> =
new DatabaseService<LogPipelineProcessor>(LogPipelineProcessor);
const loaded: Array<LoadedPipeline> = [];
for (const pipeline of pipelines) {
const processors: Array<LogPipelineProcessor> =
await processorService.findBy({
query: {
logPipelineId: pipeline._id,
isEnabled: true,
},
skip: 0,
limit: LIMIT_MAX,
sort: {
sortOrder: SortOrder.Ascending,
},
select: {
_id: true,
name: true,
processorType: true,
configuration: true,
sortOrder: true,
},
props: {
isRoot: true,
},
});
loaded.push({ pipeline, processors });
}
pipelineCache.set(cacheKey, { pipelines: loaded, loadedAt: Date.now() });
return loaded;
}
public static processLog(
logRow: JSONObject,
pipelines: Array<LoadedPipeline>,
): JSONObject {
let result: JSONObject = { ...logRow };
for (const { pipeline, processors } of pipelines) {
// Check if this pipeline's filter matches the log
const filterQuery: string = (pipeline.filterQuery as string) || "";
if (!evaluateFilter(result, filterQuery)) {
continue;
}
// Apply each processor in order
for (const processor of processors) {
try {
result = LogPipelineService.applyProcessor(result, processor);
} catch (err) {
logger.error(
`Error applying processor "${processor.name}" in pipeline "${pipeline.name}": ${err}`,
);
}
}
}
return result;
}
private static applyProcessor(
logRow: JSONObject,
processor: LogPipelineProcessor,
): JSONObject {
const config: JSONObject =
(processor.configuration as JSONObject) || {};
switch (processor.processorType) {
case LogPipelineProcessorType.AttributeRemapper:
return LogPipelineService.applyAttributeRemapper(
logRow,
config as unknown as AttributeRemapperConfig,
);
case LogPipelineProcessorType.SeverityRemapper:
return LogPipelineService.applySeverityRemapper(
logRow,
config as unknown as SeverityRemapperConfig,
);
case LogPipelineProcessorType.CategoryProcessor:
return LogPipelineService.applyCategoryProcessor(
logRow,
config as unknown as CategoryProcessorConfig,
);
default:
return logRow;
}
}
private static applyAttributeRemapper(
logRow: JSONObject,
config: AttributeRemapperConfig,
): JSONObject {
const attrs: Record<string, unknown> = {
...((logRow["attributes"] as Record<string, unknown>) || {}),
};
const sourceVal: unknown = attrs[config.sourceKey];
if (sourceVal === undefined) {
return logRow;
}
const overrideOnConflict: boolean = config.overrideOnConflict !== false;
if (!overrideOnConflict && attrs[config.targetKey] !== undefined) {
return logRow;
}
attrs[config.targetKey] = sourceVal;
if (!config.preserveSource) {
delete attrs[config.sourceKey];
}
// Update attributeKeys
const attributeKeys: Array<string> = Object.keys(attrs);
return { ...logRow, attributes: attrs, attributeKeys };
}
private static applySeverityRemapper(
logRow: JSONObject,
config: SeverityRemapperConfig,
): JSONObject {
const attrs: Record<string, unknown> =
(logRow["attributes"] as Record<string, unknown>) || {};
const sourceVal: unknown = attrs[config.sourceKey];
if (sourceVal === undefined || sourceVal === null) {
return logRow;
}
const sourceStr: string = String(sourceVal).toLowerCase();
for (const mapping of config.mappings || []) {
if (mapping.matchValue.toLowerCase() === sourceStr) {
return {
...logRow,
severityText: mapping.severityText,
severityNumber: mapping.severityNumber,
};
}
}
return logRow;
}
private static applyCategoryProcessor(
logRow: JSONObject,
config: CategoryProcessorConfig,
): JSONObject {
for (const category of config.categories || []) {
if (evaluateFilter(logRow, category.filterQuery)) {
const attrs: Record<string, unknown> = {
...((logRow["attributes"] as Record<string, unknown>) || {}),
};
attrs[config.targetKey] = category.name;
const attributeKeys: Array<string> = Object.keys(attrs);
return { ...logRow, attributes: attrs, attributeKeys };
}
}
return logRow;
}
}
export default LogPipelineService;

View File

@@ -24,6 +24,9 @@ import LogsQueueService from "./Queue/LogsQueueService";
import OtelIngestBaseService from "./OtelIngestBaseService";
import { TELEMETRY_LOG_FLUSH_BATCH_SIZE } from "../Config";
import LogService from "Common/Server/Services/LogService";
import LogPipelineService, { LoadedPipeline } from "./LogPipelineService";
import LogDropFilterService from "./LogDropFilterService";
import LogDropFilter from "Common/Models/DatabaseModels/LogDropFilter";
export default class OtelLogsIngestService extends OtelIngestBaseService {
private static async flushLogsBuffer(
@@ -92,6 +95,19 @@ export default class OtelLogsIngestService extends OtelIngestBaseService {
const serviceDictionary: Dictionary<TelemetryServiceMetadata> = {};
let totalLogsProcessed: number = 0;
// Load pipelines and drop filters once per batch
const projectId: ObjectID = (req as TelemetryRequest).projectId;
let loadedPipelines: Array<LoadedPipeline> = [];
let loadedDropFilters: Array<LogDropFilter> = [];
try {
loadedPipelines = await LogPipelineService.loadPipelines(projectId);
loadedDropFilters =
await LogDropFilterService.loadDropFilters(projectId);
} catch (loadError) {
logger.error("Error loading pipelines/drop filters:");
logger.error(loadError);
}
let resourceLogCounter: number = 0;
for (const resourceLog of resourceLogs) {
try {
@@ -316,7 +332,7 @@ export default class OtelLogsIngestService extends OtelIngestBaseService {
serviceDictionary[serviceName]!.dataRententionInDays || 15,
);
const logRow: JSONObject = {
let logRow: JSONObject = {
_id: ObjectID.generate().toString(),
createdAt: ingestionTimestamp,
updatedAt: ingestionTimestamp,
@@ -339,6 +355,25 @@ export default class OtelLogsIngestService extends OtelIngestBaseService {
OneUptimeDate.toClickhouseDateTime(retentionDate),
};
// Drop filter check (before pipeline processing)
if (
loadedDropFilters.length > 0 &&
LogDropFilterService.shouldDropLog(
logRow,
loadedDropFilters,
)
) {
continue;
}
// Pipeline processing
if (loadedPipelines.length > 0) {
logRow = LogPipelineService.processLog(
logRow,
loadedPipelines,
);
}
dbLogs.push(logRow);
totalLogsProcessed++;

View File

@@ -0,0 +1,371 @@
import { JSONObject } from "Common/Types/JSON";
// Simple filter evaluator for log rows used by pipelines and drop filters.
// Supports: =, !=, LIKE, IN, AND, OR, NOT, parentheses
// Field paths: severityText, body, serviceId, attributes.<key>
interface Token {
type:
| "field"
| "operator"
| "value"
| "and"
| "or"
| "not"
| "lparen"
| "rparen";
value: string;
}
function getFieldValue(logRow: JSONObject, fieldPath: string): string {
if (fieldPath.startsWith("attributes.")) {
const attrKey: string = fieldPath.slice("attributes.".length);
const attrs: Record<string, unknown> =
(logRow["attributes"] as Record<string, unknown>) || {};
const val: unknown = attrs[attrKey];
if (val === undefined || val === null) {
return "";
}
if (typeof val === "object") {
return JSON.stringify(val);
}
return String(val);
}
const val: unknown = logRow[fieldPath];
if (val === undefined || val === null) {
return "";
}
return String(val);
}
function tokenize(query: string): Array<Token> {
const tokens: Array<Token> = [];
let i: number = 0;
const len: number = query.length;
while (i < len) {
// Skip whitespace
if (/\s/.test(query[i]!)) {
i++;
continue;
}
// Parentheses
if (query[i] === "(") {
tokens.push({ type: "lparen", value: "(" });
i++;
continue;
}
if (query[i] === ")") {
tokens.push({ type: "rparen", value: ")" });
i++;
continue;
}
// Check for keywords (AND, OR, NOT, LIKE, IN)
const remaining: string = query.slice(i);
const keywordMatch: RegExpMatchArray | null = remaining.match(
/^(AND|OR|NOT|LIKE|IN|!=)\b/i,
);
if (keywordMatch) {
const kw: string = keywordMatch[1]!.toUpperCase();
if (kw === "AND") {
tokens.push({ type: "and", value: "AND" });
} else if (kw === "OR") {
tokens.push({ type: "or", value: "OR" });
} else if (kw === "NOT") {
tokens.push({ type: "not", value: "NOT" });
} else if (kw === "LIKE" || kw === "IN") {
tokens.push({ type: "operator", value: kw });
} else if (kw === "!=") {
tokens.push({ type: "operator", value: "!=" });
}
i += keywordMatch[0]!.length;
continue;
}
// != operator (check before = to avoid conflict)
if (query[i] === "!" && i + 1 < len && query[i + 1] === "=") {
tokens.push({ type: "operator", value: "!=" });
i += 2;
continue;
}
// = operator
if (query[i] === "=") {
tokens.push({ type: "operator", value: "=" });
i++;
continue;
}
// Quoted string value
if (query[i] === "'" || query[i] === '"') {
const quote: string = query[i]!;
i++;
let val: string = "";
while (i < len && query[i] !== quote) {
if (query[i] === "\\" && i + 1 < len) {
i++;
val += query[i];
} else {
val += query[i];
}
i++;
}
i++; // skip closing quote
tokens.push({ type: "value", value: val });
continue;
}
// Field name or unquoted value
let word: string = "";
while (i < len && !/[\s()=!]/.test(query[i]!)) {
word += query[i];
i++;
}
if (word.length > 0) {
// Determine if this is a field or a value based on context
const lastToken: Token | undefined = tokens[tokens.length - 1];
if (lastToken && lastToken.type === "operator") {
tokens.push({ type: "value", value: word });
} else {
tokens.push({ type: "field", value: word });
}
}
}
return tokens;
}
interface FilterExpression {
type: "comparison" | "and" | "or" | "not";
}
interface ComparisonExpr extends FilterExpression {
type: "comparison";
field: string;
operator: string;
value: string | Array<string>;
}
interface AndExpr extends FilterExpression {
type: "and";
left: FilterExpression;
right: FilterExpression;
}
interface OrExpr extends FilterExpression {
type: "or";
left: FilterExpression;
right: FilterExpression;
}
interface NotExpr extends FilterExpression {
type: "not";
expr: FilterExpression;
}
class Parser {
private tokens: Array<Token>;
private pos: number;
public constructor(tokens: Array<Token>) {
this.tokens = tokens;
this.pos = 0;
}
public parse(): FilterExpression {
const expr: FilterExpression = this.parseOr();
return expr;
}
private parseOr(): FilterExpression {
let left: FilterExpression = this.parseAnd();
while (this.pos < this.tokens.length && this.tokens[this.pos]!.type === "or") {
this.pos++;
const right: FilterExpression = this.parseAnd();
left = { type: "or", left, right } as OrExpr;
}
return left;
}
private parseAnd(): FilterExpression {
let left: FilterExpression = this.parseUnary();
while (this.pos < this.tokens.length && this.tokens[this.pos]!.type === "and") {
this.pos++;
const right: FilterExpression = this.parseUnary();
left = { type: "and", left, right } as AndExpr;
}
return left;
}
private parseUnary(): FilterExpression {
if (
this.pos < this.tokens.length &&
this.tokens[this.pos]!.type === "not"
) {
this.pos++;
const expr: FilterExpression = this.parseUnary();
return { type: "not", expr } as NotExpr;
}
return this.parsePrimary();
}
private parsePrimary(): FilterExpression {
if (
this.pos < this.tokens.length &&
this.tokens[this.pos]!.type === "lparen"
) {
this.pos++; // skip (
const expr: FilterExpression = this.parseOr();
if (
this.pos < this.tokens.length &&
this.tokens[this.pos]!.type === "rparen"
) {
this.pos++; // skip )
}
return expr;
}
// Comparison: field operator value
const fieldToken: Token | undefined = this.tokens[this.pos];
if (!fieldToken || fieldToken.type !== "field") {
throw new Error(
`Expected field at position ${this.pos}, got: ${fieldToken?.type || "end of input"}`,
);
}
this.pos++;
const opToken: Token | undefined = this.tokens[this.pos];
if (!opToken || opToken.type !== "operator") {
throw new Error(
`Expected operator at position ${this.pos}, got: ${opToken?.type || "end of input"}`,
);
}
this.pos++;
if (opToken.value === "IN") {
// Parse IN (val1, val2, val3)
const values: Array<string> = this.parseInValues();
return {
type: "comparison",
field: fieldToken.value,
operator: "IN",
value: values,
} as ComparisonExpr;
}
const valToken: Token | undefined = this.tokens[this.pos];
if (!valToken || valToken.type !== "value") {
throw new Error(
`Expected value at position ${this.pos}, got: ${valToken?.type || "end of input"}`,
);
}
this.pos++;
return {
type: "comparison",
field: fieldToken.value,
operator: opToken.value,
value: valToken.value,
} as ComparisonExpr;
}
private parseInValues(): Array<string> {
const values: Array<string> = [];
// Expect (
if (
this.pos < this.tokens.length &&
this.tokens[this.pos]!.type === "lparen"
) {
this.pos++;
}
while (this.pos < this.tokens.length) {
if (this.tokens[this.pos]!.type === "rparen") {
this.pos++;
break;
}
if (this.tokens[this.pos]!.type === "value") {
values.push(this.tokens[this.pos]!.value);
this.pos++;
} else {
this.pos++; // skip commas etc.
}
}
return values;
}
}
function evaluateExpr(logRow: JSONObject, expr: FilterExpression): boolean {
switch (expr.type) {
case "comparison": {
const comp: ComparisonExpr = expr as ComparisonExpr;
const fieldVal: string = getFieldValue(logRow, comp.field);
switch (comp.operator) {
case "=":
return fieldVal === comp.value;
case "!=":
return fieldVal !== comp.value;
case "LIKE": {
// Convert SQL LIKE pattern to regex: % -> .*, _ -> .
const pattern: string = String(comp.value)
.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") // escape regex chars first
.replace(/%/g, ".*")
.replace(/_/g, ".");
return new RegExp(`^${pattern}$`, "i").test(fieldVal);
}
case "IN": {
const values: Array<string> = comp.value as Array<string>;
return values.includes(fieldVal);
}
default:
return false;
}
}
case "and": {
const andExpr: AndExpr = expr as AndExpr;
return (
evaluateExpr(logRow, andExpr.left) &&
evaluateExpr(logRow, andExpr.right)
);
}
case "or": {
const orExpr: OrExpr = expr as OrExpr;
return (
evaluateExpr(logRow, orExpr.left) || evaluateExpr(logRow, orExpr.right)
);
}
case "not": {
const notExpr: NotExpr = expr as NotExpr;
return !evaluateExpr(logRow, notExpr.expr);
}
default:
return false;
}
}
export function evaluateFilter(logRow: JSONObject, filterQuery: string): boolean {
if (!filterQuery || filterQuery.trim().length === 0) {
return true; // empty filter matches everything
}
try {
const tokens: Array<Token> = tokenize(filterQuery);
if (tokens.length === 0) {
return true;
}
const parser: Parser = new Parser(tokens);
const expr: FilterExpression = parser.parse();
return evaluateExpr(logRow, expr);
} catch {
// If filter can't be parsed, don't match (safe default)
return false;
}
}
export default evaluateFilter;