Files
oneuptime/Common/Models/DatabaseModels/LogPipeline.ts
Nawaz Dhandala 0e93929a3f feat: add LogPipelineProcessor model and related services
- Implemented LogPipelineProcessor model with necessary fields and access controls.
- Created LogDropFilterAction enum for defining actions on log drop filters.
- Introduced LogPipelineProcessorType enum and configuration interfaces for various processor types.
- Developed LogExport utility for exporting logs in CSV and JSON formats.
- Added LogDropFilterService for managing drop filters with caching.
- Implemented LogPipelineService for loading and processing log pipelines with processors.
- Created LogFilterEvaluator for evaluating filter queries against log entries.
2026-03-13 11:50:07 +00:00

413 lines
10 KiB
TypeScript

import Project from "./Project";
import User from "./User";
import BaseModel from "./DatabaseBaseModel/DatabaseBaseModel";
import Route from "../../Types/API/Route";
import ColumnAccessControl from "../../Types/Database/AccessControl/ColumnAccessControl";
import TableAccessControl from "../../Types/Database/AccessControl/TableAccessControl";
import TableBillingAccessControl from "../../Types/Database/AccessControl/TableBillingAccessControl";
import ColumnLength from "../../Types/Database/ColumnLength";
import ColumnType from "../../Types/Database/ColumnType";
import CrudApiEndpoint from "../../Types/Database/CrudApiEndpoint";
import EnableDocumentation from "../../Types/Database/EnableDocumentation";
import TableColumn from "../../Types/Database/TableColumn";
import TableColumnType from "../../Types/Database/TableColumnType";
import TableMetadata from "../../Types/Database/TableMetadata";
import TenantColumn from "../../Types/Database/TenantColumn";
import IconProp from "../../Types/Icon/IconProp";
import ObjectID from "../../Types/ObjectID";
import Permission from "../../Types/Permission";
import { PlanType } from "../../Types/Billing/SubscriptionPlan";
import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
@EnableDocumentation()
@TableBillingAccessControl({
create: PlanType.Free,
read: PlanType.Free,
update: PlanType.Free,
delete: PlanType.Free,
})
@TenantColumn("projectId")
@CrudApiEndpoint(new Route("/log-pipeline"))
@Entity({
name: "LogPipeline",
})
@TableMetadata({
tableName: "LogPipeline",
singularName: "Log Pipeline",
pluralName: "Log Pipelines",
icon: IconProp.Logs,
tableDescription:
"Configure server-side log processing pipelines that transform logs at ingest time.",
})
@TableAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
delete: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.DeleteProjectLogPipeline,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipeline,
],
})
export default class LogPipeline extends BaseModel {
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
manyToOneRelationColumn: "projectId",
type: TableColumnType.Entity,
modelType: Project,
title: "Project",
description: "Relation to the project this log pipeline belongs to.",
})
@ManyToOne(
() => {
return Project;
},
{
eager: false,
nullable: true,
onDelete: "CASCADE",
orphanedRowAction: "nullify",
},
)
@JoinColumn({ name: "projectId" })
public project?: Project = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [],
})
@Index()
@TableColumn({
type: TableColumnType.ObjectID,
required: true,
canReadOnRelationQuery: true,
title: "Project ID",
description: "ID of the project this log pipeline belongs to.",
})
@Column({
type: ColumnType.ObjectID,
nullable: false,
transformer: ObjectID.getDatabaseTransformer(),
})
public projectId?: ObjectID = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipeline,
],
})
@TableColumn({
required: true,
type: TableColumnType.Name,
canReadOnRelationQuery: true,
title: "Name",
description: "Friendly name for this log pipeline.",
})
@Column({
nullable: false,
type: ColumnType.Name,
length: ColumnLength.Name,
})
public name?: string = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipeline,
],
})
@TableColumn({
required: false,
type: TableColumnType.LongText,
canReadOnRelationQuery: true,
title: "Description",
description: "Description of what this log pipeline does.",
})
@Column({
nullable: true,
type: ColumnType.LongText,
length: ColumnLength.LongText,
})
public description?: string = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipeline,
],
})
@TableColumn({
required: false,
type: TableColumnType.LongText,
canReadOnRelationQuery: true,
title: "Filter Query",
description:
"Filter expression that determines which logs this pipeline applies to.",
})
@Column({
nullable: true,
type: ColumnType.LongText,
length: ColumnLength.LongText,
})
public filterQuery?: string = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipeline,
],
})
@Index()
@TableColumn({
required: true,
type: TableColumnType.Boolean,
canReadOnRelationQuery: true,
title: "Enabled",
description: "Whether this log pipeline is active.",
defaultValue: true,
})
@Column({
nullable: false,
type: ColumnType.Boolean,
default: true,
})
public isEnabled?: boolean = undefined;
@ColumnAccessControl({
create: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.CreateProjectLogPipeline,
],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.EditProjectLogPipeline,
],
})
@TableColumn({
title: "Sort Order",
required: true,
type: TableColumnType.Number,
canReadOnRelationQuery: true,
description:
"Determines the execution order of this pipeline relative to others.",
defaultValue: 0,
})
@Column({
type: ColumnType.Number,
nullable: false,
default: 0,
})
public sortOrder?: number = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
manyToOneRelationColumn: "createdByUserId",
type: TableColumnType.Entity,
modelType: User,
title: "Created By User",
description: "Relation to the user who created this log pipeline.",
})
@ManyToOne(
() => {
return User;
},
{
eager: false,
nullable: true,
onDelete: "SET NULL",
orphanedRowAction: "nullify",
},
)
@JoinColumn({ name: "createdByUserId" })
public createdByUser?: User = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
type: TableColumnType.ObjectID,
title: "Created By User ID",
description: "ID of the user who created this log pipeline.",
})
@Column({
type: ColumnType.ObjectID,
nullable: true,
transformer: ObjectID.getDatabaseTransformer(),
})
public createdByUserId?: ObjectID = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
manyToOneRelationColumn: "deletedByUserId",
type: TableColumnType.Entity,
modelType: User,
title: "Deleted By User",
description: "Relation to the user who deleted this log pipeline.",
})
@ManyToOne(
() => {
return User;
},
{
eager: false,
nullable: true,
onDelete: "SET NULL",
orphanedRowAction: "nullify",
},
)
@JoinColumn({ name: "deletedByUserId" })
public deletedByUser?: User = undefined;
@ColumnAccessControl({
create: [],
read: [
Permission.ProjectOwner,
Permission.ProjectAdmin,
Permission.ProjectMember,
Permission.ReadProjectLogPipeline,
Permission.ReadAllProjectResources,
],
update: [],
})
@TableColumn({
type: TableColumnType.ObjectID,
title: "Deleted By User ID",
description: "ID of the user who deleted this log pipeline.",
})
@Column({
type: ColumnType.ObjectID,
nullable: true,
transformer: ObjectID.getDatabaseTransformer(),
})
public deletedByUserId?: ObjectID = undefined;
}