diff --git a/.env.example b/.env.example index 42ca561..d307d03 100644 --- a/.env.example +++ b/.env.example @@ -104,3 +104,24 @@ ENCRYPTION_KEY= # Apache Tika Integration # ONLY active if TIKA_URL is set TIKA_URL=http://tika:9998 + + +# Enterprise features (Skip this part if you are using the open-source version) + +# Batch size for managing retention policy lifecycle. (This number of emails will be checked each time when retention policy scans the database. Adjust based on your system capability.) +RETENTION_BATCH_SIZE=1000 + +# --- SMTP Journaling (Enterprise only) --- +# The port the embedded SMTP journaling listener binds to inside the container. +# This is the port your MTA (Exchange, MS365, Postfix, etc.) will send journal reports to. +# The docker-compose.yml maps this same port on the host side by default. +SMTP_JOURNALING_PORT=2525 +# The domain used to generate routing addresses for journaling sources. +# Each source gets a unique address like journal-@. +# Set this to the domain/subdomain whose MX record points to this server. +SMTP_JOURNALING_DOMAIN=journal.yourdomain.com +# Maximum number of waiting jobs in the journal queue before the SMTP listener +# returns 4xx temporary failures (backpressure). The MTA will retry automatically. +JOURNAL_QUEUE_BACKPRESSURE_THRESHOLD=10000 +#BullMQ worker concurrency for processing journaled emails. Increase on servers with more CPU cores. +JOURNAL_WORKER_CONCURRENCY=3 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index e73b9bc..ec77e67 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: container_name: open-archiver restart: unless-stopped ports: - - '3000:3000' # Frontend + - '${PORT_FRONTEND:-3000}:3000' # Frontend env_file: - .env volumes: @@ -42,7 +42,7 @@ services: - open-archiver-net meilisearch: - image: getmeili/meilisearch:v1.15 + image: getmeili/meilisearch:v1.38 container_name: meilisearch restart: unless-stopped environment: diff --git a/packages/backend/src/database/migrations/0030_strong_ultron.sql b/packages/backend/src/database/migrations/0030_strong_ultron.sql new file mode 100644 index 0000000..4127e0d --- /dev/null +++ b/packages/backend/src/database/migrations/0030_strong_ultron.sql @@ -0,0 +1,20 @@ +CREATE TYPE "public"."journaling_source_status" AS ENUM('active', 'paused');--> statement-breakpoint +ALTER TYPE "public"."ingestion_provider" ADD VALUE 'smtp_journaling';--> statement-breakpoint +ALTER TYPE "public"."audit_log_target_type" ADD VALUE 'JournalingSource' BEFORE 'RetentionPolicy';--> statement-breakpoint +CREATE TABLE "journaling_sources" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "name" text NOT NULL, + "allowed_ips" jsonb NOT NULL, + "require_tls" boolean DEFAULT true NOT NULL, + "smtp_username" text, + "smtp_password_hash" text, + "status" "journaling_source_status" DEFAULT 'active' NOT NULL, + "ingestion_source_id" uuid NOT NULL, + "routing_address" text NOT NULL, + "total_received" integer DEFAULT 0 NOT NULL, + "last_received_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "journaling_sources" ADD CONSTRAINT "journaling_sources_ingestion_source_id_ingestion_sources_id_fk" FOREIGN KEY ("ingestion_source_id") REFERENCES "public"."ingestion_sources"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/packages/backend/src/database/migrations/meta/0030_snapshot.json b/packages/backend/src/database/migrations/meta/0030_snapshot.json new file mode 100644 index 0000000..358d257 --- /dev/null +++ b/packages/backend/src/database/migrations/meta/0030_snapshot.json @@ -0,0 +1,1727 @@ +{ + "id": "a9094976-87e1-4a52-b5a5-ddec968bbecd", + "prevId": "5b69110d-3df3-41e0-982c-57413d5956f5", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.archived_emails": { + "name": "archived_emails", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ingestion_source_id": { + "name": "ingestion_source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_email": { + "name": "user_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id_header": { + "name": "message_id_header", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_message_id": { + "name": "provider_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sender_name": { + "name": "sender_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sender_email": { + "name": "sender_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipients": { + "name": "recipients", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "storage_path": { + "name": "storage_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_hash_sha256": { + "name": "storage_hash_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "is_indexed": { + "name": "is_indexed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "has_attachments": { + "name": "has_attachments", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "is_on_legal_hold": { + "name": "is_on_legal_hold", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tags": { + "name": "tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "thread_id_idx": { + "name": "thread_id_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_msg_source_idx": { + "name": "provider_msg_source_idx", + "columns": [ + { + "expression": "provider_message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "ingestion_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "archived_emails_ingestion_source_id_ingestion_sources_id_fk": { + "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk", + "tableFrom": "archived_emails", + "tableTo": "ingestion_sources", + "columnsFrom": ["ingestion_source_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.attachments": { + "name": "attachments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "content_hash_sha256": { + "name": "content_hash_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_path": { + "name": "storage_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ingestion_source_id": { + "name": "ingestion_source_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "source_hash_idx": { + "name": "source_hash_idx", + "columns": [ + { + "expression": "ingestion_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "content_hash_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "attachments_ingestion_source_id_ingestion_sources_id_fk": { + "name": "attachments_ingestion_source_id_ingestion_sources_id_fk", + "tableFrom": "attachments", + "tableTo": "ingestion_sources", + "columnsFrom": ["ingestion_source_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.email_attachments": { + "name": "email_attachments", + "schema": "", + "columns": { + "email_id": { + "name": "email_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "attachment_id": { + "name": "attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "email_attachments_email_id_archived_emails_id_fk": { + "name": "email_attachments_email_id_archived_emails_id_fk", + "tableFrom": "email_attachments", + "tableTo": "archived_emails", + "columnsFrom": ["email_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "email_attachments_attachment_id_attachments_id_fk": { + "name": "email_attachments_attachment_id_attachments_id_fk", + "tableFrom": "email_attachments", + "tableTo": "attachments", + "columnsFrom": ["attachment_id"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "email_attachments_email_id_attachment_id_pk": { + "name": "email_attachments_email_id_attachment_id_pk", + "columns": ["email_id", "attachment_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_logs": { + "name": "audit_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "previous_hash": { + "name": "previous_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "timestamp": { + "name": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "actor_identifier": { + "name": "actor_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_ip": { + "name": "actor_ip", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action_type": { + "name": "action_type", + "type": "audit_log_action", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "audit_log_target_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "current_hash": { + "name": "current_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ediscovery_cases": { + "name": "ediscovery_cases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "created_by_identifier": { + "name": "created_by_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ediscovery_cases_name_unique": { + "name": "ediscovery_cases_name_unique", + "nullsNotDistinct": false, + "columns": ["name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.email_legal_holds": { + "name": "email_legal_holds", + "schema": "", + "columns": { + "email_id": { + "name": "email_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "legal_hold_id": { + "name": "legal_hold_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "applied_at": { + "name": "applied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "applied_by_user_id": { + "name": "applied_by_user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "email_legal_holds_email_id_archived_emails_id_fk": { + "name": "email_legal_holds_email_id_archived_emails_id_fk", + "tableFrom": "email_legal_holds", + "tableTo": "archived_emails", + "columnsFrom": ["email_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "email_legal_holds_legal_hold_id_legal_holds_id_fk": { + "name": "email_legal_holds_legal_hold_id_legal_holds_id_fk", + "tableFrom": "email_legal_holds", + "tableTo": "legal_holds", + "columnsFrom": ["legal_hold_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "email_legal_holds_applied_by_user_id_users_id_fk": { + "name": "email_legal_holds_applied_by_user_id_users_id_fk", + "tableFrom": "email_legal_holds", + "tableTo": "users", + "columnsFrom": ["applied_by_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "email_legal_holds_email_id_legal_hold_id_pk": { + "name": "email_legal_holds_email_id_legal_hold_id_pk", + "columns": ["email_id", "legal_hold_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.email_retention_labels": { + "name": "email_retention_labels", + "schema": "", + "columns": { + "email_id": { + "name": "email_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "label_id": { + "name": "label_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "applied_at": { + "name": "applied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "applied_by_user_id": { + "name": "applied_by_user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "email_retention_labels_email_id_archived_emails_id_fk": { + "name": "email_retention_labels_email_id_archived_emails_id_fk", + "tableFrom": "email_retention_labels", + "tableTo": "archived_emails", + "columnsFrom": ["email_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "email_retention_labels_label_id_retention_labels_id_fk": { + "name": "email_retention_labels_label_id_retention_labels_id_fk", + "tableFrom": "email_retention_labels", + "tableTo": "retention_labels", + "columnsFrom": ["label_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "email_retention_labels_applied_by_user_id_users_id_fk": { + "name": "email_retention_labels_applied_by_user_id_users_id_fk", + "tableFrom": "email_retention_labels", + "tableTo": "users", + "columnsFrom": ["applied_by_user_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "email_retention_labels_email_id_label_id_pk": { + "name": "email_retention_labels_email_id_label_id_pk", + "columns": ["email_id", "label_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.export_jobs": { + "name": "export_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "case_id": { + "name": "case_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "query": { + "name": "query", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by_identifier": { + "name": "created_by_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "export_jobs_case_id_ediscovery_cases_id_fk": { + "name": "export_jobs_case_id_ediscovery_cases_id_fk", + "tableFrom": "export_jobs", + "tableTo": "ediscovery_cases", + "columnsFrom": ["case_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.legal_holds": { + "name": "legal_holds", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "case_id": { + "name": "case_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "legal_holds_case_id_ediscovery_cases_id_fk": { + "name": "legal_holds_case_id_ediscovery_cases_id_fk", + "tableFrom": "legal_holds", + "tableTo": "ediscovery_cases", + "columnsFrom": ["case_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.retention_events": { + "name": "retention_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "event_name": { + "name": "event_name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "event_timestamp": { + "name": "event_timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "target_criteria": { + "name": "target_criteria", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.retention_labels": { + "name": "retention_labels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "retention_period_days": { + "name": "retention_period_days", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_disabled": { + "name": "is_disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.retention_policies": { + "name": "retention_policies", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "retention_period_days": { + "name": "retention_period_days", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "action_on_expiry": { + "name": "action_on_expiry", + "type": "retention_action", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "conditions": { + "name": "conditions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ingestion_scope": { + "name": "ingestion_scope", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "retention_policies_name_unique": { + "name": "retention_policies_name_unique", + "nullsNotDistinct": false, + "columns": ["name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custodians": { + "name": "custodians", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_type": { + "name": "source_type", + "type": "ingestion_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "custodians_email_unique": { + "name": "custodians_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ingestion_sources": { + "name": "ingestion_sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "ingestion_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "ingestion_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending_auth'" + }, + "last_sync_started_at": { + "name": "last_sync_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_finished_at": { + "name": "last_sync_finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_status_message": { + "name": "last_sync_status_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_state": { + "name": "sync_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "ingestion_sources_user_id_users_id_fk": { + "name": "ingestion_sources_user_id_users_id_fk", + "tableFrom": "ingestion_sources", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.roles": { + "name": "roles", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "policies": { + "name": "policies", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "roles_name_unique": { + "name": "roles_name_unique", + "nullsNotDistinct": false, + "columns": ["name"] + }, + "roles_slug_unique": { + "name": "roles_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_roles_role_id_roles_id_fk": { + "name": "user_roles_role_id_roles_id_fk", + "tableFrom": "user_roles", + "tableTo": "roles", + "columnsFrom": ["role_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_id_pk": { + "name": "user_roles_user_id_role_id_pk", + "columns": ["user_id", "role_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "first_name": { + "name": "first_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_name": { + "name": "last_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'local'" + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "api_keys_user_id_users_id_fk": { + "name": "api_keys_user_id_users_id_fk", + "tableFrom": "api_keys", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sync_sessions": { + "name": "sync_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "ingestion_source_id": { + "name": "ingestion_source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "is_initial_import": { + "name": "is_initial_import", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "total_mailboxes": { + "name": "total_mailboxes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "completed_mailboxes": { + "name": "completed_mailboxes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "failed_mailboxes": { + "name": "failed_mailboxes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error_messages": { + "name": "error_messages", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_activity_at": { + "name": "last_activity_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sync_sessions_ingestion_source_id_ingestion_sources_id_fk": { + "name": "sync_sessions_ingestion_source_id_ingestion_sources_id_fk", + "tableFrom": "sync_sessions", + "tableTo": "ingestion_sources", + "columnsFrom": ["ingestion_source_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.journaling_sources": { + "name": "journaling_sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "allowed_ips": { + "name": "allowed_ips", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "require_tls": { + "name": "require_tls", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "smtp_username": { + "name": "smtp_username", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "smtp_password_hash": { + "name": "smtp_password_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "journaling_source_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "ingestion_source_id": { + "name": "ingestion_source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "routing_address": { + "name": "routing_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "total_received": { + "name": "total_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_received_at": { + "name": "last_received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "journaling_sources_ingestion_source_id_ingestion_sources_id_fk": { + "name": "journaling_sources_ingestion_source_id_ingestion_sources_id_fk", + "tableFrom": "journaling_sources", + "tableTo": "ingestion_sources", + "columnsFrom": ["ingestion_source_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.retention_action": { + "name": "retention_action", + "schema": "public", + "values": ["delete_permanently", "notify_admin"] + }, + "public.ingestion_provider": { + "name": "ingestion_provider", + "schema": "public", + "values": [ + "google_workspace", + "microsoft_365", + "generic_imap", + "pst_import", + "eml_import", + "mbox_import", + "smtp_journaling" + ] + }, + "public.ingestion_status": { + "name": "ingestion_status", + "schema": "public", + "values": [ + "active", + "paused", + "error", + "pending_auth", + "syncing", + "importing", + "auth_success", + "imported" + ] + }, + "public.audit_log_action": { + "name": "audit_log_action", + "schema": "public", + "values": [ + "CREATE", + "READ", + "UPDATE", + "DELETE", + "LOGIN", + "LOGOUT", + "SETUP", + "IMPORT", + "PAUSE", + "SYNC", + "UPLOAD", + "SEARCH", + "DOWNLOAD", + "GENERATE" + ] + }, + "public.audit_log_target_type": { + "name": "audit_log_target_type", + "schema": "public", + "values": [ + "ApiKey", + "ArchivedEmail", + "Dashboard", + "IngestionSource", + "JournalingSource", + "RetentionPolicy", + "RetentionLabel", + "LegalHold", + "Role", + "SystemEvent", + "SystemSettings", + "User", + "File" + ] + }, + "public.journaling_source_status": { + "name": "journaling_source_status", + "schema": "public", + "values": ["active", "paused"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/backend/src/database/migrations/meta/_journal.json b/packages/backend/src/database/migrations/meta/_journal.json index 561be85..4d26115 100644 --- a/packages/backend/src/database/migrations/meta/_journal.json +++ b/packages/backend/src/database/migrations/meta/_journal.json @@ -211,6 +211,13 @@ "when": 1773927678269, "tag": "0029_lethal_brood", "breakpoints": true + }, + { + "idx": 30, + "version": "7", + "when": 1774440788278, + "tag": "0030_strong_ultron", + "breakpoints": true } ] } diff --git a/packages/backend/src/database/schema.ts b/packages/backend/src/database/schema.ts index 444423b..3af887f 100644 --- a/packages/backend/src/database/schema.ts +++ b/packages/backend/src/database/schema.ts @@ -10,3 +10,4 @@ export * from './schema/api-keys'; export * from './schema/audit-logs'; export * from './schema/enums'; export * from './schema/sync-sessions'; +export * from './schema/journaling-sources'; diff --git a/packages/backend/src/database/schema/ingestion-sources.ts b/packages/backend/src/database/schema/ingestion-sources.ts index 98d937d..5907a0c 100644 --- a/packages/backend/src/database/schema/ingestion-sources.ts +++ b/packages/backend/src/database/schema/ingestion-sources.ts @@ -9,6 +9,7 @@ export const ingestionProviderEnum = pgEnum('ingestion_provider', [ 'pst_import', 'eml_import', 'mbox_import', + 'smtp_journaling', ]); export const ingestionStatusEnum = pgEnum('ingestion_status', [ diff --git a/packages/backend/src/database/schema/journaling-sources.ts b/packages/backend/src/database/schema/journaling-sources.ts new file mode 100644 index 0000000..bc12591 --- /dev/null +++ b/packages/backend/src/database/schema/journaling-sources.ts @@ -0,0 +1,47 @@ +import { + boolean, + integer, + jsonb, + pgEnum, + pgTable, + text, + timestamp, + uuid, +} from 'drizzle-orm/pg-core'; +import { relations } from 'drizzle-orm'; +import { ingestionSources } from './ingestion-sources'; + +export const journalingSourceStatusEnum = pgEnum('journaling_source_status', ['active', 'paused']); + +export const journalingSources = pgTable('journaling_sources', { + id: uuid('id').primaryKey().defaultRandom(), + name: text('name').notNull(), + /** CIDR blocks or IP addresses allowed to send journal reports */ + allowedIps: jsonb('allowed_ips').notNull().$type(), + /** Whether to reject non-TLS connections (GDPR compliance) */ + requireTls: boolean('require_tls').notNull().default(true), + /** Optional SMTP AUTH username */ + smtpUsername: text('smtp_username'), + /** Bcrypt-hashed SMTP AUTH password */ + smtpPasswordHash: text('smtp_password_hash'), + status: journalingSourceStatusEnum('status').notNull().default('active'), + /** The backing ingestion source that owns all archived emails */ + ingestionSourceId: uuid('ingestion_source_id') + .notNull() + .references(() => ingestionSources.id, { onDelete: 'cascade' }), + /** Persisted SMTP routing address generated at creation time (immutable unless regenerated) */ + routingAddress: text('routing_address').notNull(), + /** Running count of emails received via this journaling endpoint */ + totalReceived: integer('total_received').notNull().default(0), + /** Timestamp of the last email received */ + lastReceivedAt: timestamp('last_received_at', { withTimezone: true }), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(), +}); + +export const journalingSourcesRelations = relations(journalingSources, ({ one }) => ({ + ingestionSource: one(ingestionSources, { + fields: [journalingSources.ingestionSourceId], + references: [ingestionSources.id], + }), +})); diff --git a/packages/frontend/src/lib/translations/en.json b/packages/frontend/src/lib/translations/en.json index acaccb9..e17a7bc 100644 --- a/packages/frontend/src/lib/translations/en.json +++ b/packages/frontend/src/lib/translations/en.json @@ -609,6 +609,64 @@ "next": "Next", "ingestion_source": "Ingestion Source" }, + "journaling": { + "title": "SMTP Journaling", + "header": "SMTP Journaling Sources", + "meta_description": "Configure real-time SMTP journaling endpoints for zero-gap email archiving from corporate MTAs.", + "header_description": "Receive a real-time copy of every email directly from your mail server via SMTP journaling, ensuring zero data loss.", + "create_new": "Create Source", + "no_sources_found": "No journaling sources configured.", + "name": "Name", + "allowed_ips": "Allowed IPs / CIDR", + "require_tls": "Require TLS", + "smtp_username": "SMTP Username", + "smtp_password": "SMTP Password", + "status": "Status", + "active": "Active", + "paused": "Paused", + "total_received": "Emails Received", + "last_received_at": "Last Received", + "created_at": "Created At", + "actions": "Actions", + "create": "Create", + "edit": "Edit", + "delete": "Delete", + "pause": "Pause", + "activate": "Activate", + "save": "Save Changes", + "cancel": "Cancel", + "confirm": "Confirm Delete", + "name_placeholder": "e.g. MS365 Journal Receiver", + "allowed_ips_placeholder": "e.g. 40.107.0.0/16, 52.100.0.0/14", + "allowed_ips_hint": "Comma-separated IP addresses or CIDR blocks of your mail server(s) that are permitted to send journal reports.", + "smtp_username_placeholder": "e.g. journal-tenant-123", + "smtp_password_placeholder": "Enter a strong password for SMTP AUTH", + "smtp_auth_hint": "Optional. If set, the MTA must authenticate with these credentials when connecting.", + "create_description": "Configure a new SMTP journaling endpoint. Your MTA will send journal reports to this endpoint for real-time archiving.", + "edit_description": "Update the configuration for this journaling source.", + "delete_confirmation_title": "Delete this journaling source?", + "delete_confirmation_description": "This will permanently delete the journaling endpoint and all associated archived emails. Your MTA will no longer be able to deliver journal reports to this endpoint.", + "deleting": "Deleting", + "smtp_connection_info": "SMTP Connection Info", + "smtp_host": "Host", + "smtp_port": "Port", + "routing_address": "Routing Address", + "routing_address_hint": "Configure this address as the journal recipient in your MTA (Exchange, MS365, Postfix).", + "regenerate_address": "Regenerate Address", + "regenerate_address_warning": "This will invalidate the current address. You must update your MTA configuration to use the new address.", + "regenerate_address_confirm": "Are you sure you want to regenerate the routing address? The current address will stop working immediately and you will need to update your MTA configuration.", + "regenerate_address_success": "Routing address regenerated successfully. Update your MTA configuration with the new address.", + "regenerate_address_error": "Failed to regenerate routing address.", + "create_success": "Journaling source created successfully.", + "update_success": "Journaling source updated successfully.", + "delete_success": "Journaling source deleted successfully.", + "create_error": "Failed to create journaling source.", + "update_error": "Failed to update journaling source.", + "delete_error": "Failed to delete journaling source.", + "health_listening": "SMTP Listener: Active", + "health_down": "SMTP Listener: Down", + "never": "Never" + }, "license_page": { "title": "Enterprise License Status", "meta_description": "View the current status of your Open Archiver Enterprise license.", diff --git a/packages/frontend/src/routes/dashboard/+layout.svelte b/packages/frontend/src/routes/dashboard/+layout.svelte index 3cab2e6..4600d29 100644 --- a/packages/frontend/src/routes/dashboard/+layout.svelte +++ b/packages/frontend/src/routes/dashboard/+layout.svelte @@ -75,6 +75,16 @@ ]; const enterpriseNavItems: NavItem[] = [ + { + label: $t('app.archive.title'), + subMenu: [ + { + href: '/dashboard/ingestions/journaling', + label: $t('app.journaling.title'), + }, + ], + position: 1, + }, { label: 'Compliance', subMenu: [ diff --git a/packages/frontend/src/routes/dashboard/ingestions/journaling/+page.server.ts b/packages/frontend/src/routes/dashboard/ingestions/journaling/+page.server.ts new file mode 100644 index 0000000..49a81e5 --- /dev/null +++ b/packages/frontend/src/routes/dashboard/ingestions/journaling/+page.server.ts @@ -0,0 +1,168 @@ +import { api } from '$lib/server/api'; +import { error } from '@sveltejs/kit'; +import type { Actions, PageServerLoad } from './$types'; +import type { JournalingSource } from '@open-archiver/types'; + +export const load: PageServerLoad = async (event) => { + if (!event.locals.enterpriseMode) { + throw error( + 403, + 'This feature is only available in the Enterprise Edition. Please contact Open Archiver to upgrade.' + ); + } + + const sourcesRes = await api('/enterprise/journaling', event); + const sourcesJson = await sourcesRes.json(); + + if (!sourcesRes.ok) { + throw error(sourcesRes.status, sourcesJson.message || JSON.stringify(sourcesJson)); + } + + const sources: JournalingSource[] = sourcesJson; + + // Fetch SMTP listener health status + const healthRes = await api('/enterprise/journaling/health', event); + const healthJson = (await healthRes.json()) as { smtp: string; port: string }; + + return { + sources, + smtpHealth: healthRes.ok ? healthJson : { smtp: 'down', port: '2525' }, + }; +}; + +export const actions: Actions = { + create: async (event) => { + const data = await event.request.formData(); + + const rawIps = (data.get('allowedIps') as string) || ''; + const allowedIps = rawIps + .split(',') + .map((ip) => ip.trim()) + .filter(Boolean); + + const body: Record = { + name: data.get('name') as string, + allowedIps, + requireTls: data.get('requireTls') === 'on', + }; + + const smtpUsername = data.get('smtpUsername') as string; + const smtpPassword = data.get('smtpPassword') as string; + if (smtpUsername) body.smtpUsername = smtpUsername; + if (smtpPassword) body.smtpPassword = smtpPassword; + + const response = await api('/enterprise/journaling', event, { + method: 'POST', + body: JSON.stringify(body), + }); + + const res = await response.json(); + + if (!response.ok) { + return { + success: false, + message: res.message || 'Failed to create journaling source.', + }; + } + + return { success: true }; + }, + + update: async (event) => { + const data = await event.request.formData(); + const id = data.get('id') as string; + + const rawIps = (data.get('allowedIps') as string) || ''; + const allowedIps = rawIps + .split(',') + .map((ip) => ip.trim()) + .filter(Boolean); + + const body: Record = { + name: data.get('name') as string, + allowedIps, + requireTls: data.get('requireTls') === 'on', + }; + + const smtpUsername = data.get('smtpUsername') as string; + const smtpPassword = data.get('smtpPassword') as string; + if (smtpUsername) body.smtpUsername = smtpUsername; + if (smtpPassword) body.smtpPassword = smtpPassword; + + const response = await api(`/enterprise/journaling/${id}`, event, { + method: 'PUT', + body: JSON.stringify(body), + }); + + const res = await response.json(); + + if (!response.ok) { + return { + success: false, + message: res.message || 'Failed to update journaling source.', + }; + } + + return { success: true }; + }, + + toggleStatus: async (event) => { + const data = await event.request.formData(); + const id = data.get('id') as string; + const status = data.get('status') as string; + + const response = await api(`/enterprise/journaling/${id}`, event, { + method: 'PUT', + body: JSON.stringify({ status }), + }); + + const res = await response.json(); + + if (!response.ok) { + return { success: false, message: res.message || 'Failed to update status.' }; + } + + return { success: true, status }; + }, + + regenerateAddress: async (event) => { + const data = await event.request.formData(); + const id = data.get('id') as string; + + const response = await api(`/enterprise/journaling/${id}/regenerate-address`, event, { + method: 'POST', + }); + + if (!response.ok) { + const res = await response.json().catch(() => ({})); + return { + success: false, + message: + (res as { message?: string }).message || + 'Failed to regenerate routing address.', + }; + } + + return { success: true }; + }, + + delete: async (event) => { + const data = await event.request.formData(); + const id = data.get('id') as string; + + const response = await api(`/enterprise/journaling/${id}`, event, { + method: 'DELETE', + }); + + if (!response.ok) { + const res = await response.json().catch(() => ({})); + return { + success: false, + message: + (res as { message?: string }).message || 'Failed to delete journaling source.', + }; + } + + return { success: true }; + }, +}; diff --git a/packages/frontend/src/routes/dashboard/ingestions/journaling/+page.svelte b/packages/frontend/src/routes/dashboard/ingestions/journaling/+page.svelte new file mode 100644 index 0000000..ac2d4fd --- /dev/null +++ b/packages/frontend/src/routes/dashboard/ingestions/journaling/+page.svelte @@ -0,0 +1,746 @@ + + + + {$t('app.journaling.title')} - Open Archiver + + + + +
+
+

{$t('app.journaling.header')}

+

+ {$t('app.journaling.header_description')} +

+
+ +
+ + +
+
+ + + {smtpHealth.smtp === 'listening' + ? $t('app.journaling.health_listening') + : $t('app.journaling.health_down')} + +
+ + {$t('app.journaling.smtp_port')}: {smtpHealth.port} + +
+ +
+ + + + {$t('app.journaling.name')} + {$t('app.journaling.allowed_ips')} + {$t('app.journaling.total_received')} + {$t('app.journaling.status')} + {$t('app.journaling.last_received_at')} + {$t('app.journaling.actions')} + + + + {#if sources && sources.length > 0} + {#each sources as source (source.id)} + + +
+
{source.name}
+
+ {source.routingAddress} + +
+
+
+ +
+ {#each source.allowedIps.slice(0, 3) as ip} + + {ip} + + {/each} + {#if source.allowedIps.length > 3} + + +{source.allowedIps.length - 3} + + {/if} +
+
+ +
+ + 0 ? 'secondary' : 'outline'}> + {source.totalReceived} + +
+
+ + {#if source.status === 'active'} + + {$t('app.journaling.active')} + + {:else} + + {$t('app.journaling.paused')} + + {/if} + + + {#if source.lastReceivedAt} + {new Date(source.lastReceivedAt).toLocaleString()} + {:else} + + {$t('app.journaling.never')} + + {/if} + + + + + {#snippet child({ props })} + + {/snippet} + + + openEdit(source)}> + {$t('app.journaling.edit')} + + +
{ + return async ({ result, update }) => { + if ( + result.type === 'success' && + result.data?.success !== false + ) { + setAlert({ + type: 'success', + title: $t('app.journaling.update_success'), + message: '', + duration: 3000, + show: true, + }); + } else if ( + result.type === 'success' && + result.data?.success === false + ) { + setAlert({ + type: 'error', + title: $t('app.journaling.update_error'), + message: String(result.data?.message ?? ''), + duration: 5000, + show: true, + }); + } + await update(); + }; + }} + > + + + + + +
+ + openDelete(source)} + > + {$t('app.journaling.delete')} + +
+
+
+
+ {/each} + {:else} + + + {$t('app.journaling.no_sources_found')} + + + {/if} +
+
+
+ + + + + + {$t('app.journaling.create')} + + {$t('app.journaling.create_description')} + + +
{ + isFormLoading = true; + return async ({ result, update }) => { + isFormLoading = false; + if (result.type === 'success' && result.data?.success !== false) { + isCreateOpen = false; + setAlert({ + type: 'success', + title: $t('app.journaling.create_success'), + message: '', + duration: 3000, + show: true, + }); + } else if (result.type === 'success' && result.data?.success === false) { + setAlert({ + type: 'error', + title: $t('app.journaling.create_error'), + message: String(result.data?.message ?? ''), + duration: 5000, + show: true, + }); + } + await update(); + }; + }} + > +
+ + +
+
+ + +

+ {$t('app.journaling.allowed_ips_hint')} +

+
+
+ + +
+
+

+ {$t('app.journaling.smtp_auth_hint')} +

+
+ + +
+
+ + +
+
+
+ + +
+
+
+
+ + + + + + {$t('app.journaling.edit')} + + {$t('app.journaling.edit_description')} + + + {#if selectedSource} +
{ + isFormLoading = true; + return async ({ result, update }) => { + isFormLoading = false; + if (result.type === 'success' && result.data?.success !== false) { + isEditOpen = false; + selectedSource = null; + setAlert({ + type: 'success', + title: $t('app.journaling.update_success'), + message: '', + duration: 3000, + show: true, + }); + } else if (result.type === 'success' && result.data?.success === false) { + setAlert({ + type: 'error', + title: $t('app.journaling.update_error'), + message: String(result.data?.message ?? ''), + duration: 5000, + show: true, + }); + } + await update(); + }; + }} + > + + + +
+

+ {$t('app.journaling.smtp_connection_info')} +

+ + +
+ {$t('app.journaling.routing_address')} +
+ {selectedSource.routingAddress} + +
+

+ {$t('app.journaling.routing_address_hint')} +

+ +
+ +

+ {$t('app.journaling.regenerate_address_warning')} +

+
+
+ +
+
+ {$t('app.journaling.smtp_host')} +
+ {typeof window !== 'undefined' + ? window.location.hostname + : 'localhost'} + +
+
+
+ {$t('app.journaling.smtp_port')} +
+ {smtpHealth.port} + +
+
+
+
+ +
+ + +
+
+ + +

+ {$t('app.journaling.allowed_ips_hint')} +

+
+
+ + +
+
+

+ {$t('app.journaling.smtp_auth_hint')} +

+
+ + +
+
+ + +
+
+
+ + +
+
+ {/if} +
+
+ + + + + + {$t('app.journaling.delete_confirmation_title')} + + {$t('app.journaling.delete_confirmation_description')} + + + + + {#if selectedSource} +
{ + isFormLoading = true; + return async ({ result, update }) => { + isFormLoading = false; + if (result.type === 'success' && result.data?.success !== false) { + isDeleteOpen = false; + setAlert({ + type: 'success', + title: $t('app.journaling.delete_success'), + message: '', + duration: 3000, + show: true, + }); + selectedSource = null; + } else { + setAlert({ + type: 'error', + title: $t('app.journaling.delete_error'), + message: + result.type === 'success' + ? String(result.data?.message ?? '') + : '', + duration: 5000, + show: true, + }); + } + await update(); + }; + }} + > + + +
+ {/if} +
+
+
+ + + + + + {$t('app.journaling.regenerate_address')} + + {$t('app.journaling.regenerate_address_confirm')} + + + + + + + + diff --git a/packages/types/src/audit-log.enums.ts b/packages/types/src/audit-log.enums.ts index 9407f7d..63d3572 100644 --- a/packages/types/src/audit-log.enums.ts +++ b/packages/types/src/audit-log.enums.ts @@ -27,6 +27,7 @@ export const AuditLogTargetTypes = [ 'ArchivedEmail', 'Dashboard', 'IngestionSource', + 'JournalingSource', 'RetentionPolicy', 'RetentionLabel', 'LegalHold', diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index a0b08aa..4423354 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -14,3 +14,4 @@ export * from './integrity.types'; export * from './jobs.types'; export * from './license.types'; export * from './retention.types'; +export * from './journaling.types'; diff --git a/packages/types/src/ingestion.types.ts b/packages/types/src/ingestion.types.ts index 180233e..72bd836 100644 --- a/packages/types/src/ingestion.types.ts +++ b/packages/types/src/ingestion.types.ts @@ -24,7 +24,8 @@ export type IngestionProvider = | 'generic_imap' | 'pst_import' | 'eml_import' - | 'mbox_import'; + | 'mbox_import' + | 'smtp_journaling'; export type IngestionStatus = | 'active' @@ -91,6 +92,12 @@ export interface MboxImportCredentials extends BaseIngestionCredentials { localFilePath?: string; } +export interface SmtpJournalingCredentials extends BaseIngestionCredentials { + type: 'smtp_journaling'; + /** The ID of the journaling_sources row that owns this ingestion source */ + journalingSourceId: string; +} + // Discriminated union for all possible credential types export type IngestionCredentials = | GenericImapCredentials @@ -98,7 +105,8 @@ export type IngestionCredentials = | Microsoft365Credentials | PSTImportCredentials | EMLImportCredentials - | MboxImportCredentials; + | MboxImportCredentials + | SmtpJournalingCredentials; export interface IngestionSource { id: string; diff --git a/packages/types/src/journaling.types.ts b/packages/types/src/journaling.types.ts new file mode 100644 index 0000000..bca01fd --- /dev/null +++ b/packages/types/src/journaling.types.ts @@ -0,0 +1,65 @@ +/** Status of a journaling source's SMTP listener */ +export type JournalingSourceStatus = 'active' | 'paused'; + +/** Represents a configured journaling source */ +export interface JournalingSource { + id: string; + name: string; + /** CIDR blocks or IP addresses allowed to send journal reports */ + allowedIps: string[]; + /** Whether to reject plain-text (non-TLS) connections */ + requireTls: boolean; + /** Optional SMTP AUTH username for the journal endpoint */ + smtpUsername: string | null; + status: JournalingSourceStatus; + /** The backing ingestion source ID that owns archived emails */ + ingestionSourceId: string; + /** + * The SMTP routing address the admin must configure in their MTA + * (e.g. journal-abc12345@archive.yourdomain.com). + * Computed server-side from the source ID and SMTP_JOURNALING_DOMAIN. + */ + routingAddress: string; + /** Total number of emails received via this journaling source */ + totalReceived: number; + /** Timestamp of the last email received */ + lastReceivedAt: Date | null; + createdAt: Date; + updatedAt: Date; +} + +/** DTO for creating a new journaling source */ +export interface CreateJournalingSourceDto { + name: string; + allowedIps: string[]; + requireTls?: boolean; + smtpUsername?: string; + smtpPassword?: string; +} + +/** DTO for updating an existing journaling source */ +export interface UpdateJournalingSourceDto { + name?: string; + allowedIps?: string[]; + requireTls?: boolean; + status?: JournalingSourceStatus; + smtpUsername?: string; + smtpPassword?: string; +} + +/** Job data for the journal-inbound BullMQ job */ +export interface IJournalInboundJob { + /** The journaling source ID that received the email */ + journalingSourceId: string; + /** + * Path to the temp file containing the raw email data on the local filesystem. + * Raw emails are written to disk instead of embedded in the Redis job payload + * to avoid Redis memory pressure (base64 inflates 50MB → ~67MB per job). + * The worker is responsible for deleting this file after processing. + */ + tempFilePath: string; + /** IP address of the sending MTA */ + remoteAddress: string; + /** Timestamp when the SMTP listener received the email */ + receivedAt: string; +} diff --git a/packages/types/src/license.types.ts b/packages/types/src/license.types.ts index b2f269c..04b93a0 100644 --- a/packages/types/src/license.types.ts +++ b/packages/types/src/license.types.ts @@ -6,6 +6,7 @@ export enum OpenArchiverFeature { RETENTION_POLICY = 'retention-policy', LEGAL_HOLDS = 'legal-holds', INTEGRITY_REPORT = 'integrity-report', + JOURNALING = 'journaling', SSO = 'sso', STATUS = 'status', ALL = 'all',