diff --git a/packages/backend/src/database/migrations/0029_lethal_brood.sql b/packages/backend/src/database/migrations/0029_lethal_brood.sql new file mode 100644 index 0000000..6b9e6b8 --- /dev/null +++ b/packages/backend/src/database/migrations/0029_lethal_brood.sql @@ -0,0 +1,2 @@ +ALTER TABLE "archived_emails" ADD COLUMN "provider_message_id" text;--> statement-breakpoint +CREATE INDEX "provider_msg_source_idx" ON "archived_emails" USING btree ("provider_message_id","ingestion_source_id"); \ No newline at end of file diff --git a/packages/backend/src/database/migrations/meta/0029_snapshot.json b/packages/backend/src/database/migrations/meta/0029_snapshot.json new file mode 100644 index 0000000..d644b24 --- /dev/null +++ b/packages/backend/src/database/migrations/meta/0029_snapshot.json @@ -0,0 +1,1711 @@ +{ + "id": "5b69110d-3df3-41e0-982c-57413d5956f5", + "prevId": "3a1b6200-f229-4fd9-af5a-3b9ceddc0bd8", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.archived_emails": { + "name": "archived_emails", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ingestion_source_id": { + "name": "ingestion_source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_email": { + "name": "user_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id_header": { + "name": "message_id_header", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_message_id": { + "name": "provider_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sender_name": { + "name": "sender_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sender_email": { + "name": "sender_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipients": { + "name": "recipients", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "storage_path": { + "name": "storage_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_hash_sha256": { + "name": "storage_hash_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "is_indexed": { + "name": "is_indexed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "has_attachments": { + "name": "has_attachments", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "is_on_legal_hold": { + "name": "is_on_legal_hold", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tags": { + "name": "tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "thread_id_idx": { + "name": "thread_id_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "provider_msg_source_idx": { + "name": "provider_msg_source_idx", + "columns": [ + { + "expression": "provider_message_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "ingestion_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "archived_emails_ingestion_source_id_ingestion_sources_id_fk": { + "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk", + "tableFrom": "archived_emails", + "tableTo": "ingestion_sources", + "columnsFrom": [ + "ingestion_source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.attachments": { + "name": "attachments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "content_hash_sha256": { + "name": "content_hash_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_path": { + "name": "storage_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ingestion_source_id": { + "name": "ingestion_source_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "source_hash_idx": { + "name": "source_hash_idx", + "columns": [ + { + "expression": "ingestion_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "content_hash_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "attachments_ingestion_source_id_ingestion_sources_id_fk": { + "name": "attachments_ingestion_source_id_ingestion_sources_id_fk", + "tableFrom": "attachments", + "tableTo": "ingestion_sources", + "columnsFrom": [ + "ingestion_source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.email_attachments": { + "name": "email_attachments", + "schema": "", + "columns": { + "email_id": { + "name": "email_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "attachment_id": { + "name": "attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "email_attachments_email_id_archived_emails_id_fk": { + "name": "email_attachments_email_id_archived_emails_id_fk", + "tableFrom": "email_attachments", + "tableTo": "archived_emails", + "columnsFrom": [ + "email_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "email_attachments_attachment_id_attachments_id_fk": { + "name": "email_attachments_attachment_id_attachments_id_fk", + "tableFrom": "email_attachments", + "tableTo": "attachments", + "columnsFrom": [ + "attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "email_attachments_email_id_attachment_id_pk": { + "name": "email_attachments_email_id_attachment_id_pk", + "columns": [ + "email_id", + "attachment_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_logs": { + "name": "audit_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "previous_hash": { + "name": "previous_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "timestamp": { + "name": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "actor_identifier": { + "name": "actor_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_ip": { + "name": "actor_ip", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action_type": { + "name": "action_type", + "type": "audit_log_action", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "audit_log_target_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "current_hash": { + "name": "current_hash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ediscovery_cases": { + "name": "ediscovery_cases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "created_by_identifier": { + "name": "created_by_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ediscovery_cases_name_unique": { + "name": "ediscovery_cases_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.email_legal_holds": { + "name": "email_legal_holds", + "schema": "", + "columns": { + "email_id": { + "name": "email_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "legal_hold_id": { + "name": "legal_hold_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "applied_at": { + "name": "applied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "applied_by_user_id": { + "name": "applied_by_user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "email_legal_holds_email_id_archived_emails_id_fk": { + "name": "email_legal_holds_email_id_archived_emails_id_fk", + "tableFrom": "email_legal_holds", + "tableTo": "archived_emails", + "columnsFrom": [ + "email_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "email_legal_holds_legal_hold_id_legal_holds_id_fk": { + "name": "email_legal_holds_legal_hold_id_legal_holds_id_fk", + "tableFrom": "email_legal_holds", + "tableTo": "legal_holds", + "columnsFrom": [ + "legal_hold_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "email_legal_holds_applied_by_user_id_users_id_fk": { + "name": "email_legal_holds_applied_by_user_id_users_id_fk", + "tableFrom": "email_legal_holds", + "tableTo": "users", + "columnsFrom": [ + "applied_by_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "email_legal_holds_email_id_legal_hold_id_pk": { + "name": "email_legal_holds_email_id_legal_hold_id_pk", + "columns": [ + "email_id", + "legal_hold_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.email_retention_labels": { + "name": "email_retention_labels", + "schema": "", + "columns": { + "email_id": { + "name": "email_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "label_id": { + "name": "label_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "applied_at": { + "name": "applied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "applied_by_user_id": { + "name": "applied_by_user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "email_retention_labels_email_id_archived_emails_id_fk": { + "name": "email_retention_labels_email_id_archived_emails_id_fk", + "tableFrom": "email_retention_labels", + "tableTo": "archived_emails", + "columnsFrom": [ + "email_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "email_retention_labels_label_id_retention_labels_id_fk": { + "name": "email_retention_labels_label_id_retention_labels_id_fk", + "tableFrom": "email_retention_labels", + "tableTo": "retention_labels", + "columnsFrom": [ + "label_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "email_retention_labels_applied_by_user_id_users_id_fk": { + "name": "email_retention_labels_applied_by_user_id_users_id_fk", + "tableFrom": "email_retention_labels", + "tableTo": "users", + "columnsFrom": [ + "applied_by_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "email_retention_labels_email_id_label_id_pk": { + "name": "email_retention_labels_email_id_label_id_pk", + "columns": [ + "email_id", + "label_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.export_jobs": { + "name": "export_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "case_id": { + "name": "case_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "query": { + "name": "query", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by_identifier": { + "name": "created_by_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "export_jobs_case_id_ediscovery_cases_id_fk": { + "name": "export_jobs_case_id_ediscovery_cases_id_fk", + "tableFrom": "export_jobs", + "tableTo": "ediscovery_cases", + "columnsFrom": [ + "case_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.legal_holds": { + "name": "legal_holds", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "case_id": { + "name": "case_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "legal_holds_case_id_ediscovery_cases_id_fk": { + "name": "legal_holds_case_id_ediscovery_cases_id_fk", + "tableFrom": "legal_holds", + "tableTo": "ediscovery_cases", + "columnsFrom": [ + "case_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.retention_events": { + "name": "retention_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "event_name": { + "name": "event_name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "event_timestamp": { + "name": "event_timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "target_criteria": { + "name": "target_criteria", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.retention_labels": { + "name": "retention_labels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "retention_period_days": { + "name": "retention_period_days", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_disabled": { + "name": "is_disabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.retention_policies": { + "name": "retention_policies", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "retention_period_days": { + "name": "retention_period_days", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "action_on_expiry": { + "name": "action_on_expiry", + "type": "retention_action", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "conditions": { + "name": "conditions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ingestion_scope": { + "name": "ingestion_scope", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "retention_policies_name_unique": { + "name": "retention_policies_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custodians": { + "name": "custodians", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_type": { + "name": "source_type", + "type": "ingestion_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "custodians_email_unique": { + "name": "custodians_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ingestion_sources": { + "name": "ingestion_sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "ingestion_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "ingestion_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending_auth'" + }, + "last_sync_started_at": { + "name": "last_sync_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_finished_at": { + "name": "last_sync_finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_status_message": { + "name": "last_sync_status_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_state": { + "name": "sync_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "ingestion_sources_user_id_users_id_fk": { + "name": "ingestion_sources_user_id_users_id_fk", + "tableFrom": "ingestion_sources", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.roles": { + "name": "roles", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "policies": { + "name": "policies", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "roles_name_unique": { + "name": "roles_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + }, + "roles_slug_unique": { + "name": "roles_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_roles_role_id_roles_id_fk": { + "name": "user_roles_role_id_roles_id_fk", + "tableFrom": "user_roles", + "tableTo": "roles", + "columnsFrom": [ + "role_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_id_pk": { + "name": "user_roles_user_id_role_id_pk", + "columns": [ + "user_id", + "role_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "first_name": { + "name": "first_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_name": { + "name": "last_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'local'" + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "api_keys_user_id_users_id_fk": { + "name": "api_keys_user_id_users_id_fk", + "tableFrom": "api_keys", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sync_sessions": { + "name": "sync_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "ingestion_source_id": { + "name": "ingestion_source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "is_initial_import": { + "name": "is_initial_import", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "total_mailboxes": { + "name": "total_mailboxes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "completed_mailboxes": { + "name": "completed_mailboxes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "failed_mailboxes": { + "name": "failed_mailboxes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error_messages": { + "name": "error_messages", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_activity_at": { + "name": "last_activity_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sync_sessions_ingestion_source_id_ingestion_sources_id_fk": { + "name": "sync_sessions_ingestion_source_id_ingestion_sources_id_fk", + "tableFrom": "sync_sessions", + "tableTo": "ingestion_sources", + "columnsFrom": [ + "ingestion_source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.retention_action": { + "name": "retention_action", + "schema": "public", + "values": [ + "delete_permanently", + "notify_admin" + ] + }, + "public.ingestion_provider": { + "name": "ingestion_provider", + "schema": "public", + "values": [ + "google_workspace", + "microsoft_365", + "generic_imap", + "pst_import", + "eml_import", + "mbox_import" + ] + }, + "public.ingestion_status": { + "name": "ingestion_status", + "schema": "public", + "values": [ + "active", + "paused", + "error", + "pending_auth", + "syncing", + "importing", + "auth_success", + "imported" + ] + }, + "public.audit_log_action": { + "name": "audit_log_action", + "schema": "public", + "values": [ + "CREATE", + "READ", + "UPDATE", + "DELETE", + "LOGIN", + "LOGOUT", + "SETUP", + "IMPORT", + "PAUSE", + "SYNC", + "UPLOAD", + "SEARCH", + "DOWNLOAD", + "GENERATE" + ] + }, + "public.audit_log_target_type": { + "name": "audit_log_target_type", + "schema": "public", + "values": [ + "ApiKey", + "ArchivedEmail", + "Dashboard", + "IngestionSource", + "RetentionPolicy", + "RetentionLabel", + "LegalHold", + "Role", + "SystemEvent", + "SystemSettings", + "User", + "File" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/backend/src/database/migrations/meta/_journal.json b/packages/backend/src/database/migrations/meta/_journal.json index 3af986c..010556e 100644 --- a/packages/backend/src/database/migrations/meta/_journal.json +++ b/packages/backend/src/database/migrations/meta/_journal.json @@ -204,6 +204,13 @@ "when": 1773770326402, "tag": "0028_youthful_kitty_pryde", "breakpoints": true + }, + { + "idx": 29, + "version": "7", + "when": 1773927678269, + "tag": "0029_lethal_brood", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/backend/src/database/schema/archived-emails.ts b/packages/backend/src/database/schema/archived-emails.ts index ef90f46..42aa2bb 100644 --- a/packages/backend/src/database/schema/archived-emails.ts +++ b/packages/backend/src/database/schema/archived-emails.ts @@ -12,6 +12,9 @@ export const archivedEmails = pgTable( .references(() => ingestionSources.id, { onDelete: 'cascade' }), userEmail: text('user_email').notNull(), messageIdHeader: text('message_id_header'), + /** The provider-specific message ID (e.g., Gmail API ID, Graph API ID). + * Used by the pre-fetch duplicate check to avoid unnecessary API calls during retries. */ + providerMessageId: text('provider_message_id'), sentAt: timestamp('sent_at', { withTimezone: true }).notNull(), subject: text('subject'), senderName: text('sender_name'), @@ -27,7 +30,10 @@ export const archivedEmails = pgTable( path: text('path'), tags: jsonb('tags'), }, - (table) => [index('thread_id_idx').on(table.threadId)] + (table) => [ + index('thread_id_idx').on(table.threadId), + index('provider_msg_source_idx').on(table.providerMessageId, table.ingestionSourceId), + ] ); export const archivedEmailsRelations = relations(archivedEmails, ({ one }) => ({ diff --git a/packages/backend/src/services/IngestionService.ts b/packages/backend/src/services/IngestionService.ts index ddfc299..e3388af 100644 --- a/packages/backend/src/services/IngestionService.ts +++ b/packages/backend/src/services/IngestionService.ts @@ -8,7 +8,7 @@ import type { IngestionProvider, PendingEmail, } from '@open-archiver/types'; -import { and, desc, eq } from 'drizzle-orm'; +import { and, desc, eq, or } from 'drizzle-orm'; import { CryptoService } from './CryptoService'; import { EmailProviderFactory } from './EmailProviderFactory'; import { ingestionQueue } from '../jobs/queues'; @@ -392,8 +392,9 @@ export class IngestionService { } /** - * Quickly checks if an email exists in the database by its Message-ID header. - * This is used to skip downloading duplicate emails during ingestion. + * Pre-fetch duplicate check to avoid unnecessary API calls during ingestion. + * Checks both providerMessageId (for Google/Microsoft API IDs) and + * messageIdHeader (for IMAP/PST/EML/Mbox RFC Message-IDs and pre-migration rows). */ public static async doesEmailExist( messageId: string, @@ -401,12 +402,14 @@ export class IngestionService { ): Promise { const existingEmail = await db.query.archivedEmails.findFirst({ where: and( - eq(archivedEmails.messageIdHeader, messageId), - eq(archivedEmails.ingestionSourceId, ingestionSourceId) + eq(archivedEmails.ingestionSourceId, ingestionSourceId), + or( + eq(archivedEmails.providerMessageId, messageId), + eq(archivedEmails.messageIdHeader, messageId) + ) ), columns: { id: true }, }); - return !!existingEmail; } @@ -463,6 +466,7 @@ export class IngestionService { userEmail, threadId: email.threadId, messageIdHeader: messageId, + providerMessageId: email.id, sentAt: email.receivedAt, subject: email.subject, senderName: email.from[0]?.name, diff --git a/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts b/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts index d742932..cd704f1 100644 --- a/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts @@ -225,7 +225,6 @@ export class GoogleWorkspaceConnector implements IEmailConnector { ); }; const threadId = getThreadId(parsedEmail.headers); - console.log('threadId', threadId); yield { id: msgResponse.data.id!, threadId, @@ -348,7 +347,6 @@ export class GoogleWorkspaceConnector implements IEmailConnector { ); }; const threadId = getThreadId(parsedEmail.headers); - console.log('threadId', threadId); yield { id: msgResponse.data.id!, threadId, diff --git a/packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts b/packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts index 911e202..83e9457 100644 --- a/packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts @@ -136,7 +136,8 @@ export class MicrosoftConnector implements IEmailConnector { */ public async *fetchEmails( userEmail: string, - syncState?: SyncState | null + syncState?: SyncState | null, + checkDuplicate?: (messageId: string) => Promise ): AsyncGenerator { this.newDeltaTokens = syncState?.microsoft?.[userEmail]?.deltaTokens || {}; @@ -152,7 +153,8 @@ export class MicrosoftConnector implements IEmailConnector { userEmail, folder.id, folder.path, - this.newDeltaTokens[folder.id] + this.newDeltaTokens[folder.id], + checkDuplicate ); } } @@ -214,7 +216,8 @@ export class MicrosoftConnector implements IEmailConnector { userEmail: string, folderId: string, path: string, - deltaToken?: string + deltaToken?: string, + checkDuplicate?: (messageId: string) => Promise ): AsyncGenerator { let requestUrl: string | undefined; @@ -235,6 +238,15 @@ export class MicrosoftConnector implements IEmailConnector { for (const message of response.value) { if (message.id && !message['@removed']) { + // Skip fetching raw content for already-imported messages + if (checkDuplicate && (await checkDuplicate(message.id))) { + logger.debug( + { messageId: message.id, userEmail }, + 'Skipping duplicate email (pre-check)' + ); + continue; + } + const rawEmail = await this.getRawEmail(userEmail, message.id); if (rawEmail) { const emailObject = await this.parseEmail( @@ -243,7 +255,7 @@ export class MicrosoftConnector implements IEmailConnector { userEmail, path ); - emailObject.threadId = message.conversationId; // Add conversationId as threadId + emailObject.threadId = message.conversationId; yield emailObject; } }