From f10bf93d1bbfb3e1ef847221a7f111fceb05f1f9 Mon Sep 17 00:00:00 2001 From: Wayne <5291640+ringoinca@users.noreply.github.com> Date: Mon, 11 Aug 2025 10:55:50 +0300 Subject: [PATCH] eml import support --- README.md | 11 +- docs/.vitepress/config.mts | 6 +- docs/user-guides/email-providers/eml.md | 36 + docs/user-guides/email-providers/index.md | 2 + docs/user-guides/email-providers/pst.md | 21 + packages/backend/package.json | 6 +- .../database/migrations/0014_foamy_vapor.sql | 1 + .../migrations/meta/0014_snapshot.json | 1108 +++++++++++++++++ .../database/migrations/meta/_journal.json | 7 + .../src/database/schema/ingestion-sources.ts | 3 +- .../processors/initial-import.processor.ts | 5 +- .../sync-cycle-finished.processor.ts | 6 +- .../src/services/EmailProviderFactory.ts | 4 + .../backend/src/services/IngestionService.ts | 34 +- .../ingestion-connectors/EMLConnector.ts | 199 +++ .../ingestion-connectors/ImapConnector.ts | 2 +- .../custom/IngestionSourceForm.svelte | 13 +- .../dashboard/archived-emails/+page.svelte | 4 +- packages/types/src/ingestion.types.ts | 11 +- pnpm-lock.yaml | 354 ++++++ 20 files changed, 1800 insertions(+), 33 deletions(-) create mode 100644 docs/user-guides/email-providers/eml.md create mode 100644 docs/user-guides/email-providers/pst.md create mode 100644 packages/backend/src/database/migrations/0014_foamy_vapor.sql create mode 100644 packages/backend/src/database/migrations/meta/0014_snapshot.json create mode 100644 packages/backend/src/services/ingestion-connectors/EMLConnector.ts diff --git a/README.md b/README.md index 2fbd3d6..46db7f1 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ **A secure, sovereign, and open-source platform for email archiving and eDiscovery.** -Open Archiver provides a robust, self-hosted solution for archiving, storing, indexing, and searching emails from major platforms, including Google Workspace (Gmail), Microsoft 365, as well as generic IMAP-enabled email inboxes. Use Open Archiver to keep a permanent, tamper-proof record of your communication history, free from vendor lock-in. +Open Archiver provides a robust, self-hosted solution for archiving, storing, indexing, and searching emails from major platforms, including Google Workspace (Gmail), Microsoft 365, PST files, as well as generic IMAP-enabled email inboxes. Use Open Archiver to keep a permanent, tamper-proof record of your communication history, free from vendor lock-in. ## 📸 Screenshots @@ -40,7 +40,14 @@ Password: openarchiver_demo ## ✨ Key Features -- **Universal Ingestion**: Connect to Google Workspace, Microsoft 365, and standard IMAP servers to perform initial bulk imports and maintain continuous, real-time synchronization. +- **Universal Ingestion**: Connect to any email provider to perform initial bulk imports and maintain continuous, real-time synchronization. Ingestion sources include: + + - IMAP connection + - Google Workspace + - Microsoft 365 + - PST files + - Zipped .eml files + - **Secure & Efficient Storage**: Emails are stored in the standard `.eml` format. The system uses deduplication and compression to minimize storage costs. All data is encrypted at rest. - **Pluggable Storage Backends**: Support both local filesystem storage and S3-compatible object storage (like AWS S3 or MinIO). - **Powerful Search & eDiscovery**: A high-performance search engine indexes the full text of emails and attachments (PDF, DOCX, etc.). diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 3b671a4..c0ddfb3 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -37,9 +37,11 @@ export default defineConfig({ link: '/user-guides/email-providers/', collapsed: true, items: [ - { text: 'Google Workspace', link: '/user-guides/email-providers/google-workspace' }, { text: 'Generic IMAP Server', link: '/user-guides/email-providers/imap' }, - { text: 'Microsoft 365', link: '/user-guides/email-providers/microsoft-365' } + { text: 'Google Workspace', link: '/user-guides/email-providers/google-workspace' }, + { text: 'Microsoft 365', link: '/user-guides/email-providers/microsoft-365' }, + { text: 'EML Import', link: '/user-guides/email-providers/eml' }, + { text: 'PST Import', link: '/user-guides/email-providers/pst' } ] } ] diff --git a/docs/user-guides/email-providers/eml.md b/docs/user-guides/email-providers/eml.md new file mode 100644 index 0000000..e151b1b --- /dev/null +++ b/docs/user-guides/email-providers/eml.md @@ -0,0 +1,36 @@ +# EML Import + +OpenArchiver allows you to import EML files from a zip archive. This is useful for importing emails from a variety of sources, including other email clients and services. + +## Preparing the Zip File + +To ensure a successful import, you should compress your .eml files to one zip file according to the following guidelines: + +- **Structure:** The zip file can contain any number of `.eml` files, organized in any folder structure. The folder structure will be preserved in OpenArchiver, so you can use it to organize your emails. +- **Compression:** The zip file should be compressed using standard zip compression. + +Here's an example of a valid folder structure: + +``` +archive.zip +├── inbox +│ ├── email-01.eml +│ └── email-02.eml +├── sent +│ └── email-03.eml +└── drafts + ├── nested-folder + │ └── email-04.eml + └── email-05.eml +``` + +## Creating an EML Ingestion Source + +1. Go to the **Ingestion Sources** page in the OpenArchiver dashboard. +2. Click the **Create New** button. +3. Select **EML Import** as the provider. +4. Enter a name for the ingestion source. +5. Click the **Choose File** button and select the zip archive containing your EML files. +6. Click the **Submit** button. + +OpenArchiver will then start importing the EML files from the zip archive. The ingestion process may take some time, depending on the size of the archive. diff --git a/docs/user-guides/email-providers/index.md b/docs/user-guides/email-providers/index.md index 771b706..dfa8afd 100644 --- a/docs/user-guides/email-providers/index.md +++ b/docs/user-guides/email-providers/index.md @@ -7,3 +7,5 @@ Choose your provider from the list below to get started: - [Google Workspace](./google-workspace.md) - [Microsoft 365](./microsoft-365.md) - [Generic IMAP Server](./imap.md) +- [EML Import](./eml.md) +- [PST Import](./pst.md) diff --git a/docs/user-guides/email-providers/pst.md b/docs/user-guides/email-providers/pst.md new file mode 100644 index 0000000..a029450 --- /dev/null +++ b/docs/user-guides/email-providers/pst.md @@ -0,0 +1,21 @@ +# PST Import + +OpenArchiver allows you to import PST files. This is useful for importing emails from a variety of sources, including Microsoft Outlook. + +## Preparing the PST File + +To ensure a successful import, you should prepare your PST file according to the following guidelines: + +- **Structure:** The PST file can contain any number of emails, organized in any folder structure. The folder structure will be preserved in OpenArchiver, so you can use it to organize your emails. +- **Password Protection:** OpenArchiver does not support password-protected PST files. Please remove the password from your PST file before importing it. + +## Creating a PST Ingestion Source + +1. Go to the **Ingestion Sources** page in the OpenArchiver dashboard. +2. Click the **Create New** button. +3. Select **PST Import** as the provider. +4. Enter a name for the ingestion source. +5. Click the **Choose File** button and select the PST file. +6. Click the **Submit** button. + +OpenArchiver will then start importing the emails from the PST file. The ingestion process may take some time, depending on the size of the file. diff --git a/packages/backend/package.json b/packages/backend/package.json index b714147..47693e6 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -24,6 +24,7 @@ "@azure/msal-node": "^3.6.3", "@microsoft/microsoft-graph-client": "^3.0.7", "@open-archiver/types": "workspace:*", + "archiver": "^7.0.1", "axios": "^1.10.0", "bcryptjs": "^3.0.2", "bullmq": "^5.56.3", @@ -53,17 +54,20 @@ "reflect-metadata": "^0.2.2", "sqlite3": "^5.1.7", "tsconfig-paths": "^4.2.0", - "xlsx": "^0.18.5" + "xlsx": "^0.18.5", + "yauzl": "^3.2.0" }, "devDependencies": { "@bull-board/api": "^6.11.0", "@bull-board/express": "^6.11.0", + "@types/archiver": "^6.0.3", "@types/busboy": "^1.5.4", "@types/express": "^5.0.3", "@types/mailparser": "^3.4.6", "@types/microsoft-graph": "^2.40.1", "@types/multer": "^2.0.0", "@types/node": "^24.0.12", + "@types/yauzl": "^2.10.3", "bull-board": "^2.1.3", "ts-node-dev": "^2.0.0", "typescript": "^5.8.3" diff --git a/packages/backend/src/database/migrations/0014_foamy_vapor.sql b/packages/backend/src/database/migrations/0014_foamy_vapor.sql new file mode 100644 index 0000000..804fee8 --- /dev/null +++ b/packages/backend/src/database/migrations/0014_foamy_vapor.sql @@ -0,0 +1 @@ +ALTER TYPE "public"."ingestion_provider" ADD VALUE 'eml_import'; \ No newline at end of file diff --git a/packages/backend/src/database/migrations/meta/0014_snapshot.json b/packages/backend/src/database/migrations/meta/0014_snapshot.json new file mode 100644 index 0000000..50f5d01 --- /dev/null +++ b/packages/backend/src/database/migrations/meta/0014_snapshot.json @@ -0,0 +1,1108 @@ +{ + "id": "ad5204da-bb82-4a19-abfa-d30cc284ab27", + "prevId": "c397c819-e69f-42c7-966c-7b2969741c56", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.archived_emails": { + "name": "archived_emails", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ingestion_source_id": { + "name": "ingestion_source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_email": { + "name": "user_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id_header": { + "name": "message_id_header", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sender_name": { + "name": "sender_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sender_email": { + "name": "sender_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipients": { + "name": "recipients", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "storage_path": { + "name": "storage_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_hash_sha256": { + "name": "storage_hash_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "is_indexed": { + "name": "is_indexed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "has_attachments": { + "name": "has_attachments", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "is_on_legal_hold": { + "name": "is_on_legal_hold", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tags": { + "name": "tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "thread_id_idx": { + "name": "thread_id_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "archived_emails_ingestion_source_id_ingestion_sources_id_fk": { + "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk", + "tableFrom": "archived_emails", + "tableTo": "ingestion_sources", + "columnsFrom": [ + "ingestion_source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.attachments": { + "name": "attachments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "content_hash_sha256": { + "name": "content_hash_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_path": { + "name": "storage_path", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "attachments_content_hash_sha256_unique": { + "name": "attachments_content_hash_sha256_unique", + "nullsNotDistinct": false, + "columns": [ + "content_hash_sha256" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.email_attachments": { + "name": "email_attachments", + "schema": "", + "columns": { + "email_id": { + "name": "email_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "attachment_id": { + "name": "attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "email_attachments_email_id_archived_emails_id_fk": { + "name": "email_attachments_email_id_archived_emails_id_fk", + "tableFrom": "email_attachments", + "tableTo": "archived_emails", + "columnsFrom": [ + "email_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "email_attachments_attachment_id_attachments_id_fk": { + "name": "email_attachments_attachment_id_attachments_id_fk", + "tableFrom": "email_attachments", + "tableTo": "attachments", + "columnsFrom": [ + "attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "email_attachments_email_id_attachment_id_pk": { + "name": "email_attachments_email_id_attachment_id_pk", + "columns": [ + "email_id", + "attachment_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_logs": { + "name": "audit_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "timestamp": { + "name": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "actor_identifier": { + "name": "actor_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "is_tamper_evident": { + "name": "is_tamper_evident", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ediscovery_cases": { + "name": "ediscovery_cases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "created_by_identifier": { + "name": "created_by_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ediscovery_cases_name_unique": { + "name": "ediscovery_cases_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.export_jobs": { + "name": "export_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "case_id": { + "name": "case_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "query": { + "name": "query", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by_identifier": { + "name": "created_by_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "export_jobs_case_id_ediscovery_cases_id_fk": { + "name": "export_jobs_case_id_ediscovery_cases_id_fk", + "tableFrom": "export_jobs", + "tableTo": "ediscovery_cases", + "columnsFrom": [ + "case_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.legal_holds": { + "name": "legal_holds", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "case_id": { + "name": "case_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "custodian_id": { + "name": "custodian_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "hold_criteria": { + "name": "hold_criteria", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "applied_by_identifier": { + "name": "applied_by_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "applied_at": { + "name": "applied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "removed_at": { + "name": "removed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "legal_holds_case_id_ediscovery_cases_id_fk": { + "name": "legal_holds_case_id_ediscovery_cases_id_fk", + "tableFrom": "legal_holds", + "tableTo": "ediscovery_cases", + "columnsFrom": [ + "case_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "legal_holds_custodian_id_custodians_id_fk": { + "name": "legal_holds_custodian_id_custodians_id_fk", + "tableFrom": "legal_holds", + "tableTo": "custodians", + "columnsFrom": [ + "custodian_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.retention_policies": { + "name": "retention_policies", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "retention_period_days": { + "name": "retention_period_days", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "action_on_expiry": { + "name": "action_on_expiry", + "type": "retention_action", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "conditions": { + "name": "conditions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "retention_policies_name_unique": { + "name": "retention_policies_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custodians": { + "name": "custodians", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_type": { + "name": "source_type", + "type": "ingestion_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "custodians_email_unique": { + "name": "custodians_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ingestion_sources": { + "name": "ingestion_sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "ingestion_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "ingestion_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending_auth'" + }, + "last_sync_started_at": { + "name": "last_sync_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_finished_at": { + "name": "last_sync_finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_status_message": { + "name": "last_sync_status_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_state": { + "name": "sync_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.roles": { + "name": "roles", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "policies": { + "name": "policies", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "roles_name_unique": { + "name": "roles_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_roles_role_id_roles_id_fk": { + "name": "user_roles_role_id_roles_id_fk", + "tableFrom": "user_roles", + "tableTo": "roles", + "columnsFrom": [ + "role_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_id_pk": { + "name": "user_roles_user_id_role_id_pk", + "columns": [ + "user_id", + "role_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "first_name": { + "name": "first_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_name": { + "name": "last_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'local'" + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.retention_action": { + "name": "retention_action", + "schema": "public", + "values": [ + "delete_permanently", + "notify_admin" + ] + }, + "public.ingestion_provider": { + "name": "ingestion_provider", + "schema": "public", + "values": [ + "google_workspace", + "microsoft_365", + "generic_imap", + "pst_import", + "eml_import" + ] + }, + "public.ingestion_status": { + "name": "ingestion_status", + "schema": "public", + "values": [ + "active", + "paused", + "error", + "pending_auth", + "syncing", + "importing", + "auth_success", + "imported" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/backend/src/database/migrations/meta/_journal.json b/packages/backend/src/database/migrations/meta/_journal.json index 2950921..a969db7 100644 --- a/packages/backend/src/database/migrations/meta/_journal.json +++ b/packages/backend/src/database/migrations/meta/_journal.json @@ -99,6 +99,13 @@ "when": 1754659373517, "tag": "0013_classy_talkback", "breakpoints": true + }, + { + "idx": 14, + "version": "7", + "when": 1754831765718, + "tag": "0014_foamy_vapor", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/backend/src/database/schema/ingestion-sources.ts b/packages/backend/src/database/schema/ingestion-sources.ts index a3801de..863d9c7 100644 --- a/packages/backend/src/database/schema/ingestion-sources.ts +++ b/packages/backend/src/database/schema/ingestion-sources.ts @@ -4,7 +4,8 @@ export const ingestionProviderEnum = pgEnum('ingestion_provider', [ 'google_workspace', 'microsoft_365', 'generic_imap', - 'pst_import' + 'pst_import', + 'eml_import' ]); export const ingestionStatusEnum = pgEnum('ingestion_status', [ diff --git a/packages/backend/src/jobs/processors/initial-import.processor.ts b/packages/backend/src/jobs/processors/initial-import.processor.ts index ac9e5de..adb03a6 100644 --- a/packages/backend/src/jobs/processors/initial-import.processor.ts +++ b/packages/backend/src/jobs/processors/initial-import.processor.ts @@ -1,6 +1,6 @@ import { Job, FlowChildJob } from 'bullmq'; import { IngestionService } from '../../services/IngestionService'; -import { IInitialImportJob } from '@open-archiver/types'; +import { IInitialImportJob, IngestionProvider } from '@open-archiver/types'; import { EmailProviderFactory } from '../../services/EmailProviderFactory'; import { flowProducer } from '../queues'; import { logger } from '../../config/logger'; @@ -67,7 +67,8 @@ export default async (job: Job) => { } }); } else { - const finalStatus = source.provider === 'pst_import' ? 'imported' : 'active'; + const fileBasedIngestions = IngestionService.returnFileBasedIngestions(); + const finalStatus = fileBasedIngestions.includes(source.provider) ? 'imported' : 'active'; // If there are no users, we can consider the import finished and set to active await IngestionService.update(ingestionSourceId, { status: finalStatus, diff --git a/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts b/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts index d032e12..d4c3b43 100644 --- a/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts +++ b/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts @@ -1,7 +1,7 @@ import { Job } from 'bullmq'; import { IngestionService } from '../../services/IngestionService'; import { logger } from '../../config/logger'; -import { SyncState, ProcessMailboxError, IngestionStatus } from '@open-archiver/types'; +import { SyncState, ProcessMailboxError, IngestionStatus, IngestionProvider } from '@open-archiver/types'; import { db } from '../../database'; import { ingestionSources } from '../../database/schema'; import { eq } from 'drizzle-orm'; @@ -43,7 +43,9 @@ export default async (job: Job) => { const source = await IngestionService.findById(ingestionSourceId); let status: IngestionStatus = 'active'; - if (source.provider === 'pst_import') { + const fileBasedIngestions = IngestionService.returnFileBasedIngestions(); + + if (fileBasedIngestions.includes(source.provider)) { status = 'imported'; } let message: string; diff --git a/packages/backend/src/services/EmailProviderFactory.ts b/packages/backend/src/services/EmailProviderFactory.ts index 4d664c9..a3fe61f 100644 --- a/packages/backend/src/services/EmailProviderFactory.ts +++ b/packages/backend/src/services/EmailProviderFactory.ts @@ -4,6 +4,7 @@ import type { Microsoft365Credentials, GenericImapCredentials, PSTImportCredentials, + EMLImportCredentials, EmailObject, SyncState, MailboxUser @@ -12,6 +13,7 @@ import { GoogleWorkspaceConnector } from './ingestion-connectors/GoogleWorkspace import { MicrosoftConnector } from './ingestion-connectors/MicrosoftConnector'; import { ImapConnector } from './ingestion-connectors/ImapConnector'; import { PSTConnector } from './ingestion-connectors/PSTConnector'; +import { EMLConnector } from './ingestion-connectors/EMLConnector'; // Define a common interface for all connectors export interface IEmailConnector { @@ -36,6 +38,8 @@ export class EmailProviderFactory { return new ImapConnector(credentials as GenericImapCredentials); case 'pst_import': return new PSTConnector(credentials as PSTImportCredentials); + case 'eml_import': + return new EMLConnector(credentials as EMLImportCredentials); default: throw new Error(`Unsupported provider: ${source.provider}`); } diff --git a/packages/backend/src/services/IngestionService.ts b/packages/backend/src/services/IngestionService.ts index d1f2bac..156cb76 100644 --- a/packages/backend/src/services/IngestionService.ts +++ b/packages/backend/src/services/IngestionService.ts @@ -4,7 +4,8 @@ import type { CreateIngestionSourceDto, UpdateIngestionSourceDto, IngestionSource, - IngestionCredentials + IngestionCredentials, + IngestionProvider } from '@open-archiver/types'; import { and, desc, eq } from 'drizzle-orm'; import { CryptoService } from './CryptoService'; @@ -36,9 +37,12 @@ export class IngestionService { return { ...source, credentials: decryptedCredentials } as IngestionSource; } + public static returnFileBasedIngestions(): IngestionProvider[] { + return ['pst_import', 'eml_import']; + } + public static async create(dto: CreateIngestionSourceDto): Promise { const { providerConfig, ...rest } = dto; - console.log(providerConfig); const encryptedCredentials = CryptoService.encryptObject(providerConfig); const valuesToInsert = { @@ -140,7 +144,11 @@ export class IngestionService { const emailPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/`; await storage.delete(emailPath); - if (source.credentials.type === 'pst_import' && source.credentials.uploadedFilePath && await storage.exists(source.credentials.uploadedFilePath)) { + if ( + (source.credentials.type === 'pst_import' || source.credentials.type === 'eml_import') && + source.credentials.uploadedFilePath && + (await storage.exists(source.credentials.uploadedFilePath)) + ) { await storage.delete(source.credentials.uploadedFilePath); } @@ -204,14 +212,13 @@ export class IngestionService { } public async performBulkImport(job: IInitialImportJob): Promise { - console.log('performing bulk import'); const { ingestionSourceId } = job; const source = await IngestionService.findById(ingestionSourceId); if (!source) { throw new Error(`Ingestion source ${ingestionSourceId} not found.`); } - console.log(`Starting bulk import for source: ${source.name} (${source.id})`); + logger.info(`Starting bulk import for source: ${source.name} (${source.id})`); await IngestionService.update(ingestionSourceId, { status: 'importing', lastSyncStartedAt: new Date() @@ -233,22 +240,13 @@ export class IngestionService { } } else { // For single-mailbox providers, dispatch a single job - // console.log('source.credentials ', source.credentials); await ingestionQueue.add('process-mailbox', { ingestionSourceId: source.id, userEmail: source.credentials.type === 'generic_imap' ? source.credentials.username : 'Default' }); } - - - // await IngestionService.update(ingestionSourceId, { - // status: 'active', - // lastSyncFinishedAt: new Date(), - // lastSyncStatusMessage: 'Successfully initiated bulk import for all mailboxes.' - // }); - // console.log(`Bulk import job dispatch finished for source: ${source.name} (${source.id})`); } catch (error) { - console.error(`Bulk import failed for source: ${source.name} (${source.id})`, error); + logger.error(`Bulk import failed for source: ${source.name} (${source.id})`, error); await IngestionService.update(ingestionSourceId, { status: 'error', lastSyncFinishedAt: new Date(), @@ -290,10 +288,10 @@ export class IngestionService { return; } - console.log('processing email, ', email.id, email.subject); const emlBuffer = email.eml ?? Buffer.from(email.body, 'utf-8'); const emailHash = createHash('sha256').update(emlBuffer).digest('hex'); - const emailPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${email.path ? `${email.path}/` : ''}${email.id}.eml`; + const sanitizedPath = email.path ? email.path : ''; + const emailPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${sanitizedPath}${email.id}.eml`; await storage.put(emailPath, emlBuffer); const [archivedEmail] = await db @@ -354,7 +352,7 @@ export class IngestionService { } // adding to indexing queue //Instead: index by email (raw email object, ingestion id) - console.log('Indexing email: ', email.subject); + logger.info({ emailId: archivedEmail.id }, 'Indexing email'); // await indexingQueue.add('index-email', { // emailId: archivedEmail.id, // }); diff --git a/packages/backend/src/services/ingestion-connectors/EMLConnector.ts b/packages/backend/src/services/ingestion-connectors/EMLConnector.ts new file mode 100644 index 0000000..afd7d5e --- /dev/null +++ b/packages/backend/src/services/ingestion-connectors/EMLConnector.ts @@ -0,0 +1,199 @@ +import type { EMLImportCredentials, EmailObject, EmailAddress, SyncState, MailboxUser } from '@open-archiver/types'; +import type { IEmailConnector } from '../EmailProviderFactory'; +import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser'; +import { logger } from '../../config/logger'; +import { getThreadId } from './helpers/utils'; +import { StorageService } from '../StorageService'; +import { Readable } from 'stream'; +import { createHash } from 'crypto'; +import { join, dirname } from 'path'; +import { createReadStream, promises as fs, createWriteStream } from 'fs'; +import * as yauzl from 'yauzl'; + +const streamToBuffer = (stream: Readable): Promise => { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on('data', (chunk) => chunks.push(chunk)); + stream.on('error', reject); + stream.on('end', () => resolve(Buffer.concat(chunks))); + }); +}; + +export class EMLConnector implements IEmailConnector { + private storage: StorageService; + + constructor(private credentials: EMLImportCredentials) { + this.storage = new StorageService(); + } + + public async testConnection(): Promise { + try { + if (!this.credentials.uploadedFilePath) { + throw Error("EML file path not provided."); + } + if (!this.credentials.uploadedFilePath.includes('.zip')) { + throw Error("Provided file is not in the ZIP format."); + } + const fileExist = await this.storage.exists(this.credentials.uploadedFilePath); + if (!fileExist) { + throw Error("EML file upload not finished yet, please wait."); + } + + return true; + } catch (error) { + logger.error({ error, credentials: this.credentials }, 'EML file validation failed.'); + throw error; + } + } + + public async *listAllUsers(): AsyncGenerator { + const displayName = this.credentials.uploadedFileName || `eml-import-${new Date().getTime()}`; + logger.info(`Found potential mailbox: ${displayName}`); + const constructedPrimaryEmail = `${displayName.replace(/ /g, '.').toLowerCase()}@eml.local`; + yield { + id: constructedPrimaryEmail, + primaryEmail: constructedPrimaryEmail, + displayName: displayName, + }; + } + + public async *fetchEmails(userEmail: string, syncState?: SyncState | null): AsyncGenerator { + const fileStream = await this.storage.get(this.credentials.uploadedFilePath); + const tempDir = await fs.mkdtemp(join('/tmp', 'eml-import-')); + const unzippedPath = join(tempDir, 'unzipped'); + await fs.mkdir(unzippedPath); + const zipFilePath = join(tempDir, 'eml.zip'); + + try { + await new Promise((resolve, reject) => { + const dest = createWriteStream(zipFilePath); + (fileStream as Readable).pipe(dest); + dest.on('finish', () => resolve()); + dest.on('error', reject); + }); + + await this.extract(zipFilePath, unzippedPath); + + const files = await this.getAllFiles(unzippedPath); + + for (const file of files) { + if (file.endsWith('.eml')) { + try { + // logger.info({ file }, 'Processing EML file.'); + const stream = createReadStream(file); + const content = await streamToBuffer(stream); + // logger.info({ file, size: content.length }, 'Read file to buffer.'); + let relativePath = file.substring(unzippedPath.length + 1); + if (dirname(relativePath) === '.') { + relativePath = ''; + } else { + relativePath = dirname(relativePath); + } + const emailObject = await this.parseMessage(content, relativePath); + // logger.info({ file, messageId: emailObject.id }, 'Parsed email message.'); + yield emailObject; + } catch (error) { + logger.error({ error, file }, 'Failed to process a single EML file. Skipping.'); + } + } + } + } catch (error) { + logger.error({ error }, 'Failed to fetch email.'); + throw error; + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + } + + private extract(zipFilePath: string, dest: string): Promise { + return new Promise((resolve, reject) => { + yauzl.open(zipFilePath, { lazyEntries: true, decodeStrings: false }, (err, zipfile) => { + if (err) reject(err); + zipfile.on('error', reject); + zipfile.readEntry(); + zipfile.on('entry', (entry) => { + const fileName = entry.fileName.toString('utf8'); + // Ignore macOS-specific metadata files. + if (fileName.startsWith('__MACOSX/')) { + zipfile.readEntry(); + return; + } + const entryPath = join(dest, fileName); + if (/\/$/.test(fileName)) { + fs.mkdir(entryPath, { recursive: true }).then(() => zipfile.readEntry()).catch(reject); + } else { + zipfile.openReadStream(entry, (err, readStream) => { + if (err) reject(err); + const writeStream = createWriteStream(entryPath); + readStream.pipe(writeStream); + writeStream.on('finish', () => zipfile.readEntry()); + writeStream.on('error', reject); + }); + } + }); + zipfile.on('end', () => resolve()); + }); + }); + } + + private async getAllFiles(dirPath: string, arrayOfFiles: string[] = []): Promise { + const files = await fs.readdir(dirPath); + + for (const file of files) { + const fullPath = join(dirPath, file); + if ((await fs.stat(fullPath)).isDirectory()) { + await this.getAllFiles(fullPath, arrayOfFiles); + } else { + arrayOfFiles.push(fullPath); + } + } + + return arrayOfFiles; + } + + private async parseMessage(emlBuffer: Buffer, path: string): Promise { + const parsedEmail: ParsedMail = await simpleParser(emlBuffer); + + const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({ + filename: attachment.filename || 'untitled', + contentType: attachment.contentType, + size: attachment.size, + content: attachment.content as Buffer + })); + + const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => { + if (!addresses) return []; + const addressArray = Array.isArray(addresses) ? addresses : [addresses]; + return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address?.replaceAll(`'`, '') || '' }))); + }; + + const threadId = getThreadId(parsedEmail.headers); + let messageId = parsedEmail.messageId; + + if (!messageId) { + messageId = `generated-${createHash('sha256').update(emlBuffer).digest('hex')}`; + } + + + return { + id: messageId, + threadId: threadId, + from: mapAddresses(parsedEmail.from), + to: mapAddresses(parsedEmail.to), + cc: mapAddresses(parsedEmail.cc), + bcc: mapAddresses(parsedEmail.bcc), + subject: parsedEmail.subject || '', + body: parsedEmail.text || '', + html: parsedEmail.html || '', + headers: parsedEmail.headers, + attachments, + receivedAt: parsedEmail.date || new Date(), + eml: emlBuffer, + path + }; + } + + public getUpdatedSyncState(): SyncState { + return {}; + } +} diff --git a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts index 5d40bfd..7977122 100644 --- a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts @@ -240,7 +240,7 @@ export class ImapConnector implements IEmailConnector { const threadId = getThreadId(parsedEmail.headers); return { - id: msg.uid.toString(), + id: parsedEmail.messageId || msg.uid.toString(), threadId: threadId, from: mapAddresses(parsedEmail.from), to: mapAddresses(parsedEmail.to), diff --git a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte index 2c051cd..1a91530 100644 --- a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte +++ b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte @@ -23,7 +23,8 @@ { value: 'generic_imap', label: 'Generic IMAP' }, { value: 'google_workspace', label: 'Google Workspace' }, { value: 'microsoft_365', label: 'Microsoft 365' }, - { value: 'pst_import', label: 'PST Import' } + { value: 'pst_import', label: 'PST Import' }, + { value: 'eml_import', label: 'EML Import' } ]; let formData: CreateIngestionSourceDto = $state({ @@ -190,6 +191,16 @@ {/if} + {:else if formData.provider === 'eml_import'} +
+ +
+ + {#if fileUploading} + + {/if} +
+
{/if} {#if formData.provider === 'google_workspace' || formData.provider === 'microsoft_365'} diff --git a/packages/frontend/src/routes/dashboard/archived-emails/+page.svelte b/packages/frontend/src/routes/dashboard/archived-emails/+page.svelte index 33fb3c7..9814d4d 100644 --- a/packages/frontend/src/routes/dashboard/archived-emails/+page.svelte +++ b/packages/frontend/src/routes/dashboard/archived-emails/+page.svelte @@ -124,7 +124,9 @@ {email.userEmail} - {email.path} + {#if email.path} + {email.path} + {/if} diff --git a/packages/types/src/ingestion.types.ts b/packages/types/src/ingestion.types.ts index 1b8c7a6..715f90d 100644 --- a/packages/types/src/ingestion.types.ts +++ b/packages/types/src/ingestion.types.ts @@ -18,7 +18,7 @@ export type SyncState = { statusMessage?: string; }; -export type IngestionProvider = 'google_workspace' | 'microsoft_365' | 'generic_imap' | 'pst_import'; +export type IngestionProvider = 'google_workspace' | 'microsoft_365' | 'generic_imap' | 'pst_import' | 'eml_import'; export type IngestionStatus = | 'active' @@ -69,12 +69,19 @@ export interface PSTImportCredentials extends BaseIngestionCredentials { uploadedFilePath: string; } +export interface EMLImportCredentials extends BaseIngestionCredentials { + type: 'eml_import'; + uploadedFileName: string; + uploadedFilePath: string; +} + // Discriminated union for all possible credential types export type IngestionCredentials = | GenericImapCredentials | GoogleWorkspaceCredentials | Microsoft365Credentials - | PSTImportCredentials; + | PSTImportCredentials + | EMLImportCredentials; export interface IngestionSource { id: string; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 50d3f73..e158389 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -39,6 +39,9 @@ importers: '@open-archiver/types': specifier: workspace:* version: link:../types + archiver: + specifier: ^7.0.1 + version: 7.0.1 axios: specifier: ^1.10.0 version: 1.10.0 @@ -129,6 +132,9 @@ importers: xlsx: specifier: ^0.18.5 version: 0.18.5 + yauzl: + specifier: ^3.2.0 + version: 3.2.0 devDependencies: '@bull-board/api': specifier: ^6.11.0 @@ -136,6 +142,9 @@ importers: '@bull-board/express': specifier: ^6.11.0 version: 6.11.0 + '@types/archiver': + specifier: ^6.0.3 + version: 6.0.3 '@types/busboy': specifier: ^1.5.4 version: 1.5.4 @@ -154,6 +163,9 @@ importers: '@types/node': specifier: ^24.0.12 version: 24.0.13 + '@types/yauzl': + specifier: ^2.10.3 + version: 2.10.3 bull-board: specifier: ^2.1.3 version: 2.1.3 @@ -1052,6 +1064,10 @@ packages: '@ioredis/commands@1.2.0': resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} + '@isaacs/cliui@8.0.2': + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + '@isaacs/fs-minipass@4.0.1': resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==} engines: {node: '>=18.0.0'} @@ -1145,6 +1161,10 @@ packages: engines: {node: '>=10'} deprecated: This functionality has been moved to @npmcli/fs + '@pkgjs/parseargs@0.11.0': + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + '@polka/url@1.0.0-next.29': resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==} @@ -1672,6 +1692,9 @@ packages: '@tsconfig/node16@1.0.4': resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} + '@types/archiver@6.0.3': + resolution: {integrity: sha512-a6wUll6k3zX6qs5KlxIggs1P1JcYJaTCx2gnlr+f0S1yd2DoaEwoIK10HmBaLnZwWneBz+JBm0dwcZu0zECBcQ==} + '@types/body-parser@1.19.6': resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==} @@ -1744,6 +1767,9 @@ packages: '@types/range-parser@1.2.7': resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} + '@types/readdir-glob@1.1.5': + resolution: {integrity: sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==} + '@types/resolve@1.20.2': resolution: {integrity: sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==} @@ -1768,6 +1794,9 @@ packages: '@types/web-bluetooth@0.0.21': resolution: {integrity: sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA==} + '@types/yauzl@2.10.3': + resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} + '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} @@ -1873,6 +1902,10 @@ packages: abbrev@1.1.1: resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + accepts@1.3.8: resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} engines: {node: '>= 0.6'} @@ -1918,10 +1951,18 @@ packages: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} + ansi-regex@6.1.0: + resolution: {integrity: sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==} + engines: {node: '>=12'} + ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} + ansi-styles@6.2.1: + resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==} + engines: {node: '>=12'} + anymatch@3.1.3: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} @@ -1932,6 +1973,14 @@ packages: aproba@2.0.0: resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==} + archiver-utils@5.0.2: + resolution: {integrity: sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==} + engines: {node: '>= 14'} + + archiver@7.0.1: + resolution: {integrity: sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==} + engines: {node: '>= 14'} + are-we-there-yet@3.0.1: resolution: {integrity: sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} @@ -1967,9 +2016,15 @@ packages: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} + b4a@1.6.7: + resolution: {integrity: sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==} + balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + bare-events@2.6.1: + resolution: {integrity: sha512-AuTJkq9XmE6Vk0FJVNq5QxETrSA/vKHarWVBG5l/JbdCL1prJemiyJqUS0jrlXO0MftuPq4m3YVYhoNc5+aE/g==} + base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} @@ -2024,6 +2079,13 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} + buffer-crc32@0.2.13: + resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} + + buffer-crc32@1.0.0: + resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==} + engines: {node: '>=8.0.0'} + buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} @@ -2036,6 +2098,9 @@ packages: buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + bull-board@2.1.3: resolution: {integrity: sha512-SrmGzrC024OGtK5Wvv/6VhK4s/iq1h0XUrThc0jla8XhEBUdC79UrG24SOXs68zj7yZnFG0/EG330nPf1Pt5UQ==} deprecated: 2.x is no longer supported, we moved to use @bull-board scope @@ -2151,6 +2216,10 @@ packages: commondir@1.0.1: resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} + compress-commons@6.0.2: + resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==} + engines: {node: '>= 14'} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -2209,6 +2278,10 @@ packages: engines: {node: '>=0.8'} hasBin: true + crc32-stream@6.0.0: + resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==} + engines: {node: '>= 14'} + create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} @@ -2550,6 +2623,9 @@ packages: dynamic-dedupe@0.3.0: resolution: {integrity: sha512-ssuANeD+z97meYOqd50e04Ze5qp4bPqo8cCkI4TRjZkzAUgIDTrXV1R8QCdINpiI+hw14+rYazvTRdQrz0/rFQ==} + eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + ecdsa-sig-formatter@1.0.11: resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} @@ -2572,6 +2648,9 @@ packages: emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + encodeurl@1.0.2: resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} engines: {node: '>= 0.8'} @@ -2661,6 +2740,10 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + events@3.3.0: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} @@ -2693,6 +2776,9 @@ packages: fast-copy@3.0.2: resolution: {integrity: sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ==} + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fast-redact@3.5.0: resolution: {integrity: sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==} engines: {node: '>=6'} @@ -2746,6 +2832,10 @@ packages: debug: optional: true + foreground-child@3.3.1: + resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} + engines: {node: '>=14'} + form-data@4.0.3: resolution: {integrity: sha512-qsITQPfmvMOSAdeyZ+12I1c+CKSstAFAwu+97zrnWAbIr5u8wfsExUzCesVLC8NgHuRUqNN4Zy6UPWUTRGslcA==} engines: {node: '>= 6'} @@ -2823,6 +2913,10 @@ packages: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} + glob@10.4.5: + resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} + hasBin: true + glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} deprecated: Glob versions prior to v9 are no longer supported @@ -3035,6 +3129,10 @@ packages: is-reference@3.0.3: resolution: {integrity: sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==} + is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + is-what@4.1.16: resolution: {integrity: sha512-ZhMwEosbFJkA0YhFnNDgTM4ZxDRsS6HqTo7qsZM08fehyRYIYa0yHu5R6mgo1n/8MgaPBXiPimPD77baVFYg+A==} engines: {node: '>=12.13'} @@ -3045,6 +3143,9 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jake@10.9.2: resolution: {integrity: sha512-2P4SQ0HrLQ+fw6llpLnOaGAvN2Zu6778SJMrCUwns4fOoG9ayrTiZk3VV8sCPkVZF8ab0zksVpS8FDY5pRCNBA==} engines: {node: '>=10'} @@ -3100,6 +3201,10 @@ packages: peerDependencies: svelte: ^5.0.0 + lazystream@1.0.1: + resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==} + engines: {node: '>= 0.6.3'} + leac@0.6.0: resolution: {integrity: sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==} @@ -3224,6 +3329,9 @@ packages: lop@0.4.2: resolution: {integrity: sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==} + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lru-cache@6.0.0: resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} engines: {node: '>=10'} @@ -3345,6 +3453,10 @@ packages: resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==} engines: {node: '>=10'} + minimatch@9.0.5: + resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} + engines: {node: '>=16 || 14 >=14.17'} + minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} @@ -3556,6 +3668,9 @@ packages: resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==} engines: {node: '>=10'} + package-json-from-dist@1.0.1: + resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + pako@1.0.11: resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==} @@ -3577,6 +3692,10 @@ packages: path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + path-to-regexp@0.1.7: resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} @@ -3592,6 +3711,9 @@ packages: peberminta@0.9.0: resolution: {integrity: sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==} + pend@1.2.0: + resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + perfect-debounce@1.0.0: resolution: {integrity: sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==} @@ -3767,6 +3889,10 @@ packages: process-warning@5.0.0: resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==} + process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + promise-inflight@1.0.1: resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==} peerDependencies: @@ -3834,6 +3960,13 @@ packages: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} + readable-stream@4.7.0: + resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + readdir-glob@1.1.3: + resolution: {integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==} + readdirp@3.6.0: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} @@ -4032,6 +4165,10 @@ packages: signal-exit@3.0.7: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + simple-concat@1.0.1: resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} @@ -4118,10 +4255,17 @@ packages: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} + streamx@2.22.1: + resolution: {integrity: sha512-znKXEBxfatz2GBNK02kRnCXjV+AA4kjZIUxeWSr3UGirZMJfTE9uiwKHobnbgxWyL/JWro8tTq+vOqAK1/qbSA==} + string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} + string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + string_decoder@1.1.1: resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} @@ -4135,6 +4279,10 @@ packages: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} + strip-ansi@7.1.0: + resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} + engines: {node: '>=12'} + strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -4233,6 +4381,9 @@ packages: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} engines: {node: '>=6'} + tar-stream@3.1.7: + resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==} + tar@6.2.1: resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} engines: {node: '>=10'} @@ -4241,6 +4392,9 @@ packages: resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==} engines: {node: '>=18'} + text-decoder@1.2.3: + resolution: {integrity: sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==} + thread-stream@3.1.0: resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==} @@ -4536,6 +4690,10 @@ packages: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} + wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} @@ -4571,6 +4729,10 @@ packages: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} + yauzl@3.2.0: + resolution: {integrity: sha512-Ow9nuGZE+qp1u4JIPvg+uCiUr7xGQWdff7JQSk5VGYTAZMDe2q8lxJ10ygv10qmSj031Ty/6FNJpLO4o1Sgc+w==} + engines: {node: '>=12'} + yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} @@ -4578,6 +4740,10 @@ packages: zimmerframe@1.1.2: resolution: {integrity: sha512-rAbqEGa8ovJy4pyBxZM70hg4pE6gDgaQ0Sl9M3enG3I0d6H4XSAM3GeNGLKnsBpuijUow064sf7ww1nutC5/3w==} + zip-stream@6.0.1: + resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==} + engines: {node: '>= 14'} + zwitch@2.0.4: resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} @@ -5501,6 +5667,15 @@ snapshots: '@ioredis/commands@1.2.0': {} + '@isaacs/cliui@8.0.2': + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.1.0 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + '@isaacs/fs-minipass@4.0.1': dependencies: minipass: 7.1.2 @@ -5588,6 +5763,9 @@ snapshots: rimraf: 3.0.2 optional: true + '@pkgjs/parseargs@0.11.0': + optional: true + '@polka/url@1.0.0-next.29': {} '@rollup/plugin-commonjs@28.0.6(rollup@4.44.2)': @@ -6206,6 +6384,10 @@ snapshots: '@tsconfig/node16@1.0.4': {} + '@types/archiver@6.0.3': + dependencies: + '@types/readdir-glob': 1.1.5 + '@types/body-parser@1.19.6': dependencies: '@types/connect': 3.4.38 @@ -6296,6 +6478,10 @@ snapshots: '@types/range-parser@1.2.7': {} + '@types/readdir-glob@1.1.5': + dependencies: + '@types/node': 24.0.13 + '@types/resolve@1.20.2': {} '@types/send@0.17.5': @@ -6319,6 +6505,10 @@ snapshots: '@types/web-bluetooth@0.0.21': {} + '@types/yauzl@2.10.3': + dependencies: + '@types/node': 24.0.13 + '@ungap/structured-clone@1.3.0': {} '@vitejs/plugin-vue@5.2.4(vite@5.4.19(@types/node@24.0.13)(lightningcss@1.30.1))(vue@3.5.18(typescript@5.8.3))': @@ -6431,6 +6621,10 @@ snapshots: abbrev@1.1.1: optional: true + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + accepts@1.3.8: dependencies: mime-types: 2.1.35 @@ -6487,10 +6681,14 @@ snapshots: ansi-regex@5.0.1: {} + ansi-regex@6.1.0: {} + ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 + ansi-styles@6.2.1: {} + anymatch@3.1.3: dependencies: normalize-path: 3.0.0 @@ -6501,6 +6699,26 @@ snapshots: aproba@2.0.0: optional: true + archiver-utils@5.0.2: + dependencies: + glob: 10.4.5 + graceful-fs: 4.2.11 + is-stream: 2.0.1 + lazystream: 1.0.1 + lodash: 4.17.21 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + + archiver@7.0.1: + dependencies: + archiver-utils: 5.0.2 + async: 3.2.6 + buffer-crc32: 1.0.0 + readable-stream: 4.7.0 + readdir-glob: 1.1.3 + tar-stream: 3.1.7 + zip-stream: 6.0.1 + are-we-there-yet@3.0.1: dependencies: delegates: 1.0.0 @@ -6533,8 +6751,13 @@ snapshots: axobject-query@4.1.0: {} + b4a@1.6.7: {} + balanced-match@1.0.2: {} + bare-events@2.6.1: + optional: true + base64-js@1.5.1: {} bcryptjs@3.0.2: {} @@ -6612,6 +6835,10 @@ snapshots: dependencies: fill-range: 7.1.1 + buffer-crc32@0.2.13: {} + + buffer-crc32@1.0.0: {} + buffer-equal-constant-time@1.0.1: {} buffer-from@1.1.2: {} @@ -6626,6 +6853,11 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + bull-board@2.1.3: dependencies: '@types/express': 4.17.23 @@ -6764,6 +6996,14 @@ snapshots: commondir@1.0.1: {} + compress-commons@6.0.2: + dependencies: + crc-32: 1.2.2 + crc32-stream: 6.0.0 + is-stream: 2.0.1 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + concat-map@0.0.1: {} concat-stream@2.0.0: @@ -6814,6 +7054,11 @@ snapshots: crc-32@1.2.2: {} + crc32-stream@6.0.0: + dependencies: + crc-32: 1.2.2 + readable-stream: 4.7.0 + create-require@1.1.1: {} cron-parser@4.9.0: @@ -7047,6 +7292,8 @@ snapshots: dependencies: xtend: 4.0.2 + eastasianwidth@0.2.0: {} + ecdsa-sig-formatter@1.0.11: dependencies: safe-buffer: 5.2.1 @@ -7065,6 +7312,8 @@ snapshots: emoji-regex@8.0.0: {} + emoji-regex@9.2.2: {} + encodeurl@1.0.2: {} encodeurl@2.0.0: {} @@ -7209,6 +7458,8 @@ snapshots: etag@1.8.1: {} + event-target-shim@5.0.1: {} + events@3.3.0: {} expand-template@2.0.3: {} @@ -7294,6 +7545,8 @@ snapshots: fast-copy@3.0.2: {} + fast-fifo@1.3.2: {} + fast-redact@3.5.0: {} fast-safe-stringify@2.1.1: {} @@ -7350,6 +7603,11 @@ snapshots: follow-redirects@1.15.9: {} + foreground-child@3.3.1: + dependencies: + cross-spawn: 7.0.6 + signal-exit: 4.1.0 + form-data@4.0.3: dependencies: asynckit: 0.4.0 @@ -7441,6 +7699,15 @@ snapshots: dependencies: is-glob: 4.0.3 + glob@10.4.5: + dependencies: + foreground-child: 3.3.1 + jackspeak: 3.4.3 + minimatch: 9.0.5 + minipass: 7.1.2 + package-json-from-dist: 1.0.1 + path-scurry: 1.11.1 + glob@7.2.3: dependencies: fs.realpath: 1.0.0 @@ -7710,12 +7977,20 @@ snapshots: dependencies: '@types/estree': 1.0.8 + is-stream@2.0.1: {} + is-what@4.1.16: {} isarray@1.0.0: {} isexe@2.0.0: {} + jackspeak@3.4.3: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + jake@10.9.2: dependencies: async: 3.2.6 @@ -7812,6 +8087,10 @@ snapshots: runed: 0.28.0(svelte@5.35.5) svelte: 5.35.5 + lazystream@1.0.1: + dependencies: + readable-stream: 2.3.8 + leac@0.6.0: {} libbase64@1.3.0: {} @@ -7910,6 +8189,8 @@ snapshots: option: 0.2.4 underscore: 1.13.7 + lru-cache@10.4.3: {} + lru-cache@6.0.0: dependencies: yallist: 4.0.0 @@ -8057,6 +8338,10 @@ snapshots: dependencies: brace-expansion: 2.0.2 + minimatch@9.0.5: + dependencies: + brace-expansion: 2.0.2 + minimist@1.2.8: {} minipass-collect@1.0.2: @@ -8266,6 +8551,8 @@ snapshots: aggregate-error: 3.1.0 optional: true + package-json-from-dist@1.0.1: {} + pako@1.0.11: {} parseley@0.12.1: @@ -8281,6 +8568,11 @@ snapshots: path-parse@1.0.7: {} + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.2 + path-to-regexp@0.1.7: {} path-to-regexp@8.2.0: {} @@ -8289,6 +8581,8 @@ snapshots: peberminta@0.9.0: {} + pend@1.2.0: {} + perfect-debounce@1.0.0: {} pg-cloudflare@1.2.7: @@ -8422,6 +8716,8 @@ snapshots: process-warning@5.0.0: {} + process@0.11.10: {} + promise-inflight@1.0.1: optional: true @@ -8500,6 +8796,18 @@ snapshots: string_decoder: 1.3.0 util-deprecate: 1.0.2 + readable-stream@4.7.0: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + readdir-glob@1.1.3: + dependencies: + minimatch: 5.1.6 + readdirp@3.6.0: dependencies: picomatch: 2.3.1 @@ -8753,6 +9061,8 @@ snapshots: signal-exit@3.0.7: optional: true + signal-exit@4.1.0: {} + simple-concat@1.0.1: {} simple-get@4.0.1: @@ -8842,12 +9152,25 @@ snapshots: streamsearch@1.1.0: {} + streamx@2.22.1: + dependencies: + fast-fifo: 1.3.2 + text-decoder: 1.2.3 + optionalDependencies: + bare-events: 2.6.1 + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 is-fullwidth-code-point: 3.0.0 strip-ansi: 6.0.1 + string-width@5.1.2: + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.1.0 + string_decoder@1.1.1: dependencies: safe-buffer: 5.1.2 @@ -8865,6 +9188,10 @@ snapshots: dependencies: ansi-regex: 5.0.1 + strip-ansi@7.1.0: + dependencies: + ansi-regex: 6.1.0 + strip-bom@3.0.0: {} strip-json-comments@2.0.1: {} @@ -8973,6 +9300,12 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + tar-stream@3.1.7: + dependencies: + b4a: 1.6.7 + fast-fifo: 1.3.2 + streamx: 2.22.1 + tar@6.2.1: dependencies: chownr: 2.0.0 @@ -8991,6 +9324,10 @@ snapshots: mkdirp: 3.0.1 yallist: 5.0.0 + text-decoder@1.2.3: + dependencies: + b4a: 1.6.7 + thread-stream@3.1.0: dependencies: real-require: 0.2.0 @@ -9274,6 +9611,12 @@ snapshots: string-width: 4.2.3 strip-ansi: 6.0.1 + wrap-ansi@8.1.0: + dependencies: + ansi-styles: 6.2.1 + string-width: 5.1.2 + strip-ansi: 7.1.0 + wrappy@1.0.2: {} xlsx@0.18.5: @@ -9308,8 +9651,19 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 + yauzl@3.2.0: + dependencies: + buffer-crc32: 0.2.13 + pend: 1.2.0 + yn@3.1.1: {} zimmerframe@1.1.2: {} + zip-stream@6.0.1: + dependencies: + archiver-utils: 5.0.2 + compress-commons: 6.0.2 + readable-stream: 4.7.0 + zwitch@2.0.4: {}