From e9a65f967248193c7c474e4e387203cca6657178 Mon Sep 17 00:00:00 2001 From: "Wei S." <5291640+wayneshn@users.noreply.github.com> Date: Tue, 16 Sep 2025 20:30:22 +0300 Subject: [PATCH] feat: Add Mbox ingestion (#117) This commit introduces two major features: 1. **Mbox File Ingestion:** Users can now ingest emails from Mbox files (`.mbox`). A new Mbox connector has been implemented on the backend, and the user interface has been updated to support creating Mbox ingestion sources. Documentation for this new provider has also been added. Additionally, this commit includes new documentation for upgrading and migrating Open Archiver. Co-authored-by: Wayne <5291640+ringoinca@users.noreply.github.com> --- README.md | 2 + docs/.vitepress/config.mts | 15 + docs/user-guides/email-providers/index.md | 1 + docs/user-guides/email-providers/mbox.md | 29 + docs/user-guides/installation.md | 100 +- .../meilisearch-upgrade.md | 93 ++ .../upgrade-and-migration/upgrade.md | 42 + open-archiver.yml | 77 + package.json | 2 +- .../src/api/controllers/upload.controller.ts | 6 +- packages/backend/src/config/logger.ts | 1 + .../migrations/0020_panoramic_wolverine.sql | 1 + .../migrations/meta/0020_snapshot.json | 1245 +++++++++++++++++ .../database/migrations/meta/_journal.json | 297 ++-- .../src/database/schema/ingestion-sources.ts | 1 + .../src/services/EmailProviderFactory.ts | 4 + .../backend/src/services/IngestionService.ts | 11 +- .../ingestion-connectors/EMLConnector.ts | 10 +- .../ingestion-connectors/MboxConnector.ts | 174 +++ .../ingestion-connectors/PSTConnector.ts | 12 +- packages/frontend/package.json | 1 + .../lib/components/custom/EmailPreview.svelte | 7 +- .../custom/IngestionSourceForm.svelte | 31 +- .../frontend/src/lib/translations/en.json | 2 + .../routes/dashboard/ingestions/+page.svelte | 7 +- packages/types/src/ingestion.types.ts | 12 +- pnpm-lock.yaml | 8 + 27 files changed, 1979 insertions(+), 212 deletions(-) create mode 100644 docs/user-guides/email-providers/mbox.md create mode 100644 docs/user-guides/upgrade-and-migration/meilisearch-upgrade.md create mode 100644 docs/user-guides/upgrade-and-migration/upgrade.md create mode 100644 open-archiver.yml create mode 100644 packages/backend/src/database/migrations/0020_panoramic_wolverine.sql create mode 100644 packages/backend/src/database/migrations/meta/0020_snapshot.json create mode 100644 packages/backend/src/services/ingestion-connectors/MboxConnector.ts diff --git a/README.md b/README.md index 26ba21c..70150d7 100644 --- a/README.md +++ b/README.md @@ -46,12 +46,14 @@ Password: openarchiver_demo - Microsoft 365 - PST files - Zipped .eml files + - Mbox files - **Secure & Efficient Storage**: Emails are stored in the standard `.eml` format. The system uses deduplication and compression to minimize storage costs. All data is encrypted at rest. - **Pluggable Storage Backends**: Support both local filesystem storage and S3-compatible object storage (like AWS S3 or MinIO). - **Powerful Search & eDiscovery**: A high-performance search engine indexes the full text of emails and attachments (PDF, DOCX, etc.). - **Thread discovery**: The ability to discover if an email belongs to a thread/conversation and present the context. - **Compliance & Retention**: Define granular retention policies to automatically manage the lifecycle of your data. Place legal holds on communications to prevent deletion during litigation (TBD). +- **File Hash and Encryption**: Email and attachment file hash values are stored in the meta database upon ingestion, meaning any attempt to alter the file content will be identified, ensuring legal and regulatory compliance. - **Comprehensive Auditing**: An immutable audit trail logs all system activities, ensuring you have a clear record of who accessed what and when (TBD). ## 🛠️ Tech Stack diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 001a9dc..b8c6848 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -52,6 +52,7 @@ export default defineConfig({ }, { text: 'EML Import', link: '/user-guides/email-providers/eml' }, { text: 'PST Import', link: '/user-guides/email-providers/pst' }, + { text: 'Mbox Import', link: '/user-guides/email-providers/mbox' }, ], }, { @@ -64,6 +65,20 @@ export default defineConfig({ }, ], }, + { + text: 'Upgrading and Migration', + collapsed: true, + items: [ + { + text: 'Upgrading', + link: '/user-guides/upgrade-and-migration/upgrade', + }, + { + text: 'Meilisearch Upgrade', + link: '/user-guides/upgrade-and-migration/meilisearch-upgrade', + }, + ], + }, ], }, { diff --git a/docs/user-guides/email-providers/index.md b/docs/user-guides/email-providers/index.md index 1b1c8f7..7fa816c 100644 --- a/docs/user-guides/email-providers/index.md +++ b/docs/user-guides/email-providers/index.md @@ -9,3 +9,4 @@ Choose your provider from the list below to get started: - [Generic IMAP Server](./imap.md) - [EML Import](./eml.md) - [PST Import](./pst.md) +- [Mbox Import](./mbox.md) diff --git a/docs/user-guides/email-providers/mbox.md b/docs/user-guides/email-providers/mbox.md new file mode 100644 index 0000000..bced06c --- /dev/null +++ b/docs/user-guides/email-providers/mbox.md @@ -0,0 +1,29 @@ +# Mbox Ingestion + +Mbox is a common format for storing email messages. This guide will walk you through the process of ingesting mbox files into OpenArchiver. + +## 1. Exporting from Your Email Client + +Most email clients that support mbox exports will allow you to export a folder of emails as a single `.mbox` file. Here are the general steps: + +- **Mozilla Thunderbird**: Right-click on a folder, select **ImportExportTools NG**, and then choose **Export folder**. +- **Gmail**: You can use Google Takeout to export your emails in mbox format. +- **Other Clients**: Refer to your email client's documentation for instructions on how to export emails to an mbox file. + +## 2. Uploading to OpenArchiver + +Once you have your `.mbox` file, you can upload it to OpenArchiver through the web interface. + +1. Navigate to the **Ingestion** page. +2. Click on the **New Ingestion** button. +3. Select **Mbox** as the source type. +4. Upload your `.mbox` file. + +## 3. Folder Structure + +OpenArchiver will attempt to preserve the original folder structure of your emails. This is done by inspecting the following email headers: + +- `X-Gmail-Labels`: Used by Gmail to store labels. +- `X-Folder`: A custom header used by some email clients like Thunderbird. + +If neither of these headers is present, the emails will be ingested into the root of the archive. diff --git a/docs/user-guides/installation.md b/docs/user-guides/installation.md index 16ac1e6..abb342e 100644 --- a/docs/user-guides/installation.md +++ b/docs/user-guides/installation.md @@ -138,7 +138,9 @@ docker compose ps Once the services are running, you can access the Open Archiver web interface by navigating to `http://localhost:3000` in your web browser. -You can log in with the `ADMIN_EMAIL` and `ADMIN_PASSWORD` you configured in your `.env` file. +Upon first visit, you will be redirected to the `/setup` page where you can set up your admin account. Make sure you are the first person who accesses the instance. + +If you are not redirected to the `/setup` page but instead see the login page, there might be something wrong with the database. Restart the service and try again. ## 5. Next Steps @@ -212,9 +214,9 @@ If you are using local storage to store your emails, based on your `docker-compo Run this command to see all the volumes on your system: - ```bash - docker volume ls - ``` +```bash +docker volume ls +``` 2. **Identify the correct volume**: @@ -224,28 +226,28 @@ Look through the list for a volume name that ends with `_archiver-data`. The par Once you've identified the correct volume name, use it in the `inspect` command. For example: - ```bash - docker volume inspect - ``` +```bash +docker volume inspect +``` This will give you the correct `Mountpoint` path where your data is being stored. It will look something like this (the exact path will vary depending on your system): - ```json - { - "CreatedAt": "2025-07-25T11:22:19Z", - "Driver": "local", - "Labels": { - "com.docker.compose.config-hash": "---", - "com.docker.compose.project": "---", - "com.docker.compose.version": "2.38.2", - "com.docker.compose.volume": "us8wwos0o4ok4go4gc8cog84_archiver-data" - }, - "Mountpoint": "/var/lib/docker/volumes/us8wwos0o4ok4go4gc8cog84_archiver-data/_data", - "Name": "us8wwos0o4ok4go4gc8cog84_archiver-data", - "Options": null, - "Scope": "local" - } - ``` +```json +{ + "CreatedAt": "2025-07-25T11:22:19Z", + "Driver": "local", + "Labels": { + "com.docker.compose.config-hash": "---", + "com.docker.compose.project": "---", + "com.docker.compose.version": "2.38.2", + "com.docker.compose.volume": "us8wwos0o4ok4go4gc8cog84_archiver-data" + }, + "Mountpoint": "/var/lib/docker/volumes/us8wwos0o4ok4go4gc8cog84_archiver-data/_data", + "Name": "us8wwos0o4ok4go4gc8cog84_archiver-data", + "Options": null, + "Scope": "local" +} +``` In this example, the data is located at `/var/lib/docker/volumes/us8wwos0o4ok4go4gc8cog84_archiver-data/_data`. You can then `cd` into that directory to see your files. @@ -259,44 +261,44 @@ Here’s how you can do it: Open the `docker-compose.yml` file and find the `open-archiver` service. You're going to change the `volumes` section. - **Change this:** +**Change this:** - ```yaml - services: - open-archiver: - # ... other config - volumes: - - archiver-data:/var/data/open-archiver - ``` +```yaml +services: + open-archiver: + # ... other config + volumes: + - archiver-data:/var/data/open-archiver +``` - **To this:** +**To this:** - ```yaml - services: - open-archiver: - # ... other config - volumes: - - ./data/open-archiver:/var/data/open-archiver - ``` +```yaml +services: + open-archiver: + # ... other config + volumes: + - ./data/open-archiver:/var/data/open-archiver +``` You'll also want to remove the `archiver-data` volume definition at the bottom of the file, since it's no longer needed. - **Remove this whole block:** +**Remove this whole block:** - ```yaml - volumes: - # ... other volumes - archiver-data: - driver: local - ``` +```yaml +volumes: + # ... other volumes + archiver-data: + driver: local +``` 2. **Restart your containers**: After you've saved the changes, run the following command in your terminal to apply them. The `--force-recreate` flag will ensure the container is recreated with the new volume settings. - ```bash - docker-compose up -d --force-recreate - ``` +```bash +docker-compose up -d --force-recreate +``` After this, any new data will be saved directly into the `./data/open-archiver` folder in your project directory. diff --git a/docs/user-guides/upgrade-and-migration/meilisearch-upgrade.md b/docs/user-guides/upgrade-and-migration/meilisearch-upgrade.md new file mode 100644 index 0000000..eb4d0ee --- /dev/null +++ b/docs/user-guides/upgrade-and-migration/meilisearch-upgrade.md @@ -0,0 +1,93 @@ +# Upgrading Meilisearch + +Meilisearch, the search engine used by Open Archiver, requires a manual data migration process when upgrading to a new version. This is because Meilisearch databases are only compatible with the specific version that created them. + +If an Open Archiver upgrade includes a major Meilisearch version change, you will need to migrate your search index by following the process below. + +## Migration Process Overview + +For self-hosted instances using Docker Compose (as recommended), the migration process involves creating a data dump from your current Meilisearch instance, upgrading the Docker image, and then importing that dump into the new version. + +### Step 1: Create a Dump + +Before upgrading, you must create a dump of your existing Meilisearch data. You can do this by sending a POST request to the `/dumps` endpoint of the Meilisearch API. + +1. **Find your Meilisearch container name**: + + ```bash + docker compose ps + ``` + + Look for the service name that corresponds to Meilisearch, usually `meilisearch`. + +2. **Execute the dump command**: + You will need your Meilisearch Admin API key, which can be found in your `.env` file as `MEILI_MASTER_KEY`. + + ```bash + curl -X POST 'http://localhost:7700/dumps' \ + -H "Authorization: Bearer YOUR_MEILI_MASTER_KEY" + ``` + + This will start the dump creation process. The dump file will be created inside the `meili_data` volume used by the Meilisearch container. + +3. **Monitor the dump status**: + The dump creation request returns a `taskUid`. You can use this to check the status of the dump. + + For more details on dump and import, see the [official Meilisearch documentation](https://www.meilisearch.com/docs/learn/update_and_migration/updating). + +### Step 2: Upgrade Your Open Archiver Instance + +Once the dump is successfully created, you can proceed with the standard Open Archiver upgrade process. + +1. **Pull the latest changes and Docker images**: + + ```bash + git pull + docker compose pull + ``` + +2. **Stop the running services**: + ```bash + docker compose down + ``` + +### Step 3: Import the Dump + +Now, you need to restart the services while telling Meilisearch to import from your dump file. + +1. **Modify `docker-compose.yml`**: + You need to temporarily add the `--import-dump` flag to the Meilisearch service command. Find the `meilisearch` service in your `docker-compose.yml` and modify the `command` section. + + You will need the name of your dump file. It will be a `.dump` file located in the directory mapped to `/meili_data` inside the container. + + ```yaml + services: + meilisearch: + # ... other service config + command: + [ + '--master-key=${MEILI_MASTER_KEY}', + '--env=production', + '--import-dump=/meili_data/dumps/YOUR_DUMP_FILE.dump', + ] + ``` + +2. **Restart the services**: + ```bash + docker compose up -d + ``` + Meilisearch will now start and import the data from the dump file. This may take some time depending on the size of your index. + +### Step 4: Clean Up + +Once the import is complete and you have verified that your search is working correctly, you should remove the `--import-dump` flag from your `docker-compose.yml` to prevent it from running on every startup. + +1. **Remove the `--import-dump` line** from the `command` section of the `meilisearch` service in `docker-compose.yml`. +2. **Restart the services** one last time: + ```bash + docker compose up -d + ``` + +Your Meilisearch instance is now upgraded and running with your migrated data. + +For more advanced scenarios or troubleshooting, please refer to the **[official Meilisearch migration guide](https://www.meilisearch.com/docs/learn/update_and_migration/updating)**. diff --git a/docs/user-guides/upgrade-and-migration/upgrade.md b/docs/user-guides/upgrade-and-migration/upgrade.md new file mode 100644 index 0000000..4f38cd5 --- /dev/null +++ b/docs/user-guides/upgrade-and-migration/upgrade.md @@ -0,0 +1,42 @@ +# Upgrading Your Instance + +This guide provides instructions for upgrading your Open Archiver instance to the latest version. + +## Checking for New Versions + +Open Archiver automatically checks for new versions and will display a notification in the footer of the web interface when an update is available. You can find a list of all releases and their release notes on the [GitHub Releases](https://github.com/LogicLabs-OU/OpenArchiver/releases) page. + +## Upgrading Your Instance + +To upgrade your Open Archiver instance, follow these steps: + +1. **Pull the latest changes from the repository**: + + ```bash + git pull + ``` + +2. **Pull the latest Docker images**: + + ```bash + docker compose pull + ``` + +3. **Restart the services with the new images**: + ```bash + docker compose up -d + ``` + +This will restart your Open Archiver instance with the latest version of the application. + +## Migrating Data + +When you upgrade to a new version, database migrations are applied automatically when the application starts up. This ensures that your database schema is always up-to-date with the latest version of the application. + +No manual intervention is required for database migrations. + +## Upgrading Meilisearch + +When an Open Archiver update includes a major version change for Meilisearch, you will need to manually migrate your search data. This process is not covered by the standard upgrade commands. + +For detailed instructions, please see the [Meilisearch Upgrade Guide](./meilisearch-upgrade.md). diff --git a/open-archiver.yml b/open-archiver.yml new file mode 100644 index 0000000..9fdbb42 --- /dev/null +++ b/open-archiver.yml @@ -0,0 +1,77 @@ +# documentation: https://openarchiver.com +# slogan: A self-hosted, open-source email archiving solution with full-text search capability. +# tags: email archiving,email,compliance,search +# logo: svgs/openarchiver.svg +# port: 3000 + +services: + open-archiver: + image: logiclabshq/open-archiver:latest + environment: + - SERVICE_URL_3000 + - SERVICE_URL=${SERVICE_URL_3000} + - PORT_BACKEND=${PORT_BACKEND:-4000} + - PORT_FRONTEND=${PORT_FRONTEND:-3000} + - NODE_ENV=${NODE_ENV:-production} + - SYNC_FREQUENCY=${SYNC_FREQUENCY:-* * * * *} + - POSTGRES_DB=${POSTGRES_DB:-open_archive} + - POSTGRES_USER=${POSTGRES_USER:-admin} + - POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRES} + - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB} + - MEILI_MASTER_KEY=${SERVICE_PASSWORD_MEILISEARCH} + - MEILI_HOST=http://meilisearch:7700 + - REDIS_HOST=valkey + - REDIS_PORT=6379 + - REDIS_PASSWORD=${SERVICE_PASSWORD_VALKEY} + - REDIS_TLS_ENABLED=false + - STORAGE_TYPE=${STORAGE_TYPE:-local} + - STORAGE_LOCAL_ROOT_PATH=${STORAGE_LOCAL_ROOT_PATH:-/var/data/open-archiver} + - BODY_SIZE_LIMIT=${BODY_SIZE_LIMIT:-100M} + - STORAGE_S3_ENDPOINT=${STORAGE_S3_ENDPOINT} + - STORAGE_S3_BUCKET=${STORAGE_S3_BUCKET} + - STORAGE_S3_ACCESS_KEY_ID=${STORAGE_S3_ACCESS_KEY_ID} + - STORAGE_S3_SECRET_ACCESS_KEY=${STORAGE_S3_SECRET_ACCESS_KEY} + - STORAGE_S3_REGION=${STORAGE_S3_REGION} + - STORAGE_S3_FORCE_PATH_STYLE=${STORAGE_S3_FORCE_PATH_STYLE:-false} + - JWT_SECRET=${SERVICE_BASE64_128_JWT} + - JWT_EXPIRES_IN=${JWT_EXPIRES_IN:-7d} + - ENCRYPTION_KEY=${SERVICE_BASE64_64_ENCRYPTIONKEY} + - RATE_LIMIT_WINDOW_MS=${RATE_LIMIT_WINDOW_MS:-60000} + - RATE_LIMIT_MAX_REQUESTS=${RATE_LIMIT_MAX_REQUESTS:-100} + volumes: + - archiver-data:/var/data/open-archiver + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_started + meilisearch: + condition: service_started + + postgres: + image: postgres:17-alpine + environment: + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRES} + - LC_ALL=C + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}'] + interval: 10s + timeout: 20s + retries: 10 + + valkey: + image: valkey/valkey:8-alpine + command: valkey-server --requirepass ${SERVICE_PASSWORD_VALKEY} + volumes: + - valkeydata:/data + + meilisearch: + image: getmeili/meilisearch:v1.15 + environment: + - MEILI_MASTER_KEY=${SERVICE_PASSWORD_MEILISEARCH} + volumes: + - meilidata:/meili_data diff --git a/package.json b/package.json index 8e91bba..6c4674c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "open-archiver", - "version": "0.3.1", + "version": "0.3.3", "private": true, "scripts": { "dev": "dotenv -- pnpm --filter \"./packages/*\" --parallel dev", diff --git a/packages/backend/src/api/controllers/upload.controller.ts b/packages/backend/src/api/controllers/upload.controller.ts index 3d8cad5..5ebc5d9 100644 --- a/packages/backend/src/api/controllers/upload.controller.ts +++ b/packages/backend/src/api/controllers/upload.controller.ts @@ -7,6 +7,7 @@ import { config } from '../../config/index'; export const uploadFile = async (req: Request, res: Response) => { const storage = new StorageService(); const bb = busboy({ headers: req.headers }); + const uploads: Promise[] = []; let filePath = ''; let originalFilename = ''; @@ -14,10 +15,11 @@ export const uploadFile = async (req: Request, res: Response) => { originalFilename = filename.filename; const uuid = randomUUID(); filePath = `${config.storage.openArchiverFolderName}/tmp/${uuid}-${originalFilename}`; - storage.put(filePath, file); + uploads.push(storage.put(filePath, file)); }); - bb.on('finish', () => { + bb.on('finish', async () => { + await Promise.all(uploads); res.json({ filePath }); }); diff --git a/packages/backend/src/config/logger.ts b/packages/backend/src/config/logger.ts index 98cd83e..2a5e360 100644 --- a/packages/backend/src/config/logger.ts +++ b/packages/backend/src/config/logger.ts @@ -2,6 +2,7 @@ import pino from 'pino'; export const logger = pino({ level: process.env.LOG_LEVEL || 'info', + redact: ['password'], transport: { target: 'pino-pretty', options: { diff --git a/packages/backend/src/database/migrations/0020_panoramic_wolverine.sql b/packages/backend/src/database/migrations/0020_panoramic_wolverine.sql new file mode 100644 index 0000000..dc2c6a0 --- /dev/null +++ b/packages/backend/src/database/migrations/0020_panoramic_wolverine.sql @@ -0,0 +1 @@ +ALTER TYPE "public"."ingestion_provider" ADD VALUE 'mbox_import'; \ No newline at end of file diff --git a/packages/backend/src/database/migrations/meta/0020_snapshot.json b/packages/backend/src/database/migrations/meta/0020_snapshot.json new file mode 100644 index 0000000..57eeb9e --- /dev/null +++ b/packages/backend/src/database/migrations/meta/0020_snapshot.json @@ -0,0 +1,1245 @@ +{ + "id": "ed44e48c-b43b-402b-bf9d-5b5312edf42e", + "prevId": "a83bff5b-47ef-43e9-8e7d-667af35a206f", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.archived_emails": { + "name": "archived_emails", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ingestion_source_id": { + "name": "ingestion_source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_email": { + "name": "user_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message_id_header": { + "name": "message_id_header", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sender_name": { + "name": "sender_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sender_email": { + "name": "sender_email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recipients": { + "name": "recipients", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "storage_path": { + "name": "storage_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_hash_sha256": { + "name": "storage_hash_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "is_indexed": { + "name": "is_indexed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "has_attachments": { + "name": "has_attachments", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "is_on_legal_hold": { + "name": "is_on_legal_hold", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tags": { + "name": "tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "thread_id_idx": { + "name": "thread_id_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "archived_emails_ingestion_source_id_ingestion_sources_id_fk": { + "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk", + "tableFrom": "archived_emails", + "tableTo": "ingestion_sources", + "columnsFrom": [ + "ingestion_source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.attachments": { + "name": "attachments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "content_hash_sha256": { + "name": "content_hash_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_path": { + "name": "storage_path", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "attachments_content_hash_sha256_unique": { + "name": "attachments_content_hash_sha256_unique", + "nullsNotDistinct": false, + "columns": [ + "content_hash_sha256" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.email_attachments": { + "name": "email_attachments", + "schema": "", + "columns": { + "email_id": { + "name": "email_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "attachment_id": { + "name": "attachment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "email_attachments_email_id_archived_emails_id_fk": { + "name": "email_attachments_email_id_archived_emails_id_fk", + "tableFrom": "email_attachments", + "tableTo": "archived_emails", + "columnsFrom": [ + "email_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "email_attachments_attachment_id_attachments_id_fk": { + "name": "email_attachments_attachment_id_attachments_id_fk", + "tableFrom": "email_attachments", + "tableTo": "attachments", + "columnsFrom": [ + "attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "email_attachments_email_id_attachment_id_pk": { + "name": "email_attachments_email_id_attachment_id_pk", + "columns": [ + "email_id", + "attachment_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.audit_logs": { + "name": "audit_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "timestamp": { + "name": "timestamp", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "actor_identifier": { + "name": "actor_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "is_tamper_evident": { + "name": "is_tamper_evident", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ediscovery_cases": { + "name": "ediscovery_cases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "created_by_identifier": { + "name": "created_by_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ediscovery_cases_name_unique": { + "name": "ediscovery_cases_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.export_jobs": { + "name": "export_jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "case_id": { + "name": "case_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "format": { + "name": "format", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "query": { + "name": "query", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by_identifier": { + "name": "created_by_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "export_jobs_case_id_ediscovery_cases_id_fk": { + "name": "export_jobs_case_id_ediscovery_cases_id_fk", + "tableFrom": "export_jobs", + "tableTo": "ediscovery_cases", + "columnsFrom": [ + "case_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.legal_holds": { + "name": "legal_holds", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "case_id": { + "name": "case_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "custodian_id": { + "name": "custodian_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "hold_criteria": { + "name": "hold_criteria", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "applied_by_identifier": { + "name": "applied_by_identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "applied_at": { + "name": "applied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "removed_at": { + "name": "removed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "legal_holds_case_id_ediscovery_cases_id_fk": { + "name": "legal_holds_case_id_ediscovery_cases_id_fk", + "tableFrom": "legal_holds", + "tableTo": "ediscovery_cases", + "columnsFrom": [ + "case_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "legal_holds_custodian_id_custodians_id_fk": { + "name": "legal_holds_custodian_id_custodians_id_fk", + "tableFrom": "legal_holds", + "tableTo": "custodians", + "columnsFrom": [ + "custodian_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.retention_policies": { + "name": "retention_policies", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "retention_period_days": { + "name": "retention_period_days", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "action_on_expiry": { + "name": "action_on_expiry", + "type": "retention_action", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "conditions": { + "name": "conditions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "retention_policies_name_unique": { + "name": "retention_policies_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.custodians": { + "name": "custodians", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_type": { + "name": "source_type", + "type": "ingestion_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "custodians_email_unique": { + "name": "custodians_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ingestion_sources": { + "name": "ingestion_sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "ingestion_provider", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "ingestion_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending_auth'" + }, + "last_sync_started_at": { + "name": "last_sync_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_finished_at": { + "name": "last_sync_finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_status_message": { + "name": "last_sync_status_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_state": { + "name": "sync_state", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "ingestion_sources_user_id_users_id_fk": { + "name": "ingestion_sources_user_id_users_id_fk", + "tableFrom": "ingestion_sources", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.roles": { + "name": "roles", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "policies": { + "name": "policies", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "roles_name_unique": { + "name": "roles_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + }, + "roles_slug_unique": { + "name": "roles_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_roles": { + "name": "user_roles", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role_id": { + "name": "role_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "user_roles_user_id_users_id_fk": { + "name": "user_roles_user_id_users_id_fk", + "tableFrom": "user_roles", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "user_roles_role_id_roles_id_fk": { + "name": "user_roles_role_id_roles_id_fk", + "tableFrom": "user_roles", + "tableTo": "roles", + "columnsFrom": [ + "role_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_roles_user_id_role_id_pk": { + "name": "user_roles_user_id_role_id_pk", + "columns": [ + "user_id", + "role_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "first_name": { + "name": "first_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_name": { + "name": "last_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'local'" + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "api_keys_user_id_users_id_fk": { + "name": "api_keys_user_id_users_id_fk", + "tableFrom": "api_keys", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.retention_action": { + "name": "retention_action", + "schema": "public", + "values": [ + "delete_permanently", + "notify_admin" + ] + }, + "public.ingestion_provider": { + "name": "ingestion_provider", + "schema": "public", + "values": [ + "google_workspace", + "microsoft_365", + "generic_imap", + "pst_import", + "eml_import", + "mbox_import" + ] + }, + "public.ingestion_status": { + "name": "ingestion_status", + "schema": "public", + "values": [ + "active", + "paused", + "error", + "pending_auth", + "syncing", + "importing", + "auth_success", + "imported" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/backend/src/database/migrations/meta/_journal.json b/packages/backend/src/database/migrations/meta/_journal.json index 47a50b3..79ac072 100644 --- a/packages/backend/src/database/migrations/meta/_journal.json +++ b/packages/backend/src/database/migrations/meta/_journal.json @@ -1,146 +1,153 @@ { - "version": "7", - "dialect": "postgresql", - "entries": [ - { - "idx": 0, - "version": "7", - "when": 1752225352591, - "tag": "0000_amusing_namora", - "breakpoints": true - }, - { - "idx": 1, - "version": "7", - "when": 1752326803882, - "tag": "0001_odd_night_thrasher", - "breakpoints": true - }, - { - "idx": 2, - "version": "7", - "when": 1752332648392, - "tag": "0002_lethal_quentin_quire", - "breakpoints": true - }, - { - "idx": 3, - "version": "7", - "when": 1752332967084, - "tag": "0003_petite_wrecker", - "breakpoints": true - }, - { - "idx": 4, - "version": "7", - "when": 1752606108876, - "tag": "0004_sleepy_paper_doll", - "breakpoints": true - }, - { - "idx": 5, - "version": "7", - "when": 1752606327253, - "tag": "0005_chunky_sue_storm", - "breakpoints": true - }, - { - "idx": 6, - "version": "7", - "when": 1753112018514, - "tag": "0006_majestic_caretaker", - "breakpoints": true - }, - { - "idx": 7, - "version": "7", - "when": 1753190159356, - "tag": "0007_handy_archangel", - "breakpoints": true - }, - { - "idx": 8, - "version": "7", - "when": 1753370737317, - "tag": "0008_eminent_the_spike", - "breakpoints": true - }, - { - "idx": 9, - "version": "7", - "when": 1754337938241, - "tag": "0009_late_lenny_balinger", - "breakpoints": true - }, - { - "idx": 10, - "version": "7", - "when": 1754420780849, - "tag": "0010_perpetual_lightspeed", - "breakpoints": true - }, - { - "idx": 11, - "version": "7", - "when": 1754422064158, - "tag": "0011_tan_blackheart", - "breakpoints": true - }, - { - "idx": 12, - "version": "7", - "when": 1754476962901, - "tag": "0012_warm_the_stranger", - "breakpoints": true - }, - { - "idx": 13, - "version": "7", - "when": 1754659373517, - "tag": "0013_classy_talkback", - "breakpoints": true - }, - { - "idx": 14, - "version": "7", - "when": 1754831765718, - "tag": "0014_foamy_vapor", - "breakpoints": true - }, - { - "idx": 15, - "version": "7", - "when": 1755443936046, - "tag": "0015_wakeful_norman_osborn", - "breakpoints": true - }, - { - "idx": 16, - "version": "7", - "when": 1755780572342, - "tag": "0016_lonely_mariko_yashida", - "breakpoints": true - }, - { - "idx": 17, - "version": "7", - "when": 1755961566627, - "tag": "0017_tranquil_shooting_star", - "breakpoints": true - }, - { - "idx": 18, - "version": "7", - "when": 1756911118035, - "tag": "0018_flawless_owl", - "breakpoints": true - }, - { - "idx": 19, - "version": "7", - "when": 1756937533843, - "tag": "0019_confused_scream", - "breakpoints": true - } - ] -} + "version": "7", + "dialect": "postgresql", + "entries": [ + { + "idx": 0, + "version": "7", + "when": 1752225352591, + "tag": "0000_amusing_namora", + "breakpoints": true + }, + { + "idx": 1, + "version": "7", + "when": 1752326803882, + "tag": "0001_odd_night_thrasher", + "breakpoints": true + }, + { + "idx": 2, + "version": "7", + "when": 1752332648392, + "tag": "0002_lethal_quentin_quire", + "breakpoints": true + }, + { + "idx": 3, + "version": "7", + "when": 1752332967084, + "tag": "0003_petite_wrecker", + "breakpoints": true + }, + { + "idx": 4, + "version": "7", + "when": 1752606108876, + "tag": "0004_sleepy_paper_doll", + "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1752606327253, + "tag": "0005_chunky_sue_storm", + "breakpoints": true + }, + { + "idx": 6, + "version": "7", + "when": 1753112018514, + "tag": "0006_majestic_caretaker", + "breakpoints": true + }, + { + "idx": 7, + "version": "7", + "when": 1753190159356, + "tag": "0007_handy_archangel", + "breakpoints": true + }, + { + "idx": 8, + "version": "7", + "when": 1753370737317, + "tag": "0008_eminent_the_spike", + "breakpoints": true + }, + { + "idx": 9, + "version": "7", + "when": 1754337938241, + "tag": "0009_late_lenny_balinger", + "breakpoints": true + }, + { + "idx": 10, + "version": "7", + "when": 1754420780849, + "tag": "0010_perpetual_lightspeed", + "breakpoints": true + }, + { + "idx": 11, + "version": "7", + "when": 1754422064158, + "tag": "0011_tan_blackheart", + "breakpoints": true + }, + { + "idx": 12, + "version": "7", + "when": 1754476962901, + "tag": "0012_warm_the_stranger", + "breakpoints": true + }, + { + "idx": 13, + "version": "7", + "when": 1754659373517, + "tag": "0013_classy_talkback", + "breakpoints": true + }, + { + "idx": 14, + "version": "7", + "when": 1754831765718, + "tag": "0014_foamy_vapor", + "breakpoints": true + }, + { + "idx": 15, + "version": "7", + "when": 1755443936046, + "tag": "0015_wakeful_norman_osborn", + "breakpoints": true + }, + { + "idx": 16, + "version": "7", + "when": 1755780572342, + "tag": "0016_lonely_mariko_yashida", + "breakpoints": true + }, + { + "idx": 17, + "version": "7", + "when": 1755961566627, + "tag": "0017_tranquil_shooting_star", + "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1756911118035, + "tag": "0018_flawless_owl", + "breakpoints": true + }, + { + "idx": 19, + "version": "7", + "when": 1756937533843, + "tag": "0019_confused_scream", + "breakpoints": true + }, + { + "idx": 20, + "version": "7", + "when": 1757860242528, + "tag": "0020_panoramic_wolverine", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/packages/backend/src/database/schema/ingestion-sources.ts b/packages/backend/src/database/schema/ingestion-sources.ts index 15f9bc3..98d937d 100644 --- a/packages/backend/src/database/schema/ingestion-sources.ts +++ b/packages/backend/src/database/schema/ingestion-sources.ts @@ -8,6 +8,7 @@ export const ingestionProviderEnum = pgEnum('ingestion_provider', [ 'generic_imap', 'pst_import', 'eml_import', + 'mbox_import', ]); export const ingestionStatusEnum = pgEnum('ingestion_status', [ diff --git a/packages/backend/src/services/EmailProviderFactory.ts b/packages/backend/src/services/EmailProviderFactory.ts index f3cea9b..2d0a20e 100644 --- a/packages/backend/src/services/EmailProviderFactory.ts +++ b/packages/backend/src/services/EmailProviderFactory.ts @@ -5,6 +5,7 @@ import type { GenericImapCredentials, PSTImportCredentials, EMLImportCredentials, + MboxImportCredentials, EmailObject, SyncState, MailboxUser, @@ -14,6 +15,7 @@ import { MicrosoftConnector } from './ingestion-connectors/MicrosoftConnector'; import { ImapConnector } from './ingestion-connectors/ImapConnector'; import { PSTConnector } from './ingestion-connectors/PSTConnector'; import { EMLConnector } from './ingestion-connectors/EMLConnector'; +import { MboxConnector } from './ingestion-connectors/MboxConnector'; // Define a common interface for all connectors export interface IEmailConnector { @@ -43,6 +45,8 @@ export class EmailProviderFactory { return new PSTConnector(credentials as PSTImportCredentials); case 'eml_import': return new EMLConnector(credentials as EMLImportCredentials); + case 'mbox_import': + return new MboxConnector(credentials as MboxImportCredentials); default: throw new Error(`Unsupported provider: ${source.provider}`); } diff --git a/packages/backend/src/services/IngestionService.ts b/packages/backend/src/services/IngestionService.ts index d516a2e..1818c8e 100644 --- a/packages/backend/src/services/IngestionService.ts +++ b/packages/backend/src/services/IngestionService.ts @@ -26,6 +26,7 @@ import { SearchService } from './SearchService'; import { DatabaseService } from './DatabaseService'; import { config } from '../config/index'; import { FilterBuilder } from './FilterBuilder'; +import e from 'express'; export class IngestionService { private static decryptSource( @@ -47,7 +48,7 @@ export class IngestionService { } public static returnFileBasedIngestions(): IngestionProvider[] { - return ['pst_import', 'eml_import']; + return ['pst_import', 'eml_import', 'mbox_import']; } public static async create( @@ -76,9 +77,13 @@ export class IngestionService { const connector = EmailProviderFactory.createConnector(decryptedSource); try { - await connector.testConnection(); + const connectionValid = await connector.testConnection(); // If connection succeeds, update status to auth_success, which triggers the initial import. - return await this.update(decryptedSource.id, { status: 'auth_success' }); + if (connectionValid) { + return await this.update(decryptedSource.id, { status: 'auth_success' }); + } else { + throw Error('Ingestion authentication failed.') + } } catch (error) { // If connection fails, delete the newly created source and throw the error. await this.delete(decryptedSource.id); diff --git a/packages/backend/src/services/ingestion-connectors/EMLConnector.ts b/packages/backend/src/services/ingestion-connectors/EMLConnector.ts index e985aec..4209ab3 100644 --- a/packages/backend/src/services/ingestion-connectors/EMLConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/EMLConnector.ts @@ -69,7 +69,7 @@ export class EMLConnector implements IEmailConnector { syncState?: SyncState | null ): AsyncGenerator { const fileStream = await this.storage.get(this.credentials.uploadedFilePath); - const tempDir = await fs.mkdtemp(join('/tmp', 'eml-import-')); + const tempDir = await fs.mkdtemp(join('/tmp', `eml-import-${new Date().getTime()}`)); const unzippedPath = join(tempDir, 'unzipped'); await fs.mkdir(unzippedPath); const zipFilePath = join(tempDir, 'eml.zip'); @@ -115,6 +115,14 @@ export class EMLConnector implements IEmailConnector { throw error; } finally { await fs.rm(tempDir, { recursive: true, force: true }); + try { + await this.storage.delete(this.credentials.uploadedFilePath); + } catch (error) { + logger.error( + { error, file: this.credentials.uploadedFilePath }, + 'Failed to delete EML file after processing.' + ); + } } } diff --git a/packages/backend/src/services/ingestion-connectors/MboxConnector.ts b/packages/backend/src/services/ingestion-connectors/MboxConnector.ts new file mode 100644 index 0000000..b193d03 --- /dev/null +++ b/packages/backend/src/services/ingestion-connectors/MboxConnector.ts @@ -0,0 +1,174 @@ +import type { + MboxImportCredentials, + EmailObject, + EmailAddress, + SyncState, + MailboxUser, +} from '@open-archiver/types'; +import type { IEmailConnector } from '../EmailProviderFactory'; +import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser'; +import { logger } from '../../config/logger'; +import { getThreadId } from './helpers/utils'; +import { StorageService } from '../StorageService'; +import { Readable } from 'stream'; +import { createHash } from 'crypto'; +import { streamToBuffer } from '../../helpers/streamToBuffer'; + +export class MboxConnector implements IEmailConnector { + private storage: StorageService; + + constructor(private credentials: MboxImportCredentials) { + this.storage = new StorageService(); + } + + public async testConnection(): Promise { + try { + if (!this.credentials.uploadedFilePath) { + throw Error('Mbox file path not provided.'); + } + if (!this.credentials.uploadedFilePath.includes('.mbox')) { + throw Error('Provided file is not in the MBOX format.'); + } + const fileExist = await this.storage.exists(this.credentials.uploadedFilePath); + if (!fileExist) { + throw Error('Mbox file upload not finished yet, please wait.'); + } + + return true; + } catch (error) { + logger.error({ error, credentials: this.credentials }, 'Mbox file validation failed.'); + throw error; + } + } + + public async *listAllUsers(): AsyncGenerator { + const displayName = + this.credentials.uploadedFileName || `mbox-import-${new Date().getTime()}`; + logger.info(`Found potential mailbox: ${displayName}`); + const constructedPrimaryEmail = `${displayName.replace(/ /g, '.').toLowerCase()}@mbox.local`; + yield { + id: constructedPrimaryEmail, + primaryEmail: constructedPrimaryEmail, + displayName: displayName, + }; + } + + public async *fetchEmails( + userEmail: string, + syncState?: SyncState | null + ): AsyncGenerator { + try { + const fileStream = await this.storage.get(this.credentials.uploadedFilePath); + const fileBuffer = await streamToBuffer(fileStream as Readable); + const mboxContent = fileBuffer.toString('utf-8'); + const emailDelimiter = '\nFrom '; + const emails = mboxContent.split(emailDelimiter); + + // The first split part might be empty or part of the first email's header, so we adjust. + if (emails.length > 0 && !mboxContent.startsWith('From ')) { + emails.shift(); // Adjust if the file doesn't start with "From " + } + + logger.info(`Found ${emails.length} potential emails in the mbox file.`); + let emailCount = 0; + + for (const email of emails) { + try { + // Re-add the "From " delimiter for the parser, except for the very first email + const emailWithDelimiter = + emailCount > 0 || mboxContent.startsWith('From ') ? `From ${email}` : email; + const emailBuffer = Buffer.from(emailWithDelimiter, 'utf-8'); + const emailObject = await this.parseMessage(emailBuffer, ''); + yield emailObject; + emailCount++; + } catch (error) { + logger.error( + { error, file: this.credentials.uploadedFilePath }, + 'Failed to process a single message from mbox file. Skipping.' + ); + } + } + logger.info(`Finished processing mbox file. Total emails processed: ${emailCount}`); + } finally { + try { + await this.storage.delete(this.credentials.uploadedFilePath); + } catch (error) { + logger.error( + { error, file: this.credentials.uploadedFilePath }, + 'Failed to delete mbox file after processing.' + ); + } + } + } + + private async parseMessage(emlBuffer: Buffer, path: string): Promise { + const parsedEmail: ParsedMail = await simpleParser(emlBuffer); + + const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({ + filename: attachment.filename || 'untitled', + contentType: attachment.contentType, + size: attachment.size, + content: attachment.content as Buffer, + })); + + const mapAddresses = ( + addresses: AddressObject | AddressObject[] | undefined + ): EmailAddress[] => { + if (!addresses) return []; + const addressArray = Array.isArray(addresses) ? addresses : [addresses]; + return addressArray.flatMap((a) => + a.value.map((v) => ({ + name: v.name, + address: v.address?.replaceAll(`'`, '') || '', + })) + ); + }; + + const threadId = getThreadId(parsedEmail.headers); + let messageId = parsedEmail.messageId; + + if (!messageId) { + messageId = `generated-${createHash('sha256').update(emlBuffer).digest('hex')}`; + } + + const from = mapAddresses(parsedEmail.from); + if (from.length === 0) { + from.push({ name: 'No Sender', address: 'No Sender' }); + } + + // Extract folder path from headers. Mbox files don't have a standard folder structure, so we rely on custom headers added by email clients. + // Gmail uses 'X-Gmail-Labels', and other clients like Thunderbird may use 'X-Folder'. + const gmailLabels = parsedEmail.headers.get('x-gmail-labels'); + const folderHeader = parsedEmail.headers.get('x-folder'); + let finalPath = ''; + + if (gmailLabels && typeof gmailLabels === 'string') { + // We take the first label as the primary folder. + // Gmail labels can be hierarchical, but we'll simplify to the first label. + finalPath = gmailLabels.split(',')[0]; + } else if (folderHeader && typeof folderHeader === 'string') { + finalPath = folderHeader; + } + + return { + id: messageId, + threadId: threadId, + from, + to: mapAddresses(parsedEmail.to), + cc: mapAddresses(parsedEmail.cc), + bcc: mapAddresses(parsedEmail.bcc), + subject: parsedEmail.subject || '', + body: parsedEmail.text || '', + html: parsedEmail.html || '', + headers: parsedEmail.headers, + attachments, + receivedAt: parsedEmail.date || new Date(), + eml: emlBuffer, + path: finalPath, + }; + } + + public getUpdatedSyncState(): SyncState { + return {}; + } +} diff --git a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts index f3a0376..1199b32 100644 --- a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts @@ -193,6 +193,14 @@ export class PSTConnector implements IEmailConnector { throw error; } finally { pstFile?.close(); + try { + await this.storage.delete(this.credentials.uploadedFilePath); + } catch (error) { + logger.error( + { error, file: this.credentials.uploadedFilePath }, + 'Failed to delete PST file after processing.' + ); + } } } @@ -273,8 +281,8 @@ export class PSTConnector implements IEmailConnector { emlBuffer ?? Buffer.from(parsedEmail.text || parsedEmail.html || '', 'utf-8') ) .digest('hex')}-${createHash('sha256') - .update(emlBuffer ?? Buffer.from(msg.subject || '', 'utf-8')) - .digest('hex')}-${msg.clientSubmitTime?.getTime()}`; + .update(emlBuffer ?? Buffer.from(msg.subject || '', 'utf-8')) + .digest('hex')}-${msg.clientSubmitTime?.getTime()}`; } return { id: messageId, diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 98fa812..bdeb6bf 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -19,6 +19,7 @@ "bits-ui": "^2.8.10", "clsx": "^2.1.1", "d3-shape": "^3.2.0", + "html-entities": "^2.6.0", "jose": "^6.0.1", "lucide-svelte": "^0.525.0", "postal-mime": "^2.4.4", diff --git a/packages/frontend/src/lib/components/custom/EmailPreview.svelte b/packages/frontend/src/lib/components/custom/EmailPreview.svelte index e86c6ae..3effe6c 100644 --- a/packages/frontend/src/lib/components/custom/EmailPreview.svelte +++ b/packages/frontend/src/lib/components/custom/EmailPreview.svelte @@ -2,6 +2,7 @@ import PostalMime, { type Email } from 'postal-mime'; import type { Buffer } from 'buffer'; import { t } from '$lib/translations'; + import { encode } from 'html-entities'; let { raw, @@ -18,7 +19,9 @@ if (parsedEmail && parsedEmail.html) { return `${parsedEmail.html}`; } else if (parsedEmail && parsedEmail.text) { - return `${parsedEmail.text}`; + // display raw text email body in html + const safeHtmlContent: string = encode(parsedEmail.text); + return `
${safeHtmlContent.replaceAll('\n', '
')}
`; } else if (rawHtml) { return `${rawHtml}`; } @@ -53,7 +56,7 @@
{#if isLoading}

{$t('app.components.email_preview.loading')}

- {:else if emailHtml} + {:else if emailHtml()}