-
- {#if thread}
- {#each thread as item, i (item.id)}
-
-
-
+ {#if thread}
+ {#each thread as item, i (item.id)}
+
+
-
-
-
-
From: {item.senderEmail}
-
{new Date(item.sentAt).toLocaleString()}
+
+
+
+ From: {item.senderEmail}
+ {new Date(item.sentAt).toLocaleString()}
+
-
- {/each}
- {/if}
-
+
diff --git a/packages/frontend/src/lib/components/ui/scroll-area/index.ts b/packages/frontend/src/lib/components/ui/scroll-area/index.ts
new file mode 100644
index 0000000..e86a25b
--- /dev/null
+++ b/packages/frontend/src/lib/components/ui/scroll-area/index.ts
@@ -0,0 +1,10 @@
+import Scrollbar from "./scroll-area-scrollbar.svelte";
+import Root from "./scroll-area.svelte";
+
+export {
+ Root,
+ Scrollbar,
+ //,
+ Root as ScrollArea,
+ Scrollbar as ScrollAreaScrollbar,
+};
diff --git a/packages/frontend/src/lib/components/ui/scroll-area/scroll-area-scrollbar.svelte b/packages/frontend/src/lib/components/ui/scroll-area/scroll-area-scrollbar.svelte
new file mode 100644
index 0000000..4127444
--- /dev/null
+++ b/packages/frontend/src/lib/components/ui/scroll-area/scroll-area-scrollbar.svelte
@@ -0,0 +1,31 @@
+
+
+
+ {@render children?.()}
+
diff --git a/packages/frontend/src/lib/components/ui/scroll-area/scroll-area.svelte b/packages/frontend/src/lib/components/ui/scroll-area/scroll-area.svelte
new file mode 100644
index 0000000..38a1847
--- /dev/null
+++ b/packages/frontend/src/lib/components/ui/scroll-area/scroll-area.svelte
@@ -0,0 +1,40 @@
+
+
+
+
+ {@render children?.()}
+
+ {#if orientation === "vertical" || orientation === "both"}
+
From 842f8092d6aac1517db9df19d3341e7ffde7faa2 Mon Sep 17 00:00:00 2001
From: Wayne <5291640+ringoinca@users.noreply.github.com>
Date: Wed, 6 Aug 2025 00:01:15 +0300
Subject: [PATCH 2/9] Migrating user service to database, sunsetting admin user
---
docs/developer-guides/iam-policies.md | 111 ++
.../src/api/controllers/auth.controller.ts | 49 +-
.../src/api/controllers/iam.controller.ts | 71 ++
.../backend/src/api/routes/auth.routes.ts | 14 +
packages/backend/src/api/routes/iam.routes.ts | 37 +
.../migrations/0010_perpetual_lightspeed.sql | 36 +
.../migrations/0011_tan_blackheart.sql | 2 +
.../migrations/meta/0010_snapshot.json | 1087 ++++++++++++++++
.../migrations/meta/0011_snapshot.json | 1093 +++++++++++++++++
.../database/migrations/meta/_journal.json | 14 +
packages/backend/src/database/schema.ts | 1 +
.../src/database/schema/archived-emails.ts | 6 +-
packages/backend/src/database/schema/users.ts | 89 ++
.../backend/src/iam-policy/iam-definitions.ts | 155 +++
.../src/iam-policy/policy-validator.ts | 100 ++
packages/backend/src/index.ts | 13 +-
packages/backend/src/services/AuthService.ts | 58 +-
packages/backend/src/services/IamService.ts | 24 +
packages/backend/src/services/UserService.ts | 98 +-
.../frontend/src/routes/+layout.server.ts | 20 +-
.../frontend/src/routes/setup/+page.server.ts | 5 +
.../frontend/src/routes/setup/+page.svelte | 111 ++
.../src/routes/signin/+page.server.ts | 11 +
packages/types/src/auth.types.ts | 6 +-
packages/types/src/iam.types.ts | 9 +
packages/types/src/index.ts | 1 +
packages/types/src/user.types.ts | 44 +-
27 files changed, 3174 insertions(+), 91 deletions(-)
create mode 100644 docs/developer-guides/iam-policies.md
create mode 100644 packages/backend/src/api/controllers/iam.controller.ts
create mode 100644 packages/backend/src/api/routes/iam.routes.ts
create mode 100644 packages/backend/src/database/migrations/0010_perpetual_lightspeed.sql
create mode 100644 packages/backend/src/database/migrations/0011_tan_blackheart.sql
create mode 100644 packages/backend/src/database/migrations/meta/0010_snapshot.json
create mode 100644 packages/backend/src/database/migrations/meta/0011_snapshot.json
create mode 100644 packages/backend/src/database/schema/users.ts
create mode 100644 packages/backend/src/iam-policy/iam-definitions.ts
create mode 100644 packages/backend/src/iam-policy/policy-validator.ts
create mode 100644 packages/backend/src/services/IamService.ts
create mode 100644 packages/frontend/src/routes/setup/+page.server.ts
create mode 100644 packages/frontend/src/routes/setup/+page.svelte
create mode 100644 packages/frontend/src/routes/signin/+page.server.ts
create mode 100644 packages/types/src/iam.types.ts
diff --git a/docs/developer-guides/iam-policies.md b/docs/developer-guides/iam-policies.md
new file mode 100644
index 0000000..edbcea8
--- /dev/null
+++ b/docs/developer-guides/iam-policies.md
@@ -0,0 +1,111 @@
+# IAM Policies Guide
+
+This document provides a comprehensive guide to the Identity and Access Management (IAM) policies in Open Archiver. Our policy structure is inspired by AWS IAM, providing a powerful and flexible way to manage permissions.
+
+## 1. Policy Structure
+
+A policy is a JSON object that consists of one or more statements. Each statement includes an `Effect`, `Action`, and `Resource`.
+
+```json
+{
+ "Effect": "Allow",
+ "Action": ["archive:read", "archive:search"],
+ "Resource": ["archive/all"]
+}
+```
+
+- **`Effect`**: Specifies whether the statement results in an `Allow` or `Deny`. An explicit `Deny` always overrides an `Allow`.
+- **`Action`**: A list of operations that the policy grants or denies permission to perform. Actions are formatted as `service:operation`.
+- **`Resource`**: A list of resources to which the actions apply. Resources are specified in a hierarchical format. Wildcards (`*`) can be used.
+
+## 2. Actions and Resources by Service
+
+The following sections define the available actions and resources, categorized by their respective services.
+
+### Service: `archive`
+
+The `archive` service pertains to all actions related to accessing and managing archived emails.
+
+**Actions:**
+
+| Action | Description |
+| :--------------- | :--------------------------------------------------------------------- |
+| `archive:read` | Grants permission to read the content and metadata of archived emails. |
+| `archive:search` | Grants permission to perform search queries against the email archive. |
+| `archive:export` | Grants permission to export search results or individual emails. |
+
+**Resources:**
+
+| Resource | Description |
+| :------------------------------------ | :------------------------------------------------------------- |
+| `archive/all` | Represents the entire email archive. |
+| `archive/ingestion-source/{sourceId}` | Scopes the action to emails from a specific ingestion source. |
+| `archive/email/{emailId}` | Scopes the action to a single, specific email. |
+| `archive/custodian/{custodianId}` | Scopes the action to emails belonging to a specific custodian. |
+
+---
+
+### Service: `ingestion`
+
+The `ingestion` service covers the management of email ingestion sources.
+
+**Actions:**
+
+| Action | Description |
+| :----------------------- | :--------------------------------------------------------------------------- |
+| `ingestion:createSource` | Grants permission to create a new ingestion source. |
+| `ingestion:readSource` | Grants permission to view the details of ingestion sources. |
+| `ingestion:updateSource` | Grants permission to modify the configuration of an ingestion source. |
+| `ingestion:deleteSource` | Grants permission to delete an ingestion source. |
+| `ingestion:manageSync` | Grants permission to trigger, pause, or force a sync on an ingestion source. |
+
+**Resources:**
+
+| Resource | Description |
+| :---------------------------- | :-------------------------------------------------------- |
+| `ingestion-source/*` | Represents all ingestion sources. |
+| `ingestion-source/{sourceId}` | Scopes the action to a single, specific ingestion source. |
+
+---
+
+### Service: `system`
+
+The `system` service is for managing system-level settings, users, and roles.
+
+**Actions:**
+
+| Action | Description |
+| :---------------------- | :-------------------------------------------------- |
+| `system:readSettings` | Grants permission to view system settings. |
+| `system:updateSettings` | Grants permission to modify system settings. |
+| `system:readUsers` | Grants permission to list and view user accounts. |
+| `system:createUser` | Grants permission to create new user accounts. |
+| `system:updateUser` | Grants permission to modify existing user accounts. |
+| `system:deleteUser` | Grants permission to delete user accounts. |
+| `system:assignRole` | Grants permission to assign roles to users. |
+
+**Resources:**
+
+| Resource | Description |
+| :--------------------- | :---------------------------------------------------- |
+| `system/settings` | Represents the system configuration. |
+| `system/users` | Represents all user accounts within the system. |
+| `system/user/{userId}` | Scopes the action to a single, specific user account. |
+
+---
+
+### Service: `dashboard`
+
+The `dashboard` service relates to viewing analytics and overview information.
+
+**Actions:**
+
+| Action | Description |
+| :--------------- | :-------------------------------------------------------------- |
+| `dashboard:read` | Grants permission to view all dashboard widgets and statistics. |
+
+**Resources:**
+
+| Resource | Description |
+| :------------ | :------------------------------------------ |
+| `dashboard/*` | Represents all components of the dashboard. |
diff --git a/packages/backend/src/api/controllers/auth.controller.ts b/packages/backend/src/api/controllers/auth.controller.ts
index 557ccca..3ddd3ca 100644
--- a/packages/backend/src/api/controllers/auth.controller.ts
+++ b/packages/backend/src/api/controllers/auth.controller.ts
@@ -1,13 +1,44 @@
import type { Request, Response } from 'express';
-import type { IAuthService } from '../../services/AuthService';
+import { AuthService } from '../../services/AuthService';
+import { UserService } from '../../services/UserService';
+import { db } from '../../database';
+import * as schema from '../../database/schema';
+import { sql } from 'drizzle-orm';
export class AuthController {
- #authService: IAuthService;
+ #authService: AuthService;
+ #userService: UserService;
- constructor(authService: IAuthService) {
+ constructor(authService: AuthService, userService: UserService) {
this.#authService = authService;
+ this.#userService = userService;
}
+ public setup = async (req: Request, res: Response): Promise
=> {
+ const { email, password, first_name, last_name } = req.body;
+
+ if (!email || !password || !first_name || !last_name) {
+ return res.status(400).json({ message: 'Email, password, and name are required' });
+ }
+
+ try {
+ const userCountResult = await db.select({ count: sql`count(*)` }).from(schema.users);
+ const userCount = Number(userCountResult[0].count);
+
+ if (userCount > 0) {
+ return res.status(403).json({ message: 'Setup has already been completed.' });
+ }
+
+ const newUser = await this.#userService.createUser({ email, password, first_name, last_name });
+ const result = await this.#authService.login(email, password);
+
+ return res.status(201).json(result);
+ } catch (error) {
+ console.error('Setup error:', error);
+ return res.status(500).json({ message: 'An internal server error occurred' });
+ }
+ };
+
public login = async (req: Request, res: Response): Promise => {
const { email, password } = req.body;
@@ -28,4 +59,16 @@ export class AuthController {
return res.status(500).json({ message: 'An internal server error occurred' });
}
};
+
+ public status = async (req: Request, res: Response): Promise => {
+ try {
+ const userCountResult = await db.select({ count: sql`count(*)` }).from(schema.users);
+ const userCount = Number(userCountResult[0].count);
+ const needsSetup = userCount === 0;
+ return res.status(200).json({ needsSetup });
+ } catch (error) {
+ console.error('Status check error:', error);
+ return res.status(500).json({ message: 'An internal server error occurred' });
+ }
+ };
}
diff --git a/packages/backend/src/api/controllers/iam.controller.ts b/packages/backend/src/api/controllers/iam.controller.ts
new file mode 100644
index 0000000..f24d1f8
--- /dev/null
+++ b/packages/backend/src/api/controllers/iam.controller.ts
@@ -0,0 +1,71 @@
+import { Request, Response } from 'express';
+import { IamService } from '../../services/IamService';
+import { PolicyValidator } from '../../iam-policy/policy-validator';
+import type { PolicyStatement } from '@open-archiver/types';
+
+export class IamController {
+ #iamService: IamService;
+
+ constructor(iamService: IamService) {
+ this.#iamService = iamService;
+ }
+
+ public getRoles = async (req: Request, res: Response): Promise => {
+ try {
+ const roles = await this.#iamService.getRoles();
+ res.status(200).json(roles);
+ } catch (error) {
+ res.status(500).json({ error: 'Failed to get roles.' });
+ }
+ };
+
+ public getRoleById = async (req: Request, res: Response): Promise => {
+ const { id } = req.params;
+
+ try {
+ const role = await this.#iamService.getRoleById(id);
+ if (role) {
+ res.status(200).json(role);
+ } else {
+ res.status(404).json({ error: 'Role not found.' });
+ }
+ } catch (error) {
+ res.status(500).json({ error: 'Failed to get role.' });
+ }
+ };
+
+ public createRole = async (req: Request, res: Response): Promise => {
+ const { name, policy } = req.body;
+
+ if (!name || !policy) {
+ res.status(400).json({ error: 'Missing required fields: name and policy.' });
+ return;
+ }
+
+ for (const statement of policy) {
+ const { valid, reason } = PolicyValidator.isValid(statement as PolicyStatement);
+ if (!valid) {
+ res.status(400).json({ error: `Invalid policy statement: ${reason}` });
+ return;
+ }
+ }
+
+ try {
+ const role = await this.#iamService.createRole(name, policy);
+ res.status(201).json(role);
+ } catch (error) {
+ res.status(500).json({ error: 'Failed to create role.' });
+ }
+ };
+
+ public deleteRole = async (req: Request, res: Response): Promise => {
+ const { id } = req.params;
+
+ try {
+ await this.#iamService.deleteRole(id);
+ res.status(204).send();
+ } catch (error) {
+ res.status(500).json({ error: 'Failed to delete role.' });
+ }
+ };
+}
diff --git a/packages/backend/src/api/routes/auth.routes.ts b/packages/backend/src/api/routes/auth.routes.ts
index 288fdb8..3d42e2e 100644
--- a/packages/backend/src/api/routes/auth.routes.ts
+++ b/packages/backend/src/api/routes/auth.routes.ts
@@ -5,6 +5,13 @@ import type { AuthController } from '../controllers/auth.controller';
export const createAuthRouter = (authController: AuthController): Router => {
const router = Router();
+ /**
+ * @route POST /api/v1/auth/setup
+ * @description Creates the initial administrator user.
+ * @access Public
+ */
+ router.post('/setup', loginRateLimiter, authController.setup);
+
/**
* @route POST /api/v1/auth/login
* @description Authenticates a user and returns a JWT.
@@ -12,5 +19,12 @@ export const createAuthRouter = (authController: AuthController): Router => {
*/
router.post('/login', loginRateLimiter, authController.login);
+ /**
+ * @route GET /api/v1/auth/status
+ * @description Checks if the application has been set up.
+ * @access Public
+ */
+ router.get('/status', authController.status);
+
return router;
};
diff --git a/packages/backend/src/api/routes/iam.routes.ts b/packages/backend/src/api/routes/iam.routes.ts
new file mode 100644
index 0000000..0c5cbd3
--- /dev/null
+++ b/packages/backend/src/api/routes/iam.routes.ts
@@ -0,0 +1,37 @@
+import { Router } from 'express';
+import { requireAuth } from '../middleware/requireAuth';
+import type { IamController } from '../controllers/iam.controller';
+
+export const createIamRouter = (iamController: IamController): Router => {
+ const router = Router();
+
+ /**
+ * @route GET /api/v1/iam/roles
+ * @description Gets all roles.
+ * @access Private
+ */
+ router.get('/roles', requireAuth, iamController.getRoles);
+
+ /**
+ * @route GET /api/v1/iam/roles/:id
+ * @description Gets a role by ID.
+ * @access Private
+ */
+ router.get('/roles/:id', requireAuth, iamController.getRoleById);
+
+ /**
+ * @route POST /api/v1/iam/roles
+ * @description Creates a new role.
+ * @access Private
+ */
+ router.post('/roles', requireAuth, iamController.createRole);
+
+ /**
+ * @route DELETE /api/v1/iam/roles/:id
+ * @description Deletes a role.
+ * @access Private
+ */
+ router.delete('/roles/:id', requireAuth, iamController.deleteRole);
+
+ return router;
+};
diff --git a/packages/backend/src/database/migrations/0010_perpetual_lightspeed.sql b/packages/backend/src/database/migrations/0010_perpetual_lightspeed.sql
new file mode 100644
index 0000000..4a406d9
--- /dev/null
+++ b/packages/backend/src/database/migrations/0010_perpetual_lightspeed.sql
@@ -0,0 +1,36 @@
+CREATE TABLE "roles" (
+ "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+ "name" text NOT NULL,
+ "policies" jsonb DEFAULT '[]'::jsonb NOT NULL,
+ "created_at" timestamp DEFAULT now() NOT NULL,
+ "updated_at" timestamp DEFAULT now() NOT NULL,
+ CONSTRAINT "roles_name_unique" UNIQUE("name")
+);
+--> statement-breakpoint
+CREATE TABLE "sessions" (
+ "id" text PRIMARY KEY NOT NULL,
+ "user_id" uuid NOT NULL,
+ "expires_at" timestamp with time zone NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE "user_roles" (
+ "user_id" uuid NOT NULL,
+ "role_id" uuid NOT NULL,
+ CONSTRAINT "user_roles_user_id_role_id_pk" PRIMARY KEY("user_id","role_id")
+);
+--> statement-breakpoint
+CREATE TABLE "users" (
+ "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+ "email" text NOT NULL,
+ "name" text,
+ "password" text,
+ "provider" text DEFAULT 'local',
+ "provider_id" text,
+ "created_at" timestamp DEFAULT now() NOT NULL,
+ "updated_at" timestamp DEFAULT now() NOT NULL,
+ CONSTRAINT "users_email_unique" UNIQUE("email")
+);
+--> statement-breakpoint
+ALTER TABLE "sessions" ADD CONSTRAINT "sessions_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "user_roles" ADD CONSTRAINT "user_roles_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "user_roles" ADD CONSTRAINT "user_roles_role_id_roles_id_fk" FOREIGN KEY ("role_id") REFERENCES "public"."roles"("id") ON DELETE cascade ON UPDATE no action;
\ No newline at end of file
diff --git a/packages/backend/src/database/migrations/0011_tan_blackheart.sql b/packages/backend/src/database/migrations/0011_tan_blackheart.sql
new file mode 100644
index 0000000..18e42f8
--- /dev/null
+++ b/packages/backend/src/database/migrations/0011_tan_blackheart.sql
@@ -0,0 +1,2 @@
+ALTER TABLE "users" RENAME COLUMN "name" TO "first_name";--> statement-breakpoint
+ALTER TABLE "users" ADD COLUMN "last_name" text;
\ No newline at end of file
diff --git a/packages/backend/src/database/migrations/meta/0010_snapshot.json b/packages/backend/src/database/migrations/meta/0010_snapshot.json
new file mode 100644
index 0000000..087f867
--- /dev/null
+++ b/packages/backend/src/database/migrations/meta/0010_snapshot.json
@@ -0,0 +1,1087 @@
+{
+ "id": "ab45f75d-f50c-457f-ad40-e62ca27a4e2b",
+ "prevId": "701eda75-451a-4a6d-87e3-b6658fca65da",
+ "version": "7",
+ "dialect": "postgresql",
+ "tables": {
+ "public.archived_emails": {
+ "name": "archived_emails",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "thread_id": {
+ "name": "thread_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "ingestion_source_id": {
+ "name": "ingestion_source_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "user_email": {
+ "name": "user_email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "message_id_header": {
+ "name": "message_id_header",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sent_at": {
+ "name": "sent_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "subject": {
+ "name": "subject",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sender_name": {
+ "name": "sender_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sender_email": {
+ "name": "sender_email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "recipients": {
+ "name": "recipients",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "storage_path": {
+ "name": "storage_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "storage_hash_sha256": {
+ "name": "storage_hash_sha256",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "size_bytes": {
+ "name": "size_bytes",
+ "type": "bigint",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "is_indexed": {
+ "name": "is_indexed",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "has_attachments": {
+ "name": "has_attachments",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "is_on_legal_hold": {
+ "name": "is_on_legal_hold",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "archived_at": {
+ "name": "archived_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "thread_id_idx": {
+ "name": "thread_id_idx",
+ "columns": [
+ {
+ "expression": "thread_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "archived_emails_ingestion_source_id_ingestion_sources_id_fk": {
+ "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
+ "tableFrom": "archived_emails",
+ "tableTo": "ingestion_sources",
+ "columnsFrom": [
+ "ingestion_source_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.attachments": {
+ "name": "attachments",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "filename": {
+ "name": "filename",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "mime_type": {
+ "name": "mime_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "size_bytes": {
+ "name": "size_bytes",
+ "type": "bigint",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "content_hash_sha256": {
+ "name": "content_hash_sha256",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "storage_path": {
+ "name": "storage_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "attachments_content_hash_sha256_unique": {
+ "name": "attachments_content_hash_sha256_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "content_hash_sha256"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.email_attachments": {
+ "name": "email_attachments",
+ "schema": "",
+ "columns": {
+ "email_id": {
+ "name": "email_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "attachment_id": {
+ "name": "attachment_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "email_attachments_email_id_archived_emails_id_fk": {
+ "name": "email_attachments_email_id_archived_emails_id_fk",
+ "tableFrom": "email_attachments",
+ "tableTo": "archived_emails",
+ "columnsFrom": [
+ "email_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "email_attachments_attachment_id_attachments_id_fk": {
+ "name": "email_attachments_attachment_id_attachments_id_fk",
+ "tableFrom": "email_attachments",
+ "tableTo": "attachments",
+ "columnsFrom": [
+ "attachment_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "restrict",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "email_attachments_email_id_attachment_id_pk": {
+ "name": "email_attachments_email_id_attachment_id_pk",
+ "columns": [
+ "email_id",
+ "attachment_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.audit_logs": {
+ "name": "audit_logs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "bigserial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "timestamp": {
+ "name": "timestamp",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "actor_identifier": {
+ "name": "actor_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "action": {
+ "name": "action",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "target_type": {
+ "name": "target_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "target_id": {
+ "name": "target_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "details": {
+ "name": "details",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "is_tamper_evident": {
+ "name": "is_tamper_evident",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": false,
+ "default": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.ediscovery_cases": {
+ "name": "ediscovery_cases",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'open'"
+ },
+ "created_by_identifier": {
+ "name": "created_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "ediscovery_cases_name_unique": {
+ "name": "ediscovery_cases_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.export_jobs": {
+ "name": "export_jobs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "case_id": {
+ "name": "case_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "format": {
+ "name": "format",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending'"
+ },
+ "query": {
+ "name": "query",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "file_path": {
+ "name": "file_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_by_identifier": {
+ "name": "created_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "export_jobs_case_id_ediscovery_cases_id_fk": {
+ "name": "export_jobs_case_id_ediscovery_cases_id_fk",
+ "tableFrom": "export_jobs",
+ "tableTo": "ediscovery_cases",
+ "columnsFrom": [
+ "case_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.legal_holds": {
+ "name": "legal_holds",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "case_id": {
+ "name": "case_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "custodian_id": {
+ "name": "custodian_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "hold_criteria": {
+ "name": "hold_criteria",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "reason": {
+ "name": "reason",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "applied_by_identifier": {
+ "name": "applied_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "applied_at": {
+ "name": "applied_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "removed_at": {
+ "name": "removed_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "legal_holds_case_id_ediscovery_cases_id_fk": {
+ "name": "legal_holds_case_id_ediscovery_cases_id_fk",
+ "tableFrom": "legal_holds",
+ "tableTo": "ediscovery_cases",
+ "columnsFrom": [
+ "case_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "legal_holds_custodian_id_custodians_id_fk": {
+ "name": "legal_holds_custodian_id_custodians_id_fk",
+ "tableFrom": "legal_holds",
+ "tableTo": "custodians",
+ "columnsFrom": [
+ "custodian_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.retention_policies": {
+ "name": "retention_policies",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "priority": {
+ "name": "priority",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "retention_period_days": {
+ "name": "retention_period_days",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "action_on_expiry": {
+ "name": "action_on_expiry",
+ "type": "retention_action",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "is_enabled": {
+ "name": "is_enabled",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": true
+ },
+ "conditions": {
+ "name": "conditions",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "retention_policies_name_unique": {
+ "name": "retention_policies_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.custodians": {
+ "name": "custodians",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "display_name": {
+ "name": "display_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "source_type": {
+ "name": "source_type",
+ "type": "ingestion_provider",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "custodians_email_unique": {
+ "name": "custodians_email_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "email"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.ingestion_sources": {
+ "name": "ingestion_sources",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "provider": {
+ "name": "provider",
+ "type": "ingestion_provider",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "credentials": {
+ "name": "credentials",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "ingestion_status",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending_auth'"
+ },
+ "last_sync_started_at": {
+ "name": "last_sync_started_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_sync_finished_at": {
+ "name": "last_sync_finished_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_sync_status_message": {
+ "name": "last_sync_status_message",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sync_state": {
+ "name": "sync_state",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.roles": {
+ "name": "roles",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "policies": {
+ "name": "policies",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'[]'::jsonb"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "roles_name_unique": {
+ "name": "roles_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.sessions": {
+ "name": "sessions",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "sessions_user_id_users_id_fk": {
+ "name": "sessions_user_id_users_id_fk",
+ "tableFrom": "sessions",
+ "tableTo": "users",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.user_roles": {
+ "name": "user_roles",
+ "schema": "",
+ "columns": {
+ "user_id": {
+ "name": "user_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "role_id": {
+ "name": "role_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "user_roles_user_id_users_id_fk": {
+ "name": "user_roles_user_id_users_id_fk",
+ "tableFrom": "user_roles",
+ "tableTo": "users",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "user_roles_role_id_roles_id_fk": {
+ "name": "user_roles_role_id_roles_id_fk",
+ "tableFrom": "user_roles",
+ "tableTo": "roles",
+ "columnsFrom": [
+ "role_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "user_roles_user_id_role_id_pk": {
+ "name": "user_roles_user_id_role_id_pk",
+ "columns": [
+ "user_id",
+ "role_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.users": {
+ "name": "users",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "password": {
+ "name": "password",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "provider": {
+ "name": "provider",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "'local'"
+ },
+ "provider_id": {
+ "name": "provider_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "users_email_unique": {
+ "name": "users_email_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "email"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ }
+ },
+ "enums": {
+ "public.retention_action": {
+ "name": "retention_action",
+ "schema": "public",
+ "values": [
+ "delete_permanently",
+ "notify_admin"
+ ]
+ },
+ "public.ingestion_provider": {
+ "name": "ingestion_provider",
+ "schema": "public",
+ "values": [
+ "google_workspace",
+ "microsoft_365",
+ "generic_imap"
+ ]
+ },
+ "public.ingestion_status": {
+ "name": "ingestion_status",
+ "schema": "public",
+ "values": [
+ "active",
+ "paused",
+ "error",
+ "pending_auth",
+ "syncing",
+ "importing",
+ "auth_success"
+ ]
+ }
+ },
+ "schemas": {},
+ "sequences": {},
+ "roles": {},
+ "policies": {},
+ "views": {},
+ "_meta": {
+ "columns": {},
+ "schemas": {},
+ "tables": {}
+ }
+}
\ No newline at end of file
diff --git a/packages/backend/src/database/migrations/meta/0011_snapshot.json b/packages/backend/src/database/migrations/meta/0011_snapshot.json
new file mode 100644
index 0000000..bc8a473
--- /dev/null
+++ b/packages/backend/src/database/migrations/meta/0011_snapshot.json
@@ -0,0 +1,1093 @@
+{
+ "id": "6252768a-7c7f-4dae-9dbd-d3ea9f647cea",
+ "prevId": "ab45f75d-f50c-457f-ad40-e62ca27a4e2b",
+ "version": "7",
+ "dialect": "postgresql",
+ "tables": {
+ "public.archived_emails": {
+ "name": "archived_emails",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "thread_id": {
+ "name": "thread_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "ingestion_source_id": {
+ "name": "ingestion_source_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "user_email": {
+ "name": "user_email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "message_id_header": {
+ "name": "message_id_header",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sent_at": {
+ "name": "sent_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "subject": {
+ "name": "subject",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sender_name": {
+ "name": "sender_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sender_email": {
+ "name": "sender_email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "recipients": {
+ "name": "recipients",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "storage_path": {
+ "name": "storage_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "storage_hash_sha256": {
+ "name": "storage_hash_sha256",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "size_bytes": {
+ "name": "size_bytes",
+ "type": "bigint",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "is_indexed": {
+ "name": "is_indexed",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "has_attachments": {
+ "name": "has_attachments",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "is_on_legal_hold": {
+ "name": "is_on_legal_hold",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "archived_at": {
+ "name": "archived_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "thread_id_idx": {
+ "name": "thread_id_idx",
+ "columns": [
+ {
+ "expression": "thread_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "archived_emails_ingestion_source_id_ingestion_sources_id_fk": {
+ "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
+ "tableFrom": "archived_emails",
+ "tableTo": "ingestion_sources",
+ "columnsFrom": [
+ "ingestion_source_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.attachments": {
+ "name": "attachments",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "filename": {
+ "name": "filename",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "mime_type": {
+ "name": "mime_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "size_bytes": {
+ "name": "size_bytes",
+ "type": "bigint",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "content_hash_sha256": {
+ "name": "content_hash_sha256",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "storage_path": {
+ "name": "storage_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "attachments_content_hash_sha256_unique": {
+ "name": "attachments_content_hash_sha256_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "content_hash_sha256"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.email_attachments": {
+ "name": "email_attachments",
+ "schema": "",
+ "columns": {
+ "email_id": {
+ "name": "email_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "attachment_id": {
+ "name": "attachment_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "email_attachments_email_id_archived_emails_id_fk": {
+ "name": "email_attachments_email_id_archived_emails_id_fk",
+ "tableFrom": "email_attachments",
+ "tableTo": "archived_emails",
+ "columnsFrom": [
+ "email_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "email_attachments_attachment_id_attachments_id_fk": {
+ "name": "email_attachments_attachment_id_attachments_id_fk",
+ "tableFrom": "email_attachments",
+ "tableTo": "attachments",
+ "columnsFrom": [
+ "attachment_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "restrict",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "email_attachments_email_id_attachment_id_pk": {
+ "name": "email_attachments_email_id_attachment_id_pk",
+ "columns": [
+ "email_id",
+ "attachment_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.audit_logs": {
+ "name": "audit_logs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "bigserial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "timestamp": {
+ "name": "timestamp",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "actor_identifier": {
+ "name": "actor_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "action": {
+ "name": "action",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "target_type": {
+ "name": "target_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "target_id": {
+ "name": "target_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "details": {
+ "name": "details",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "is_tamper_evident": {
+ "name": "is_tamper_evident",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": false,
+ "default": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.ediscovery_cases": {
+ "name": "ediscovery_cases",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'open'"
+ },
+ "created_by_identifier": {
+ "name": "created_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "ediscovery_cases_name_unique": {
+ "name": "ediscovery_cases_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.export_jobs": {
+ "name": "export_jobs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "case_id": {
+ "name": "case_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "format": {
+ "name": "format",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending'"
+ },
+ "query": {
+ "name": "query",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "file_path": {
+ "name": "file_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_by_identifier": {
+ "name": "created_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "export_jobs_case_id_ediscovery_cases_id_fk": {
+ "name": "export_jobs_case_id_ediscovery_cases_id_fk",
+ "tableFrom": "export_jobs",
+ "tableTo": "ediscovery_cases",
+ "columnsFrom": [
+ "case_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.legal_holds": {
+ "name": "legal_holds",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "case_id": {
+ "name": "case_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "custodian_id": {
+ "name": "custodian_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "hold_criteria": {
+ "name": "hold_criteria",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "reason": {
+ "name": "reason",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "applied_by_identifier": {
+ "name": "applied_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "applied_at": {
+ "name": "applied_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "removed_at": {
+ "name": "removed_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "legal_holds_case_id_ediscovery_cases_id_fk": {
+ "name": "legal_holds_case_id_ediscovery_cases_id_fk",
+ "tableFrom": "legal_holds",
+ "tableTo": "ediscovery_cases",
+ "columnsFrom": [
+ "case_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "legal_holds_custodian_id_custodians_id_fk": {
+ "name": "legal_holds_custodian_id_custodians_id_fk",
+ "tableFrom": "legal_holds",
+ "tableTo": "custodians",
+ "columnsFrom": [
+ "custodian_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.retention_policies": {
+ "name": "retention_policies",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "priority": {
+ "name": "priority",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "retention_period_days": {
+ "name": "retention_period_days",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "action_on_expiry": {
+ "name": "action_on_expiry",
+ "type": "retention_action",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "is_enabled": {
+ "name": "is_enabled",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": true
+ },
+ "conditions": {
+ "name": "conditions",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "retention_policies_name_unique": {
+ "name": "retention_policies_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.custodians": {
+ "name": "custodians",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "display_name": {
+ "name": "display_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "source_type": {
+ "name": "source_type",
+ "type": "ingestion_provider",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "custodians_email_unique": {
+ "name": "custodians_email_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "email"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.ingestion_sources": {
+ "name": "ingestion_sources",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "provider": {
+ "name": "provider",
+ "type": "ingestion_provider",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "credentials": {
+ "name": "credentials",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "ingestion_status",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending_auth'"
+ },
+ "last_sync_started_at": {
+ "name": "last_sync_started_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_sync_finished_at": {
+ "name": "last_sync_finished_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_sync_status_message": {
+ "name": "last_sync_status_message",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sync_state": {
+ "name": "sync_state",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.roles": {
+ "name": "roles",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "policies": {
+ "name": "policies",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'[]'::jsonb"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "roles_name_unique": {
+ "name": "roles_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.sessions": {
+ "name": "sessions",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "sessions_user_id_users_id_fk": {
+ "name": "sessions_user_id_users_id_fk",
+ "tableFrom": "sessions",
+ "tableTo": "users",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.user_roles": {
+ "name": "user_roles",
+ "schema": "",
+ "columns": {
+ "user_id": {
+ "name": "user_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "role_id": {
+ "name": "role_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "user_roles_user_id_users_id_fk": {
+ "name": "user_roles_user_id_users_id_fk",
+ "tableFrom": "user_roles",
+ "tableTo": "users",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "user_roles_role_id_roles_id_fk": {
+ "name": "user_roles_role_id_roles_id_fk",
+ "tableFrom": "user_roles",
+ "tableTo": "roles",
+ "columnsFrom": [
+ "role_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "user_roles_user_id_role_id_pk": {
+ "name": "user_roles_user_id_role_id_pk",
+ "columns": [
+ "user_id",
+ "role_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.users": {
+ "name": "users",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "first_name": {
+ "name": "first_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_name": {
+ "name": "last_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "password": {
+ "name": "password",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "provider": {
+ "name": "provider",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "'local'"
+ },
+ "provider_id": {
+ "name": "provider_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "users_email_unique": {
+ "name": "users_email_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "email"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ }
+ },
+ "enums": {
+ "public.retention_action": {
+ "name": "retention_action",
+ "schema": "public",
+ "values": [
+ "delete_permanently",
+ "notify_admin"
+ ]
+ },
+ "public.ingestion_provider": {
+ "name": "ingestion_provider",
+ "schema": "public",
+ "values": [
+ "google_workspace",
+ "microsoft_365",
+ "generic_imap"
+ ]
+ },
+ "public.ingestion_status": {
+ "name": "ingestion_status",
+ "schema": "public",
+ "values": [
+ "active",
+ "paused",
+ "error",
+ "pending_auth",
+ "syncing",
+ "importing",
+ "auth_success"
+ ]
+ }
+ },
+ "schemas": {},
+ "sequences": {},
+ "roles": {},
+ "policies": {},
+ "views": {},
+ "_meta": {
+ "columns": {},
+ "schemas": {},
+ "tables": {}
+ }
+}
\ No newline at end of file
diff --git a/packages/backend/src/database/migrations/meta/_journal.json b/packages/backend/src/database/migrations/meta/_journal.json
index 0ca8395..8d05fd2 100644
--- a/packages/backend/src/database/migrations/meta/_journal.json
+++ b/packages/backend/src/database/migrations/meta/_journal.json
@@ -71,6 +71,20 @@
"when": 1754337938241,
"tag": "0009_late_lenny_balinger",
"breakpoints": true
+ },
+ {
+ "idx": 10,
+ "version": "7",
+ "when": 1754420780849,
+ "tag": "0010_perpetual_lightspeed",
+ "breakpoints": true
+ },
+ {
+ "idx": 11,
+ "version": "7",
+ "when": 1754422064158,
+ "tag": "0011_tan_blackheart",
+ "breakpoints": true
}
]
}
\ No newline at end of file
diff --git a/packages/backend/src/database/schema.ts b/packages/backend/src/database/schema.ts
index 571a02b..557f1e8 100644
--- a/packages/backend/src/database/schema.ts
+++ b/packages/backend/src/database/schema.ts
@@ -4,3 +4,4 @@ export * from './schema/audit-logs';
export * from './schema/compliance';
export * from './schema/custodians';
export * from './schema/ingestion-sources';
+export * from './schema/users';
diff --git a/packages/backend/src/database/schema/archived-emails.ts b/packages/backend/src/database/schema/archived-emails.ts
index 0ec8072..01043fc 100644
--- a/packages/backend/src/database/schema/archived-emails.ts
+++ b/packages/backend/src/database/schema/archived-emails.ts
@@ -25,11 +25,7 @@ export const archivedEmails = pgTable(
isOnLegalHold: boolean('is_on_legal_hold').notNull().default(false),
archivedAt: timestamp('archived_at', { withTimezone: true }).notNull().defaultNow(),
},
- (table) => {
- return {
- threadIdIdx: index('thread_id_idx').on(table.threadId)
- };
- }
+ (table) => [index('thread_id_idx').on(table.threadId)]
);
export const archivedEmailsRelations = relations(archivedEmails, ({ one }) => ({
diff --git a/packages/backend/src/database/schema/users.ts b/packages/backend/src/database/schema/users.ts
new file mode 100644
index 0000000..9e67661
--- /dev/null
+++ b/packages/backend/src/database/schema/users.ts
@@ -0,0 +1,89 @@
+import { relations, sql } from 'drizzle-orm';
+import {
+ pgTable,
+ text,
+ timestamp,
+ uuid,
+ primaryKey,
+ jsonb
+} from 'drizzle-orm/pg-core';
+import type { PolicyStatement } from '@open-archiver/types';
+
+/**
+ * The `users` table stores the core user information for authentication and identification.
+ */
+export const users = pgTable('users', {
+ id: uuid('id').primaryKey().defaultRandom(),
+ email: text('email').notNull().unique(),
+ first_name: text('first_name'),
+ last_name: text('last_name'),
+ password: text('password'),
+ provider: text('provider').default('local'),
+ providerId: text('provider_id'),
+ createdAt: timestamp('created_at').defaultNow().notNull(),
+ updatedAt: timestamp('updated_at').defaultNow().notNull()
+});
+
+/**
+ * The `sessions` table stores user session information for managing login state.
+ * It links a session to a user and records its expiration time.
+ */
+export const sessions = pgTable('sessions', {
+ id: text('id').primaryKey(),
+ userId: uuid('user_id')
+ .notNull()
+ .references(() => users.id, { onDelete: 'cascade' }),
+ expiresAt: timestamp('expires_at', {
+ withTimezone: true,
+ mode: 'date'
+ }).notNull()
+});
+
+/**
+ * The `roles` table defines the roles that can be assigned to users.
+ * Each role has a name and a set of policies that define its permissions.
+ */
+export const roles = pgTable('roles', {
+ id: uuid('id').primaryKey().defaultRandom(),
+ name: text('name').notNull().unique(),
+ policies: jsonb('policies').$type().notNull().default(sql`'[]'::jsonb`),
+ createdAt: timestamp('created_at').defaultNow().notNull(),
+ updatedAt: timestamp('updated_at').defaultNow().notNull()
+});
+
+/**
+ * The `user_roles` table is a join table that maps users to their assigned roles.
+ * This many-to-many relationship allows a user to have multiple roles.
+ */
+export const userRoles = pgTable(
+ 'user_roles',
+ {
+ userId: uuid('user_id')
+ .notNull()
+ .references(() => users.id, { onDelete: 'cascade' }),
+ roleId: uuid('role_id')
+ .notNull()
+ .references(() => roles.id, { onDelete: 'cascade' })
+ },
+ (t) => [primaryKey({ columns: [t.userId, t.roleId] })]
+);
+
+// Define relationships for Drizzle ORM
+export const usersRelations = relations(users, ({ many }) => ({
+ userRoles: many(userRoles)
+}));
+
+export const rolesRelations = relations(roles, ({ many }) => ({
+ userRoles: many(userRoles)
+}));
+
+export const userRolesRelations = relations(userRoles, ({ one }) => ({
+ role: one(roles, {
+ fields: [userRoles.roleId],
+ references: [roles.id]
+ }),
+ user: one(users, {
+ fields: [userRoles.userId],
+ references: [users.id]
+ })
+}));
diff --git a/packages/backend/src/iam-policy/iam-definitions.ts b/packages/backend/src/iam-policy/iam-definitions.ts
new file mode 100644
index 0000000..2ba8bd7
--- /dev/null
+++ b/packages/backend/src/iam-policy/iam-definitions.ts
@@ -0,0 +1,155 @@
+/**
+ * @file This file serves as the single source of truth for all Identity and Access Management (IAM)
+ * definitions within Open Archiver. Centralizing these definitions is an industry-standard practice
+ * that offers several key benefits:
+ *
+ * 1. **Prevents "Magic Strings"**: Avoids the use of hardcoded strings for actions and resources
+ * throughout the codebase, reducing the risk of typos and inconsistencies.
+ * 2. **Single Source of Truth**: Provides a clear, comprehensive, and maintainable list of all
+ * possible permissions in the system.
+ * 3. **Enables Validation**: Allows for the creation of a robust validation function that can
+ * programmatically check if a policy statement is valid before it is saved.
+ * 4. **Simplifies Auditing**: Makes it easy to audit and understand the scope of permissions
+ * that can be granted.
+ *
+ * The structure is inspired by AWS IAM, using a `service:operation` format for actions and a
+ * hierarchical, slash-separated path for resources.
+ */
+
+// ===================================================================================
+// SERVICE: archive
+// ===================================================================================
+
+const ARCHIVE_ACTIONS = {
+ READ: 'archive:read',
+ SEARCH: 'archive:search',
+ EXPORT: 'archive:export',
+} as const;
+
+const ARCHIVE_RESOURCES = {
+ ALL: 'archive/all',
+ INGESTION_SOURCE: 'archive/ingestion-source/*',
+ EMAIL: 'archive/email/*',
+ CUSTODIAN: 'archive/custodian/*',
+} as const;
+
+
+// ===================================================================================
+// SERVICE: ingestion
+// ===================================================================================
+
+const INGESTION_ACTIONS = {
+ CREATE_SOURCE: 'ingestion:createSource',
+ READ_SOURCE: 'ingestion:readSource',
+ UPDATE_SOURCE: 'ingestion:updateSource',
+ DELETE_SOURCE: 'ingestion:deleteSource',
+ MANAGE_SYNC: 'ingestion:manageSync', // Covers triggering, pausing, and forcing syncs
+} as const;
+
+const INGESTION_RESOURCES = {
+ ALL: 'ingestion-source/*',
+ SOURCE: 'ingestion-source/{sourceId}',
+} as const;
+
+
+// ===================================================================================
+// SERVICE: system
+// ===================================================================================
+
+const SYSTEM_ACTIONS = {
+ READ_SETTINGS: 'system:readSettings',
+ UPDATE_SETTINGS: 'system:updateSettings',
+ READ_USERS: 'system:readUsers',
+ CREATE_USER: 'system:createUser',
+ UPDATE_USER: 'system:updateUser',
+ DELETE_USER: 'system:deleteUser',
+ ASSIGN_ROLE: 'system:assignRole',
+} as const;
+
+const SYSTEM_RESOURCES = {
+ SETTINGS: 'system/settings',
+ USERS: 'system/users',
+ USER: 'system/user/{userId}',
+} as const;
+
+
+// ===================================================================================
+// SERVICE: dashboard
+// ===================================================================================
+
+const DASHBOARD_ACTIONS = {
+ READ: 'dashboard:read',
+} as const;
+
+const DASHBOARD_RESOURCES = {
+ ALL: 'dashboard/*',
+} as const;
+
+
+// ===================================================================================
+// EXPORTED DEFINITIONS
+// ===================================================================================
+
+/**
+ * A comprehensive set of all valid IAM actions in the system.
+ * This is used by the policy validator to ensure that any action in a policy is recognized.
+ */
+export const ValidActions: Set = new Set([
+ ...Object.values(ARCHIVE_ACTIONS),
+ ...Object.values(INGESTION_ACTIONS),
+ ...Object.values(SYSTEM_ACTIONS),
+ ...Object.values(DASHBOARD_ACTIONS),
+]);
+
+/**
+ * An object containing regular expressions for validating resource formats.
+ * The validator uses these patterns to ensure that resource strings in a policy
+ * conform to the expected structure.
+ *
+ * Logic:
+ * - The key represents the service (e.g., 'archive').
+ * - The value is a RegExp that matches all valid resource formats for that service.
+ * - This allows for flexible validation. For example, `archive/*` is a valid pattern,
+ * as is `archive/email/123-abc`.
+ */
+export const ValidResourcePatterns = {
+ archive: /^archive\/(all|ingestion-source\/[^\/]+|email\/[^\/]+|custodian\/[^\/]+)$/,
+ ingestion: /^ingestion-source\/(\*|[^\/]+)$/,
+ system: /^system\/(settings|users|user\/[^\/]+)$/,
+ dashboard: /^dashboard\/\*$/,
+};
+
+
+/**
+ * --- How to Use These Definitions for Validation (Conceptual) ---
+ *
+ * A validator function would be created, likely in an `AuthorizationService`,
+ * that accepts a `PolicyStatement` object.
+ *
+ * export function isPolicyStatementValid(statement: PolicyStatement): boolean {
+ * // 1. Validate Actions
+ * for (const action of statement.Action) {
+ * if (action.endsWith('*')) {
+ * // For wildcards, check if the service prefix is valid
+ * const service = action.split(':')[0];
+ * if (!Object.keys(ValidResourcePatterns).includes(service)) {
+ * return false; // Invalid service
+ * }
+ * } else if (!ValidActions.has(action)) {
+ * return false; // Action is not in the set of known actions
+ * }
+ * }
+ *
+ * // 2. Validate Resources
+ * for (const resource of statement.Resource) {
+ * const service = resource.split('/')[0];
+ * const pattern = ValidResourcePatterns[service];
+ *
+ * if (!pattern || !pattern.test(resource)) {
+ * return false; // Resource format is invalid for the specified service
+ * }
+ * }
+ *
+ * return true;
+ * }
+ */
diff --git a/packages/backend/src/iam-policy/policy-validator.ts b/packages/backend/src/iam-policy/policy-validator.ts
new file mode 100644
index 0000000..da910b1
--- /dev/null
+++ b/packages/backend/src/iam-policy/policy-validator.ts
@@ -0,0 +1,100 @@
+import type { PolicyStatement } from '@open-archiver/types';
+import { ValidActions, ValidResourcePatterns } from './iam-definitions';
+
+/**
+ * @class PolicyValidator
+ *
+ * This class provides a static method to validate an IAM policy statement.
+ * It is designed to be used before a policy is saved to the database, ensuring that
+ * only valid and well-formed policies are stored.
+ *
+ * The verification logic is based on the centralized definitions in `iam-definitions.ts`.
+ */
+export class PolicyValidator {
+ /**
+ * Validates a single policy statement to ensure its actions and resources are valid.
+ *
+ * @param {PolicyStatement} statement - The policy statement to validate.
+ * @returns {{valid: boolean; reason?: string}} - An object containing a boolean `valid` property
+ * and an optional `reason` string if validation fails.
+ */
+ public static isValid(statement: PolicyStatement): { valid: boolean; reason: string; } {
+ if (!statement || !statement.Action || !statement.Resource || !statement.Effect) {
+ return { valid: false, reason: 'Policy statement is missing required fields.' };
+ }
+
+ // 1. Validate Actions
+ for (const action of statement.Action) {
+ const { valid, reason } = this.isActionValid(action);
+ if (!valid) {
+ return { valid: false, reason };
+ }
+ }
+
+ // 2. Validate Resources
+ for (const resource of statement.Resource) {
+ const { valid, reason } = this.isResourceValid(resource);
+ if (!valid) {
+ return { valid: false, reason };
+ }
+ }
+
+ return { valid: true, reason: 'valid' };
+ }
+
+ /**
+ * Checks if a single action string is valid.
+ *
+ * Logic:
+ * - If the action contains a wildcard (e.g., 'archive:*'), it checks if the service part
+ * (e.g., 'archive') is a recognized service.
+ * - If there is no wildcard, it checks if the full action string (e.g., 'archive:read')
+ * exists in the `ValidActions` set.
+ *
+ * @param {string} action - The action string to validate.
+ * @returns {{valid: boolean; reason?: string}} - An object indicating validity and a reason for failure.
+ */
+ private static isActionValid(action: string): { valid: boolean; reason: string; } {
+ if (action === '*') {
+ return { valid: true, reason: 'valid' };
+ }
+ if (action.endsWith(':*')) {
+ const service = action.split(':')[0];
+ if (service in ValidResourcePatterns) {
+ return { valid: true, reason: 'valid' };
+ }
+ return { valid: false, reason: `Invalid service '${service}' in action wildcard '${action}'.` };
+ }
+ if (ValidActions.has(action)) {
+ return { valid: true, reason: 'valid' };
+ }
+ return { valid: false, reason: `Action '${action}' is not a valid action.` };
+ }
+
+ /**
+ * Checks if a single resource string has a valid format.
+ *
+ * Logic:
+ * - It extracts the service name from the resource string (e.g., 'archive' from 'archive/all').
+ * - It looks up the corresponding regular expression for that service in `ValidResourcePatterns`.
+ * - It tests the resource string against the pattern. If the service does not exist or the
+ * pattern does not match, the resource is considered invalid.
+ *
+ * @param {string} resource - The resource string to validate.
+ * @returns {{valid: boolean; reason?: string}} - An object indicating validity and a reason for failure.
+ */
+ private static isResourceValid(resource: string): { valid: boolean; reason: string; } {
+ const service = resource.split('/')[0];
+ if (service === '*') {
+ return { valid: true, reason: 'valid' };
+ }
+ if (service in ValidResourcePatterns) {
+ const pattern = ValidResourcePatterns[service as keyof typeof ValidResourcePatterns];
+ if (pattern.test(resource)) {
+ return { valid: true, reason: 'valid' };
+ }
+ return { valid: false, reason: `Resource '${resource}' does not match the expected format for the '${service}' service.` };
+ }
+ return { valid: false, reason: `Invalid service '${service}' in resource '${resource}'.` };
+ }
+}
diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts
index 2b3f3fb..f23b382 100644
--- a/packages/backend/src/index.ts
+++ b/packages/backend/src/index.ts
@@ -5,8 +5,10 @@ import { IngestionController } from './api/controllers/ingestion.controller';
import { ArchivedEmailController } from './api/controllers/archived-email.controller';
import { StorageController } from './api/controllers/storage.controller';
import { SearchController } from './api/controllers/search.controller';
+import { IamController } from './api/controllers/iam.controller';
import { requireAuth } from './api/middleware/requireAuth';
import { createAuthRouter } from './api/routes/auth.routes';
+import { createIamRouter } from './api/routes/iam.routes';
import { createIngestionRouter } from './api/routes/ingestion.routes';
import { createArchivedEmailRouter } from './api/routes/archived-email.routes';
import { createStorageRouter } from './api/routes/storage.routes';
@@ -14,7 +16,8 @@ import { createSearchRouter } from './api/routes/search.routes';
import { createDashboardRouter } from './api/routes/dashboard.routes';
import testRouter from './api/routes/test.routes';
import { AuthService } from './services/AuthService';
-import { AdminUserService } from './services/UserService';
+import { UserService } from './services/UserService';
+import { IamService } from './services/IamService';
import { StorageService } from './services/StorageService';
import { SearchService } from './services/SearchService';
@@ -37,15 +40,17 @@ if (!PORT_BACKEND || !JWT_SECRET || !JWT_EXPIRES_IN) {
// --- Dependency Injection Setup ---
-const userService = new AdminUserService();
+const userService = new UserService();
const authService = new AuthService(userService, JWT_SECRET, JWT_EXPIRES_IN);
-const authController = new AuthController(authService);
+const authController = new AuthController(authService, userService);
const ingestionController = new IngestionController();
const archivedEmailController = new ArchivedEmailController();
const storageService = new StorageService();
const storageController = new StorageController(storageService);
const searchService = new SearchService();
const searchController = new SearchController();
+const iamService = new IamService();
+const iamController = new IamController(iamService);
// --- Express App Initialization ---
const app = express();
@@ -60,7 +65,9 @@ const archivedEmailRouter = createArchivedEmailRouter(archivedEmailController, a
const storageRouter = createStorageRouter(storageController, authService);
const searchRouter = createSearchRouter(searchController, authService);
const dashboardRouter = createDashboardRouter(authService);
+const iamRouter = createIamRouter(iamController);
app.use('/v1/auth', authRouter);
+app.use('/v1/iam', iamRouter);
app.use('/v1/ingestion-sources', ingestionRouter);
app.use('/v1/archived-emails', archivedEmailRouter);
app.use('/v1/storage', storageRouter);
diff --git a/packages/backend/src/services/AuthService.ts b/packages/backend/src/services/AuthService.ts
index d05b42d..ed331f9 100644
--- a/packages/backend/src/services/AuthService.ts
+++ b/packages/backend/src/services/AuthService.ts
@@ -1,38 +1,23 @@
-import { compare, hash } from 'bcryptjs';
-import type { SignJWT, jwtVerify } from 'jose';
-import type { AuthTokenPayload, User, LoginResponse } from '@open-archiver/types';
+import { compare } from 'bcryptjs';
+import { SignJWT, jwtVerify } from 'jose';
+import type { AuthTokenPayload, LoginResponse } from '@open-archiver/types';
+import { UserService } from './UserService';
+import { db } from '../database';
+import * as schema from '../database/schema';
+import { eq } from 'drizzle-orm';
-// This interface defines the contract for a service that manages users.
-// The AuthService will depend on this abstraction, not a concrete implementation.
-export interface IUserService {
- findByEmail(email: string): Promise;
-}
-
-// This interface defines the contract for our AuthService.
-export interface IAuthService {
- verifyPassword(password: string, hash: string): Promise;
- login(email: string, password: string): Promise;
- verifyToken(token: string): Promise;
-}
-
-export class AuthService implements IAuthService {
- #userService: IUserService;
+export class AuthService {
+ #userService: UserService;
#jwtSecret: Uint8Array;
#jwtExpiresIn: string;
- #jose: Promise<{ SignJWT: typeof SignJWT; jwtVerify: typeof jwtVerify; }>;
- constructor(userService: IUserService, jwtSecret: string, jwtExpiresIn: string) {
+ constructor(userService: UserService, jwtSecret: string, jwtExpiresIn: string) {
this.#userService = userService;
this.#jwtSecret = new TextEncoder().encode(jwtSecret);
this.#jwtExpiresIn = jwtExpiresIn;
- this.#jose = import('jose');
}
- #hashPassword(password: string): Promise {
- return hash(password, 10);
- }
-
- public verifyPassword(password: string, hash: string): Promise {
+ public async verifyPassword(password: string, hash: string): Promise {
return compare(password, hash);
}
@@ -40,7 +25,6 @@ export class AuthService implements IAuthService {
if (!payload.sub) {
throw new Error('JWT payload must have a subject (sub) claim.');
}
- const { SignJWT } = await this.#jose;
return new SignJWT(payload)
.setProtectedHeader({ alg: 'HS256' })
.setIssuedAt()
@@ -52,22 +36,31 @@ export class AuthService implements IAuthService {
public async login(email: string, password: string): Promise {
const user = await this.#userService.findByEmail(email);
- if (!user) {
- return null; // User not found
+ if (!user || !user.password) {
+ return null; // User not found or password not set
}
- const isPasswordValid = await this.verifyPassword(password, user.passwordHash);
+ const isPasswordValid = await this.verifyPassword(password, user.password);
if (!isPasswordValid) {
return null; // Invalid password
}
- const { passwordHash, ...userWithoutPassword } = user;
+ const userRoles = await db.query.userRoles.findMany({
+ where: eq(schema.userRoles.userId, user.id),
+ with: {
+ role: true
+ }
+ });
+
+ const roles = userRoles.map(ur => ur.role.name);
+
+ const { password: _, ...userWithoutPassword } = user;
const accessToken = await this.#generateAccessToken({
sub: user.id,
email: user.email,
- role: user.role,
+ roles: roles,
});
return { accessToken, user: userWithoutPassword };
@@ -75,7 +68,6 @@ export class AuthService implements IAuthService {
public async verifyToken(token: string): Promise {
try {
- const { jwtVerify } = await this.#jose;
const { payload } = await jwtVerify(token, this.#jwtSecret);
return payload;
} catch (error) {
diff --git a/packages/backend/src/services/IamService.ts b/packages/backend/src/services/IamService.ts
new file mode 100644
index 0000000..eec25d5
--- /dev/null
+++ b/packages/backend/src/services/IamService.ts
@@ -0,0 +1,24 @@
+import { db } from '../database';
+import { roles } from '../database/schema/users';
+import type { Role, PolicyStatement } from '@open-archiver/types';
+import { eq } from 'drizzle-orm';
+
+export class IamService {
+ public async getRoles(): Promise {
+ return db.select().from(roles);
+ }
+
+ public async getRoleById(id: string): Promise {
+ const [role] = await db.select().from(roles).where(eq(roles.id, id));
+ return role;
+ }
+
+ public async createRole(name: string, policy: PolicyStatement[]): Promise {
+ const [role] = await db.insert(roles).values({ name, policies: policy }).returning();
+ return role;
+ }
+
+ public async deleteRole(id: string): Promise {
+ await db.delete(roles).where(eq(roles.id, id));
+ }
+}
diff --git a/packages/backend/src/services/UserService.ts b/packages/backend/src/services/UserService.ts
index a3b9e69..6f4ddc0 100644
--- a/packages/backend/src/services/UserService.ts
+++ b/packages/backend/src/services/UserService.ts
@@ -1,31 +1,79 @@
+import { db } from '../database';
+import * as schema from '../database/schema';
+import { and, eq, asc, sql } from 'drizzle-orm';
import { hash } from 'bcryptjs';
-import type { User } from '@open-archiver/types';
-import type { IUserService } from './AuthService';
+import type { PolicyStatement, User } from '@open-archiver/types';
+import { PolicyValidator } from '../iam-policy/policy-validator';
-// This is a mock implementation of the IUserService.
-// Later on, this service would interact with a database.
-export class AdminUserService implements IUserService {
- #users: User[] = [];
-
- constructor() {
- // Immediately seed the user when the service is instantiated.
- this.seed();
- }
-
- // use .env admin user
- private async seed() {
- const passwordHash = await hash(process.env.ADMIN_PASSWORD as string, 10);
- this.#users.push({
- id: '1',
- email: process.env.ADMIN_EMAIL as string,
- role: 'Super Administrator',
- passwordHash: passwordHash,
+export class UserService {
+ /**
+ * Finds a user by their email address.
+ * @param email The email address of the user to find.
+ * @returns The user object if found, otherwise null.
+ */
+ public async findByEmail(email: string): Promise<(typeof schema.users.$inferSelect) | null> {
+ const user = await db.query.users.findFirst({
+ where: eq(schema.users.email, email)
});
- }
-
- public async findByEmail(email: string): Promise {
- // once user service is ready, this would be a database query.
- const user = this.#users.find(u => u.email === email);
return user || null;
}
+
+ /**
+ * Finds a user by their ID.
+ * @param id The ID of the user to find.
+ * @returns The user object if found, otherwise null.
+ */
+ public async findById(id: string): Promise<(typeof schema.users.$inferSelect) | null> {
+ const user = await db.query.users.findFirst({
+ where: eq(schema.users.id, id)
+ });
+ return user || null;
+ }
+
+ /**
+ * Creates a new user in the database.
+ * The first user created will be assigned the 'Super Admin' role.
+ * @param userDetails The details of the user to create.
+ * @returns The newly created user object.
+ */
+ public async createUser(userDetails: Pick & { password?: string; }): Promise<(typeof schema.users.$inferSelect)> {
+ const { email, first_name, last_name, password } = userDetails;
+
+ const userCountResult = await db.select({ count: sql`count(*)` }).from(schema.users);
+ const isFirstUser = Number(userCountResult[0].count) === 0;
+
+ const hashedPassword = password ? await hash(password, 10) : undefined;
+
+ const newUser = await db.insert(schema.users).values({
+ email,
+ first_name,
+ last_name,
+ password: hashedPassword,
+ }).returning();
+
+ if (isFirstUser) {
+ let superAdminRole = await db.query.roles.findFirst({
+ where: eq(schema.roles.name, 'Super Admin')
+ });
+
+ if (!superAdminRole) {
+ const suerAdminPolicies: PolicyStatement[] = [{
+ Effect: 'Allow',
+ Action: ['*'],
+ Resource: ['*']
+ }];
+ superAdminRole = (await db.insert(schema.roles).values({
+ name: 'Super Admin',
+ policies: suerAdminPolicies
+ }).returning())[0];
+ }
+
+ await db.insert(schema.userRoles).values({
+ userId: newUser[0].id,
+ roleId: superAdminRole.id
+ });
+ }
+
+ return newUser[0];
+ }
}
diff --git a/packages/frontend/src/routes/+layout.server.ts b/packages/frontend/src/routes/+layout.server.ts
index 82f5bf6..1e4bb8a 100644
--- a/packages/frontend/src/routes/+layout.server.ts
+++ b/packages/frontend/src/routes/+layout.server.ts
@@ -1,9 +1,27 @@
import { redirect } from '@sveltejs/kit';
import type { LayoutServerLoad } from './$types';
import 'dotenv/config';
+import { api } from '$lib/server/api';
+
+export const load: LayoutServerLoad = async (event) => {
+ const { locals, url } = event;
+ try {
+ const response = await api('/auth/status', event);
+ const { needsSetup } = await response.json();
+
+ if (needsSetup && url.pathname !== '/setup') {
+ throw redirect(307, '/setup');
+ }
+
+ if (!needsSetup && url.pathname === '/setup') {
+ throw redirect(307, '/signin');
+ }
+ } catch (error) {
+ throw error;
+
+ }
-export const load: LayoutServerLoad = async ({ locals }) => {
return {
user: locals.user,
accessToken: locals.accessToken,
diff --git a/packages/frontend/src/routes/setup/+page.server.ts b/packages/frontend/src/routes/setup/+page.server.ts
new file mode 100644
index 0000000..8baf2b9
--- /dev/null
+++ b/packages/frontend/src/routes/setup/+page.server.ts
@@ -0,0 +1,5 @@
+import { redirect } from '@sveltejs/kit';
+import type { PageServerLoad } from "./$types";
+
+
+export const load = (async (event) => { }) satisfies PageServerLoad;
\ No newline at end of file
diff --git a/packages/frontend/src/routes/setup/+page.svelte b/packages/frontend/src/routes/setup/+page.svelte
new file mode 100644
index 0000000..ab4fa15
--- /dev/null
+++ b/packages/frontend/src/routes/setup/+page.svelte
@@ -0,0 +1,111 @@
+
+
+
+ Setup - Open Archiver
+
+
+
+
+
+
+ Welcome
+ Create the first administrator account to get started.
+
+
+
+
+
+
;
+ user: Omit;
}
diff --git a/packages/types/src/iam.types.ts b/packages/types/src/iam.types.ts
new file mode 100644
index 0000000..e6e26d0
--- /dev/null
+++ b/packages/types/src/iam.types.ts
@@ -0,0 +1,9 @@
+export type Action = string;
+
+export type Resource = string;
+
+export interface PolicyStatement {
+ Effect: 'Allow' | 'Deny';
+ Action: Action[];
+ Resource: Resource[];
+}
diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts
index e16b6cc..d1a16af 100644
--- a/packages/types/src/index.ts
+++ b/packages/types/src/index.ts
@@ -6,3 +6,4 @@ export * from './email.types';
export * from './archived-emails.types';
export * from './search.types';
export * from './dashboard.types';
+export * from './iam.types';
diff --git a/packages/types/src/user.types.ts b/packages/types/src/user.types.ts
index 1647a45..7bc54ac 100644
--- a/packages/types/src/user.types.ts
+++ b/packages/types/src/user.types.ts
@@ -1,26 +1,34 @@
-/**
- * Defines the possible roles a user can have within the system.
- */
-export type UserRole = 'Super Administrator' | 'Auditor/Compliance Officer' | 'End User';
+import { PolicyStatement } from './iam.types';
/**
* Represents a user account in the system.
+ * This is the core user object that will be stored in the database.
*/
export interface User {
- /**
- * The unique identifier for the user.
- */
id: string;
- /**
- * The user's email address, used for login.
- */
+ first_name: string | null;
+ last_name: string | null;
email: string;
- /**
- * The user's assigned role, which determines their permissions.
- */
- role: UserRole;
- /**
- * The hashed password for the user. This should never be exposed to the client.
- */
- passwordHash: string;
+}
+
+/**
+ * Represents a user's session.
+ * This is used to track a user's login status.
+ */
+export interface Session {
+ id: string;
+ userId: string;
+ expiresAt: Date;
+}
+
+/**
+ * Defines a role that can be assigned to users.
+ * Roles are used to group a set of permissions together.
+ */
+export interface Role {
+ id: string;
+ name: string;
+ policies: PolicyStatement[];
+ createdAt: Date;
+ updatedAt: Date;
}
From 23ebe942b24463191f6ed88afce5a23fa37def78 Mon Sep 17 00:00:00 2001
From: Wayne <5291640+ringoinca@users.noreply.github.com>
Date: Wed, 6 Aug 2025 01:12:33 +0300
Subject: [PATCH 3/9] IAM policies
---
.../IAM-service}/iam-policies.md | 44 ++++++++++++++++---
.../src/api/controllers/auth.controller.ts | 10 +++--
packages/backend/src/api/routes/iam.routes.ts | 1 -
.../backend/src/iam-policy/iam-definitions.ts | 39 +---------------
packages/backend/src/index.ts | 2 +-
packages/backend/src/services/UserService.ts | 42 +++++++++---------
6 files changed, 68 insertions(+), 70 deletions(-)
rename docs/{developer-guides => services/IAM-service}/iam-policies.md (75%)
diff --git a/docs/developer-guides/iam-policies.md b/docs/services/IAM-service/iam-policies.md
similarity index 75%
rename from docs/developer-guides/iam-policies.md
rename to docs/services/IAM-service/iam-policies.md
index edbcea8..08a1891 100644
--- a/docs/developer-guides/iam-policies.md
+++ b/docs/services/IAM-service/iam-policies.md
@@ -18,7 +18,37 @@ A policy is a JSON object that consists of one or more statements. Each statemen
- **`Action`**: A list of operations that the policy grants or denies permission to perform. Actions are formatted as `service:operation`.
- **`Resource`**: A list of resources to which the actions apply. Resources are specified in a hierarchical format. Wildcards (`*`) can be used.
-## 2. Actions and Resources by Service
+## 2. Wildcard Support
+
+Our IAM system supports wildcards (`*`) in both `Action` and `Resource` fields to provide flexible permission management, as defined in the `PolicyValidator`.
+
+### Action Wildcards
+
+You can use wildcards to grant broad permissions for actions:
+
+- **Global Wildcard (`*`)**: A standalone `*` in the `Action` field grants permission for all possible actions across all services.
+ ```json
+ "Action": ["*"]
+ ```
+- **Service-Level Wildcard (`service:*`)**: A wildcard at the end of an action string grants permission for all actions within that specific service.
+ ```json
+ "Action": ["archive:*"]
+ ```
+
+### Resource Wildcards
+
+Wildcards can also be used to specify resources:
+
+- **Global Wildcard (`*`)**: A standalone `*` in the `Resource` field applies the policy to all resources in the system.
+ ```json
+ "Resource": ["*"]
+ ```
+- **Partial Wildcards**: Some services allow wildcards at specific points in the resource path to refer to all resources of a certain type. For example, to target all ingestion sources:
+ ```json
+ "Resource": ["ingestion-source/*"]
+ ```
+
+## 3. Actions and Resources by Service
The following sections define the available actions and resources, categorized by their respective services.
@@ -36,12 +66,12 @@ The `archive` service pertains to all actions related to accessing and managing
**Resources:**
-| Resource | Description |
-| :------------------------------------ | :------------------------------------------------------------- |
-| `archive/all` | Represents the entire email archive. |
-| `archive/ingestion-source/{sourceId}` | Scopes the action to emails from a specific ingestion source. |
-| `archive/email/{emailId}` | Scopes the action to a single, specific email. |
-| `archive/custodian/{custodianId}` | Scopes the action to emails belonging to a specific custodian. |
+| Resource | Description |
+| :------------------------------------ | :--------------------------------------------------------------------------------------- |
+| `archive/all` | Represents the entire email archive. |
+| `archive/ingestion-source/{sourceId}` | Scopes the action to emails from a specific ingestion source. |
+| `archive/mailbox/{email}` | Scopes the action to a single, specific mailbox, usually identified by an email address. |
+| `archive/custodian/{custodianId}` | Scopes the action to emails belonging to a specific custodian. |
---
diff --git a/packages/backend/src/api/controllers/auth.controller.ts b/packages/backend/src/api/controllers/auth.controller.ts
index 3ddd3ca..b7f7d98 100644
--- a/packages/backend/src/api/controllers/auth.controller.ts
+++ b/packages/backend/src/api/controllers/auth.controller.ts
@@ -13,7 +13,12 @@ export class AuthController {
this.#authService = authService;
this.#userService = userService;
}
-
+ /**
+ * Only used for setting up the instance, should only be displayed once upon instance set up.
+ * @param req
+ * @param res
+ * @returns
+ */
public setup = async (req: Request, res: Response): Promise => {
const { email, password, first_name, last_name } = req.body;
@@ -29,9 +34,8 @@ export class AuthController {
return res.status(403).json({ message: 'Setup has already been completed.' });
}
- const newUser = await this.#userService.createUser({ email, password, first_name, last_name });
+ const newUser = await this.#userService.createAdminUser({ email, password, first_name, last_name });
const result = await this.#authService.login(email, password);
-
return res.status(201).json(result);
} catch (error) {
console.error('Setup error:', error);
diff --git a/packages/backend/src/api/routes/iam.routes.ts b/packages/backend/src/api/routes/iam.routes.ts
index 0c5cbd3..ad000a7 100644
--- a/packages/backend/src/api/routes/iam.routes.ts
+++ b/packages/backend/src/api/routes/iam.routes.ts
@@ -32,6 +32,5 @@ export const createIamRouter = (iamController: IamController): Router => {
* @access Private
*/
router.delete('/roles/:id', requireAuth, iamController.deleteRole);
-
return router;
};
diff --git a/packages/backend/src/iam-policy/iam-definitions.ts b/packages/backend/src/iam-policy/iam-definitions.ts
index 2ba8bd7..bf16cfb 100644
--- a/packages/backend/src/iam-policy/iam-definitions.ts
+++ b/packages/backend/src/iam-policy/iam-definitions.ts
@@ -29,7 +29,7 @@ const ARCHIVE_ACTIONS = {
const ARCHIVE_RESOURCES = {
ALL: 'archive/all',
INGESTION_SOURCE: 'archive/ingestion-source/*',
- EMAIL: 'archive/email/*',
+ MAILBOX: 'archive/mailbox/*',
CUSTODIAN: 'archive/custodian/*',
} as const;
@@ -113,43 +113,8 @@ export const ValidActions: Set = new Set([
* as is `archive/email/123-abc`.
*/
export const ValidResourcePatterns = {
- archive: /^archive\/(all|ingestion-source\/[^\/]+|email\/[^\/]+|custodian\/[^\/]+)$/,
+ archive: /^archive\/(all|ingestion-source\/[^\/]+|mailbox\/[^\/]+|custodian\/[^\/]+)$/,
ingestion: /^ingestion-source\/(\*|[^\/]+)$/,
system: /^system\/(settings|users|user\/[^\/]+)$/,
dashboard: /^dashboard\/\*$/,
};
-
-
-/**
- * --- How to Use These Definitions for Validation (Conceptual) ---
- *
- * A validator function would be created, likely in an `AuthorizationService`,
- * that accepts a `PolicyStatement` object.
- *
- * export function isPolicyStatementValid(statement: PolicyStatement): boolean {
- * // 1. Validate Actions
- * for (const action of statement.Action) {
- * if (action.endsWith('*')) {
- * // For wildcards, check if the service prefix is valid
- * const service = action.split(':')[0];
- * if (!Object.keys(ValidResourcePatterns).includes(service)) {
- * return false; // Invalid service
- * }
- * } else if (!ValidActions.has(action)) {
- * return false; // Action is not in the set of known actions
- * }
- * }
- *
- * // 2. Validate Resources
- * for (const resource of statement.Resource) {
- * const service = resource.split('/')[0];
- * const pattern = ValidResourcePatterns[service];
- *
- * if (!pattern || !pattern.test(resource)) {
- * return false; // Resource format is invalid for the specified service
- * }
- * }
- *
- * return true;
- * }
- */
diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts
index f23b382..582917d 100644
--- a/packages/backend/src/index.ts
+++ b/packages/backend/src/index.ts
@@ -35,7 +35,7 @@ const {
if (!PORT_BACKEND || !JWT_SECRET || !JWT_EXPIRES_IN) {
- throw new Error('Missing required environment variables for the backend.');
+ throw new Error('Missing required environment variables for the backend: PORT_BACKEND, JWT_SECRET, JWT_EXPIRES_IN.');
}
// --- Dependency Injection Setup ---
diff --git a/packages/backend/src/services/UserService.ts b/packages/backend/src/services/UserService.ts
index 6f4ddc0..c8a2f6e 100644
--- a/packages/backend/src/services/UserService.ts
+++ b/packages/backend/src/services/UserService.ts
@@ -36,7 +36,7 @@ export class UserService {
* @param userDetails The details of the user to create.
* @returns The newly created user object.
*/
- public async createUser(userDetails: Pick & { password?: string; }): Promise<(typeof schema.users.$inferSelect)> {
+ public async createAdminUser(userDetails: Pick & { password?: string; }): Promise<(typeof schema.users.$inferSelect)> {
const { email, first_name, last_name, password } = userDetails;
const userCountResult = await db.select({ count: sql`count(*)` }).from(schema.users);
@@ -51,29 +51,29 @@ export class UserService {
password: hashedPassword,
}).returning();
- if (isFirstUser) {
- let superAdminRole = await db.query.roles.findFirst({
- where: eq(schema.roles.name, 'Super Admin')
- });
+ // find super admin role
+ let superAdminRole = await db.query.roles.findFirst({
+ where: eq(schema.roles.name, 'Super Admin')
+ });
- if (!superAdminRole) {
- const suerAdminPolicies: PolicyStatement[] = [{
- Effect: 'Allow',
- Action: ['*'],
- Resource: ['*']
- }];
- superAdminRole = (await db.insert(schema.roles).values({
- name: 'Super Admin',
- policies: suerAdminPolicies
- }).returning())[0];
- }
-
- await db.insert(schema.userRoles).values({
- userId: newUser[0].id,
- roleId: superAdminRole.id
- });
+ if (!superAdminRole) {
+ const suerAdminPolicies: PolicyStatement[] = [{
+ Effect: 'Allow',
+ Action: ['*'],
+ Resource: ['*']
+ }];
+ superAdminRole = (await db.insert(schema.roles).values({
+ name: 'Super Admin',
+ policies: suerAdminPolicies
+ }).returning())[0];
}
+ await db.insert(schema.userRoles).values({
+ userId: newUser[0].id,
+ roleId: superAdminRole.id
+ });
+
+
return newUser[0];
}
}
From e02ad0355e57fd6e4c46ca42ec330ab0c561c0f2 Mon Sep 17 00:00:00 2001
From: Wayne <5291640+ringoinca@users.noreply.github.com>
Date: Wed, 6 Aug 2025 11:06:09 +0300
Subject: [PATCH 4/9] Restrict createAdminUser to initial setup only
---
.../src/api/controllers/auth.controller.ts | 2 +-
packages/backend/src/services/UserService.ts | 16 +++++++++++-----
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/packages/backend/src/api/controllers/auth.controller.ts b/packages/backend/src/api/controllers/auth.controller.ts
index b7f7d98..54df7e3 100644
--- a/packages/backend/src/api/controllers/auth.controller.ts
+++ b/packages/backend/src/api/controllers/auth.controller.ts
@@ -34,7 +34,7 @@ export class AuthController {
return res.status(403).json({ message: 'Setup has already been completed.' });
}
- const newUser = await this.#userService.createAdminUser({ email, password, first_name, last_name });
+ const newUser = await this.#userService.createAdminUser({ email, password, first_name, last_name }, true);
const result = await this.#authService.login(email, password);
return res.status(201).json(result);
} catch (error) {
diff --git a/packages/backend/src/services/UserService.ts b/packages/backend/src/services/UserService.ts
index c8a2f6e..951a615 100644
--- a/packages/backend/src/services/UserService.ts
+++ b/packages/backend/src/services/UserService.ts
@@ -31,17 +31,23 @@ export class UserService {
}
/**
- * Creates a new user in the database.
- * The first user created will be assigned the 'Super Admin' role.
+ * Creates an admin user in the database.
+ * The user created will be assigned the 'Super Admin' role.
+ * Caution โ ๏ธ: This action can only be allowed in the initial setup
* @param userDetails The details of the user to create.
+ * @param isSetup Is this an initial setup?
* @returns The newly created user object.
*/
- public async createAdminUser(userDetails: Pick & { password?: string; }): Promise<(typeof schema.users.$inferSelect)> {
+ public async createAdminUser(userDetails: Pick & { password?: string; }, isSetup: boolean): Promise<(typeof schema.users.$inferSelect)> {
+ if (!isSetup) {
+ throw Error('This operation is only allowed upon initial setup.');
+ }
const { email, first_name, last_name, password } = userDetails;
-
const userCountResult = await db.select({ count: sql`count(*)` }).from(schema.users);
const isFirstUser = Number(userCountResult[0].count) === 0;
-
+ if (!isFirstUser) {
+ throw Error('This operation is only allowed upon initial setup.');
+ }
const hashedPassword = password ? await hash(password, 10) : undefined;
const newUser = await db.insert(schema.users).values({
From 4872ed597f4543b847dfd877786f848c2c6a18e6 Mon Sep 17 00:00:00 2001
From: Wayne <5291640+ringoinca@users.noreply.github.com>
Date: Thu, 7 Aug 2025 17:03:08 +0300
Subject: [PATCH 5/9] PST ingestion
---
.env.example | 3 -
docs/user-guides/installation.md | 111 +-
packages/backend/package.json | 5 +
.../src/api/controllers/auth.controller.ts | 15 +
.../src/api/controllers/upload.controller.ts | 24 +
.../backend/src/api/middleware/requireAuth.ts | 4 +-
.../src/api/routes/archived-email.routes.ts | 4 +-
.../src/api/routes/dashboard.routes.ts | 4 +-
.../src/api/routes/ingestion.routes.ts | 4 +-
.../backend/src/api/routes/search.routes.ts | 4 +-
.../backend/src/api/routes/storage.routes.ts | 4 +-
.../backend/src/api/routes/upload.routes.ts | 14 +
.../migrations/0012_warm_the_stranger.sql | 2 +
.../migrations/meta/0012_snapshot.json | 1095 +++++++++++++++++
.../database/migrations/meta/_journal.json | 7 +
.../src/database/schema/ingestion-sources.ts | 6 +-
packages/backend/src/index.ts | 12 +-
.../processors/initial-import.processor.ts | 3 +-
.../sync-cycle-finished.processor.ts | 8 +-
.../src/services/ArchivedEmailService.ts | 7 +-
.../src/services/EmailProviderFactory.ts | 4 +
.../backend/src/services/IngestionService.ts | 2 +-
packages/backend/src/services/UserService.ts | 5 +-
.../GoogleWorkspaceConnector.ts | 2 +-
.../ingestion-connectors/ImapConnector.ts | 2 +-
.../ingestion-connectors/PSTConnector.ts | 330 +++++
.../{ => helpers}/utils.ts | 9 +
packages/frontend/src/lib/api.client.ts | 8 +-
.../lib/components/custom/EmailPreview.svelte | 1 +
.../lib/components/custom/EmailThread.svelte | 2 +-
.../custom/IngestionSourceForm.svelte | 60 +-
.../src/routes/dashboard/+page.svelte | 3 +-
.../routes/dashboard/ingestions/+page.svelte | 138 ++-
packages/types/src/ingestion.types.ts | 20 +-
pnpm-lock.yaml | 112 ++
35 files changed, 1982 insertions(+), 52 deletions(-)
create mode 100644 packages/backend/src/api/controllers/upload.controller.ts
create mode 100644 packages/backend/src/api/routes/upload.routes.ts
create mode 100644 packages/backend/src/database/migrations/0012_warm_the_stranger.sql
create mode 100644 packages/backend/src/database/migrations/meta/0012_snapshot.json
create mode 100644 packages/backend/src/services/ingestion-connectors/PSTConnector.ts
rename packages/backend/src/services/ingestion-connectors/{ => helpers}/utils.ts (82%)
diff --git a/.env.example b/.env.example
index 3b4e416..5055822 100644
--- a/.env.example
+++ b/.env.example
@@ -57,10 +57,7 @@ STORAGE_S3_FORCE_PATH_STYLE=false
JWT_SECRET=a-very-secret-key-that-you-should-change
JWT_EXPIRES_IN="7d"
-# Admin User
# Set the credentials for the initial admin user.
-ADMIN_EMAIL=admin@local.com
-ADMIN_PASSWORD=a_strong_password_that_you_should_change
SUPER_API_KEY=
# Master Encryption Key for sensitive data (Such as Ingestion source credentials and passwords)
diff --git a/docs/user-guides/installation.md b/docs/user-guides/installation.md
index 0ff8388..4bac769 100644
--- a/docs/user-guides/installation.md
+++ b/docs/user-guides/installation.md
@@ -37,7 +37,6 @@ You must change the following placeholder values to secure your instance:
- `REDIS_PASSWORD`: A strong, unique password for the Valkey/Redis service.
- `MEILI_MASTER_KEY`: A complex key for Meilisearch.
- `JWT_SECRET`: A long, random string for signing authentication tokens.
-- `ADMIN_PASSWORD`: A strong password for the initial admin user.
- `ENCRYPTION_KEY`: A 32-byte hex string for encrypting sensitive data in the database. You can generate one with the following command:
```bash
openssl rand -hex 32
@@ -104,14 +103,12 @@ These variables are used by `docker-compose.yml` to configure the services.
#### Security & Authentication
-| Variable | Description | Default Value |
-| ---------------- | --------------------------------------------------- | ------------------------------------------ |
-| `JWT_SECRET` | A secret key for signing JWT tokens. | `a-very-secret-key-that-you-should-change` |
-| `JWT_EXPIRES_IN` | The expiration time for JWT tokens. | `7d` |
-| `ADMIN_EMAIL` | The email for the initial admin user. | `admin@local.com` |
-| `ADMIN_PASSWORD` | The password for the initial admin user. | `a_strong_password_that_you_should_change` |
-| `SUPER_API_KEY` | An API key with super admin privileges. | |
-| `ENCRYPTION_KEY` | A 32-byte hex string for encrypting sensitive data. | |
+| Variable | Description | Default Value |
+| ---------------- | ------------------------------------------------------------------- | ------------------------------------------ |
+| `JWT_SECRET` | A secret key for signing JWT tokens. | `a-very-secret-key-that-you-should-change` |
+| `JWT_EXPIRES_IN` | The expiration time for JWT tokens. | `7d` |
+| `SUPER_API_KEY` | An API key with super admin privileges. | |
+| `ENCRYPTION_KEY` | A 32-byte hex string for encrypting sensitive data in the database. | |
## 3. Run the Application
@@ -203,3 +200,99 @@ To do this, you will need to make a small modification to your `docker-compose.y
By removing these sections, you allow Coolify to automatically create and manage the necessary networks, ensuring that all services can communicate with each other and are correctly exposed through Coolify's reverse proxy.
After making these changes, you can proceed with deploying your application on Coolify as you normally would.
+
+## Where is my data stored (When using local storage and Docker)?
+
+If you are using local storage to store your emails, based on your `docker-compose.yml` file, your data is being stored in what's called a "named volume" (`archiver-data`). That's why you're not seeing the files in the `./data/open-archiver` directory you created.
+
+1. **List all Docker volumes**:
+
+Run this command to see all the volumes on your system:
+
+ ```bash
+ docker volume ls
+ ```
+
+2. **Identify the correct volume**:
+
+Look through the list for a volume name that ends with `_archiver-data`. The part before that will be your project's directory name. For example, if your project is in a folder named `OpenArchiver`, the volume will be `openarchiver_archiver-data` But it can be a randomly generated hash.
+
+3. **Inspect the correct volume**:
+
+Once you've identified the correct volume name, use it in the `inspect` command. For example:
+
+ ```bash
+ docker volume inspect
+ ```
+
+This will give you the correct `Mountpoint` path where your data is being stored. It will look something like this (the exact path will vary depending on your system):
+
+ ```json
+ {
+ "CreatedAt": "2025-07-25T11:22:19Z",
+ "Driver": "local",
+ "Labels": {
+ "com.docker.compose.config-hash": "---",
+ "com.docker.compose.project": "---",
+ "com.docker.compose.version": "2.38.2",
+ "com.docker.compose.volume": "us8wwos0o4ok4go4gc8cog84_archiver-data"
+ },
+ "Mountpoint": "/var/lib/docker/volumes/us8wwos0o4ok4go4gc8cog84_archiver-data/_data",
+ "Name": "us8wwos0o4ok4go4gc8cog84_archiver-data",
+ "Options": null,
+ "Scope": "local"
+ }
+ ```
+
+In this example, the data is located at `/var/lib/docker/volumes/us8wwos0o4ok4go4gc8cog84_archiver-data/_data`. You can then `cd` into that directory to see your files.
+
+### To save data to a specific folder
+
+To save the data to a specific folder on your machine, you'll need to make a change to your `docker-compose.yml`. You need to switch from a named volume to a "bind mount".
+
+Hereโs how you can do it:
+
+1. **Edit `docker-compose.yml`**:
+
+Open the `docker-compose.yml` file and find the `open-archiver` service. You're going to change the `volumes` section.
+
+ **Change this:**
+
+ ```yaml
+ services:
+ open-archiver:
+ # ... other config
+ volumes:
+ - archiver-data:/var/data/open-archiver
+ ```
+
+ **To this:**
+
+ ```yaml
+ services:
+ open-archiver:
+ # ... other config
+ volumes:
+ - ./data/open-archiver:/var/data/open-archiver
+ ```
+
+You'll also want to remove the `archiver-data` volume definition at the bottom of the file, since it's no longer needed.
+
+ **Remove this whole block:**
+
+ ```yaml
+ volumes:
+ # ... other volumes
+ archiver-data:
+ driver: local
+ ```
+
+2. **Restart your containers**:
+
+After you've saved the changes, run the following command in your terminal to apply them. The `--force-recreate` flag will ensure the container is recreated with the new volume settings.
+
+ ```bash
+ docker-compose up -d --force-recreate
+ ```
+
+After this, any new data will be saved directly into the `./data/open-archiver` folder in your project directory.
diff --git a/packages/backend/package.json b/packages/backend/package.json
index 0b1a984..b714147 100644
--- a/packages/backend/package.json
+++ b/packages/backend/package.json
@@ -27,6 +27,7 @@
"axios": "^1.10.0",
"bcryptjs": "^3.0.2",
"bullmq": "^5.56.3",
+ "busboy": "^1.6.0",
"cross-fetch": "^4.1.0",
"deepmerge-ts": "^7.1.5",
"dotenv": "^17.2.0",
@@ -42,11 +43,13 @@
"mailparser": "^3.7.4",
"mammoth": "^1.9.1",
"meilisearch": "^0.51.0",
+ "multer": "^2.0.2",
"pdf2json": "^3.1.6",
"pg": "^8.16.3",
"pino": "^9.7.0",
"pino-pretty": "^13.0.0",
"postgres": "^3.4.7",
+ "pst-extractor": "^1.11.0",
"reflect-metadata": "^0.2.2",
"sqlite3": "^5.1.7",
"tsconfig-paths": "^4.2.0",
@@ -55,9 +58,11 @@
"devDependencies": {
"@bull-board/api": "^6.11.0",
"@bull-board/express": "^6.11.0",
+ "@types/busboy": "^1.5.4",
"@types/express": "^5.0.3",
"@types/mailparser": "^3.4.6",
"@types/microsoft-graph": "^2.40.1",
+ "@types/multer": "^2.0.0",
"@types/node": "^24.0.12",
"bull-board": "^2.1.3",
"ts-node-dev": "^2.0.0",
diff --git a/packages/backend/src/api/controllers/auth.controller.ts b/packages/backend/src/api/controllers/auth.controller.ts
index 54df7e3..7b760af 100644
--- a/packages/backend/src/api/controllers/auth.controller.ts
+++ b/packages/backend/src/api/controllers/auth.controller.ts
@@ -4,6 +4,8 @@ import { UserService } from '../../services/UserService';
import { db } from '../../database';
import * as schema from '../../database/schema';
import { sql } from 'drizzle-orm';
+import 'dotenv/config';
+
export class AuthController {
#authService: AuthService;
@@ -66,9 +68,22 @@ export class AuthController {
public status = async (req: Request, res: Response): Promise => {
try {
+
+
+
const userCountResult = await db.select({ count: sql`count(*)` }).from(schema.users);
const userCount = Number(userCountResult[0].count);
const needsSetup = userCount === 0;
+ // in case user uses older version with admin user variables, we will create the admin user using those variables.
+ if (needsSetup && process.env.ADMIN_EMAIL && process.env.ADMIN_PASSWORD) {
+ await this.#userService.createAdminUser({
+ email: process.env.ADMIN_EMAIL,
+ password: process.env.ADMIN_PASSWORD,
+ first_name: "Admin",
+ last_name: "User"
+ }, true);
+ return res.status(200).json({ needsSetup: false });
+ }
return res.status(200).json({ needsSetup });
} catch (error) {
console.error('Status check error:', error);
diff --git a/packages/backend/src/api/controllers/upload.controller.ts b/packages/backend/src/api/controllers/upload.controller.ts
new file mode 100644
index 0000000..b682f3d
--- /dev/null
+++ b/packages/backend/src/api/controllers/upload.controller.ts
@@ -0,0 +1,24 @@
+import { Request, Response } from 'express';
+import { StorageService } from '../../services/StorageService';
+import { randomUUID } from 'crypto';
+import busboy from 'busboy';
+
+export const uploadFile = async (req: Request, res: Response) => {
+ const storage = new StorageService();
+ const bb = busboy({ headers: req.headers });
+ let filePath = '';
+ let originalFilename = '';
+
+ bb.on('file', (fieldname, file, filename) => {
+ originalFilename = filename.filename;
+ const uuid = randomUUID();
+ filePath = `temp/${uuid}-${originalFilename}`;
+ storage.put(filePath, file);
+ });
+
+ bb.on('finish', () => {
+ res.json({ filePath });
+ });
+
+ req.pipe(bb);
+};
diff --git a/packages/backend/src/api/middleware/requireAuth.ts b/packages/backend/src/api/middleware/requireAuth.ts
index 965a51b..87db157 100644
--- a/packages/backend/src/api/middleware/requireAuth.ts
+++ b/packages/backend/src/api/middleware/requireAuth.ts
@@ -1,5 +1,5 @@
import type { Request, Response, NextFunction } from 'express';
-import type { IAuthService } from '../../services/AuthService';
+import type { AuthService } from '../../services/AuthService';
import type { AuthTokenPayload } from '@open-archiver/types';
import 'dotenv/config';
// By using module augmentation, we can add our custom 'user' property
@@ -12,7 +12,7 @@ declare global {
}
}
-export const requireAuth = (authService: IAuthService) => {
+export const requireAuth = (authService: AuthService) => {
return async (req: Request, res: Response, next: NextFunction) => {
const authHeader = req.headers.authorization;
if (!authHeader || !authHeader.startsWith('Bearer ')) {
diff --git a/packages/backend/src/api/routes/archived-email.routes.ts b/packages/backend/src/api/routes/archived-email.routes.ts
index aecfe4b..b896669 100644
--- a/packages/backend/src/api/routes/archived-email.routes.ts
+++ b/packages/backend/src/api/routes/archived-email.routes.ts
@@ -1,11 +1,11 @@
import { Router } from 'express';
import { ArchivedEmailController } from '../controllers/archived-email.controller';
import { requireAuth } from '../middleware/requireAuth';
-import { IAuthService } from '../../services/AuthService';
+import { AuthService } from '../../services/AuthService';
export const createArchivedEmailRouter = (
archivedEmailController: ArchivedEmailController,
- authService: IAuthService
+ authService: AuthService
): Router => {
const router = Router();
diff --git a/packages/backend/src/api/routes/dashboard.routes.ts b/packages/backend/src/api/routes/dashboard.routes.ts
index 6e1cbbb..e34d5ea 100644
--- a/packages/backend/src/api/routes/dashboard.routes.ts
+++ b/packages/backend/src/api/routes/dashboard.routes.ts
@@ -1,9 +1,9 @@
import { Router } from 'express';
import { dashboardController } from '../controllers/dashboard.controller';
import { requireAuth } from '../middleware/requireAuth';
-import { IAuthService } from '../../services/AuthService';
+import { AuthService } from '../../services/AuthService';
-export const createDashboardRouter = (authService: IAuthService): Router => {
+export const createDashboardRouter = (authService: AuthService): Router => {
const router = Router();
router.use(requireAuth(authService));
diff --git a/packages/backend/src/api/routes/ingestion.routes.ts b/packages/backend/src/api/routes/ingestion.routes.ts
index d635b73..92956df 100644
--- a/packages/backend/src/api/routes/ingestion.routes.ts
+++ b/packages/backend/src/api/routes/ingestion.routes.ts
@@ -1,11 +1,11 @@
import { Router } from 'express';
import { IngestionController } from '../controllers/ingestion.controller';
import { requireAuth } from '../middleware/requireAuth';
-import { IAuthService } from '../../services/AuthService';
+import { AuthService } from '../../services/AuthService';
export const createIngestionRouter = (
ingestionController: IngestionController,
- authService: IAuthService
+ authService: AuthService
): Router => {
const router = Router();
diff --git a/packages/backend/src/api/routes/search.routes.ts b/packages/backend/src/api/routes/search.routes.ts
index 25140ef..674d29f 100644
--- a/packages/backend/src/api/routes/search.routes.ts
+++ b/packages/backend/src/api/routes/search.routes.ts
@@ -1,11 +1,11 @@
import { Router } from 'express';
import { SearchController } from '../controllers/search.controller';
import { requireAuth } from '../middleware/requireAuth';
-import { IAuthService } from '../../services/AuthService';
+import { AuthService } from '../../services/AuthService';
export const createSearchRouter = (
searchController: SearchController,
- authService: IAuthService
+ authService: AuthService
): Router => {
const router = Router();
diff --git a/packages/backend/src/api/routes/storage.routes.ts b/packages/backend/src/api/routes/storage.routes.ts
index b1fc4f8..f4f24b6 100644
--- a/packages/backend/src/api/routes/storage.routes.ts
+++ b/packages/backend/src/api/routes/storage.routes.ts
@@ -1,11 +1,11 @@
import { Router } from 'express';
import { StorageController } from '../controllers/storage.controller';
import { requireAuth } from '../middleware/requireAuth';
-import { IAuthService } from '../../services/AuthService';
+import { AuthService } from '../../services/AuthService';
export const createStorageRouter = (
storageController: StorageController,
- authService: IAuthService
+ authService: AuthService
): Router => {
const router = Router();
diff --git a/packages/backend/src/api/routes/upload.routes.ts b/packages/backend/src/api/routes/upload.routes.ts
new file mode 100644
index 0000000..f194c23
--- /dev/null
+++ b/packages/backend/src/api/routes/upload.routes.ts
@@ -0,0 +1,14 @@
+import { Router } from 'express';
+import { uploadFile } from '../controllers/upload.controller';
+import { requireAuth } from '../middleware/requireAuth';
+import { AuthService } from '../../services/AuthService';
+
+export const createUploadRouter = (authService: AuthService): Router => {
+ const router = Router();
+
+ router.use(requireAuth(authService));
+
+ router.post('/', uploadFile);
+
+ return router;
+};
diff --git a/packages/backend/src/database/migrations/0012_warm_the_stranger.sql b/packages/backend/src/database/migrations/0012_warm_the_stranger.sql
new file mode 100644
index 0000000..4a3d26f
--- /dev/null
+++ b/packages/backend/src/database/migrations/0012_warm_the_stranger.sql
@@ -0,0 +1,2 @@
+ALTER TYPE "public"."ingestion_provider" ADD VALUE 'pst_import';--> statement-breakpoint
+ALTER TYPE "public"."ingestion_status" ADD VALUE 'imported';
\ No newline at end of file
diff --git a/packages/backend/src/database/migrations/meta/0012_snapshot.json b/packages/backend/src/database/migrations/meta/0012_snapshot.json
new file mode 100644
index 0000000..2980aa9
--- /dev/null
+++ b/packages/backend/src/database/migrations/meta/0012_snapshot.json
@@ -0,0 +1,1095 @@
+{
+ "id": "02ed9805-d480-483a-b73c-5e03a0e526b7",
+ "prevId": "6252768a-7c7f-4dae-9dbd-d3ea9f647cea",
+ "version": "7",
+ "dialect": "postgresql",
+ "tables": {
+ "public.archived_emails": {
+ "name": "archived_emails",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "thread_id": {
+ "name": "thread_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "ingestion_source_id": {
+ "name": "ingestion_source_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "user_email": {
+ "name": "user_email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "message_id_header": {
+ "name": "message_id_header",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sent_at": {
+ "name": "sent_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "subject": {
+ "name": "subject",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sender_name": {
+ "name": "sender_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sender_email": {
+ "name": "sender_email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "recipients": {
+ "name": "recipients",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "storage_path": {
+ "name": "storage_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "storage_hash_sha256": {
+ "name": "storage_hash_sha256",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "size_bytes": {
+ "name": "size_bytes",
+ "type": "bigint",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "is_indexed": {
+ "name": "is_indexed",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "has_attachments": {
+ "name": "has_attachments",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "is_on_legal_hold": {
+ "name": "is_on_legal_hold",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "archived_at": {
+ "name": "archived_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {
+ "thread_id_idx": {
+ "name": "thread_id_idx",
+ "columns": [
+ {
+ "expression": "thread_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "archived_emails_ingestion_source_id_ingestion_sources_id_fk": {
+ "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
+ "tableFrom": "archived_emails",
+ "tableTo": "ingestion_sources",
+ "columnsFrom": [
+ "ingestion_source_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.attachments": {
+ "name": "attachments",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "filename": {
+ "name": "filename",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "mime_type": {
+ "name": "mime_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "size_bytes": {
+ "name": "size_bytes",
+ "type": "bigint",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "content_hash_sha256": {
+ "name": "content_hash_sha256",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "storage_path": {
+ "name": "storage_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "attachments_content_hash_sha256_unique": {
+ "name": "attachments_content_hash_sha256_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "content_hash_sha256"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.email_attachments": {
+ "name": "email_attachments",
+ "schema": "",
+ "columns": {
+ "email_id": {
+ "name": "email_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "attachment_id": {
+ "name": "attachment_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "email_attachments_email_id_archived_emails_id_fk": {
+ "name": "email_attachments_email_id_archived_emails_id_fk",
+ "tableFrom": "email_attachments",
+ "tableTo": "archived_emails",
+ "columnsFrom": [
+ "email_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "email_attachments_attachment_id_attachments_id_fk": {
+ "name": "email_attachments_attachment_id_attachments_id_fk",
+ "tableFrom": "email_attachments",
+ "tableTo": "attachments",
+ "columnsFrom": [
+ "attachment_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "restrict",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "email_attachments_email_id_attachment_id_pk": {
+ "name": "email_attachments_email_id_attachment_id_pk",
+ "columns": [
+ "email_id",
+ "attachment_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.audit_logs": {
+ "name": "audit_logs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "bigserial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "timestamp": {
+ "name": "timestamp",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "actor_identifier": {
+ "name": "actor_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "action": {
+ "name": "action",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "target_type": {
+ "name": "target_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "target_id": {
+ "name": "target_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "details": {
+ "name": "details",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "is_tamper_evident": {
+ "name": "is_tamper_evident",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": false,
+ "default": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.ediscovery_cases": {
+ "name": "ediscovery_cases",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'open'"
+ },
+ "created_by_identifier": {
+ "name": "created_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "ediscovery_cases_name_unique": {
+ "name": "ediscovery_cases_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.export_jobs": {
+ "name": "export_jobs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "case_id": {
+ "name": "case_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "format": {
+ "name": "format",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending'"
+ },
+ "query": {
+ "name": "query",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "file_path": {
+ "name": "file_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_by_identifier": {
+ "name": "created_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "export_jobs_case_id_ediscovery_cases_id_fk": {
+ "name": "export_jobs_case_id_ediscovery_cases_id_fk",
+ "tableFrom": "export_jobs",
+ "tableTo": "ediscovery_cases",
+ "columnsFrom": [
+ "case_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.legal_holds": {
+ "name": "legal_holds",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "case_id": {
+ "name": "case_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "custodian_id": {
+ "name": "custodian_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "hold_criteria": {
+ "name": "hold_criteria",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "reason": {
+ "name": "reason",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "applied_by_identifier": {
+ "name": "applied_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "applied_at": {
+ "name": "applied_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "removed_at": {
+ "name": "removed_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "legal_holds_case_id_ediscovery_cases_id_fk": {
+ "name": "legal_holds_case_id_ediscovery_cases_id_fk",
+ "tableFrom": "legal_holds",
+ "tableTo": "ediscovery_cases",
+ "columnsFrom": [
+ "case_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "legal_holds_custodian_id_custodians_id_fk": {
+ "name": "legal_holds_custodian_id_custodians_id_fk",
+ "tableFrom": "legal_holds",
+ "tableTo": "custodians",
+ "columnsFrom": [
+ "custodian_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.retention_policies": {
+ "name": "retention_policies",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "priority": {
+ "name": "priority",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "retention_period_days": {
+ "name": "retention_period_days",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "action_on_expiry": {
+ "name": "action_on_expiry",
+ "type": "retention_action",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "is_enabled": {
+ "name": "is_enabled",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": true
+ },
+ "conditions": {
+ "name": "conditions",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "retention_policies_name_unique": {
+ "name": "retention_policies_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.custodians": {
+ "name": "custodians",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "display_name": {
+ "name": "display_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "source_type": {
+ "name": "source_type",
+ "type": "ingestion_provider",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "custodians_email_unique": {
+ "name": "custodians_email_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "email"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.ingestion_sources": {
+ "name": "ingestion_sources",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "provider": {
+ "name": "provider",
+ "type": "ingestion_provider",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "credentials": {
+ "name": "credentials",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "ingestion_status",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending_auth'"
+ },
+ "last_sync_started_at": {
+ "name": "last_sync_started_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_sync_finished_at": {
+ "name": "last_sync_finished_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_sync_status_message": {
+ "name": "last_sync_status_message",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sync_state": {
+ "name": "sync_state",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.roles": {
+ "name": "roles",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "policies": {
+ "name": "policies",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'[]'::jsonb"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "roles_name_unique": {
+ "name": "roles_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.sessions": {
+ "name": "sessions",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "sessions_user_id_users_id_fk": {
+ "name": "sessions_user_id_users_id_fk",
+ "tableFrom": "sessions",
+ "tableTo": "users",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.user_roles": {
+ "name": "user_roles",
+ "schema": "",
+ "columns": {
+ "user_id": {
+ "name": "user_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "role_id": {
+ "name": "role_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "user_roles_user_id_users_id_fk": {
+ "name": "user_roles_user_id_users_id_fk",
+ "tableFrom": "user_roles",
+ "tableTo": "users",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "user_roles_role_id_roles_id_fk": {
+ "name": "user_roles_role_id_roles_id_fk",
+ "tableFrom": "user_roles",
+ "tableTo": "roles",
+ "columnsFrom": [
+ "role_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "user_roles_user_id_role_id_pk": {
+ "name": "user_roles_user_id_role_id_pk",
+ "columns": [
+ "user_id",
+ "role_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.users": {
+ "name": "users",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "first_name": {
+ "name": "first_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_name": {
+ "name": "last_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "password": {
+ "name": "password",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "provider": {
+ "name": "provider",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "'local'"
+ },
+ "provider_id": {
+ "name": "provider_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "users_email_unique": {
+ "name": "users_email_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "email"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ }
+ },
+ "enums": {
+ "public.retention_action": {
+ "name": "retention_action",
+ "schema": "public",
+ "values": [
+ "delete_permanently",
+ "notify_admin"
+ ]
+ },
+ "public.ingestion_provider": {
+ "name": "ingestion_provider",
+ "schema": "public",
+ "values": [
+ "google_workspace",
+ "microsoft_365",
+ "generic_imap",
+ "pst_import"
+ ]
+ },
+ "public.ingestion_status": {
+ "name": "ingestion_status",
+ "schema": "public",
+ "values": [
+ "active",
+ "paused",
+ "error",
+ "pending_auth",
+ "syncing",
+ "importing",
+ "auth_success",
+ "imported"
+ ]
+ }
+ },
+ "schemas": {},
+ "sequences": {},
+ "roles": {},
+ "policies": {},
+ "views": {},
+ "_meta": {
+ "columns": {},
+ "schemas": {},
+ "tables": {}
+ }
+}
\ No newline at end of file
diff --git a/packages/backend/src/database/migrations/meta/_journal.json b/packages/backend/src/database/migrations/meta/_journal.json
index 8d05fd2..9e29ae8 100644
--- a/packages/backend/src/database/migrations/meta/_journal.json
+++ b/packages/backend/src/database/migrations/meta/_journal.json
@@ -85,6 +85,13 @@
"when": 1754422064158,
"tag": "0011_tan_blackheart",
"breakpoints": true
+ },
+ {
+ "idx": 12,
+ "version": "7",
+ "when": 1754476962901,
+ "tag": "0012_warm_the_stranger",
+ "breakpoints": true
}
]
}
\ No newline at end of file
diff --git a/packages/backend/src/database/schema/ingestion-sources.ts b/packages/backend/src/database/schema/ingestion-sources.ts
index 4da4bb1..a3801de 100644
--- a/packages/backend/src/database/schema/ingestion-sources.ts
+++ b/packages/backend/src/database/schema/ingestion-sources.ts
@@ -3,7 +3,8 @@ import { jsonb, pgEnum, pgTable, text, timestamp, uuid } from 'drizzle-orm/pg-co
export const ingestionProviderEnum = pgEnum('ingestion_provider', [
'google_workspace',
'microsoft_365',
- 'generic_imap'
+ 'generic_imap',
+ 'pst_import'
]);
export const ingestionStatusEnum = pgEnum('ingestion_status', [
@@ -13,7 +14,8 @@ export const ingestionStatusEnum = pgEnum('ingestion_status', [
'pending_auth',
'syncing',
'importing',
- 'auth_success'
+ 'auth_success',
+ 'imported'
]);
export const ingestionSources = pgTable('ingestion_sources', {
diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts
index 582917d..cec3257 100644
--- a/packages/backend/src/index.ts
+++ b/packages/backend/src/index.ts
@@ -14,6 +14,7 @@ import { createArchivedEmailRouter } from './api/routes/archived-email.routes';
import { createStorageRouter } from './api/routes/storage.routes';
import { createSearchRouter } from './api/routes/search.routes';
import { createDashboardRouter } from './api/routes/dashboard.routes';
+import { createUploadRouter } from './api/routes/upload.routes';
import testRouter from './api/routes/test.routes';
import { AuthService } from './services/AuthService';
import { UserService } from './services/UserService';
@@ -55,9 +56,6 @@ const iamController = new IamController(iamService);
// --- Express App Initialization ---
const app = express();
-// Middleware
-app.use(express.json()); // For parsing application/json
-
// --- Routes ---
const authRouter = createAuthRouter(authController);
const ingestionRouter = createIngestionRouter(ingestionController, authService);
@@ -66,6 +64,14 @@ const storageRouter = createStorageRouter(storageController, authService);
const searchRouter = createSearchRouter(searchController, authService);
const dashboardRouter = createDashboardRouter(authService);
const iamRouter = createIamRouter(iamController);
+const uploadRouter = createUploadRouter(authService);
+// upload route is added before middleware because it doesn't use the json middleware.
+app.use('/v1/upload', uploadRouter);
+
+// Middleware for all other routes
+app.use(express.json());
+app.use(express.urlencoded({ extended: true }));
+
app.use('/v1/auth', authRouter);
app.use('/v1/iam', iamRouter);
app.use('/v1/ingestion-sources', ingestionRouter);
diff --git a/packages/backend/src/jobs/processors/initial-import.processor.ts b/packages/backend/src/jobs/processors/initial-import.processor.ts
index 2e34d31..ed31a72 100644
--- a/packages/backend/src/jobs/processors/initial-import.processor.ts
+++ b/packages/backend/src/jobs/processors/initial-import.processor.ts
@@ -67,9 +67,10 @@ export default async (job: Job) => {
}
});
} else {
+ const finalStatus = source.provider === 'pst_import' ? 'imported' : 'active';
// If there are no users, we can consider the import finished and set to active
await IngestionService.update(ingestionSourceId, {
- status: 'active',
+ status: finalStatus,
lastSyncFinishedAt: new Date(),
lastSyncStatusMessage: 'Initial import complete. No users found.'
});
diff --git a/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts b/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts
index a497663..e40910c 100644
--- a/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts
+++ b/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts
@@ -1,7 +1,7 @@
import { Job } from 'bullmq';
import { IngestionService } from '../../services/IngestionService';
import { logger } from '../../config/logger';
-import { SyncState, ProcessMailboxError } from '@open-archiver/types';
+import { SyncState, ProcessMailboxError, IngestionStatus } from '@open-archiver/types';
import { db } from '../../database';
import { ingestionSources } from '../../database/schema';
import { eq } from 'drizzle-orm';
@@ -41,7 +41,11 @@ export default async (job: Job) => {
const finalSyncState = deepmerge(...successfulJobs.filter(s => s && Object.keys(s).length > 0));
- let status: 'active' | 'error' = 'active';
+ const source = await IngestionService.findById(ingestionSourceId);
+ let status: IngestionStatus = 'active';
+ if (source.provider === 'pst_import') {
+ status = 'imported';
+ }
let message: string;
if (failedJobs.length > 0) {
diff --git a/packages/backend/src/services/ArchivedEmailService.ts b/packages/backend/src/services/ArchivedEmailService.ts
index 3492a03..a66c976 100644
--- a/packages/backend/src/services/ArchivedEmailService.ts
+++ b/packages/backend/src/services/ArchivedEmailService.ts
@@ -1,4 +1,4 @@
-import { count, desc, eq, asc } from 'drizzle-orm';
+import { count, desc, eq, asc, and } from 'drizzle-orm';
import { db } from '../database';
import { archivedEmails, attachments, emailAttachments } from '../database/schema';
import type { PaginatedArchivedEmails, ArchivedEmail, Recipient, ThreadEmail } from '@open-archiver/types';
@@ -81,7 +81,10 @@ export class ArchivedEmailService {
if (email.threadId) {
threadEmails = await db.query.archivedEmails.findMany({
- where: eq(archivedEmails.threadId, email.threadId),
+ where: and(
+ eq(archivedEmails.threadId, email.threadId),
+ eq(archivedEmails.ingestionSourceId, email.ingestionSourceId)
+ ),
orderBy: [asc(archivedEmails.sentAt)],
columns: {
id: true,
diff --git a/packages/backend/src/services/EmailProviderFactory.ts b/packages/backend/src/services/EmailProviderFactory.ts
index 920fb92..4d664c9 100644
--- a/packages/backend/src/services/EmailProviderFactory.ts
+++ b/packages/backend/src/services/EmailProviderFactory.ts
@@ -3,6 +3,7 @@ import type {
GoogleWorkspaceCredentials,
Microsoft365Credentials,
GenericImapCredentials,
+ PSTImportCredentials,
EmailObject,
SyncState,
MailboxUser
@@ -10,6 +11,7 @@ import type {
import { GoogleWorkspaceConnector } from './ingestion-connectors/GoogleWorkspaceConnector';
import { MicrosoftConnector } from './ingestion-connectors/MicrosoftConnector';
import { ImapConnector } from './ingestion-connectors/ImapConnector';
+import { PSTConnector } from './ingestion-connectors/PSTConnector';
// Define a common interface for all connectors
export interface IEmailConnector {
@@ -32,6 +34,8 @@ export class EmailProviderFactory {
return new MicrosoftConnector(credentials as Microsoft365Credentials);
case 'generic_imap':
return new ImapConnector(credentials as GenericImapCredentials);
+ case 'pst_import':
+ return new PSTConnector(credentials as PSTImportCredentials);
default:
throw new Error(`Unsupported provider: ${source.provider}`);
}
diff --git a/packages/backend/src/services/IngestionService.ts b/packages/backend/src/services/IngestionService.ts
index 4398bf7..3e6d71b 100644
--- a/packages/backend/src/services/IngestionService.ts
+++ b/packages/backend/src/services/IngestionService.ts
@@ -37,7 +37,7 @@ export class IngestionService {
public static async create(dto: CreateIngestionSourceDto): Promise {
const { providerConfig, ...rest } = dto;
-
+ console.log(providerConfig);
const encryptedCredentials = CryptoService.encryptObject(providerConfig);
const valuesToInsert = {
diff --git a/packages/backend/src/services/UserService.ts b/packages/backend/src/services/UserService.ts
index 951a615..af4ec07 100644
--- a/packages/backend/src/services/UserService.ts
+++ b/packages/backend/src/services/UserService.ts
@@ -31,9 +31,10 @@ export class UserService {
}
/**
- * Creates an admin user in the database.
- * The user created will be assigned the 'Super Admin' role.
+ * Creates an admin user in the database. The user created will be assigned the 'Super Admin' role.
+ *
* Caution โ ๏ธ: This action can only be allowed in the initial setup
+ *
* @param userDetails The details of the user to create.
* @param isSetup Is this an initial setup?
* @returns The newly created user object.
diff --git a/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts b/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts
index db1df77..b9fde23 100644
--- a/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts
+++ b/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts
@@ -10,7 +10,7 @@ import type {
import type { IEmailConnector } from '../EmailProviderFactory';
import { logger } from '../../config/logger';
import { simpleParser, ParsedMail, Attachment, AddressObject, Headers } from 'mailparser';
-import { getThreadId } from './utils';
+import { getThreadId } from './helpers/utils';
/**
* A connector for Google Workspace that uses a service account with domain-wide delegation
diff --git a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts
index 8c05bc7..57b558f 100644
--- a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts
+++ b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts
@@ -3,7 +3,7 @@ import type { IEmailConnector } from '../EmailProviderFactory';
import { ImapFlow } from 'imapflow';
import { simpleParser, ParsedMail, Attachment, AddressObject, Headers } from 'mailparser';
import { logger } from '../../config/logger';
-import { getThreadId } from './utils';
+import { getThreadId } from './helpers/utils';
export class ImapConnector implements IEmailConnector {
private client: ImapFlow;
diff --git a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts
new file mode 100644
index 0000000..93ec9a5
--- /dev/null
+++ b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts
@@ -0,0 +1,330 @@
+import type { PSTImportCredentials, EmailObject, EmailAddress, SyncState, MailboxUser } from '@open-archiver/types';
+import type { IEmailConnector } from '../EmailProviderFactory';
+import { PSTFile, PSTFolder, PSTMessage } from 'pst-extractor';
+import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
+import { logger } from '../../config/logger';
+import { getThreadId } from './helpers/utils';
+import { StorageService } from '../StorageService';
+import { Readable } from 'stream';
+import { createHash } from 'crypto';
+
+const streamToBuffer = (stream: Readable): Promise => {
+ return new Promise((resolve, reject) => {
+ const chunks: Buffer[] = [];
+ stream.on('data', (chunk) => chunks.push(chunk));
+ stream.on('error', reject);
+ stream.on('end', () => resolve(Buffer.concat(chunks)));
+ });
+};
+
+// We have to hardcode names for deleted and trash folders here as current lib doesn't support looking into PST properties.
+const DELETED_FOLDERS = new Set([
+ // English
+ 'deleted items', 'trash',
+ // Spanish
+ 'elementos eliminados', 'papelera',
+ // French
+ 'รฉlรฉments supprimรฉs', 'corbeille',
+ // German
+ 'gelรถschte elemente', 'papierkorb',
+ // Italian
+ 'posta eliminata', 'cestino',
+ // Portuguese
+ 'itens excluรญdos', 'lixo',
+ // Dutch
+ 'verwijderde items', 'prullenbak',
+ // Russian
+ 'ัะดะฐะปะตะฝะฝัะต', 'ะบะพัะทะธะฝะฐ',
+ // Polish
+ 'usuniฤte elementy', 'kosz',
+ // Japanese
+ 'ๅ้คๆธใฟใขใคใใ ',
+ // Czech
+ 'odstranฤnรก poลกta', 'koลก',
+ // Estonian
+ 'kustutatud kirjad', 'prรผgikast',
+ // Swedish
+ 'borttagna objekt', 'skrรคp',
+ // Danish
+ 'slettet post', 'papirkurv',
+ // Norwegian
+ 'slettede elementer',
+ // Finnish
+ 'poistetut', 'roskakori'
+]);
+
+const JUNK_FOLDERS = new Set([
+ // English
+ 'junk email', 'spam',
+ // Spanish
+ 'correo no deseado',
+ // French
+ 'courrier indรฉsirable',
+ // German
+ 'junk-e-mail',
+ // Italian
+ 'posta indesiderata',
+ // Portuguese
+ 'lixo eletrรดnico',
+ // Dutch
+ 'ongewenste e-mail',
+ // Russian
+ 'ะฝะตะถะตะปะฐัะตะปัะฝะฐั ะฟะพััะฐ', 'ัะฟะฐะผ',
+ // Polish
+ 'wiadomoลci-ลmieci',
+ // Japanese
+ '่ฟทๆใกใผใซ', 'ในใใ ',
+ // Czech
+ 'nevyลพรกdanรก poลกta',
+ // Estonian
+ 'rรคmpspost',
+ // Swedish
+ 'skrรคppost',
+ // Danish
+ 'uรธnsket post',
+ // Norwegian
+ 'sรธppelpost',
+ // Finnish
+ 'roskaposti'
+]);
+
+export class PSTConnector implements IEmailConnector {
+ private storage: StorageService;
+ private pstFile: PSTFile | null = null;
+
+ constructor(private credentials: PSTImportCredentials) {
+ this.storage = new StorageService();
+ }
+
+ private async loadPstFile(): Promise {
+ if (this.pstFile) {
+ return this.pstFile;
+ }
+ const fileStream = await this.storage.get(this.credentials.uploadedFilePath);
+ const buffer = await streamToBuffer(fileStream as Readable);
+ this.pstFile = new PSTFile(buffer);
+ return this.pstFile;
+ }
+
+ public async testConnection(): Promise {
+ try {
+ if (!this.credentials.uploadedFilePath) {
+ throw Error("PST file path not provided.");
+ }
+ if (!this.credentials.uploadedFilePath.includes('.pst')) {
+ throw Error("Provided file is not in the PST format.");
+ }
+ return true;
+ } catch (error) {
+ logger.error({ error, credentials: this.credentials }, 'PST file validation failed.');
+ throw error;
+ }
+ }
+
+ /**
+ * Lists mailboxes within the PST. It treats each top-level folder
+ * as a distinct mailbox, allowing it to handle PSTs that have been
+ * consolidated from multiple sources.
+ */
+ public async *listAllUsers(): AsyncGenerator {
+ let pstFile: PSTFile | null = null;
+ try {
+ pstFile = await this.loadPstFile();
+ const root = pstFile.getRootFolder();
+ const displayName = root.displayName || pstFile.pstFilename;
+ logger.info(`Found potential mailbox: ${displayName}`);
+ const constructedPrimaryEmail = `${displayName.replace(/ /g, '.').toLowerCase()}@pst.local`;
+ yield {
+ id: constructedPrimaryEmail,
+ // We will address the primaryEmail problem in the next section.
+ primaryEmail: constructedPrimaryEmail,
+ displayName: displayName,
+ };
+ } catch (error) {
+ logger.error({ error }, 'Failed to list users from PST file using top-level folder strategy.');
+ pstFile?.close();
+ throw error;
+ } finally {
+ pstFile?.close();
+ }
+ }
+
+ public async *fetchEmails(userEmail: string, syncState?: SyncState | null): AsyncGenerator {
+ let pstFile: PSTFile | null = null;
+ try {
+ pstFile = await this.loadPstFile();
+ const root = pstFile.getRootFolder();
+ yield* this.processFolder(root);
+ } catch (error) {
+ logger.error({ error }, 'Failed to list users from PST file using top-level folder strategy.');
+ pstFile?.close();
+ throw error;
+ }
+ finally {
+
+ pstFile?.close();
+ }
+ }
+
+ private async *processFolder(folder: PSTFolder): AsyncGenerator {
+ const folderName = folder.displayName.toLowerCase();
+ if (DELETED_FOLDERS.has(folderName) || JUNK_FOLDERS.has(folderName)) {
+ logger.info(`Skipping folder: ${folder.displayName}`);
+ return;
+ }
+
+ if (folder.contentCount > 0) {
+ let email: PSTMessage | null = folder.getNextChild();
+ while (email != null) {
+ yield await this.parseMessage(email);
+ try {
+ email = folder.getNextChild();
+ } catch (error) {
+ console.warn("Folder doesn't have child");
+ email = null;
+ }
+ }
+ }
+
+ if (folder.hasSubfolders) {
+ for (const subFolder of folder.getSubFolders()) {
+ yield* this.processFolder(subFolder);
+ }
+ }
+ }
+
+ private async parseMessage(msg: PSTMessage): Promise {
+ const emlContent = await this.constructEml(msg);
+ const emlBuffer = Buffer.from(emlContent, 'utf-8');
+ const parsedEmail: ParsedMail = await simpleParser(emlBuffer);
+
+ const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
+ filename: attachment.filename || 'untitled',
+ contentType: attachment.contentType,
+ size: attachment.size,
+ content: attachment.content as Buffer
+ }));
+
+ const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
+ if (!addresses) return [];
+ const addressArray = Array.isArray(addresses) ? addresses : [addresses];
+ return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address?.replaceAll(`'`, '') || '' })));
+ };
+
+ const threadId = getThreadId(parsedEmail.headers);
+ let messageId = msg.internetMessageId;
+ // generate a unique ID for this message
+
+ if (!messageId) {
+ messageId = `generated-${createHash('sha256').update(emlBuffer ?? Buffer.from(parsedEmail.text || parsedEmail.html || '', 'utf-8')).digest('hex')}-${createHash('sha256').update(emlBuffer ?? Buffer.from(msg.subject || '', 'utf-8')).digest('hex')}-${msg.clientSubmitTime?.getTime()}`;
+ }
+ return {
+ id: messageId,
+ threadId: threadId,
+ from: mapAddresses(parsedEmail.from),
+ to: mapAddresses(parsedEmail.to),
+ cc: mapAddresses(parsedEmail.cc),
+ bcc: mapAddresses(parsedEmail.bcc),
+ subject: parsedEmail.subject || '',
+ body: parsedEmail.text || '',
+ html: parsedEmail.html || '',
+ headers: parsedEmail.headers,
+ attachments,
+ receivedAt: parsedEmail.date || new Date(),
+ eml: emlBuffer
+ };
+ }
+
+ private async constructEml(msg: PSTMessage): Promise {
+ let eml = '';
+ const boundary = '----boundary-openarchiver';
+ const altBoundary = '----boundary-openarchiver_alt';
+
+ let headers = '';
+
+ if (msg.senderName || msg.senderEmailAddress) {
+ headers += `From: ${msg.senderName} <${msg.senderEmailAddress}>\n`;
+ }
+ if (msg.displayTo) {
+ headers += `To: ${msg.displayTo}\n`;
+ }
+ if (msg.displayCC) {
+ headers += `Cc: ${msg.displayCC}\n`;
+ }
+ if (msg.displayBCC) {
+ headers += `Bcc: ${msg.displayBCC}\n`;
+ }
+ if (msg.subject) {
+ headers += `Subject: ${msg.subject}\n`;
+ }
+ if (msg.clientSubmitTime) {
+ headers += `Date: ${new Date(msg.clientSubmitTime).toUTCString()}\n`;
+ }
+ if (msg.internetMessageId) {
+ headers += `Message-ID: <${msg.internetMessageId}>\n`;
+ }
+ if (msg.inReplyToId) {
+ headers += `In-Reply-To: ${msg.inReplyToId}`;
+ }
+ if (msg.conversationId) {
+ headers += `Conversation-Id: ${msg.conversationId}`;
+ }
+ headers += 'MIME-Version: 1.0\n';
+
+ console.log("headers", headers);
+ //add new headers
+ if (!/Content-Type:/i.test(headers)) {
+ if (msg.hasAttachments) {
+ headers += `Content-Type: multipart/mixed; boundary="${boundary}"\n`;
+ headers += `Content-Type: multipart/alternative; boundary="${altBoundary}"\n\n`;
+ eml += headers;
+ eml += `--${boundary}\n\n`;
+ } else {
+ eml += headers;
+ eml += `Content-Type: multipart/alternative; boundary="${altBoundary}"\n\n`;
+ }
+ }
+ // Body
+ const hasBody = !!msg.body;
+ const hasHtml = !!msg.bodyHTML;
+
+ if (hasBody) {
+ eml += `--${altBoundary}\n`;
+ eml += 'Content-Type: text/plain; charset="utf-8"\n\n';
+ eml += `${msg.body}\n\n`;
+ }
+
+ if (hasHtml) {
+ eml += `--${altBoundary}\n`;
+ eml += 'Content-Type: text/html; charset="utf-8"\n\n';
+ eml += `${msg.bodyHTML}\n\n`;
+ }
+
+ if (hasBody || hasHtml) {
+ eml += `--${altBoundary}--\n`;
+ }
+
+ if (msg.hasAttachments) {
+ for (let i = 0; i < msg.numberOfAttachments; i++) {
+ const attachment = msg.getAttachment(i);
+ const attachmentStream = attachment.fileInputStream;
+ if (attachmentStream) {
+ const attachmentBuffer = Buffer.alloc(attachment.filesize);
+ attachmentStream.readCompletely(attachmentBuffer);
+ eml += `\n--${boundary}\n`;
+ eml += `Content-Type: ${attachment.mimeTag}; name="${attachment.longFilename}"\n`;
+ eml += `Content-Disposition: attachment; filename="${attachment.longFilename}"\n`;
+ eml += 'Content-Transfer-Encoding: base64\n\n';
+ eml += `${attachmentBuffer.toString('base64')}\n`;
+ }
+ }
+ eml += `\n--${boundary}--`;
+ }
+
+ return eml;
+ }
+
+ public getUpdatedSyncState(): SyncState {
+ return {};
+ }
+}
diff --git a/packages/backend/src/services/ingestion-connectors/utils.ts b/packages/backend/src/services/ingestion-connectors/helpers/utils.ts
similarity index 82%
rename from packages/backend/src/services/ingestion-connectors/utils.ts
rename to packages/backend/src/services/ingestion-connectors/helpers/utils.ts
index a4197d2..46776d4 100644
--- a/packages/backend/src/services/ingestion-connectors/utils.ts
+++ b/packages/backend/src/services/ingestion-connectors/helpers/utils.ts
@@ -34,6 +34,15 @@ export function getThreadId(headers: Headers): string | undefined {
}
}
+ const conversationIdHeader = headers.get('conversation-id');
+
+ if (conversationIdHeader) {
+ const conversationId = getHeaderValue(conversationIdHeader);
+ if (conversationId) {
+ return conversationId.trim();
+ }
+ }
+
const messageIdHeader = headers.get('message-id');
if (messageIdHeader) {
diff --git a/packages/frontend/src/lib/api.client.ts b/packages/frontend/src/lib/api.client.ts
index f20c09a..d8732ea 100644
--- a/packages/frontend/src/lib/api.client.ts
+++ b/packages/frontend/src/lib/api.client.ts
@@ -14,9 +14,11 @@ export const api = async (
options: RequestInit = {}
): Promise => {
const { accessToken } = get(authStore);
- const defaultHeaders: HeadersInit = {
- 'Content-Type': 'application/json'
- };
+ const defaultHeaders: HeadersInit = {};
+
+ if (!(options.body instanceof FormData)) {
+ defaultHeaders['Content-Type'] = 'application/json';
+ }
if (accessToken) {
defaultHeaders['Authorization'] = `Bearer ${accessToken}`;
diff --git a/packages/frontend/src/lib/components/custom/EmailPreview.svelte b/packages/frontend/src/lib/components/custom/EmailPreview.svelte
index 6fea5b0..dc3960f 100644
--- a/packages/frontend/src/lib/components/custom/EmailPreview.svelte
+++ b/packages/frontend/src/lib/components/custom/EmailPreview.svelte
@@ -6,6 +6,7 @@
raw,
rawHtml
}: { raw?: Buffer | { type: 'Buffer'; data: number[] } | undefined; rawHtml?: string } = $props();
+
let parsedEmail: Email | null = $state(null);
let isLoading = $state(true);
diff --git a/packages/frontend/src/lib/components/custom/EmailThread.svelte b/packages/frontend/src/lib/components/custom/EmailThread.svelte
index 489df63..18d5b3f 100644
--- a/packages/frontend/src/lib/components/custom/EmailThread.svelte
+++ b/packages/frontend/src/lib/components/custom/EmailThread.svelte
@@ -19,7 +19,7 @@
{#each thread as item, i (item.id)}
{
event.preventDefault();
isSubmitting = true;
@@ -52,6 +57,45 @@
isSubmitting = false;
}
};
+
+ const handleFileChange = async (event: Event) => {
+ const target = event.target as HTMLInputElement;
+ const file = target.files?.[0];
+ fileUploading = true;
+ if (!file) {
+ fileUploading = false;
+ return;
+ }
+
+ const uploadFormData = new FormData();
+ uploadFormData.append('file', file);
+
+ try {
+ const response = await api('/upload', {
+ method: 'POST',
+ body: uploadFormData
+ });
+
+ if (!response.ok) {
+ throw new Error('File upload failed');
+ }
+
+ const result = await response.json();
+ formData.providerConfig.uploadedFilePath = result.filePath;
+ formData.providerConfig.uploadedFileName = file.name;
+ console.log(formData.providerConfig.uploadedFilePath);
+ fileUploading = false;
+ } catch (error) {
+ fileUploading = false;
+ setAlert({
+ type: 'error',
+ title: 'Upload Failed',
+ message: 'PST file upload failed. Please try again.',
+ duration: 5000,
+ show: true
+ });
+ }
+ };
+ {:else if formData.provider === 'pst_import'}
+
+
PST File
+
+
+
@@ -150,7 +204,7 @@
{/if}
-
+
{#if isSubmitting}
Submitting...
{:else}
diff --git a/packages/frontend/src/routes/dashboard/+page.svelte b/packages/frontend/src/routes/dashboard/+page.svelte
index 0f934a2..5a8640c 100644
--- a/packages/frontend/src/routes/dashboard/+page.svelte
+++ b/packages/frontend/src/routes/dashboard/+page.svelte
@@ -70,8 +70,9 @@
0}
+ class:text-green-600={data.stats.failedIngestionsLast7Days <= 0}
>
{data.stats.failedIngestionsLast7Days}
diff --git a/packages/frontend/src/routes/dashboard/ingestions/+page.svelte b/packages/frontend/src/routes/dashboard/ingestions/+page.svelte
index 433f4a9..c2157fc 100644
--- a/packages/frontend/src/routes/dashboard/ingestions/+page.svelte
+++ b/packages/frontend/src/routes/dashboard/ingestions/+page.svelte
@@ -3,9 +3,10 @@
import * as Table from '$lib/components/ui/table';
import { Button } from '$lib/components/ui/button';
import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
- import { MoreHorizontal } from 'lucide-svelte';
+ import { MoreHorizontal, Trash, RefreshCw } from 'lucide-svelte';
import * as Dialog from '$lib/components/ui/dialog';
import { Switch } from '$lib/components/ui/switch';
+ import { Checkbox } from '$lib/components/ui/checkbox';
import IngestionSourceForm from '$lib/components/custom/IngestionSourceForm.svelte';
import { api } from '$lib/api.client';
import type { IngestionSource, CreateIngestionSourceDto } from '@open-archiver/types';
@@ -20,6 +21,8 @@
let selectedSource = $state(null);
let sourceToDelete = $state(null);
let isDeleting = $state(false);
+ let selectedIds = $state([]);
+ let isBulkDeleteDialogOpen = $state(false);
const openCreateDialog = () => {
selectedSource = null;
@@ -125,6 +128,64 @@
}
};
+ const handleBulkDelete = async () => {
+ isDeleting = true;
+ try {
+ for (const id of selectedIds) {
+ const res = await api(`/ingestion-sources/${id}`, { method: 'DELETE' });
+ if (!res.ok) {
+ const errorBody = await res.json();
+ setAlert({
+ type: 'error',
+ title: `Failed to delete ingestion ${id}`,
+ message: errorBody.message || JSON.stringify(errorBody),
+ duration: 5000,
+ show: true
+ });
+ }
+ }
+ ingestionSources = ingestionSources.filter((s) => !selectedIds.includes(s.id));
+ selectedIds = [];
+ isBulkDeleteDialogOpen = false;
+ } finally {
+ isDeleting = false;
+ }
+ };
+
+ const handleBulkForceSync = async () => {
+ try {
+ for (const id of selectedIds) {
+ const res = await api(`/ingestion-sources/${id}/sync`, { method: 'POST' });
+ if (!res.ok) {
+ const errorBody = await res.json();
+ setAlert({
+ type: 'error',
+ title: `Failed to trigger force sync for ingestion ${id}`,
+ message: errorBody.message || JSON.stringify(errorBody),
+ duration: 5000,
+ show: true
+ });
+ }
+ }
+ const updatedSources = ingestionSources.map((s) => {
+ if (selectedIds.includes(s.id)) {
+ return { ...s, status: 'syncing' as const };
+ }
+ return s;
+ });
+ ingestionSources = updatedSources;
+ selectedIds = [];
+ } catch (e) {
+ setAlert({
+ type: 'error',
+ title: 'Failed to trigger force sync',
+ message: e instanceof Error ? e.message : JSON.stringify(e),
+ duration: 5000,
+ show: true
+ });
+ }
+ };
+
const handleFormSubmit = async (formData: CreateIngestionSourceDto) => {
try {
if (selectedSource) {
@@ -174,6 +235,8 @@
switch (status) {
case 'active':
return 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300';
+ case 'imported':
+ return 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300';
case 'paused':
return 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-300';
case 'error':
@@ -198,7 +261,29 @@
-
Ingestion Sources
+
+
Ingestion Sources
+ {#if selectedIds.length > 0}
+
+
+
+ Bulk Actions ({selectedIds.length})
+
+
+
+
+
+ (isBulkDeleteDialogOpen = true)}>
+
+
+
+ {/if}
+
Create New
@@ -206,6 +291,20 @@
+
+ {
+ if (checked) {
+ selectedIds = ingestionSources.map((s) => s.id);
+ } else {
+ selectedIds = [];
+ }
+ }}
+ checked={ingestionSources.length > 0 && selectedIds.length === ingestionSources.length
+ ? true
+ : ((selectedIds.length > 0 ? 'indeterminate' : false) as any)}
+ />
+
Name
Provider
Status
@@ -218,6 +317,18 @@
{#if ingestionSources.length > 0}
{#each ingestionSources as source (source.id)}
+
+ {
+ if (selectedIds.includes(source.id)) {
+ selectedIds = selectedIds.filter((id) => id !== source.id);
+ } else {
+ selectedIds = [...selectedIds, source.id];
+ }
+ }}
+ />
+
{source.name}
@@ -324,3 +435,26 @@
+
+
+
+
+ Are you sure you want to delete {selectedIds.length} selected ingestions?
+
+ This will delete all archived emails, attachments, indexing, and files associated with these
+ ingestions. If you only want to stop syncing new emails, you can pause the ingestions
+ instead.
+
+
+
+ {#if isDeleting}Deleting...{:else}Confirm{/if}
+
+ Cancel
+
+
+
+
diff --git a/packages/types/src/ingestion.types.ts b/packages/types/src/ingestion.types.ts
index 88a53c4..a00467b 100644
--- a/packages/types/src/ingestion.types.ts
+++ b/packages/types/src/ingestion.types.ts
@@ -17,7 +17,7 @@ export type SyncState = {
lastSyncTimestamp?: string;
};
-export type IngestionProvider = 'google_workspace' | 'microsoft_365' | 'generic_imap';
+export type IngestionProvider = 'google_workspace' | 'microsoft_365' | 'generic_imap' | 'pst_import';
export type IngestionStatus =
| 'active'
@@ -26,7 +26,8 @@ export type IngestionStatus =
| 'pending_auth'
| 'syncing'
| 'importing'
- | 'auth_success';
+ | 'auth_success'
+ | 'imported';
export interface BaseIngestionCredentials {
type: IngestionProvider;
@@ -61,11 +62,18 @@ export interface Microsoft365Credentials extends BaseIngestionCredentials {
tenantId: string;
}
+export interface PSTImportCredentials extends BaseIngestionCredentials {
+ type: 'pst_import';
+ uploadedFileName: string;
+ uploadedFilePath: string;
+}
+
// Discriminated union for all possible credential types
export type IngestionCredentials =
| GenericImapCredentials
| GoogleWorkspaceCredentials
- | Microsoft365Credentials;
+ | Microsoft365Credentials
+ | PSTImportCredentials;
export interface IngestionSource {
id: string;
@@ -118,6 +126,12 @@ export interface IProcessMailboxJob {
userEmail: string;
}
+export interface IPstProcessingJob {
+ ingestionSourceId: string;
+ filePath: string;
+ originalFilename: string;
+}
+
export type MailboxUser = {
id: string;
primaryEmail: string;
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index c2a522a..50d3f73 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -48,6 +48,9 @@ importers:
bullmq:
specifier: ^5.56.3
version: 5.56.3
+ busboy:
+ specifier: ^1.6.0
+ version: 1.6.0
cross-fetch:
specifier: ^4.1.0
version: 4.1.0(encoding@0.1.13)
@@ -93,6 +96,9 @@ importers:
meilisearch:
specifier: ^0.51.0
version: 0.51.0
+ multer:
+ specifier: ^2.0.2
+ version: 2.0.2
pdf2json:
specifier: ^3.1.6
version: 3.1.6
@@ -108,6 +114,9 @@ importers:
postgres:
specifier: ^3.4.7
version: 3.4.7
+ pst-extractor:
+ specifier: ^1.11.0
+ version: 1.11.0
reflect-metadata:
specifier: ^0.2.2
version: 0.2.2
@@ -127,6 +136,9 @@ importers:
'@bull-board/express':
specifier: ^6.11.0
version: 6.11.0
+ '@types/busboy':
+ specifier: ^1.5.4
+ version: 1.5.4
'@types/express':
specifier: ^5.0.3
version: 5.0.3
@@ -136,6 +148,9 @@ importers:
'@types/microsoft-graph':
specifier: ^2.40.1
version: 2.40.1
+ '@types/multer':
+ specifier: ^2.0.0
+ version: 2.0.0
'@types/node':
specifier: ^24.0.12
version: 24.0.13
@@ -1660,6 +1675,9 @@ packages:
'@types/body-parser@1.19.6':
resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==}
+ '@types/busboy@1.5.4':
+ resolution: {integrity: sha512-kG7WrUuAKK0NoyxfQHsVE6j1m01s6kMma64E+OZenQABMQyTJop1DumUWcLwAQ2JzpefU7PDYoRDKl8uZosFjw==}
+
'@types/connect@3.4.38':
resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==}
@@ -1714,6 +1732,9 @@ packages:
'@types/mime@1.3.5':
resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==}
+ '@types/multer@2.0.0':
+ resolution: {integrity: sha512-C3Z9v9Evij2yST3RSBktxP9STm6OdMc5uR1xF1SGr98uv8dUlAL2hqwrZ3GVB3uyMyiegnscEK6PGtYvNrjTjw==}
+
'@types/node@24.0.13':
resolution: {integrity: sha512-Qm9OYVOFHFYg3wJoTSrz80hoec5Lia/dPp84do3X7dZvLikQvM1YpmvTBEdIr/e+U8HTkFjLHLnl78K/qjf+jQ==}
@@ -1905,6 +1926,9 @@ packages:
resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==}
engines: {node: '>= 8'}
+ append-field@1.0.0:
+ resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==}
+
aproba@2.0.0:
resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==}
@@ -2019,6 +2043,10 @@ packages:
bullmq@5.56.3:
resolution: {integrity: sha512-03szheVTKfLsCm5EwzOjSSUTI0UIGJjTUgX91W4+a0pj6SSfiuuNzB29QJh+T3bcgUZUHuTp01Jyxa101sv0Lg==}
+ busboy@1.6.0:
+ resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==}
+ engines: {node: '>=10.16.0'}
+
bytes@3.1.0:
resolution: {integrity: sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==}
engines: {node: '>= 0.8'}
@@ -2126,6 +2154,10 @@ packages:
concat-map@0.0.1:
resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
+ concat-stream@2.0.0:
+ resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==}
+ engines: {'0': node >= 6.0}
+
concurrently@9.2.0:
resolution: {integrity: sha512-IsB/fiXTupmagMW4MNp2lx2cdSN2FfZq78vF90LBB+zZHArbIQZjQtzXCiXnvTxCZSvXanTqFLWBjw2UkLx1SQ==}
engines: {node: '>=18'}
@@ -3186,6 +3218,9 @@ packages:
lodash@4.17.21:
resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
+ long@5.3.2:
+ resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==}
+
lop@0.4.2:
resolution: {integrity: sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==}
@@ -3362,6 +3397,10 @@ packages:
mkdirp-classic@0.5.3:
resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==}
+ mkdirp@0.5.6:
+ resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==}
+ hasBin: true
+
mkdirp@1.0.4:
resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==}
engines: {node: '>=10'}
@@ -3401,6 +3440,10 @@ packages:
msgpackr@1.11.4:
resolution: {integrity: sha512-uaff7RG9VIC4jacFW9xzL3jc0iM32DNHe4jYVycBcjUePT/Klnfj7pqtWJt9khvDFizmjN2TlYniYmSS2LIaZg==}
+ multer@2.0.2:
+ resolution: {integrity: sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==}
+ engines: {node: '>= 10.16.0'}
+
nanoid@3.3.11:
resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==}
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
@@ -3480,6 +3523,10 @@ packages:
engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0}
deprecated: This package is no longer supported.
+ object-assign@4.1.1:
+ resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}
+ engines: {node: '>=0.10.0'}
+
object-inspect@1.13.4:
resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==}
engines: {node: '>= 0.4'}
@@ -3742,6 +3789,10 @@ packages:
proxy-from-env@1.1.0:
resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}
+ pst-extractor@1.11.0:
+ resolution: {integrity: sha512-y4IzdvKlXabFrbIqQiehkBok/F1+YNoNl9R4o0phamzO13g79HSLzjs/Nctz8YxHlHQ1490WP1YIlHSLtuVa/w==}
+ engines: {node: '>=10'}
+
pump@3.0.3:
resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==}
@@ -4063,6 +4114,10 @@ packages:
stream-browserify@3.0.0:
resolution: {integrity: sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==}
+ streamsearch@1.1.0:
+ resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==}
+ engines: {node: '>=10.0.0'}
+
string-width@4.2.3:
resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
engines: {node: '>=8'}
@@ -4272,6 +4327,9 @@ packages:
resolution: {integrity: sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==}
engines: {node: '>= 0.6'}
+ typedarray@0.0.6:
+ resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==}
+
typescript@5.8.3:
resolution: {integrity: sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==}
engines: {node: '>=14.17'}
@@ -4321,6 +4379,9 @@ packages:
resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==}
engines: {node: '>= 0.4.0'}
+ uuid-parse@1.1.0:
+ resolution: {integrity: sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A==}
+
uuid@8.3.2:
resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==}
hasBin: true
@@ -6150,6 +6211,10 @@ snapshots:
'@types/connect': 3.4.38
'@types/node': 24.0.13
+ '@types/busboy@1.5.4':
+ dependencies:
+ '@types/node': 24.0.13
+
'@types/connect@3.4.38':
dependencies:
'@types/node': 24.0.13
@@ -6219,6 +6284,10 @@ snapshots:
'@types/mime@1.3.5': {}
+ '@types/multer@2.0.0':
+ dependencies:
+ '@types/express': 5.0.3
+
'@types/node@24.0.13':
dependencies:
undici-types: 7.8.0
@@ -6427,6 +6496,8 @@ snapshots:
normalize-path: 3.0.0
picomatch: 2.3.1
+ append-field@1.0.0: {}
+
aproba@2.0.0:
optional: true
@@ -6577,6 +6648,10 @@ snapshots:
transitivePeerDependencies:
- supports-color
+ busboy@1.6.0:
+ dependencies:
+ streamsearch: 1.1.0
+
bytes@3.1.0: {}
bytes@3.1.2: {}
@@ -6691,6 +6766,13 @@ snapshots:
concat-map@0.0.1: {}
+ concat-stream@2.0.0:
+ dependencies:
+ buffer-from: 1.1.2
+ inherits: 2.0.4
+ readable-stream: 3.6.2
+ typedarray: 0.0.6
+
concurrently@9.2.0:
dependencies:
chalk: 4.1.2
@@ -7820,6 +7902,8 @@ snapshots:
lodash@4.17.21: {}
+ long@5.3.2: {}
+
lop@0.4.2:
dependencies:
duck: 0.1.12
@@ -8027,6 +8111,10 @@ snapshots:
mkdirp-classic@0.5.3: {}
+ mkdirp@0.5.6:
+ dependencies:
+ minimist: 1.2.8
+
mkdirp@1.0.4: {}
mkdirp@3.0.1: {}
@@ -8063,6 +8151,16 @@ snapshots:
optionalDependencies:
msgpackr-extract: 3.0.3
+ multer@2.0.2:
+ dependencies:
+ append-field: 1.0.0
+ busboy: 1.6.0
+ concat-stream: 2.0.0
+ mkdirp: 0.5.6
+ object-assign: 4.1.1
+ type-is: 1.6.18
+ xtend: 4.0.2
+
nanoid@3.3.11: {}
napi-build-utils@2.0.0: {}
@@ -8137,6 +8235,8 @@ snapshots:
set-blocking: 2.0.0
optional: true
+ object-assign@4.1.1: {}
+
object-inspect@1.13.4: {}
on-exit-leak-free@2.1.2: {}
@@ -8340,6 +8440,12 @@ snapshots:
proxy-from-env@1.1.0: {}
+ pst-extractor@1.11.0:
+ dependencies:
+ iconv-lite: 0.6.3
+ long: 5.3.2
+ uuid-parse: 1.1.0
+
pump@3.0.3:
dependencies:
end-of-stream: 1.4.5
@@ -8734,6 +8840,8 @@ snapshots:
inherits: 2.0.4
readable-stream: 3.6.2
+ streamsearch@1.1.0: {}
+
string-width@4.2.3:
dependencies:
emoji-regex: 8.0.0
@@ -8978,6 +9086,8 @@ snapshots:
media-typer: 1.1.0
mime-types: 3.0.1
+ typedarray@0.0.6: {}
+
typescript@5.8.3: {}
uc.micro@2.1.0: {}
@@ -9027,6 +9137,8 @@ snapshots:
utils-merge@1.0.1: {}
+ uuid-parse@1.1.0: {}
+
uuid@8.3.2: {}
uuid@9.0.1: {}
From a87000f9dcce947f8ee2fc29501f2c3b150e2cfc Mon Sep 17 00:00:00 2001
From: Wayne <5291640+ringoinca@users.noreply.github.com>
Date: Fri, 8 Aug 2025 13:20:33 +0300
Subject: [PATCH 6/9] PST Import improvement
---
docs/api/ingestion.md | 16 ++++++++--------
.../src/api/controllers/upload.controller.ts | 4 +++-
packages/backend/src/config/storage.ts | 4 +++-
.../jobs/processors/initial-import.processor.ts | 13 -------------
.../backend/src/services/IngestionService.ts | 10 +++++++---
.../ingestion-connectors/PSTConnector.ts | 12 ++++++++----
.../dashboard/archived-emails/[id]/+page.svelte | 2 +-
.../src/routes/dashboard/ingestions/+page.svelte | 2 +-
packages/types/src/storage.types.ts | 2 ++
9 files changed, 33 insertions(+), 32 deletions(-)
diff --git a/docs/api/ingestion.md b/docs/api/ingestion.md
index 9b81f67..247d4b2 100644
--- a/docs/api/ingestion.md
+++ b/docs/api/ingestion.md
@@ -6,7 +6,7 @@ The Ingestion Service manages ingestion sources, which are configurations for co
All endpoints in this service require authentication.
-### POST /api/v1/ingestion
+### POST /api/v1/ingestion-sources
Creates a new ingestion source.
@@ -29,7 +29,7 @@ interface CreateIngestionSourceDto {
- **201 Created:** The newly created ingestion source.
- **500 Internal Server Error:** An unexpected error occurred.
-### GET /api/v1/ingestion
+### GET /api/v1/ingestion-sources
Retrieves all ingestion sources.
@@ -40,7 +40,7 @@ Retrieves all ingestion sources.
- **200 OK:** An array of ingestion source objects.
- **500 Internal Server Error:** An unexpected error occurred.
-### GET /api/v1/ingestion/:id
+### GET /api/v1/ingestion-sources/:id
Retrieves a single ingestion source by its ID.
@@ -58,7 +58,7 @@ Retrieves a single ingestion source by its ID.
- **404 Not Found:** Ingestion source not found.
- **500 Internal Server Error:** An unexpected error occurred.
-### PUT /api/v1/ingestion/:id
+### PUT /api/v1/ingestion-sources/:id
Updates an existing ingestion source.
@@ -95,7 +95,7 @@ interface UpdateIngestionSourceDto {
- **404 Not Found:** Ingestion source not found.
- **500 Internal Server Error:** An unexpected error occurred.
-### DELETE /api/v1/ingestion/:id
+### DELETE /api/v1/ingestion-sources/:id
Deletes an ingestion source and all associated data.
@@ -113,7 +113,7 @@ Deletes an ingestion source and all associated data.
- **404 Not Found:** Ingestion source not found.
- **500 Internal Server Error:** An unexpected error occurred.
-### POST /api/v1/ingestion/:id/import
+### POST /api/v1/ingestion-sources/:id/import
Triggers the initial import process for an ingestion source.
@@ -131,7 +131,7 @@ Triggers the initial import process for an ingestion source.
- **404 Not Found:** Ingestion source not found.
- **500 Internal Server Error:** An unexpected error occurred.
-### POST /api/v1/ingestion/:id/pause
+### POST /api/v1/ingestion-sources/:id/pause
Pauses an active ingestion source.
@@ -149,7 +149,7 @@ Pauses an active ingestion source.
- **404 Not Found:** Ingestion source not found.
- **500 Internal Server Error:** An unexpected error occurred.
-### POST /api/v1/ingestion/:id/sync
+### POST /api/v1/ingestion-sources/:id/sync
Triggers a forced synchronization for an ingestion source.
diff --git a/packages/backend/src/api/controllers/upload.controller.ts b/packages/backend/src/api/controllers/upload.controller.ts
index b682f3d..9be0144 100644
--- a/packages/backend/src/api/controllers/upload.controller.ts
+++ b/packages/backend/src/api/controllers/upload.controller.ts
@@ -2,6 +2,8 @@ import { Request, Response } from 'express';
import { StorageService } from '../../services/StorageService';
import { randomUUID } from 'crypto';
import busboy from 'busboy';
+import { config } from '../../config/index';
+
export const uploadFile = async (req: Request, res: Response) => {
const storage = new StorageService();
@@ -12,7 +14,7 @@ export const uploadFile = async (req: Request, res: Response) => {
bb.on('file', (fieldname, file, filename) => {
originalFilename = filename.filename;
const uuid = randomUUID();
- filePath = `temp/${uuid}-${originalFilename}`;
+ filePath = `${config.storage.openArchiverFolderName}/tmp/${uuid}-${originalFilename}`;
storage.put(filePath, file);
});
diff --git a/packages/backend/src/config/storage.ts b/packages/backend/src/config/storage.ts
index 9d978d9..656faef 100644
--- a/packages/backend/src/config/storage.ts
+++ b/packages/backend/src/config/storage.ts
@@ -2,7 +2,7 @@ import { StorageConfig } from '@open-archiver/types';
import 'dotenv/config';
const storageType = process.env.STORAGE_TYPE;
-
+const openArchiverFolderName = 'open-archiver';
let storageConfig: StorageConfig;
if (storageType === 'local') {
@@ -12,6 +12,7 @@ if (storageType === 'local') {
storageConfig = {
type: 'local',
rootPath: process.env.STORAGE_LOCAL_ROOT_PATH,
+ openArchiverFolderName: openArchiverFolderName
};
} else if (storageType === 's3') {
if (
@@ -30,6 +31,7 @@ if (storageType === 'local') {
secretAccessKey: process.env.STORAGE_S3_SECRET_ACCESS_KEY,
region: process.env.STORAGE_S3_REGION,
forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true',
+ openArchiverFolderName: openArchiverFolderName
};
} else {
throw new Error(`Invalid STORAGE_TYPE: ${storageType}`);
diff --git a/packages/backend/src/jobs/processors/initial-import.processor.ts b/packages/backend/src/jobs/processors/initial-import.processor.ts
index ed31a72..ac9e5de 100644
--- a/packages/backend/src/jobs/processors/initial-import.processor.ts
+++ b/packages/backend/src/jobs/processors/initial-import.processor.ts
@@ -75,19 +75,6 @@ export default async (job: Job) => {
lastSyncStatusMessage: 'Initial import complete. No users found.'
});
}
- // } else {
- // // For other providers, we might trigger a simpler bulk import directly
- // await new IngestionService().performBulkImport(job.data);
- // await flowProducer.add({
- // name: 'sync-cycle-finished',
- // queueName: 'ingestion',
- // data: {
- // ingestionSourceId,
- // userCount: 1,
- // isInitialImport: true
- // }
- // });
- // }
logger.info({ ingestionSourceId }, 'Finished initial import master job');
} catch (error) {
diff --git a/packages/backend/src/services/IngestionService.ts b/packages/backend/src/services/IngestionService.ts
index 3e6d71b..d9776e4 100644
--- a/packages/backend/src/services/IngestionService.ts
+++ b/packages/backend/src/services/IngestionService.ts
@@ -19,6 +19,7 @@ import { logger } from '../config/logger';
import { IndexingService } from './IndexingService';
import { SearchService } from './SearchService';
import { DatabaseService } from './DatabaseService';
+import { config } from '../config/index';
export class IngestionService {
@@ -136,9 +137,12 @@ export class IngestionService {
// Delete all emails and attachments from storage
const storage = new StorageService();
- const emailPath = `open-archiver/${source.name.replaceAll(' ', '-')}-${source.id}/`;
+ const emailPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/`;
await storage.delete(emailPath);
+ if (source.credentials.type === 'pst_import' && source.credentials.uploadedFilePath && await storage.exists(source.credentials.uploadedFilePath)) {
+ await storage.delete(source.credentials.uploadedFilePath);
+ }
// Delete all emails from the database
// NOTE: This is done by database CASADE, change when CASADE relation no longer exists.
@@ -289,7 +293,7 @@ export class IngestionService {
console.log('processing email, ', email.id, email.subject);
const emlBuffer = email.eml ?? Buffer.from(email.body, 'utf-8');
const emailHash = createHash('sha256').update(emlBuffer).digest('hex');
- const emailPath = `open-archiver/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${email.id}.eml`;
+ const emailPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${email.id}.eml`;
await storage.put(emailPath, emlBuffer);
const [archivedEmail] = await db
@@ -319,7 +323,7 @@ export class IngestionService {
for (const attachment of email.attachments) {
const attachmentBuffer = attachment.content;
const attachmentHash = createHash('sha256').update(attachmentBuffer).digest('hex');
- const attachmentPath = `open-archiver/${source.name.replaceAll(' ', '-')}-${source.id}/attachments/${attachment.filename}`;
+ const attachmentPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/attachments/${attachment.filename}`;
await storage.put(attachmentPath, attachmentBuffer);
const [newAttachment] = await db
diff --git a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts
index 93ec9a5..a60b659 100644
--- a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts
+++ b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts
@@ -114,6 +114,11 @@ export class PSTConnector implements IEmailConnector {
if (!this.credentials.uploadedFilePath.includes('.pst')) {
throw Error("Provided file is not in the PST format.");
}
+ const fileExist = await this.storage.exists(this.credentials.uploadedFilePath);
+ if (!fileExist) {
+ throw Error("PST file upload not finished yet, please wait.");
+ }
+
return true;
} catch (error) {
logger.error({ error, credentials: this.credentials }, 'PST file validation failed.');
@@ -131,7 +136,7 @@ export class PSTConnector implements IEmailConnector {
try {
pstFile = await this.loadPstFile();
const root = pstFile.getRootFolder();
- const displayName = root.displayName || pstFile.pstFilename;
+ const displayName: string = root.displayName || pstFile.pstFilename || String(new Date().getTime());
logger.info(`Found potential mailbox: ${displayName}`);
const constructedPrimaryEmail = `${displayName.replace(/ /g, '.').toLowerCase()}@pst.local`;
yield {
@@ -141,7 +146,7 @@ export class PSTConnector implements IEmailConnector {
displayName: displayName,
};
} catch (error) {
- logger.error({ error }, 'Failed to list users from PST file using top-level folder strategy.');
+ logger.error({ error }, 'Failed to list users from PST file.');
pstFile?.close();
throw error;
} finally {
@@ -156,7 +161,7 @@ export class PSTConnector implements IEmailConnector {
const root = pstFile.getRootFolder();
yield* this.processFolder(root);
} catch (error) {
- logger.error({ error }, 'Failed to list users from PST file using top-level folder strategy.');
+ logger.error({ error }, 'Failed to fetch email.');
pstFile?.close();
throw error;
}
@@ -271,7 +276,6 @@ export class PSTConnector implements IEmailConnector {
}
headers += 'MIME-Version: 1.0\n';
- console.log("headers", headers);
//add new headers
if (!/Content-Type:/i.test(headers)) {
if (msg.hasAttachments) {
diff --git a/packages/frontend/src/routes/dashboard/archived-emails/[id]/+page.svelte b/packages/frontend/src/routes/dashboard/archived-emails/[id]/+page.svelte
index 684db3b..e4203a8 100644
--- a/packages/frontend/src/routes/dashboard/archived-emails/[id]/+page.svelte
+++ b/packages/frontend/src/routes/dashboard/archived-emails/[id]/+page.svelte
@@ -52,7 +52,7 @@
Recipients
-
To: {email.recipients.map((r) => r.email).join(', ')}
+
To: {email.recipients.map((r) => r.email || r.name).join(', ')}
Email Preview
diff --git a/packages/frontend/src/routes/dashboard/ingestions/+page.svelte b/packages/frontend/src/routes/dashboard/ingestions/+page.svelte
index c2157fc..5a9e2e2 100644
--- a/packages/frontend/src/routes/dashboard/ingestions/+page.svelte
+++ b/packages/frontend/src/routes/dashboard/ingestions/+page.svelte
@@ -387,7 +387,7 @@
{/each}
{:else}
- No ingestion sources found.
+
{/if}
diff --git a/packages/types/src/storage.types.ts b/packages/types/src/storage.types.ts
index bdde23e..7f0401f 100644
--- a/packages/types/src/storage.types.ts
+++ b/packages/types/src/storage.types.ts
@@ -44,6 +44,7 @@ export interface LocalStorageConfig {
type: 'local';
// The absolute root path on the server where the archive will be stored.
rootPath: string;
+ openArchiverFolderName: string;
}
/**
@@ -64,6 +65,7 @@ export interface S3StorageConfig {
region?: string;
// Force path-style addressing, required for MinIO.
forcePathStyle?: boolean;
+ openArchiverFolderName: string;
}
export type StorageConfig = LocalStorageConfig | S3StorageConfig;
From 29db34c5d85854d68b60b3079d1b7ccb95701c04 Mon Sep 17 00:00:00 2001
From: Wayne <5291640+ringoinca@users.noreply.github.com>
Date: Fri, 8 Aug 2025 14:56:21 +0300
Subject: [PATCH 7/9] Imap batch process, handle rate limits gracefully
---
.../sync-cycle-finished.processor.ts | 9 +-
.../ingestion-connectors/ImapConnector.ts | 120 +++++++++++++-----
packages/types/src/ingestion.types.ts | 1 +
3 files changed, 94 insertions(+), 36 deletions(-)
diff --git a/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts b/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts
index e40910c..d032e12 100644
--- a/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts
+++ b/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts
@@ -48,12 +48,19 @@ export default async (job: Job
) => {
}
let message: string;
+ // Check for a specific rate-limit message from the successful jobs
+ const rateLimitMessage = successfulJobs.find(j => j.statusMessage)?.statusMessage;
+
if (failedJobs.length > 0) {
status = 'error';
const errorMessages = failedJobs.map(j => j.message).join('\n');
message = `Sync cycle completed with ${failedJobs.length} error(s):\n${errorMessages}`;
logger.error({ ingestionSourceId, errors: errorMessages }, 'Sync cycle finished with errors.');
- } else {
+ } else if (rateLimitMessage) {
+ message = rateLimitMessage;
+ logger.warn({ ingestionSourceId, message }, 'Sync cycle paused due to rate limiting.');
+ }
+ else {
message = 'Continuous sync cycle finished successfully.';
if (isInitialImport) {
message = `Initial import finished for ${userCount} mailboxes.`;
diff --git a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts
index 57b558f..1f93789 100644
--- a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts
+++ b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts
@@ -9,9 +9,14 @@ export class ImapConnector implements IEmailConnector {
private client: ImapFlow;
private newMaxUids: { [mailboxPath: string]: number; } = {};
private isConnected = false;
+ private statusMessage: string | undefined;
constructor(private credentials: GenericImapCredentials) {
- this.client = new ImapFlow({
+ this.client = this.createClient();
+ }
+
+ private createClient(): ImapFlow {
+ const client = new ImapFlow({
host: this.credentials.host,
port: this.credentials.port,
secure: this.credentials.secure,
@@ -23,10 +28,12 @@ export class ImapConnector implements IEmailConnector {
});
// Handles client-level errors, like unexpected disconnects, to prevent crashes.
- this.client.on('error', (err) => {
+ client.on('error', (err) => {
logger.error({ err }, 'IMAP client error');
this.isConnected = false;
});
+
+ return client;
}
/**
@@ -36,6 +43,12 @@ export class ImapConnector implements IEmailConnector {
if (this.isConnected && this.client.usable) {
return;
}
+
+ // If the client is not usable (e.g., after a logout), create a new one.
+ if (!this.client.usable) {
+ this.client = this.createClient();
+ }
+
try {
await this.client.connect();
this.isConnected = true;
@@ -100,7 +113,7 @@ export class ImapConnector implements IEmailConnector {
* @param maxRetries The maximum number of retries.
* @returns The result of the action.
*/
- private async withRetry(action: () => Promise, maxRetries = 3): Promise {
+ private async withRetry(action: () => Promise, maxRetries = 5): Promise {
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
await this.connect();
@@ -113,7 +126,10 @@ export class ImapConnector implements IEmailConnector {
throw err;
}
// Wait for a short period before retrying
- await new Promise(resolve => setTimeout(resolve, 1000 * attempt));
+ const delay = Math.pow(2, attempt) * 1000;
+ const jitter = Math.random() * 1000;
+ logger.info(`Retrying in ${Math.round((delay + jitter) / 1000)}s`);
+ await new Promise(resolve => setTimeout(resolve, delay + jitter));
}
}
// This line should be unreachable
@@ -121,28 +137,32 @@ export class ImapConnector implements IEmailConnector {
}
public async *fetchEmails(userEmail: string, syncState?: SyncState | null): AsyncGenerator {
- try {
- const mailboxes = await this.withRetry(() => this.client.list());
- // console.log('fetched mailboxes:', mailboxes);
- const processableMailboxes = mailboxes.filter(mailbox => {
- // filter out trash and all mail emails
- if (mailbox.specialUse) {
- const specialUse = mailbox.specialUse.toLowerCase();
- if (specialUse === '\\junk' || specialUse === '\\trash' || specialUse === '\\all') {
- return false;
- }
- }
- // Fallback to checking flags
- if (mailbox.flags.has('\\Noselect') || mailbox.flags.has('\\Trash') || mailbox.flags.has('\\Junk') || mailbox.flags.has('\\All')) {
+ // list all mailboxes first
+ const mailboxes = await this.withRetry(async () => await this.client.list());
+ await this.disconnect();
+
+ const processableMailboxes = mailboxes.filter(mailbox => {
+ // filter out trash and all mail emails
+ if (mailbox.specialUse) {
+ const specialUse = mailbox.specialUse.toLowerCase();
+ if (specialUse === '\\junk' || specialUse === '\\trash' || specialUse === '\\all') {
return false;
}
+ }
+ // Fallback to checking flags
+ if (mailbox.flags.has('\\Noselect') || mailbox.flags.has('\\Trash') || mailbox.flags.has('\\Junk') || mailbox.flags.has('\\All')) {
+ return false;
+ }
- return true;
- });
+ return true;
+ });
- for (const mailboxInfo of processableMailboxes) {
- const mailboxPath = mailboxInfo.path;
- const mailbox = await this.withRetry(() => this.client.mailboxOpen(mailboxPath));
+ for (const mailboxInfo of processableMailboxes) {
+ const mailboxPath = mailboxInfo.path;
+ logger.info({ mailboxPath }, 'Processing mailbox');
+
+ try {
+ const mailbox = await this.withRetry(async () => await this.client.mailboxOpen(mailboxPath));
const lastUid = syncState?.imap?.[mailboxPath]?.maxUid;
let currentMaxUid = lastUid || 0;
@@ -154,27 +174,51 @@ export class ImapConnector implements IEmailConnector {
}
this.newMaxUids[mailboxPath] = currentMaxUid;
- const searchCriteria = lastUid ? { uid: `${lastUid + 1}:*` } : { all: true };
-
// Only fetch if the mailbox has messages, to avoid errors on empty mailboxes with some IMAP servers.
if (mailbox.exists > 0) {
- for await (const msg of this.client.fetch(searchCriteria, { envelope: true, source: true, bodyStructure: true, uid: true })) {
- if (lastUid && msg.uid <= lastUid) {
- continue;
+ const BATCH_SIZE = 250; // A configurable batch size
+ let startUid = (lastUid || 0) + 1;
+
+ while (true) {
+ const endUid = startUid + BATCH_SIZE - 1;
+ const searchCriteria = { uid: `${startUid}:${endUid}` };
+ let messagesInBatch = 0;
+
+ for await (const msg of this.client.fetch(searchCriteria, { envelope: true, source: true, bodyStructure: true, uid: true })) {
+ messagesInBatch++;
+
+ if (lastUid && msg.uid <= lastUid) {
+ continue;
+ }
+
+ if (msg.uid > this.newMaxUids[mailboxPath]) {
+ this.newMaxUids[mailboxPath] = msg.uid;
+ }
+
+ if (msg.envelope && msg.source) {
+ yield await this.parseMessage(msg);
+ }
}
- if (msg.uid > this.newMaxUids[mailboxPath]) {
- this.newMaxUids[mailboxPath] = msg.uid;
+ // If this batch was smaller than the batch size, we've reached the end
+ if (messagesInBatch < BATCH_SIZE) {
+ break;
}
- if (msg.envelope && msg.source) {
- yield await this.parseMessage(msg);
- }
+ // Move to the next batch
+ startUid = endUid + 1;
}
}
+ } catch (err: any) {
+ logger.error({ err, mailboxPath }, 'Failed to process mailbox');
+ // Check if the error indicates a persistent failure after retries
+ if (err.message.includes('IMAP operation failed after all retries')) {
+ this.statusMessage = 'Sync paused due to reaching the mail server rate limit. The process will automatically resume later.';
+ }
+ }
+ finally {
+ await this.disconnect();
}
- } finally {
- await this.disconnect();
}
}
@@ -217,8 +261,14 @@ export class ImapConnector implements IEmailConnector {
for (const [path, uid] of Object.entries(this.newMaxUids)) {
imapSyncState[path] = { maxUid: uid };
}
- return {
+ const syncState: SyncState = {
imap: imapSyncState
};
+
+ if (this.statusMessage) {
+ syncState.statusMessage = this.statusMessage;
+ }
+
+ return syncState;
}
}
diff --git a/packages/types/src/ingestion.types.ts b/packages/types/src/ingestion.types.ts
index a00467b..1b8c7a6 100644
--- a/packages/types/src/ingestion.types.ts
+++ b/packages/types/src/ingestion.types.ts
@@ -15,6 +15,7 @@ export type SyncState = {
};
};
lastSyncTimestamp?: string;
+ statusMessage?: string;
};
export type IngestionProvider = 'google_workspace' | 'microsoft_365' | 'generic_imap' | 'pst_import';
From 512f0312ba5b6fce1cf1f64b7565eb5c3d63b3fc Mon Sep 17 00:00:00 2001
From: Wayne <5291640+ringoinca@users.noreply.github.com>
Date: Sat, 9 Aug 2025 16:40:25 +0300
Subject: [PATCH 8/9] Preserve original email path and tags. Emails with path
now will be stored within their path
---
.../migrations/0013_classy_talkback.sql | 2 +
.../migrations/meta/0013_snapshot.json | 1107 +++++++++++++++++
.../database/migrations/meta/_journal.json | 7 +
.../src/database/schema/archived-emails.ts | 2 +
.../src/services/ArchivedEmailService.ts | 8 +-
.../backend/src/services/IngestionService.ts | 6 +-
.../GoogleWorkspaceConnector.ts | 51 +-
.../ingestion-connectors/ImapConnector.ts | 7 +-
.../MicrosoftConnector.ts | 43 +-
.../ingestion-connectors/PSTConnector.ts | 15 +-
packages/frontend/src/app.css | 4 +
.../dashboard/archived-emails/+page.svelte | 19 +-
.../archived-emails/[id]/+page.svelte | 34 +-
.../routes/dashboard/ingestions/+page.svelte | 4 +-
packages/types/src/archived-emails.types.ts | 2 +
packages/types/src/email.types.ts | 4 +
16 files changed, 1276 insertions(+), 39 deletions(-)
create mode 100644 packages/backend/src/database/migrations/0013_classy_talkback.sql
create mode 100644 packages/backend/src/database/migrations/meta/0013_snapshot.json
diff --git a/packages/backend/src/database/migrations/0013_classy_talkback.sql b/packages/backend/src/database/migrations/0013_classy_talkback.sql
new file mode 100644
index 0000000..6b7fbfe
--- /dev/null
+++ b/packages/backend/src/database/migrations/0013_classy_talkback.sql
@@ -0,0 +1,2 @@
+ALTER TABLE "archived_emails" ADD COLUMN "path" text;--> statement-breakpoint
+ALTER TABLE "archived_emails" ADD COLUMN "tags" jsonb;
\ No newline at end of file
diff --git a/packages/backend/src/database/migrations/meta/0013_snapshot.json b/packages/backend/src/database/migrations/meta/0013_snapshot.json
new file mode 100644
index 0000000..701cbde
--- /dev/null
+++ b/packages/backend/src/database/migrations/meta/0013_snapshot.json
@@ -0,0 +1,1107 @@
+{
+ "id": "c397c819-e69f-42c7-966c-7b2969741c56",
+ "prevId": "02ed9805-d480-483a-b73c-5e03a0e526b7",
+ "version": "7",
+ "dialect": "postgresql",
+ "tables": {
+ "public.archived_emails": {
+ "name": "archived_emails",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "thread_id": {
+ "name": "thread_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "ingestion_source_id": {
+ "name": "ingestion_source_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "user_email": {
+ "name": "user_email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "message_id_header": {
+ "name": "message_id_header",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sent_at": {
+ "name": "sent_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "subject": {
+ "name": "subject",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sender_name": {
+ "name": "sender_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sender_email": {
+ "name": "sender_email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "recipients": {
+ "name": "recipients",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "storage_path": {
+ "name": "storage_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "storage_hash_sha256": {
+ "name": "storage_hash_sha256",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "size_bytes": {
+ "name": "size_bytes",
+ "type": "bigint",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "is_indexed": {
+ "name": "is_indexed",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "has_attachments": {
+ "name": "has_attachments",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "is_on_legal_hold": {
+ "name": "is_on_legal_hold",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "archived_at": {
+ "name": "archived_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "path": {
+ "name": "path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "tags": {
+ "name": "tags",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "thread_id_idx": {
+ "name": "thread_id_idx",
+ "columns": [
+ {
+ "expression": "thread_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "archived_emails_ingestion_source_id_ingestion_sources_id_fk": {
+ "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
+ "tableFrom": "archived_emails",
+ "tableTo": "ingestion_sources",
+ "columnsFrom": [
+ "ingestion_source_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.attachments": {
+ "name": "attachments",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "filename": {
+ "name": "filename",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "mime_type": {
+ "name": "mime_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "size_bytes": {
+ "name": "size_bytes",
+ "type": "bigint",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "content_hash_sha256": {
+ "name": "content_hash_sha256",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "storage_path": {
+ "name": "storage_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "attachments_content_hash_sha256_unique": {
+ "name": "attachments_content_hash_sha256_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "content_hash_sha256"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.email_attachments": {
+ "name": "email_attachments",
+ "schema": "",
+ "columns": {
+ "email_id": {
+ "name": "email_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "attachment_id": {
+ "name": "attachment_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "email_attachments_email_id_archived_emails_id_fk": {
+ "name": "email_attachments_email_id_archived_emails_id_fk",
+ "tableFrom": "email_attachments",
+ "tableTo": "archived_emails",
+ "columnsFrom": [
+ "email_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "email_attachments_attachment_id_attachments_id_fk": {
+ "name": "email_attachments_attachment_id_attachments_id_fk",
+ "tableFrom": "email_attachments",
+ "tableTo": "attachments",
+ "columnsFrom": [
+ "attachment_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "restrict",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "email_attachments_email_id_attachment_id_pk": {
+ "name": "email_attachments_email_id_attachment_id_pk",
+ "columns": [
+ "email_id",
+ "attachment_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.audit_logs": {
+ "name": "audit_logs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "bigserial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "timestamp": {
+ "name": "timestamp",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "actor_identifier": {
+ "name": "actor_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "action": {
+ "name": "action",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "target_type": {
+ "name": "target_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "target_id": {
+ "name": "target_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "details": {
+ "name": "details",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "is_tamper_evident": {
+ "name": "is_tamper_evident",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": false,
+ "default": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.ediscovery_cases": {
+ "name": "ediscovery_cases",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'open'"
+ },
+ "created_by_identifier": {
+ "name": "created_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "ediscovery_cases_name_unique": {
+ "name": "ediscovery_cases_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.export_jobs": {
+ "name": "export_jobs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "case_id": {
+ "name": "case_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "format": {
+ "name": "format",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending'"
+ },
+ "query": {
+ "name": "query",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "file_path": {
+ "name": "file_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_by_identifier": {
+ "name": "created_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "export_jobs_case_id_ediscovery_cases_id_fk": {
+ "name": "export_jobs_case_id_ediscovery_cases_id_fk",
+ "tableFrom": "export_jobs",
+ "tableTo": "ediscovery_cases",
+ "columnsFrom": [
+ "case_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.legal_holds": {
+ "name": "legal_holds",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "case_id": {
+ "name": "case_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "custodian_id": {
+ "name": "custodian_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "hold_criteria": {
+ "name": "hold_criteria",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "reason": {
+ "name": "reason",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "applied_by_identifier": {
+ "name": "applied_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "applied_at": {
+ "name": "applied_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "removed_at": {
+ "name": "removed_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "legal_holds_case_id_ediscovery_cases_id_fk": {
+ "name": "legal_holds_case_id_ediscovery_cases_id_fk",
+ "tableFrom": "legal_holds",
+ "tableTo": "ediscovery_cases",
+ "columnsFrom": [
+ "case_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "legal_holds_custodian_id_custodians_id_fk": {
+ "name": "legal_holds_custodian_id_custodians_id_fk",
+ "tableFrom": "legal_holds",
+ "tableTo": "custodians",
+ "columnsFrom": [
+ "custodian_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.retention_policies": {
+ "name": "retention_policies",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "priority": {
+ "name": "priority",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "retention_period_days": {
+ "name": "retention_period_days",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "action_on_expiry": {
+ "name": "action_on_expiry",
+ "type": "retention_action",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "is_enabled": {
+ "name": "is_enabled",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": true
+ },
+ "conditions": {
+ "name": "conditions",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "retention_policies_name_unique": {
+ "name": "retention_policies_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.custodians": {
+ "name": "custodians",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "display_name": {
+ "name": "display_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "source_type": {
+ "name": "source_type",
+ "type": "ingestion_provider",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "custodians_email_unique": {
+ "name": "custodians_email_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "email"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.ingestion_sources": {
+ "name": "ingestion_sources",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "provider": {
+ "name": "provider",
+ "type": "ingestion_provider",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "credentials": {
+ "name": "credentials",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "ingestion_status",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending_auth'"
+ },
+ "last_sync_started_at": {
+ "name": "last_sync_started_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_sync_finished_at": {
+ "name": "last_sync_finished_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_sync_status_message": {
+ "name": "last_sync_status_message",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sync_state": {
+ "name": "sync_state",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.roles": {
+ "name": "roles",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "policies": {
+ "name": "policies",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'[]'::jsonb"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "roles_name_unique": {
+ "name": "roles_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.sessions": {
+ "name": "sessions",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "sessions_user_id_users_id_fk": {
+ "name": "sessions_user_id_users_id_fk",
+ "tableFrom": "sessions",
+ "tableTo": "users",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.user_roles": {
+ "name": "user_roles",
+ "schema": "",
+ "columns": {
+ "user_id": {
+ "name": "user_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "role_id": {
+ "name": "role_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "user_roles_user_id_users_id_fk": {
+ "name": "user_roles_user_id_users_id_fk",
+ "tableFrom": "user_roles",
+ "tableTo": "users",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "user_roles_role_id_roles_id_fk": {
+ "name": "user_roles_role_id_roles_id_fk",
+ "tableFrom": "user_roles",
+ "tableTo": "roles",
+ "columnsFrom": [
+ "role_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "user_roles_user_id_role_id_pk": {
+ "name": "user_roles_user_id_role_id_pk",
+ "columns": [
+ "user_id",
+ "role_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.users": {
+ "name": "users",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "first_name": {
+ "name": "first_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_name": {
+ "name": "last_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "password": {
+ "name": "password",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "provider": {
+ "name": "provider",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "'local'"
+ },
+ "provider_id": {
+ "name": "provider_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "users_email_unique": {
+ "name": "users_email_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "email"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ }
+ },
+ "enums": {
+ "public.retention_action": {
+ "name": "retention_action",
+ "schema": "public",
+ "values": [
+ "delete_permanently",
+ "notify_admin"
+ ]
+ },
+ "public.ingestion_provider": {
+ "name": "ingestion_provider",
+ "schema": "public",
+ "values": [
+ "google_workspace",
+ "microsoft_365",
+ "generic_imap",
+ "pst_import"
+ ]
+ },
+ "public.ingestion_status": {
+ "name": "ingestion_status",
+ "schema": "public",
+ "values": [
+ "active",
+ "paused",
+ "error",
+ "pending_auth",
+ "syncing",
+ "importing",
+ "auth_success",
+ "imported"
+ ]
+ }
+ },
+ "schemas": {},
+ "sequences": {},
+ "roles": {},
+ "policies": {},
+ "views": {},
+ "_meta": {
+ "columns": {},
+ "schemas": {},
+ "tables": {}
+ }
+}
\ No newline at end of file
diff --git a/packages/backend/src/database/migrations/meta/_journal.json b/packages/backend/src/database/migrations/meta/_journal.json
index 9e29ae8..2950921 100644
--- a/packages/backend/src/database/migrations/meta/_journal.json
+++ b/packages/backend/src/database/migrations/meta/_journal.json
@@ -92,6 +92,13 @@
"when": 1754476962901,
"tag": "0012_warm_the_stranger",
"breakpoints": true
+ },
+ {
+ "idx": 13,
+ "version": "7",
+ "when": 1754659373517,
+ "tag": "0013_classy_talkback",
+ "breakpoints": true
}
]
}
\ No newline at end of file
diff --git a/packages/backend/src/database/schema/archived-emails.ts b/packages/backend/src/database/schema/archived-emails.ts
index 01043fc..750848a 100644
--- a/packages/backend/src/database/schema/archived-emails.ts
+++ b/packages/backend/src/database/schema/archived-emails.ts
@@ -24,6 +24,8 @@ export const archivedEmails = pgTable(
hasAttachments: boolean('has_attachments').notNull().default(false),
isOnLegalHold: boolean('is_on_legal_hold').notNull().default(false),
archivedAt: timestamp('archived_at', { withTimezone: true }).notNull().defaultNow(),
+ path: text('path'),
+ tags: jsonb('tags'),
},
(table) => [index('thread_id_idx').on(table.threadId)]
);
diff --git a/packages/backend/src/services/ArchivedEmailService.ts b/packages/backend/src/services/ArchivedEmailService.ts
index a66c976..0e67bb6 100644
--- a/packages/backend/src/services/ArchivedEmailService.ts
+++ b/packages/backend/src/services/ArchivedEmailService.ts
@@ -59,7 +59,9 @@ export class ArchivedEmailService {
return {
items: items.map((item) => ({
...item,
- recipients: this.mapRecipients(item.recipients)
+ recipients: this.mapRecipients(item.recipients),
+ tags: (item.tags as string[] | null) || null,
+ path: item.path || null
})),
total: total.count,
page,
@@ -103,7 +105,9 @@ export class ArchivedEmailService {
...email,
recipients: this.mapRecipients(email.recipients),
raw,
- thread: threadEmails
+ thread: threadEmails,
+ tags: (email.tags as string[] | null) || null,
+ path: email.path || null
};
if (email.hasAttachments) {
diff --git a/packages/backend/src/services/IngestionService.ts b/packages/backend/src/services/IngestionService.ts
index d9776e4..d1f2bac 100644
--- a/packages/backend/src/services/IngestionService.ts
+++ b/packages/backend/src/services/IngestionService.ts
@@ -293,7 +293,7 @@ export class IngestionService {
console.log('processing email, ', email.id, email.subject);
const emlBuffer = email.eml ?? Buffer.from(email.body, 'utf-8');
const emailHash = createHash('sha256').update(emlBuffer).digest('hex');
- const emailPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${email.id}.eml`;
+ const emailPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${email.path ? `${email.path}/` : ''}${email.id}.eml`;
await storage.put(emailPath, emlBuffer);
const [archivedEmail] = await db
@@ -315,7 +315,9 @@ export class IngestionService {
storagePath: emailPath,
storageHashSha256: emailHash,
sizeBytes: emlBuffer.length,
- hasAttachments: email.attachments.length > 0
+ hasAttachments: email.attachments.length > 0,
+ path: email.path,
+ tags: email.tags
})
.returning();
diff --git a/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts b/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts
index b9fde23..013d620 100644
--- a/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts
+++ b/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts
@@ -168,9 +168,18 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
for (const messageAdded of historyRecord.messagesAdded) {
if (messageAdded.message?.id) {
try {
+ const messageId = messageAdded.message.id;
+ const metadataResponse = await gmail.users.messages.get({
+ userId: userEmail,
+ id: messageId,
+ format: 'METADATA',
+ fields: 'labelIds'
+ });
+ const labels = await this.getLabelDetails(gmail, userEmail, metadataResponse.data.labelIds || []);
+
const msgResponse = await gmail.users.messages.get({
userId: userEmail,
- id: messageAdded.message.id,
+ id: messageId,
format: 'RAW'
});
@@ -205,6 +214,8 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
headers: parsedEmail.headers,
attachments,
receivedAt: parsedEmail.date || new Date(),
+ path: labels.path,
+ tags: labels.tags
};
}
} catch (error: any) {
@@ -243,9 +254,18 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
for (const message of messages) {
if (message.id) {
try {
+ const messageId = message.id;
+ const metadataResponse = await gmail.users.messages.get({
+ userId: userEmail,
+ id: messageId,
+ format: 'METADATA',
+ fields: 'labelIds'
+ });
+ const labels = await this.getLabelDetails(gmail, userEmail, metadataResponse.data.labelIds || []);
+
const msgResponse = await gmail.users.messages.get({
userId: userEmail,
- id: message.id,
+ id: messageId,
format: 'RAW'
});
@@ -280,6 +300,8 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
headers: parsedEmail.headers,
attachments,
receivedAt: parsedEmail.date || new Date(),
+ path: labels.path,
+ tags: labels.tags
};
}
} catch (error: any) {
@@ -313,4 +335,29 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
}
};
}
+
+ private labelCache: Map = new Map();
+
+ private async getLabelDetails(gmail: gmail_v1.Gmail, userEmail: string, labelIds: string[]): Promise<{ path: string, tags: string[]; }> {
+ const tags: string[] = [];
+ let path = '';
+
+ for (const labelId of labelIds) {
+ let label = this.labelCache.get(labelId);
+ if (!label) {
+ const res = await gmail.users.labels.get({ userId: userEmail, id: labelId });
+ label = res.data;
+ this.labelCache.set(labelId, label);
+ }
+
+ if (label.name) {
+ tags.push(label.name);
+ if (label.type === 'user') {
+ path = path ? `${path}/${label.name}` : label.name;
+ }
+ }
+ }
+
+ return { path, tags };
+ }
}
diff --git a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts
index 1f93789..5d40bfd 100644
--- a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts
+++ b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts
@@ -196,7 +196,7 @@ export class ImapConnector implements IEmailConnector {
}
if (msg.envelope && msg.source) {
- yield await this.parseMessage(msg);
+ yield await this.parseMessage(msg, mailboxPath);
}
}
@@ -222,7 +222,7 @@ export class ImapConnector implements IEmailConnector {
}
}
- private async parseMessage(msg: any): Promise {
+ private async parseMessage(msg: any, mailboxPath: string): Promise {
const parsedEmail: ParsedMail = await simpleParser(msg.source);
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
filename: attachment.filename || 'untitled',
@@ -252,7 +252,8 @@ export class ImapConnector implements IEmailConnector {
headers: parsedEmail.headers,
attachments,
receivedAt: parsedEmail.date || new Date(),
- eml: msg.source
+ eml: msg.source,
+ path: mailboxPath
};
}
diff --git a/packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts b/packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts
index 072650a..a03eaa0 100644
--- a/packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts
+++ b/packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts
@@ -143,9 +143,9 @@ export class MicrosoftConnector implements IEmailConnector {
try {
const folders = this.listAllFolders(userEmail);
for await (const folder of folders) {
- if (folder.id) {
+ if (folder.id && folder.path) {
logger.info({ userEmail, folderId: folder.id, folderName: folder.displayName }, 'Syncing folder');
- yield* this.syncFolder(userEmail, folder.id, this.newDeltaTokens[folder.id]);
+ yield* this.syncFolder(userEmail, folder.id, folder.path, this.newDeltaTokens[folder.id]);
}
}
} catch (error) {
@@ -159,20 +159,33 @@ export class MicrosoftConnector implements IEmailConnector {
* @param userEmail The user principal name or ID.
* @returns An async generator that yields each mail folder.
*/
- private async *listAllFolders(userEmail: string): AsyncGenerator {
- let requestUrl: string | undefined = `/users/${userEmail}/mailFolders`;
+ private async *listAllFolders(userEmail: string, parentFolderId?: string, currentPath = ''): AsyncGenerator {
+ const requestUrl = parentFolderId
+ ? `/users/${userEmail}/mailFolders/${parentFolderId}/childFolders`
+ : `/users/${userEmail}/mailFolders`;
- while (requestUrl) {
- try {
- const response = await this.graphClient.api(requestUrl).get();
+ try {
+ let response = await this.graphClient.api(requestUrl).get();
+
+ while (response) {
for (const folder of response.value as MailFolder[]) {
- yield folder;
+ const newPath = currentPath ? `${currentPath}/${folder.displayName || ''}` : folder.displayName || '';
+ yield { ...folder, path: newPath || '' };
+
+ if (folder.childFolderCount && folder.childFolderCount > 0) {
+ yield* this.listAllFolders(userEmail, folder.id, newPath);
+ }
+ }
+
+ if (response['@odata.nextLink']) {
+ response = await this.graphClient.api(response['@odata.nextLink']).get();
+ } else {
+ break;
}
- requestUrl = response['@odata.nextLink'];
- } catch (error) {
- logger.error({ err: error, userEmail }, 'Failed to list mail folders');
- throw error; // Stop if we can't list folders
}
+ } catch (error) {
+ logger.error({ err: error, userEmail }, 'Failed to list mail folders');
+ throw error;
}
}
@@ -186,6 +199,7 @@ export class MicrosoftConnector implements IEmailConnector {
private async *syncFolder(
userEmail: string,
folderId: string,
+ path: string,
deltaToken?: string
): AsyncGenerator {
let requestUrl: string | undefined;
@@ -208,7 +222,7 @@ export class MicrosoftConnector implements IEmailConnector {
if (message.id && !(message)['@removed']) {
const rawEmail = await this.getRawEmail(userEmail, message.id);
if (rawEmail) {
- const emailObject = await this.parseEmail(rawEmail, message.id, userEmail);
+ const emailObject = await this.parseEmail(rawEmail, message.id, userEmail, path);
emailObject.threadId = message.conversationId; // Add conversationId as threadId
yield emailObject;
}
@@ -242,7 +256,7 @@ export class MicrosoftConnector implements IEmailConnector {
}
}
- private async parseEmail(rawEmail: Buffer, messageId: string, userEmail: string): Promise {
+ private async parseEmail(rawEmail: Buffer, messageId: string, userEmail: string, path: string): Promise {
const parsedEmail: ParsedMail = await simpleParser(rawEmail);
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
filename: attachment.filename || 'untitled',
@@ -270,6 +284,7 @@ export class MicrosoftConnector implements IEmailConnector {
headers: parsedEmail.headers,
attachments,
receivedAt: parsedEmail.date || new Date(),
+ path
};
}
diff --git a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts
index a60b659..459dde9 100644
--- a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts
+++ b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts
@@ -159,7 +159,7 @@ export class PSTConnector implements IEmailConnector {
try {
pstFile = await this.loadPstFile();
const root = pstFile.getRootFolder();
- yield* this.processFolder(root);
+ yield* this.processFolder(root, '');
} catch (error) {
logger.error({ error }, 'Failed to fetch email.');
pstFile?.close();
@@ -171,17 +171,19 @@ export class PSTConnector implements IEmailConnector {
}
}
- private async *processFolder(folder: PSTFolder): AsyncGenerator {
+ private async *processFolder(folder: PSTFolder, currentPath: string): AsyncGenerator {
const folderName = folder.displayName.toLowerCase();
if (DELETED_FOLDERS.has(folderName) || JUNK_FOLDERS.has(folderName)) {
logger.info(`Skipping folder: ${folder.displayName}`);
return;
}
+ const newPath = currentPath ? `${currentPath}/${folder.displayName}` : folder.displayName;
+
if (folder.contentCount > 0) {
let email: PSTMessage | null = folder.getNextChild();
while (email != null) {
- yield await this.parseMessage(email);
+ yield await this.parseMessage(email, newPath);
try {
email = folder.getNextChild();
} catch (error) {
@@ -193,12 +195,12 @@ export class PSTConnector implements IEmailConnector {
if (folder.hasSubfolders) {
for (const subFolder of folder.getSubFolders()) {
- yield* this.processFolder(subFolder);
+ yield* this.processFolder(subFolder, newPath);
}
}
}
- private async parseMessage(msg: PSTMessage): Promise {
+ private async parseMessage(msg: PSTMessage, path: string): Promise {
const emlContent = await this.constructEml(msg);
const emlBuffer = Buffer.from(emlContent, 'utf-8');
const parsedEmail: ParsedMail = await simpleParser(emlBuffer);
@@ -236,7 +238,8 @@ export class PSTConnector implements IEmailConnector {
headers: parsedEmail.headers,
attachments,
receivedAt: parsedEmail.date || new Date(),
- eml: emlBuffer
+ eml: emlBuffer,
+ path
};
}
diff --git a/packages/frontend/src/app.css b/packages/frontend/src/app.css
index e4022b1..7a5d9d7 100644
--- a/packages/frontend/src/app.css
+++ b/packages/frontend/src/app.css
@@ -111,6 +111,10 @@
--color-sidebar-ring: var(--sidebar-ring);
}
+.link {
+ @apply hover:text-primary font-medium hover:underline hover:underline-offset-2;
+}
+
@layer base {
* {
@apply border-border outline-ring/50;
diff --git a/packages/frontend/src/routes/dashboard/archived-emails/+page.svelte b/packages/frontend/src/routes/dashboard/archived-emails/+page.svelte
index ade6497..33fb3c7 100644
--- a/packages/frontend/src/routes/dashboard/archived-emails/+page.svelte
+++ b/packages/frontend/src/routes/dashboard/archived-emails/+page.svelte
@@ -99,28 +99,33 @@
Date
- Inbox
Subject
Sender
- Attachments
+ Inbox
+ Path
Actions
-
+
{#if archivedEmails.items.length > 0}
{#each archivedEmails.items as email (email.id)}
{new Date(email.sentAt).toLocaleString()}
- {email.userEmail}
+
- {email.senderEmail}
- {email.hasAttachments ? 'Yes' : 'No'}
+
+ {email.senderEmail || email.senderName}
+
+ {email.userEmail}
+
+ {email.path}
+
View
diff --git a/packages/frontend/src/routes/dashboard/archived-emails/[id]/+page.svelte b/packages/frontend/src/routes/dashboard/archived-emails/[id]/+page.svelte
index e4203a8..83b8d10 100644
--- a/packages/frontend/src/routes/dashboard/archived-emails/[id]/+page.svelte
+++ b/packages/frontend/src/routes/dashboard/archived-emails/[id]/+page.svelte
@@ -6,6 +6,7 @@
import EmailThread from '$lib/components/custom/EmailThread.svelte';
import { api } from '$lib/api.client';
import { browser } from '$app/environment';
+ import { formatBytes } from '$lib/utils';
let { data }: { data: PageData } = $props();
let email = $derived(data.email);
@@ -50,9 +51,38 @@
-
+
Recipients
-
To: {email.recipients.map((r) => r.email || r.name).join(', ')}
+
+ To: {email.recipients.map((r) => r.email || r.name).join(', ')}
+
+
+
+
Meta data
+
+ {#if email.path}
+
+ Folder:
+ {email.path || '/'}
+
+ {/if}
+ {#if email.tags && email.tags.length > 0}
+
+ Tags:
+ {#each email.tags as tag}
+ {tag}
+ {/each}
+
+ {/if}
+
+ size:
+ {formatBytes(email.sizeBytes)}
+
+
Email Preview
diff --git a/packages/frontend/src/routes/dashboard/ingestions/+page.svelte b/packages/frontend/src/routes/dashboard/ingestions/+page.svelte
index 5a9e2e2..5b51c9c 100644
--- a/packages/frontend/src/routes/dashboard/ingestions/+page.svelte
+++ b/packages/frontend/src/routes/dashboard/ingestions/+page.svelte
@@ -330,7 +330,9 @@
/>
- {source.name}
+ {source.name}
{source.provider.split('_').join(' ')}
diff --git a/packages/types/src/archived-emails.types.ts b/packages/types/src/archived-emails.types.ts
index 048fa56..4f4a3f0 100644
--- a/packages/types/src/archived-emails.types.ts
+++ b/packages/types/src/archived-emails.types.ts
@@ -48,6 +48,8 @@ export interface ArchivedEmail {
attachments?: Attachment[];
raw?: Buffer;
thread?: ThreadEmail[];
+ path: string | null;
+ tags: string[] | null;
}
/**
diff --git a/packages/types/src/email.types.ts b/packages/types/src/email.types.ts
index c067b36..5e64322 100644
--- a/packages/types/src/email.types.ts
+++ b/packages/types/src/email.types.ts
@@ -49,6 +49,10 @@ export interface EmailObject {
eml?: Buffer;
/** The email address of the user whose mailbox this email belongs to. */
userEmail?: string;
+ /** The folder path of the email in the source mailbox. */
+ path?: string;
+ /** An array of tags or labels associated with the email. */
+ tags?: string[];
}
// Define the structure of the document to be indexed in Meilisearch
From f10bf93d1bbfb3e1ef847221a7f111fceb05f1f9 Mon Sep 17 00:00:00 2001
From: Wayne <5291640+ringoinca@users.noreply.github.com>
Date: Mon, 11 Aug 2025 10:55:50 +0300
Subject: [PATCH 9/9] eml import support
---
README.md | 11 +-
docs/.vitepress/config.mts | 6 +-
docs/user-guides/email-providers/eml.md | 36 +
docs/user-guides/email-providers/index.md | 2 +
docs/user-guides/email-providers/pst.md | 21 +
packages/backend/package.json | 6 +-
.../database/migrations/0014_foamy_vapor.sql | 1 +
.../migrations/meta/0014_snapshot.json | 1108 +++++++++++++++++
.../database/migrations/meta/_journal.json | 7 +
.../src/database/schema/ingestion-sources.ts | 3 +-
.../processors/initial-import.processor.ts | 5 +-
.../sync-cycle-finished.processor.ts | 6 +-
.../src/services/EmailProviderFactory.ts | 4 +
.../backend/src/services/IngestionService.ts | 34 +-
.../ingestion-connectors/EMLConnector.ts | 199 +++
.../ingestion-connectors/ImapConnector.ts | 2 +-
.../custom/IngestionSourceForm.svelte | 13 +-
.../dashboard/archived-emails/+page.svelte | 4 +-
packages/types/src/ingestion.types.ts | 11 +-
pnpm-lock.yaml | 354 ++++++
20 files changed, 1800 insertions(+), 33 deletions(-)
create mode 100644 docs/user-guides/email-providers/eml.md
create mode 100644 docs/user-guides/email-providers/pst.md
create mode 100644 packages/backend/src/database/migrations/0014_foamy_vapor.sql
create mode 100644 packages/backend/src/database/migrations/meta/0014_snapshot.json
create mode 100644 packages/backend/src/services/ingestion-connectors/EMLConnector.ts
diff --git a/README.md b/README.md
index 2fbd3d6..46db7f1 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@
**A secure, sovereign, and open-source platform for email archiving and eDiscovery.**
-Open Archiver provides a robust, self-hosted solution for archiving, storing, indexing, and searching emails from major platforms, including Google Workspace (Gmail), Microsoft 365, as well as generic IMAP-enabled email inboxes. Use Open Archiver to keep a permanent, tamper-proof record of your communication history, free from vendor lock-in.
+Open Archiver provides a robust, self-hosted solution for archiving, storing, indexing, and searching emails from major platforms, including Google Workspace (Gmail), Microsoft 365, PST files, as well as generic IMAP-enabled email inboxes. Use Open Archiver to keep a permanent, tamper-proof record of your communication history, free from vendor lock-in.
## ๐ธ Screenshots
@@ -40,7 +40,14 @@ Password: openarchiver_demo
## โจ Key Features
-- **Universal Ingestion**: Connect to Google Workspace, Microsoft 365, and standard IMAP servers to perform initial bulk imports and maintain continuous, real-time synchronization.
+- **Universal Ingestion**: Connect to any email provider to perform initial bulk imports and maintain continuous, real-time synchronization. Ingestion sources include:
+
+ - IMAP connection
+ - Google Workspace
+ - Microsoft 365
+ - PST files
+ - Zipped .eml files
+
- **Secure & Efficient Storage**: Emails are stored in the standard `.eml` format. The system uses deduplication and compression to minimize storage costs. All data is encrypted at rest.
- **Pluggable Storage Backends**: Support both local filesystem storage and S3-compatible object storage (like AWS S3 or MinIO).
- **Powerful Search & eDiscovery**: A high-performance search engine indexes the full text of emails and attachments (PDF, DOCX, etc.).
diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts
index 3b671a4..c0ddfb3 100644
--- a/docs/.vitepress/config.mts
+++ b/docs/.vitepress/config.mts
@@ -37,9 +37,11 @@ export default defineConfig({
link: '/user-guides/email-providers/',
collapsed: true,
items: [
- { text: 'Google Workspace', link: '/user-guides/email-providers/google-workspace' },
{ text: 'Generic IMAP Server', link: '/user-guides/email-providers/imap' },
- { text: 'Microsoft 365', link: '/user-guides/email-providers/microsoft-365' }
+ { text: 'Google Workspace', link: '/user-guides/email-providers/google-workspace' },
+ { text: 'Microsoft 365', link: '/user-guides/email-providers/microsoft-365' },
+ { text: 'EML Import', link: '/user-guides/email-providers/eml' },
+ { text: 'PST Import', link: '/user-guides/email-providers/pst' }
]
}
]
diff --git a/docs/user-guides/email-providers/eml.md b/docs/user-guides/email-providers/eml.md
new file mode 100644
index 0000000..e151b1b
--- /dev/null
+++ b/docs/user-guides/email-providers/eml.md
@@ -0,0 +1,36 @@
+# EML Import
+
+OpenArchiver allows you to import EML files from a zip archive. This is useful for importing emails from a variety of sources, including other email clients and services.
+
+## Preparing the Zip File
+
+To ensure a successful import, you should compress your .eml files to one zip file according to the following guidelines:
+
+- **Structure:** The zip file can contain any number of `.eml` files, organized in any folder structure. The folder structure will be preserved in OpenArchiver, so you can use it to organize your emails.
+- **Compression:** The zip file should be compressed using standard zip compression.
+
+Here's an example of a valid folder structure:
+
+```
+archive.zip
+โโโ inbox
+โ โโโ email-01.eml
+โ โโโ email-02.eml
+โโโ sent
+โ โโโ email-03.eml
+โโโ drafts
+ โโโ nested-folder
+ โ โโโ email-04.eml
+ โโโ email-05.eml
+```
+
+## Creating an EML Ingestion Source
+
+1. Go to the **Ingestion Sources** page in the OpenArchiver dashboard.
+2. Click the **Create New** button.
+3. Select **EML Import** as the provider.
+4. Enter a name for the ingestion source.
+5. Click the **Choose File** button and select the zip archive containing your EML files.
+6. Click the **Submit** button.
+
+OpenArchiver will then start importing the EML files from the zip archive. The ingestion process may take some time, depending on the size of the archive.
diff --git a/docs/user-guides/email-providers/index.md b/docs/user-guides/email-providers/index.md
index 771b706..dfa8afd 100644
--- a/docs/user-guides/email-providers/index.md
+++ b/docs/user-guides/email-providers/index.md
@@ -7,3 +7,5 @@ Choose your provider from the list below to get started:
- [Google Workspace](./google-workspace.md)
- [Microsoft 365](./microsoft-365.md)
- [Generic IMAP Server](./imap.md)
+- [EML Import](./eml.md)
+- [PST Import](./pst.md)
diff --git a/docs/user-guides/email-providers/pst.md b/docs/user-guides/email-providers/pst.md
new file mode 100644
index 0000000..a029450
--- /dev/null
+++ b/docs/user-guides/email-providers/pst.md
@@ -0,0 +1,21 @@
+# PST Import
+
+OpenArchiver allows you to import PST files. This is useful for importing emails from a variety of sources, including Microsoft Outlook.
+
+## Preparing the PST File
+
+To ensure a successful import, you should prepare your PST file according to the following guidelines:
+
+- **Structure:** The PST file can contain any number of emails, organized in any folder structure. The folder structure will be preserved in OpenArchiver, so you can use it to organize your emails.
+- **Password Protection:** OpenArchiver does not support password-protected PST files. Please remove the password from your PST file before importing it.
+
+## Creating a PST Ingestion Source
+
+1. Go to the **Ingestion Sources** page in the OpenArchiver dashboard.
+2. Click the **Create New** button.
+3. Select **PST Import** as the provider.
+4. Enter a name for the ingestion source.
+5. Click the **Choose File** button and select the PST file.
+6. Click the **Submit** button.
+
+OpenArchiver will then start importing the emails from the PST file. The ingestion process may take some time, depending on the size of the file.
diff --git a/packages/backend/package.json b/packages/backend/package.json
index b714147..47693e6 100644
--- a/packages/backend/package.json
+++ b/packages/backend/package.json
@@ -24,6 +24,7 @@
"@azure/msal-node": "^3.6.3",
"@microsoft/microsoft-graph-client": "^3.0.7",
"@open-archiver/types": "workspace:*",
+ "archiver": "^7.0.1",
"axios": "^1.10.0",
"bcryptjs": "^3.0.2",
"bullmq": "^5.56.3",
@@ -53,17 +54,20 @@
"reflect-metadata": "^0.2.2",
"sqlite3": "^5.1.7",
"tsconfig-paths": "^4.2.0",
- "xlsx": "^0.18.5"
+ "xlsx": "^0.18.5",
+ "yauzl": "^3.2.0"
},
"devDependencies": {
"@bull-board/api": "^6.11.0",
"@bull-board/express": "^6.11.0",
+ "@types/archiver": "^6.0.3",
"@types/busboy": "^1.5.4",
"@types/express": "^5.0.3",
"@types/mailparser": "^3.4.6",
"@types/microsoft-graph": "^2.40.1",
"@types/multer": "^2.0.0",
"@types/node": "^24.0.12",
+ "@types/yauzl": "^2.10.3",
"bull-board": "^2.1.3",
"ts-node-dev": "^2.0.0",
"typescript": "^5.8.3"
diff --git a/packages/backend/src/database/migrations/0014_foamy_vapor.sql b/packages/backend/src/database/migrations/0014_foamy_vapor.sql
new file mode 100644
index 0000000..804fee8
--- /dev/null
+++ b/packages/backend/src/database/migrations/0014_foamy_vapor.sql
@@ -0,0 +1 @@
+ALTER TYPE "public"."ingestion_provider" ADD VALUE 'eml_import';
\ No newline at end of file
diff --git a/packages/backend/src/database/migrations/meta/0014_snapshot.json b/packages/backend/src/database/migrations/meta/0014_snapshot.json
new file mode 100644
index 0000000..50f5d01
--- /dev/null
+++ b/packages/backend/src/database/migrations/meta/0014_snapshot.json
@@ -0,0 +1,1108 @@
+{
+ "id": "ad5204da-bb82-4a19-abfa-d30cc284ab27",
+ "prevId": "c397c819-e69f-42c7-966c-7b2969741c56",
+ "version": "7",
+ "dialect": "postgresql",
+ "tables": {
+ "public.archived_emails": {
+ "name": "archived_emails",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "thread_id": {
+ "name": "thread_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "ingestion_source_id": {
+ "name": "ingestion_source_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "user_email": {
+ "name": "user_email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "message_id_header": {
+ "name": "message_id_header",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sent_at": {
+ "name": "sent_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "subject": {
+ "name": "subject",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sender_name": {
+ "name": "sender_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sender_email": {
+ "name": "sender_email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "recipients": {
+ "name": "recipients",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "storage_path": {
+ "name": "storage_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "storage_hash_sha256": {
+ "name": "storage_hash_sha256",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "size_bytes": {
+ "name": "size_bytes",
+ "type": "bigint",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "is_indexed": {
+ "name": "is_indexed",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "has_attachments": {
+ "name": "has_attachments",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "is_on_legal_hold": {
+ "name": "is_on_legal_hold",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": false
+ },
+ "archived_at": {
+ "name": "archived_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "path": {
+ "name": "path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "tags": {
+ "name": "tags",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {
+ "thread_id_idx": {
+ "name": "thread_id_idx",
+ "columns": [
+ {
+ "expression": "thread_id",
+ "isExpression": false,
+ "asc": true,
+ "nulls": "last"
+ }
+ ],
+ "isUnique": false,
+ "concurrently": false,
+ "method": "btree",
+ "with": {}
+ }
+ },
+ "foreignKeys": {
+ "archived_emails_ingestion_source_id_ingestion_sources_id_fk": {
+ "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
+ "tableFrom": "archived_emails",
+ "tableTo": "ingestion_sources",
+ "columnsFrom": [
+ "ingestion_source_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.attachments": {
+ "name": "attachments",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "filename": {
+ "name": "filename",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "mime_type": {
+ "name": "mime_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "size_bytes": {
+ "name": "size_bytes",
+ "type": "bigint",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "content_hash_sha256": {
+ "name": "content_hash_sha256",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "storage_path": {
+ "name": "storage_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "attachments_content_hash_sha256_unique": {
+ "name": "attachments_content_hash_sha256_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "content_hash_sha256"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.email_attachments": {
+ "name": "email_attachments",
+ "schema": "",
+ "columns": {
+ "email_id": {
+ "name": "email_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "attachment_id": {
+ "name": "attachment_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "email_attachments_email_id_archived_emails_id_fk": {
+ "name": "email_attachments_email_id_archived_emails_id_fk",
+ "tableFrom": "email_attachments",
+ "tableTo": "archived_emails",
+ "columnsFrom": [
+ "email_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "email_attachments_attachment_id_attachments_id_fk": {
+ "name": "email_attachments_attachment_id_attachments_id_fk",
+ "tableFrom": "email_attachments",
+ "tableTo": "attachments",
+ "columnsFrom": [
+ "attachment_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "restrict",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "email_attachments_email_id_attachment_id_pk": {
+ "name": "email_attachments_email_id_attachment_id_pk",
+ "columns": [
+ "email_id",
+ "attachment_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.audit_logs": {
+ "name": "audit_logs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "bigserial",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "timestamp": {
+ "name": "timestamp",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "actor_identifier": {
+ "name": "actor_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "action": {
+ "name": "action",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "target_type": {
+ "name": "target_type",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "target_id": {
+ "name": "target_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "details": {
+ "name": "details",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "is_tamper_evident": {
+ "name": "is_tamper_evident",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": false,
+ "default": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.ediscovery_cases": {
+ "name": "ediscovery_cases",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'open'"
+ },
+ "created_by_identifier": {
+ "name": "created_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "ediscovery_cases_name_unique": {
+ "name": "ediscovery_cases_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.export_jobs": {
+ "name": "export_jobs",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "case_id": {
+ "name": "case_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "format": {
+ "name": "format",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "status": {
+ "name": "status",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending'"
+ },
+ "query": {
+ "name": "query",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "file_path": {
+ "name": "file_path",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_by_identifier": {
+ "name": "created_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "completed_at": {
+ "name": "completed_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "export_jobs_case_id_ediscovery_cases_id_fk": {
+ "name": "export_jobs_case_id_ediscovery_cases_id_fk",
+ "tableFrom": "export_jobs",
+ "tableTo": "ediscovery_cases",
+ "columnsFrom": [
+ "case_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "set null",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.legal_holds": {
+ "name": "legal_holds",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "case_id": {
+ "name": "case_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "custodian_id": {
+ "name": "custodian_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "hold_criteria": {
+ "name": "hold_criteria",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "reason": {
+ "name": "reason",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "applied_by_identifier": {
+ "name": "applied_by_identifier",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "applied_at": {
+ "name": "applied_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "removed_at": {
+ "name": "removed_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "legal_holds_case_id_ediscovery_cases_id_fk": {
+ "name": "legal_holds_case_id_ediscovery_cases_id_fk",
+ "tableFrom": "legal_holds",
+ "tableTo": "ediscovery_cases",
+ "columnsFrom": [
+ "case_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "legal_holds_custodian_id_custodians_id_fk": {
+ "name": "legal_holds_custodian_id_custodians_id_fk",
+ "tableFrom": "legal_holds",
+ "tableTo": "custodians",
+ "columnsFrom": [
+ "custodian_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.retention_policies": {
+ "name": "retention_policies",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "description": {
+ "name": "description",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "priority": {
+ "name": "priority",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "retention_period_days": {
+ "name": "retention_period_days",
+ "type": "integer",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "action_on_expiry": {
+ "name": "action_on_expiry",
+ "type": "retention_action",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "is_enabled": {
+ "name": "is_enabled",
+ "type": "boolean",
+ "primaryKey": false,
+ "notNull": true,
+ "default": true
+ },
+ "conditions": {
+ "name": "conditions",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "retention_policies_name_unique": {
+ "name": "retention_policies_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.custodians": {
+ "name": "custodians",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "display_name": {
+ "name": "display_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "source_type": {
+ "name": "source_type",
+ "type": "ingestion_provider",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "custodians_email_unique": {
+ "name": "custodians_email_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "email"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.ingestion_sources": {
+ "name": "ingestion_sources",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "provider": {
+ "name": "provider",
+ "type": "ingestion_provider",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "credentials": {
+ "name": "credentials",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "status": {
+ "name": "status",
+ "type": "ingestion_status",
+ "typeSchema": "public",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'pending_auth'"
+ },
+ "last_sync_started_at": {
+ "name": "last_sync_started_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_sync_finished_at": {
+ "name": "last_sync_finished_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_sync_status_message": {
+ "name": "last_sync_status_message",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "sync_state": {
+ "name": "sync_state",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.roles": {
+ "name": "roles",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "name": {
+ "name": "name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "policies": {
+ "name": "policies",
+ "type": "jsonb",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "'[]'::jsonb"
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "roles_name_unique": {
+ "name": "roles_name_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "name"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.sessions": {
+ "name": "sessions",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "text",
+ "primaryKey": true,
+ "notNull": true
+ },
+ "user_id": {
+ "name": "user_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "expires_at": {
+ "name": "expires_at",
+ "type": "timestamp with time zone",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "sessions_user_id_users_id_fk": {
+ "name": "sessions_user_id_users_id_fk",
+ "tableFrom": "sessions",
+ "tableTo": "users",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.user_roles": {
+ "name": "user_roles",
+ "schema": "",
+ "columns": {
+ "user_id": {
+ "name": "user_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "role_id": {
+ "name": "role_id",
+ "type": "uuid",
+ "primaryKey": false,
+ "notNull": true
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {
+ "user_roles_user_id_users_id_fk": {
+ "name": "user_roles_user_id_users_id_fk",
+ "tableFrom": "user_roles",
+ "tableTo": "users",
+ "columnsFrom": [
+ "user_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ },
+ "user_roles_role_id_roles_id_fk": {
+ "name": "user_roles_role_id_roles_id_fk",
+ "tableFrom": "user_roles",
+ "tableTo": "roles",
+ "columnsFrom": [
+ "role_id"
+ ],
+ "columnsTo": [
+ "id"
+ ],
+ "onDelete": "cascade",
+ "onUpdate": "no action"
+ }
+ },
+ "compositePrimaryKeys": {
+ "user_roles_user_id_role_id_pk": {
+ "name": "user_roles_user_id_role_id_pk",
+ "columns": [
+ "user_id",
+ "role_id"
+ ]
+ }
+ },
+ "uniqueConstraints": {},
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ },
+ "public.users": {
+ "name": "users",
+ "schema": "",
+ "columns": {
+ "id": {
+ "name": "id",
+ "type": "uuid",
+ "primaryKey": true,
+ "notNull": true,
+ "default": "gen_random_uuid()"
+ },
+ "email": {
+ "name": "email",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": true
+ },
+ "first_name": {
+ "name": "first_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "last_name": {
+ "name": "last_name",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "password": {
+ "name": "password",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "provider": {
+ "name": "provider",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false,
+ "default": "'local'"
+ },
+ "provider_id": {
+ "name": "provider_id",
+ "type": "text",
+ "primaryKey": false,
+ "notNull": false
+ },
+ "created_at": {
+ "name": "created_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ },
+ "updated_at": {
+ "name": "updated_at",
+ "type": "timestamp",
+ "primaryKey": false,
+ "notNull": true,
+ "default": "now()"
+ }
+ },
+ "indexes": {},
+ "foreignKeys": {},
+ "compositePrimaryKeys": {},
+ "uniqueConstraints": {
+ "users_email_unique": {
+ "name": "users_email_unique",
+ "nullsNotDistinct": false,
+ "columns": [
+ "email"
+ ]
+ }
+ },
+ "policies": {},
+ "checkConstraints": {},
+ "isRLSEnabled": false
+ }
+ },
+ "enums": {
+ "public.retention_action": {
+ "name": "retention_action",
+ "schema": "public",
+ "values": [
+ "delete_permanently",
+ "notify_admin"
+ ]
+ },
+ "public.ingestion_provider": {
+ "name": "ingestion_provider",
+ "schema": "public",
+ "values": [
+ "google_workspace",
+ "microsoft_365",
+ "generic_imap",
+ "pst_import",
+ "eml_import"
+ ]
+ },
+ "public.ingestion_status": {
+ "name": "ingestion_status",
+ "schema": "public",
+ "values": [
+ "active",
+ "paused",
+ "error",
+ "pending_auth",
+ "syncing",
+ "importing",
+ "auth_success",
+ "imported"
+ ]
+ }
+ },
+ "schemas": {},
+ "sequences": {},
+ "roles": {},
+ "policies": {},
+ "views": {},
+ "_meta": {
+ "columns": {},
+ "schemas": {},
+ "tables": {}
+ }
+}
\ No newline at end of file
diff --git a/packages/backend/src/database/migrations/meta/_journal.json b/packages/backend/src/database/migrations/meta/_journal.json
index 2950921..a969db7 100644
--- a/packages/backend/src/database/migrations/meta/_journal.json
+++ b/packages/backend/src/database/migrations/meta/_journal.json
@@ -99,6 +99,13 @@
"when": 1754659373517,
"tag": "0013_classy_talkback",
"breakpoints": true
+ },
+ {
+ "idx": 14,
+ "version": "7",
+ "when": 1754831765718,
+ "tag": "0014_foamy_vapor",
+ "breakpoints": true
}
]
}
\ No newline at end of file
diff --git a/packages/backend/src/database/schema/ingestion-sources.ts b/packages/backend/src/database/schema/ingestion-sources.ts
index a3801de..863d9c7 100644
--- a/packages/backend/src/database/schema/ingestion-sources.ts
+++ b/packages/backend/src/database/schema/ingestion-sources.ts
@@ -4,7 +4,8 @@ export const ingestionProviderEnum = pgEnum('ingestion_provider', [
'google_workspace',
'microsoft_365',
'generic_imap',
- 'pst_import'
+ 'pst_import',
+ 'eml_import'
]);
export const ingestionStatusEnum = pgEnum('ingestion_status', [
diff --git a/packages/backend/src/jobs/processors/initial-import.processor.ts b/packages/backend/src/jobs/processors/initial-import.processor.ts
index ac9e5de..adb03a6 100644
--- a/packages/backend/src/jobs/processors/initial-import.processor.ts
+++ b/packages/backend/src/jobs/processors/initial-import.processor.ts
@@ -1,6 +1,6 @@
import { Job, FlowChildJob } from 'bullmq';
import { IngestionService } from '../../services/IngestionService';
-import { IInitialImportJob } from '@open-archiver/types';
+import { IInitialImportJob, IngestionProvider } from '@open-archiver/types';
import { EmailProviderFactory } from '../../services/EmailProviderFactory';
import { flowProducer } from '../queues';
import { logger } from '../../config/logger';
@@ -67,7 +67,8 @@ export default async (job: Job) => {
}
});
} else {
- const finalStatus = source.provider === 'pst_import' ? 'imported' : 'active';
+ const fileBasedIngestions = IngestionService.returnFileBasedIngestions();
+ const finalStatus = fileBasedIngestions.includes(source.provider) ? 'imported' : 'active';
// If there are no users, we can consider the import finished and set to active
await IngestionService.update(ingestionSourceId, {
status: finalStatus,
diff --git a/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts b/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts
index d032e12..d4c3b43 100644
--- a/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts
+++ b/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts
@@ -1,7 +1,7 @@
import { Job } from 'bullmq';
import { IngestionService } from '../../services/IngestionService';
import { logger } from '../../config/logger';
-import { SyncState, ProcessMailboxError, IngestionStatus } from '@open-archiver/types';
+import { SyncState, ProcessMailboxError, IngestionStatus, IngestionProvider } from '@open-archiver/types';
import { db } from '../../database';
import { ingestionSources } from '../../database/schema';
import { eq } from 'drizzle-orm';
@@ -43,7 +43,9 @@ export default async (job: Job) => {
const source = await IngestionService.findById(ingestionSourceId);
let status: IngestionStatus = 'active';
- if (source.provider === 'pst_import') {
+ const fileBasedIngestions = IngestionService.returnFileBasedIngestions();
+
+ if (fileBasedIngestions.includes(source.provider)) {
status = 'imported';
}
let message: string;
diff --git a/packages/backend/src/services/EmailProviderFactory.ts b/packages/backend/src/services/EmailProviderFactory.ts
index 4d664c9..a3fe61f 100644
--- a/packages/backend/src/services/EmailProviderFactory.ts
+++ b/packages/backend/src/services/EmailProviderFactory.ts
@@ -4,6 +4,7 @@ import type {
Microsoft365Credentials,
GenericImapCredentials,
PSTImportCredentials,
+ EMLImportCredentials,
EmailObject,
SyncState,
MailboxUser
@@ -12,6 +13,7 @@ import { GoogleWorkspaceConnector } from './ingestion-connectors/GoogleWorkspace
import { MicrosoftConnector } from './ingestion-connectors/MicrosoftConnector';
import { ImapConnector } from './ingestion-connectors/ImapConnector';
import { PSTConnector } from './ingestion-connectors/PSTConnector';
+import { EMLConnector } from './ingestion-connectors/EMLConnector';
// Define a common interface for all connectors
export interface IEmailConnector {
@@ -36,6 +38,8 @@ export class EmailProviderFactory {
return new ImapConnector(credentials as GenericImapCredentials);
case 'pst_import':
return new PSTConnector(credentials as PSTImportCredentials);
+ case 'eml_import':
+ return new EMLConnector(credentials as EMLImportCredentials);
default:
throw new Error(`Unsupported provider: ${source.provider}`);
}
diff --git a/packages/backend/src/services/IngestionService.ts b/packages/backend/src/services/IngestionService.ts
index d1f2bac..156cb76 100644
--- a/packages/backend/src/services/IngestionService.ts
+++ b/packages/backend/src/services/IngestionService.ts
@@ -4,7 +4,8 @@ import type {
CreateIngestionSourceDto,
UpdateIngestionSourceDto,
IngestionSource,
- IngestionCredentials
+ IngestionCredentials,
+ IngestionProvider
} from '@open-archiver/types';
import { and, desc, eq } from 'drizzle-orm';
import { CryptoService } from './CryptoService';
@@ -36,9 +37,12 @@ export class IngestionService {
return { ...source, credentials: decryptedCredentials } as IngestionSource;
}
+ public static returnFileBasedIngestions(): IngestionProvider[] {
+ return ['pst_import', 'eml_import'];
+ }
+
public static async create(dto: CreateIngestionSourceDto): Promise {
const { providerConfig, ...rest } = dto;
- console.log(providerConfig);
const encryptedCredentials = CryptoService.encryptObject(providerConfig);
const valuesToInsert = {
@@ -140,7 +144,11 @@ export class IngestionService {
const emailPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/`;
await storage.delete(emailPath);
- if (source.credentials.type === 'pst_import' && source.credentials.uploadedFilePath && await storage.exists(source.credentials.uploadedFilePath)) {
+ if (
+ (source.credentials.type === 'pst_import' || source.credentials.type === 'eml_import') &&
+ source.credentials.uploadedFilePath &&
+ (await storage.exists(source.credentials.uploadedFilePath))
+ ) {
await storage.delete(source.credentials.uploadedFilePath);
}
@@ -204,14 +212,13 @@ export class IngestionService {
}
public async performBulkImport(job: IInitialImportJob): Promise {
- console.log('performing bulk import');
const { ingestionSourceId } = job;
const source = await IngestionService.findById(ingestionSourceId);
if (!source) {
throw new Error(`Ingestion source ${ingestionSourceId} not found.`);
}
- console.log(`Starting bulk import for source: ${source.name} (${source.id})`);
+ logger.info(`Starting bulk import for source: ${source.name} (${source.id})`);
await IngestionService.update(ingestionSourceId, {
status: 'importing',
lastSyncStartedAt: new Date()
@@ -233,22 +240,13 @@ export class IngestionService {
}
} else {
// For single-mailbox providers, dispatch a single job
- // console.log('source.credentials ', source.credentials);
await ingestionQueue.add('process-mailbox', {
ingestionSourceId: source.id,
userEmail: source.credentials.type === 'generic_imap' ? source.credentials.username : 'Default'
});
}
-
-
- // await IngestionService.update(ingestionSourceId, {
- // status: 'active',
- // lastSyncFinishedAt: new Date(),
- // lastSyncStatusMessage: 'Successfully initiated bulk import for all mailboxes.'
- // });
- // console.log(`Bulk import job dispatch finished for source: ${source.name} (${source.id})`);
} catch (error) {
- console.error(`Bulk import failed for source: ${source.name} (${source.id})`, error);
+ logger.error(`Bulk import failed for source: ${source.name} (${source.id})`, error);
await IngestionService.update(ingestionSourceId, {
status: 'error',
lastSyncFinishedAt: new Date(),
@@ -290,10 +288,10 @@ export class IngestionService {
return;
}
- console.log('processing email, ', email.id, email.subject);
const emlBuffer = email.eml ?? Buffer.from(email.body, 'utf-8');
const emailHash = createHash('sha256').update(emlBuffer).digest('hex');
- const emailPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${email.path ? `${email.path}/` : ''}${email.id}.eml`;
+ const sanitizedPath = email.path ? email.path : '';
+ const emailPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${sanitizedPath}${email.id}.eml`;
await storage.put(emailPath, emlBuffer);
const [archivedEmail] = await db
@@ -354,7 +352,7 @@ export class IngestionService {
}
// adding to indexing queue
//Instead: index by email (raw email object, ingestion id)
- console.log('Indexing email: ', email.subject);
+ logger.info({ emailId: archivedEmail.id }, 'Indexing email');
// await indexingQueue.add('index-email', {
// emailId: archivedEmail.id,
// });
diff --git a/packages/backend/src/services/ingestion-connectors/EMLConnector.ts b/packages/backend/src/services/ingestion-connectors/EMLConnector.ts
new file mode 100644
index 0000000..afd7d5e
--- /dev/null
+++ b/packages/backend/src/services/ingestion-connectors/EMLConnector.ts
@@ -0,0 +1,199 @@
+import type { EMLImportCredentials, EmailObject, EmailAddress, SyncState, MailboxUser } from '@open-archiver/types';
+import type { IEmailConnector } from '../EmailProviderFactory';
+import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
+import { logger } from '../../config/logger';
+import { getThreadId } from './helpers/utils';
+import { StorageService } from '../StorageService';
+import { Readable } from 'stream';
+import { createHash } from 'crypto';
+import { join, dirname } from 'path';
+import { createReadStream, promises as fs, createWriteStream } from 'fs';
+import * as yauzl from 'yauzl';
+
+const streamToBuffer = (stream: Readable): Promise => {
+ return new Promise((resolve, reject) => {
+ const chunks: Buffer[] = [];
+ stream.on('data', (chunk) => chunks.push(chunk));
+ stream.on('error', reject);
+ stream.on('end', () => resolve(Buffer.concat(chunks)));
+ });
+};
+
+export class EMLConnector implements IEmailConnector {
+ private storage: StorageService;
+
+ constructor(private credentials: EMLImportCredentials) {
+ this.storage = new StorageService();
+ }
+
+ public async testConnection(): Promise {
+ try {
+ if (!this.credentials.uploadedFilePath) {
+ throw Error("EML file path not provided.");
+ }
+ if (!this.credentials.uploadedFilePath.includes('.zip')) {
+ throw Error("Provided file is not in the ZIP format.");
+ }
+ const fileExist = await this.storage.exists(this.credentials.uploadedFilePath);
+ if (!fileExist) {
+ throw Error("EML file upload not finished yet, please wait.");
+ }
+
+ return true;
+ } catch (error) {
+ logger.error({ error, credentials: this.credentials }, 'EML file validation failed.');
+ throw error;
+ }
+ }
+
+ public async *listAllUsers(): AsyncGenerator {
+ const displayName = this.credentials.uploadedFileName || `eml-import-${new Date().getTime()}`;
+ logger.info(`Found potential mailbox: ${displayName}`);
+ const constructedPrimaryEmail = `${displayName.replace(/ /g, '.').toLowerCase()}@eml.local`;
+ yield {
+ id: constructedPrimaryEmail,
+ primaryEmail: constructedPrimaryEmail,
+ displayName: displayName,
+ };
+ }
+
+ public async *fetchEmails(userEmail: string, syncState?: SyncState | null): AsyncGenerator {
+ const fileStream = await this.storage.get(this.credentials.uploadedFilePath);
+ const tempDir = await fs.mkdtemp(join('/tmp', 'eml-import-'));
+ const unzippedPath = join(tempDir, 'unzipped');
+ await fs.mkdir(unzippedPath);
+ const zipFilePath = join(tempDir, 'eml.zip');
+
+ try {
+ await new Promise((resolve, reject) => {
+ const dest = createWriteStream(zipFilePath);
+ (fileStream as Readable).pipe(dest);
+ dest.on('finish', () => resolve());
+ dest.on('error', reject);
+ });
+
+ await this.extract(zipFilePath, unzippedPath);
+
+ const files = await this.getAllFiles(unzippedPath);
+
+ for (const file of files) {
+ if (file.endsWith('.eml')) {
+ try {
+ // logger.info({ file }, 'Processing EML file.');
+ const stream = createReadStream(file);
+ const content = await streamToBuffer(stream);
+ // logger.info({ file, size: content.length }, 'Read file to buffer.');
+ let relativePath = file.substring(unzippedPath.length + 1);
+ if (dirname(relativePath) === '.') {
+ relativePath = '';
+ } else {
+ relativePath = dirname(relativePath);
+ }
+ const emailObject = await this.parseMessage(content, relativePath);
+ // logger.info({ file, messageId: emailObject.id }, 'Parsed email message.');
+ yield emailObject;
+ } catch (error) {
+ logger.error({ error, file }, 'Failed to process a single EML file. Skipping.');
+ }
+ }
+ }
+ } catch (error) {
+ logger.error({ error }, 'Failed to fetch email.');
+ throw error;
+ } finally {
+ await fs.rm(tempDir, { recursive: true, force: true });
+ }
+ }
+
+ private extract(zipFilePath: string, dest: string): Promise {
+ return new Promise((resolve, reject) => {
+ yauzl.open(zipFilePath, { lazyEntries: true, decodeStrings: false }, (err, zipfile) => {
+ if (err) reject(err);
+ zipfile.on('error', reject);
+ zipfile.readEntry();
+ zipfile.on('entry', (entry) => {
+ const fileName = entry.fileName.toString('utf8');
+ // Ignore macOS-specific metadata files.
+ if (fileName.startsWith('__MACOSX/')) {
+ zipfile.readEntry();
+ return;
+ }
+ const entryPath = join(dest, fileName);
+ if (/\/$/.test(fileName)) {
+ fs.mkdir(entryPath, { recursive: true }).then(() => zipfile.readEntry()).catch(reject);
+ } else {
+ zipfile.openReadStream(entry, (err, readStream) => {
+ if (err) reject(err);
+ const writeStream = createWriteStream(entryPath);
+ readStream.pipe(writeStream);
+ writeStream.on('finish', () => zipfile.readEntry());
+ writeStream.on('error', reject);
+ });
+ }
+ });
+ zipfile.on('end', () => resolve());
+ });
+ });
+ }
+
+ private async getAllFiles(dirPath: string, arrayOfFiles: string[] = []): Promise {
+ const files = await fs.readdir(dirPath);
+
+ for (const file of files) {
+ const fullPath = join(dirPath, file);
+ if ((await fs.stat(fullPath)).isDirectory()) {
+ await this.getAllFiles(fullPath, arrayOfFiles);
+ } else {
+ arrayOfFiles.push(fullPath);
+ }
+ }
+
+ return arrayOfFiles;
+ }
+
+ private async parseMessage(emlBuffer: Buffer, path: string): Promise {
+ const parsedEmail: ParsedMail = await simpleParser(emlBuffer);
+
+ const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
+ filename: attachment.filename || 'untitled',
+ contentType: attachment.contentType,
+ size: attachment.size,
+ content: attachment.content as Buffer
+ }));
+
+ const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
+ if (!addresses) return [];
+ const addressArray = Array.isArray(addresses) ? addresses : [addresses];
+ return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address?.replaceAll(`'`, '') || '' })));
+ };
+
+ const threadId = getThreadId(parsedEmail.headers);
+ let messageId = parsedEmail.messageId;
+
+ if (!messageId) {
+ messageId = `generated-${createHash('sha256').update(emlBuffer).digest('hex')}`;
+ }
+
+
+ return {
+ id: messageId,
+ threadId: threadId,
+ from: mapAddresses(parsedEmail.from),
+ to: mapAddresses(parsedEmail.to),
+ cc: mapAddresses(parsedEmail.cc),
+ bcc: mapAddresses(parsedEmail.bcc),
+ subject: parsedEmail.subject || '',
+ body: parsedEmail.text || '',
+ html: parsedEmail.html || '',
+ headers: parsedEmail.headers,
+ attachments,
+ receivedAt: parsedEmail.date || new Date(),
+ eml: emlBuffer,
+ path
+ };
+ }
+
+ public getUpdatedSyncState(): SyncState {
+ return {};
+ }
+}
diff --git a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts
index 5d40bfd..7977122 100644
--- a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts
+++ b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts
@@ -240,7 +240,7 @@ export class ImapConnector implements IEmailConnector {
const threadId = getThreadId(parsedEmail.headers);
return {
- id: msg.uid.toString(),
+ id: parsedEmail.messageId || msg.uid.toString(),
threadId: threadId,
from: mapAddresses(parsedEmail.from),
to: mapAddresses(parsedEmail.to),
diff --git a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte
index 2c051cd..1a91530 100644
--- a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte
+++ b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte
@@ -23,7 +23,8 @@
{ value: 'generic_imap', label: 'Generic IMAP' },
{ value: 'google_workspace', label: 'Google Workspace' },
{ value: 'microsoft_365', label: 'Microsoft 365' },
- { value: 'pst_import', label: 'PST Import' }
+ { value: 'pst_import', label: 'PST Import' },
+ { value: 'eml_import', label: 'EML Import' }
];
let formData: CreateIngestionSourceDto = $state({
@@ -190,6 +191,16 @@
{/if}
+ {:else if formData.provider === 'eml_import'}
+
+
EML File
+
+
+
{/if}
{#if formData.provider === 'google_workspace' || formData.provider === 'microsoft_365'}
diff --git a/packages/frontend/src/routes/dashboard/archived-emails/+page.svelte b/packages/frontend/src/routes/dashboard/archived-emails/+page.svelte
index 33fb3c7..9814d4d 100644
--- a/packages/frontend/src/routes/dashboard/archived-emails/+page.svelte
+++ b/packages/frontend/src/routes/dashboard/archived-emails/+page.svelte
@@ -124,7 +124,9 @@
{email.userEmail}
- {email.path}
+ {#if email.path}
+ {email.path}
+ {/if}
diff --git a/packages/types/src/ingestion.types.ts b/packages/types/src/ingestion.types.ts
index 1b8c7a6..715f90d 100644
--- a/packages/types/src/ingestion.types.ts
+++ b/packages/types/src/ingestion.types.ts
@@ -18,7 +18,7 @@ export type SyncState = {
statusMessage?: string;
};
-export type IngestionProvider = 'google_workspace' | 'microsoft_365' | 'generic_imap' | 'pst_import';
+export type IngestionProvider = 'google_workspace' | 'microsoft_365' | 'generic_imap' | 'pst_import' | 'eml_import';
export type IngestionStatus =
| 'active'
@@ -69,12 +69,19 @@ export interface PSTImportCredentials extends BaseIngestionCredentials {
uploadedFilePath: string;
}
+export interface EMLImportCredentials extends BaseIngestionCredentials {
+ type: 'eml_import';
+ uploadedFileName: string;
+ uploadedFilePath: string;
+}
+
// Discriminated union for all possible credential types
export type IngestionCredentials =
| GenericImapCredentials
| GoogleWorkspaceCredentials
| Microsoft365Credentials
- | PSTImportCredentials;
+ | PSTImportCredentials
+ | EMLImportCredentials;
export interface IngestionSource {
id: string;
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 50d3f73..e158389 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -39,6 +39,9 @@ importers:
'@open-archiver/types':
specifier: workspace:*
version: link:../types
+ archiver:
+ specifier: ^7.0.1
+ version: 7.0.1
axios:
specifier: ^1.10.0
version: 1.10.0
@@ -129,6 +132,9 @@ importers:
xlsx:
specifier: ^0.18.5
version: 0.18.5
+ yauzl:
+ specifier: ^3.2.0
+ version: 3.2.0
devDependencies:
'@bull-board/api':
specifier: ^6.11.0
@@ -136,6 +142,9 @@ importers:
'@bull-board/express':
specifier: ^6.11.0
version: 6.11.0
+ '@types/archiver':
+ specifier: ^6.0.3
+ version: 6.0.3
'@types/busboy':
specifier: ^1.5.4
version: 1.5.4
@@ -154,6 +163,9 @@ importers:
'@types/node':
specifier: ^24.0.12
version: 24.0.13
+ '@types/yauzl':
+ specifier: ^2.10.3
+ version: 2.10.3
bull-board:
specifier: ^2.1.3
version: 2.1.3
@@ -1052,6 +1064,10 @@ packages:
'@ioredis/commands@1.2.0':
resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}
+ '@isaacs/cliui@8.0.2':
+ resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
+ engines: {node: '>=12'}
+
'@isaacs/fs-minipass@4.0.1':
resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==}
engines: {node: '>=18.0.0'}
@@ -1145,6 +1161,10 @@ packages:
engines: {node: '>=10'}
deprecated: This functionality has been moved to @npmcli/fs
+ '@pkgjs/parseargs@0.11.0':
+ resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
+ engines: {node: '>=14'}
+
'@polka/url@1.0.0-next.29':
resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==}
@@ -1672,6 +1692,9 @@ packages:
'@tsconfig/node16@1.0.4':
resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==}
+ '@types/archiver@6.0.3':
+ resolution: {integrity: sha512-a6wUll6k3zX6qs5KlxIggs1P1JcYJaTCx2gnlr+f0S1yd2DoaEwoIK10HmBaLnZwWneBz+JBm0dwcZu0zECBcQ==}
+
'@types/body-parser@1.19.6':
resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==}
@@ -1744,6 +1767,9 @@ packages:
'@types/range-parser@1.2.7':
resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==}
+ '@types/readdir-glob@1.1.5':
+ resolution: {integrity: sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==}
+
'@types/resolve@1.20.2':
resolution: {integrity: sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==}
@@ -1768,6 +1794,9 @@ packages:
'@types/web-bluetooth@0.0.21':
resolution: {integrity: sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA==}
+ '@types/yauzl@2.10.3':
+ resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==}
+
'@ungap/structured-clone@1.3.0':
resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==}
@@ -1873,6 +1902,10 @@ packages:
abbrev@1.1.1:
resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==}
+ abort-controller@3.0.0:
+ resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}
+ engines: {node: '>=6.5'}
+
accepts@1.3.8:
resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==}
engines: {node: '>= 0.6'}
@@ -1918,10 +1951,18 @@ packages:
resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
engines: {node: '>=8'}
+ ansi-regex@6.1.0:
+ resolution: {integrity: sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==}
+ engines: {node: '>=12'}
+
ansi-styles@4.3.0:
resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}
engines: {node: '>=8'}
+ ansi-styles@6.2.1:
+ resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==}
+ engines: {node: '>=12'}
+
anymatch@3.1.3:
resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==}
engines: {node: '>= 8'}
@@ -1932,6 +1973,14 @@ packages:
aproba@2.0.0:
resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==}
+ archiver-utils@5.0.2:
+ resolution: {integrity: sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==}
+ engines: {node: '>= 14'}
+
+ archiver@7.0.1:
+ resolution: {integrity: sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==}
+ engines: {node: '>= 14'}
+
are-we-there-yet@3.0.1:
resolution: {integrity: sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==}
engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0}
@@ -1967,9 +2016,15 @@ packages:
resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==}
engines: {node: '>= 0.4'}
+ b4a@1.6.7:
+ resolution: {integrity: sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==}
+
balanced-match@1.0.2:
resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
+ bare-events@2.6.1:
+ resolution: {integrity: sha512-AuTJkq9XmE6Vk0FJVNq5QxETrSA/vKHarWVBG5l/JbdCL1prJemiyJqUS0jrlXO0MftuPq4m3YVYhoNc5+aE/g==}
+
base64-js@1.5.1:
resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
@@ -2024,6 +2079,13 @@ packages:
resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==}
engines: {node: '>=8'}
+ buffer-crc32@0.2.13:
+ resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==}
+
+ buffer-crc32@1.0.0:
+ resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==}
+ engines: {node: '>=8.0.0'}
+
buffer-equal-constant-time@1.0.1:
resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==}
@@ -2036,6 +2098,9 @@ packages:
buffer@5.7.1:
resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
+ buffer@6.0.3:
+ resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
+
bull-board@2.1.3:
resolution: {integrity: sha512-SrmGzrC024OGtK5Wvv/6VhK4s/iq1h0XUrThc0jla8XhEBUdC79UrG24SOXs68zj7yZnFG0/EG330nPf1Pt5UQ==}
deprecated: 2.x is no longer supported, we moved to use @bull-board scope
@@ -2151,6 +2216,10 @@ packages:
commondir@1.0.1:
resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==}
+ compress-commons@6.0.2:
+ resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==}
+ engines: {node: '>= 14'}
+
concat-map@0.0.1:
resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
@@ -2209,6 +2278,10 @@ packages:
engines: {node: '>=0.8'}
hasBin: true
+ crc32-stream@6.0.0:
+ resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==}
+ engines: {node: '>= 14'}
+
create-require@1.1.1:
resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==}
@@ -2550,6 +2623,9 @@ packages:
dynamic-dedupe@0.3.0:
resolution: {integrity: sha512-ssuANeD+z97meYOqd50e04Ze5qp4bPqo8cCkI4TRjZkzAUgIDTrXV1R8QCdINpiI+hw14+rYazvTRdQrz0/rFQ==}
+ eastasianwidth@0.2.0:
+ resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==}
+
ecdsa-sig-formatter@1.0.11:
resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==}
@@ -2572,6 +2648,9 @@ packages:
emoji-regex@8.0.0:
resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==}
+ emoji-regex@9.2.2:
+ resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==}
+
encodeurl@1.0.2:
resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==}
engines: {node: '>= 0.8'}
@@ -2661,6 +2740,10 @@ packages:
resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==}
engines: {node: '>= 0.6'}
+ event-target-shim@5.0.1:
+ resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==}
+ engines: {node: '>=6'}
+
events@3.3.0:
resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==}
engines: {node: '>=0.8.x'}
@@ -2693,6 +2776,9 @@ packages:
fast-copy@3.0.2:
resolution: {integrity: sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ==}
+ fast-fifo@1.3.2:
+ resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==}
+
fast-redact@3.5.0:
resolution: {integrity: sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==}
engines: {node: '>=6'}
@@ -2746,6 +2832,10 @@ packages:
debug:
optional: true
+ foreground-child@3.3.1:
+ resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==}
+ engines: {node: '>=14'}
+
form-data@4.0.3:
resolution: {integrity: sha512-qsITQPfmvMOSAdeyZ+12I1c+CKSstAFAwu+97zrnWAbIr5u8wfsExUzCesVLC8NgHuRUqNN4Zy6UPWUTRGslcA==}
engines: {node: '>= 6'}
@@ -2823,6 +2913,10 @@ packages:
resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
engines: {node: '>= 6'}
+ glob@10.4.5:
+ resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==}
+ hasBin: true
+
glob@7.2.3:
resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==}
deprecated: Glob versions prior to v9 are no longer supported
@@ -3035,6 +3129,10 @@ packages:
is-reference@3.0.3:
resolution: {integrity: sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==}
+ is-stream@2.0.1:
+ resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}
+ engines: {node: '>=8'}
+
is-what@4.1.16:
resolution: {integrity: sha512-ZhMwEosbFJkA0YhFnNDgTM4ZxDRsS6HqTo7qsZM08fehyRYIYa0yHu5R6mgo1n/8MgaPBXiPimPD77baVFYg+A==}
engines: {node: '>=12.13'}
@@ -3045,6 +3143,9 @@ packages:
isexe@2.0.0:
resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
+ jackspeak@3.4.3:
+ resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==}
+
jake@10.9.2:
resolution: {integrity: sha512-2P4SQ0HrLQ+fw6llpLnOaGAvN2Zu6778SJMrCUwns4fOoG9ayrTiZk3VV8sCPkVZF8ab0zksVpS8FDY5pRCNBA==}
engines: {node: '>=10'}
@@ -3100,6 +3201,10 @@ packages:
peerDependencies:
svelte: ^5.0.0
+ lazystream@1.0.1:
+ resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==}
+ engines: {node: '>= 0.6.3'}
+
leac@0.6.0:
resolution: {integrity: sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==}
@@ -3224,6 +3329,9 @@ packages:
lop@0.4.2:
resolution: {integrity: sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==}
+ lru-cache@10.4.3:
+ resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==}
+
lru-cache@6.0.0:
resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==}
engines: {node: '>=10'}
@@ -3345,6 +3453,10 @@ packages:
resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==}
engines: {node: '>=10'}
+ minimatch@9.0.5:
+ resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==}
+ engines: {node: '>=16 || 14 >=14.17'}
+
minimist@1.2.8:
resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}
@@ -3556,6 +3668,9 @@ packages:
resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==}
engines: {node: '>=10'}
+ package-json-from-dist@1.0.1:
+ resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==}
+
pako@1.0.11:
resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==}
@@ -3577,6 +3692,10 @@ packages:
path-parse@1.0.7:
resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==}
+ path-scurry@1.11.1:
+ resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==}
+ engines: {node: '>=16 || 14 >=14.18'}
+
path-to-regexp@0.1.7:
resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==}
@@ -3592,6 +3711,9 @@ packages:
peberminta@0.9.0:
resolution: {integrity: sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==}
+ pend@1.2.0:
+ resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==}
+
perfect-debounce@1.0.0:
resolution: {integrity: sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==}
@@ -3767,6 +3889,10 @@ packages:
process-warning@5.0.0:
resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==}
+ process@0.11.10:
+ resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==}
+ engines: {node: '>= 0.6.0'}
+
promise-inflight@1.0.1:
resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==}
peerDependencies:
@@ -3834,6 +3960,13 @@ packages:
resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
engines: {node: '>= 6'}
+ readable-stream@4.7.0:
+ resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==}
+ engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
+
+ readdir-glob@1.1.3:
+ resolution: {integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==}
+
readdirp@3.6.0:
resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
engines: {node: '>=8.10.0'}
@@ -4032,6 +4165,10 @@ packages:
signal-exit@3.0.7:
resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==}
+ signal-exit@4.1.0:
+ resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==}
+ engines: {node: '>=14'}
+
simple-concat@1.0.1:
resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==}
@@ -4118,10 +4255,17 @@ packages:
resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==}
engines: {node: '>=10.0.0'}
+ streamx@2.22.1:
+ resolution: {integrity: sha512-znKXEBxfatz2GBNK02kRnCXjV+AA4kjZIUxeWSr3UGirZMJfTE9uiwKHobnbgxWyL/JWro8tTq+vOqAK1/qbSA==}
+
string-width@4.2.3:
resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
engines: {node: '>=8'}
+ string-width@5.1.2:
+ resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==}
+ engines: {node: '>=12'}
+
string_decoder@1.1.1:
resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==}
@@ -4135,6 +4279,10 @@ packages:
resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==}
engines: {node: '>=8'}
+ strip-ansi@7.1.0:
+ resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==}
+ engines: {node: '>=12'}
+
strip-bom@3.0.0:
resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==}
engines: {node: '>=4'}
@@ -4233,6 +4381,9 @@ packages:
resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==}
engines: {node: '>=6'}
+ tar-stream@3.1.7:
+ resolution: {integrity: sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==}
+
tar@6.2.1:
resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==}
engines: {node: '>=10'}
@@ -4241,6 +4392,9 @@ packages:
resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==}
engines: {node: '>=18'}
+ text-decoder@1.2.3:
+ resolution: {integrity: sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==}
+
thread-stream@3.1.0:
resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==}
@@ -4536,6 +4690,10 @@ packages:
resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==}
engines: {node: '>=10'}
+ wrap-ansi@8.1.0:
+ resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==}
+ engines: {node: '>=12'}
+
wrappy@1.0.2:
resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
@@ -4571,6 +4729,10 @@ packages:
resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==}
engines: {node: '>=12'}
+ yauzl@3.2.0:
+ resolution: {integrity: sha512-Ow9nuGZE+qp1u4JIPvg+uCiUr7xGQWdff7JQSk5VGYTAZMDe2q8lxJ10ygv10qmSj031Ty/6FNJpLO4o1Sgc+w==}
+ engines: {node: '>=12'}
+
yn@3.1.1:
resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==}
engines: {node: '>=6'}
@@ -4578,6 +4740,10 @@ packages:
zimmerframe@1.1.2:
resolution: {integrity: sha512-rAbqEGa8ovJy4pyBxZM70hg4pE6gDgaQ0Sl9M3enG3I0d6H4XSAM3GeNGLKnsBpuijUow064sf7ww1nutC5/3w==}
+ zip-stream@6.0.1:
+ resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==}
+ engines: {node: '>= 14'}
+
zwitch@2.0.4:
resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==}
@@ -5501,6 +5667,15 @@ snapshots:
'@ioredis/commands@1.2.0': {}
+ '@isaacs/cliui@8.0.2':
+ dependencies:
+ string-width: 5.1.2
+ string-width-cjs: string-width@4.2.3
+ strip-ansi: 7.1.0
+ strip-ansi-cjs: strip-ansi@6.0.1
+ wrap-ansi: 8.1.0
+ wrap-ansi-cjs: wrap-ansi@7.0.0
+
'@isaacs/fs-minipass@4.0.1':
dependencies:
minipass: 7.1.2
@@ -5588,6 +5763,9 @@ snapshots:
rimraf: 3.0.2
optional: true
+ '@pkgjs/parseargs@0.11.0':
+ optional: true
+
'@polka/url@1.0.0-next.29': {}
'@rollup/plugin-commonjs@28.0.6(rollup@4.44.2)':
@@ -6206,6 +6384,10 @@ snapshots:
'@tsconfig/node16@1.0.4': {}
+ '@types/archiver@6.0.3':
+ dependencies:
+ '@types/readdir-glob': 1.1.5
+
'@types/body-parser@1.19.6':
dependencies:
'@types/connect': 3.4.38
@@ -6296,6 +6478,10 @@ snapshots:
'@types/range-parser@1.2.7': {}
+ '@types/readdir-glob@1.1.5':
+ dependencies:
+ '@types/node': 24.0.13
+
'@types/resolve@1.20.2': {}
'@types/send@0.17.5':
@@ -6319,6 +6505,10 @@ snapshots:
'@types/web-bluetooth@0.0.21': {}
+ '@types/yauzl@2.10.3':
+ dependencies:
+ '@types/node': 24.0.13
+
'@ungap/structured-clone@1.3.0': {}
'@vitejs/plugin-vue@5.2.4(vite@5.4.19(@types/node@24.0.13)(lightningcss@1.30.1))(vue@3.5.18(typescript@5.8.3))':
@@ -6431,6 +6621,10 @@ snapshots:
abbrev@1.1.1:
optional: true
+ abort-controller@3.0.0:
+ dependencies:
+ event-target-shim: 5.0.1
+
accepts@1.3.8:
dependencies:
mime-types: 2.1.35
@@ -6487,10 +6681,14 @@ snapshots:
ansi-regex@5.0.1: {}
+ ansi-regex@6.1.0: {}
+
ansi-styles@4.3.0:
dependencies:
color-convert: 2.0.1
+ ansi-styles@6.2.1: {}
+
anymatch@3.1.3:
dependencies:
normalize-path: 3.0.0
@@ -6501,6 +6699,26 @@ snapshots:
aproba@2.0.0:
optional: true
+ archiver-utils@5.0.2:
+ dependencies:
+ glob: 10.4.5
+ graceful-fs: 4.2.11
+ is-stream: 2.0.1
+ lazystream: 1.0.1
+ lodash: 4.17.21
+ normalize-path: 3.0.0
+ readable-stream: 4.7.0
+
+ archiver@7.0.1:
+ dependencies:
+ archiver-utils: 5.0.2
+ async: 3.2.6
+ buffer-crc32: 1.0.0
+ readable-stream: 4.7.0
+ readdir-glob: 1.1.3
+ tar-stream: 3.1.7
+ zip-stream: 6.0.1
+
are-we-there-yet@3.0.1:
dependencies:
delegates: 1.0.0
@@ -6533,8 +6751,13 @@ snapshots:
axobject-query@4.1.0: {}
+ b4a@1.6.7: {}
+
balanced-match@1.0.2: {}
+ bare-events@2.6.1:
+ optional: true
+
base64-js@1.5.1: {}
bcryptjs@3.0.2: {}
@@ -6612,6 +6835,10 @@ snapshots:
dependencies:
fill-range: 7.1.1
+ buffer-crc32@0.2.13: {}
+
+ buffer-crc32@1.0.0: {}
+
buffer-equal-constant-time@1.0.1: {}
buffer-from@1.1.2: {}
@@ -6626,6 +6853,11 @@ snapshots:
base64-js: 1.5.1
ieee754: 1.2.1
+ buffer@6.0.3:
+ dependencies:
+ base64-js: 1.5.1
+ ieee754: 1.2.1
+
bull-board@2.1.3:
dependencies:
'@types/express': 4.17.23
@@ -6764,6 +6996,14 @@ snapshots:
commondir@1.0.1: {}
+ compress-commons@6.0.2:
+ dependencies:
+ crc-32: 1.2.2
+ crc32-stream: 6.0.0
+ is-stream: 2.0.1
+ normalize-path: 3.0.0
+ readable-stream: 4.7.0
+
concat-map@0.0.1: {}
concat-stream@2.0.0:
@@ -6814,6 +7054,11 @@ snapshots:
crc-32@1.2.2: {}
+ crc32-stream@6.0.0:
+ dependencies:
+ crc-32: 1.2.2
+ readable-stream: 4.7.0
+
create-require@1.1.1: {}
cron-parser@4.9.0:
@@ -7047,6 +7292,8 @@ snapshots:
dependencies:
xtend: 4.0.2
+ eastasianwidth@0.2.0: {}
+
ecdsa-sig-formatter@1.0.11:
dependencies:
safe-buffer: 5.2.1
@@ -7065,6 +7312,8 @@ snapshots:
emoji-regex@8.0.0: {}
+ emoji-regex@9.2.2: {}
+
encodeurl@1.0.2: {}
encodeurl@2.0.0: {}
@@ -7209,6 +7458,8 @@ snapshots:
etag@1.8.1: {}
+ event-target-shim@5.0.1: {}
+
events@3.3.0: {}
expand-template@2.0.3: {}
@@ -7294,6 +7545,8 @@ snapshots:
fast-copy@3.0.2: {}
+ fast-fifo@1.3.2: {}
+
fast-redact@3.5.0: {}
fast-safe-stringify@2.1.1: {}
@@ -7350,6 +7603,11 @@ snapshots:
follow-redirects@1.15.9: {}
+ foreground-child@3.3.1:
+ dependencies:
+ cross-spawn: 7.0.6
+ signal-exit: 4.1.0
+
form-data@4.0.3:
dependencies:
asynckit: 0.4.0
@@ -7441,6 +7699,15 @@ snapshots:
dependencies:
is-glob: 4.0.3
+ glob@10.4.5:
+ dependencies:
+ foreground-child: 3.3.1
+ jackspeak: 3.4.3
+ minimatch: 9.0.5
+ minipass: 7.1.2
+ package-json-from-dist: 1.0.1
+ path-scurry: 1.11.1
+
glob@7.2.3:
dependencies:
fs.realpath: 1.0.0
@@ -7710,12 +7977,20 @@ snapshots:
dependencies:
'@types/estree': 1.0.8
+ is-stream@2.0.1: {}
+
is-what@4.1.16: {}
isarray@1.0.0: {}
isexe@2.0.0: {}
+ jackspeak@3.4.3:
+ dependencies:
+ '@isaacs/cliui': 8.0.2
+ optionalDependencies:
+ '@pkgjs/parseargs': 0.11.0
+
jake@10.9.2:
dependencies:
async: 3.2.6
@@ -7812,6 +8087,10 @@ snapshots:
runed: 0.28.0(svelte@5.35.5)
svelte: 5.35.5
+ lazystream@1.0.1:
+ dependencies:
+ readable-stream: 2.3.8
+
leac@0.6.0: {}
libbase64@1.3.0: {}
@@ -7910,6 +8189,8 @@ snapshots:
option: 0.2.4
underscore: 1.13.7
+ lru-cache@10.4.3: {}
+
lru-cache@6.0.0:
dependencies:
yallist: 4.0.0
@@ -8057,6 +8338,10 @@ snapshots:
dependencies:
brace-expansion: 2.0.2
+ minimatch@9.0.5:
+ dependencies:
+ brace-expansion: 2.0.2
+
minimist@1.2.8: {}
minipass-collect@1.0.2:
@@ -8266,6 +8551,8 @@ snapshots:
aggregate-error: 3.1.0
optional: true
+ package-json-from-dist@1.0.1: {}
+
pako@1.0.11: {}
parseley@0.12.1:
@@ -8281,6 +8568,11 @@ snapshots:
path-parse@1.0.7: {}
+ path-scurry@1.11.1:
+ dependencies:
+ lru-cache: 10.4.3
+ minipass: 7.1.2
+
path-to-regexp@0.1.7: {}
path-to-regexp@8.2.0: {}
@@ -8289,6 +8581,8 @@ snapshots:
peberminta@0.9.0: {}
+ pend@1.2.0: {}
+
perfect-debounce@1.0.0: {}
pg-cloudflare@1.2.7:
@@ -8422,6 +8716,8 @@ snapshots:
process-warning@5.0.0: {}
+ process@0.11.10: {}
+
promise-inflight@1.0.1:
optional: true
@@ -8500,6 +8796,18 @@ snapshots:
string_decoder: 1.3.0
util-deprecate: 1.0.2
+ readable-stream@4.7.0:
+ dependencies:
+ abort-controller: 3.0.0
+ buffer: 6.0.3
+ events: 3.3.0
+ process: 0.11.10
+ string_decoder: 1.3.0
+
+ readdir-glob@1.1.3:
+ dependencies:
+ minimatch: 5.1.6
+
readdirp@3.6.0:
dependencies:
picomatch: 2.3.1
@@ -8753,6 +9061,8 @@ snapshots:
signal-exit@3.0.7:
optional: true
+ signal-exit@4.1.0: {}
+
simple-concat@1.0.1: {}
simple-get@4.0.1:
@@ -8842,12 +9152,25 @@ snapshots:
streamsearch@1.1.0: {}
+ streamx@2.22.1:
+ dependencies:
+ fast-fifo: 1.3.2
+ text-decoder: 1.2.3
+ optionalDependencies:
+ bare-events: 2.6.1
+
string-width@4.2.3:
dependencies:
emoji-regex: 8.0.0
is-fullwidth-code-point: 3.0.0
strip-ansi: 6.0.1
+ string-width@5.1.2:
+ dependencies:
+ eastasianwidth: 0.2.0
+ emoji-regex: 9.2.2
+ strip-ansi: 7.1.0
+
string_decoder@1.1.1:
dependencies:
safe-buffer: 5.1.2
@@ -8865,6 +9188,10 @@ snapshots:
dependencies:
ansi-regex: 5.0.1
+ strip-ansi@7.1.0:
+ dependencies:
+ ansi-regex: 6.1.0
+
strip-bom@3.0.0: {}
strip-json-comments@2.0.1: {}
@@ -8973,6 +9300,12 @@ snapshots:
inherits: 2.0.4
readable-stream: 3.6.2
+ tar-stream@3.1.7:
+ dependencies:
+ b4a: 1.6.7
+ fast-fifo: 1.3.2
+ streamx: 2.22.1
+
tar@6.2.1:
dependencies:
chownr: 2.0.0
@@ -8991,6 +9324,10 @@ snapshots:
mkdirp: 3.0.1
yallist: 5.0.0
+ text-decoder@1.2.3:
+ dependencies:
+ b4a: 1.6.7
+
thread-stream@3.1.0:
dependencies:
real-require: 0.2.0
@@ -9274,6 +9611,12 @@ snapshots:
string-width: 4.2.3
strip-ansi: 6.0.1
+ wrap-ansi@8.1.0:
+ dependencies:
+ ansi-styles: 6.2.1
+ string-width: 5.1.2
+ strip-ansi: 7.1.0
+
wrappy@1.0.2: {}
xlsx@0.18.5:
@@ -9308,8 +9651,19 @@ snapshots:
y18n: 5.0.8
yargs-parser: 21.1.1
+ yauzl@3.2.0:
+ dependencies:
+ buffer-crc32: 0.2.13
+ pend: 1.2.0
+
yn@3.1.1: {}
zimmerframe@1.1.2: {}
+ zip-stream@6.0.1:
+ dependencies:
+ archiver-utils: 5.0.2
+ compress-commons: 6.0.2
+ readable-stream: 4.7.0
+
zwitch@2.0.4: {}
+