Docker deployment

.dockerignore

@@ -0,0 +1,46 @@
+# Git
+.git
+.gitignore
+
+# Node
+node_modules
+.pnpm-store
+
+# Env
+.env
+.env.*
+!/.env.example
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# IDEs
+.idea
+.vscode
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
+
+# Docker
+docker-compose.yml
+.dockerignore
+Dockerfile
+
+# Local data
+meili_data

.env.example

@@ -3,6 +3,7 @@ NODE_ENV=development
 PORT_BACKEND=4000
 PORT_FRONTEND=3000
 
+
 # PostgreSQL
 DATABASE_URL="postgresql://admin:password@postgres:5432/open_archive?schema=public"
 

docker-compose.yml

@@ -1,77 +0,0 @@
-version: '3.8'
-
-services:
-    frontend:
-        build:
-            context: ./packages/frontend
-            dockerfile: Dockerfile
-        ports:
-            - '3000:3000'
-        depends_on:
-            - backend-api
-        env_file:
-            - ./.env
-
-    backend-api:
-        build:
-            context: ./packages/backend
-            dockerfile: Dockerfile
-        ports:
-            - '4000:4000'
-        depends_on:
-            - postgres
-            - redis
-        env_file:
-            - ./.env
-
-    ingestion-worker:
-        build:
-            context: ./packages/backend
-            dockerfile: Dockerfile
-        command: 'pnpm ts-node-dev --respawn --transpile-only src/workers/ingestion.worker.ts'
-        depends_on:
-            - postgres
-            - redis
-        env_file:
-            - ./.env
-
-    indexing-worker:
-        build:
-            context: ./packages/backend
-            dockerfile: Dockerfile
-        command: 'pnpm ts-node-dev --respawn --transpile-only src/workers/indexing.worker.ts'
-        depends_on:
-            - postgres
-            - redis
-        env_file:
-            - ./.env
-
-    postgres:
-        image: postgres:15
-        ports:
-            - '5432:5432'
-        volumes:
-            - postgres_data:/var/lib/postgresql/data
-        env_file:
-            - ./.env
-
-    redis:
-        image: redis:7
-        ports:
-            - '6379:6379'
-        volumes:
-            - redis_data:/data
-
-    meilisearch:
-        image: getmeili/meilisearch:v1.3
-        ports:
-            - '7700:7700'
-        volumes:
-            - meili_data:/meili_data
-        env_file:
-            - ./.env
-
-volumes:
-    postgres_data:
-    redis_data:
-    meili_data:

docker/Dockerfile

@@ -0,0 +1,53 @@
+# Dockerfile for Open Archiver
+
+# 1. Build Stage: Install all dependencies and build the project
+FROM node:22-alpine AS build
+WORKDIR /app
+
+# Install pnpm
+RUN npm install -g pnpm
+
+# Copy manifests and lockfile
+COPY package.json pnpm-workspace.yaml pnpm-lock.yaml* ./
+COPY packages/backend/package.json ./packages/backend/
+COPY packages/frontend/package.json ./packages/frontend/
+COPY packages/types/package.json ./packages/types/
+
+# Install all dependencies 
+RUN pnpm install --frozen-lockfile --prod=false
+
+# Copy the rest of the source code
+COPY . .
+
+# Build all packages
+RUN pnpm build
+
+# 2. Production Stage: Install only production dependencies and copy built artifacts
+FROM node:22-alpine AS production
+WORKDIR /app
+
+# Install pnpm
+RUN npm install -g pnpm
+
+# Copy manifests and lockfile
+COPY package.json pnpm-workspace.yaml pnpm-lock.yaml* ./
+COPY packages/backend/package.json ./packages/backend/
+COPY packages/frontend/package.json ./packages/frontend/
+COPY packages/types/package.json ./packages/types/
+
+# Install only production dependencies
+RUN pnpm install --frozen-lockfile --prod
+
+# Copy built application from build stage
+COPY --from=build /app/packages/backend/dist ./packages/backend/dist
+COPY --from=build /app/packages/frontend/build ./packages/frontend/build
+COPY --from=build /app/packages/types/dist ./packages/types/dist
+COPY --from=build /app/packages/backend/drizzle.config.ts ./packages/backend/drizzle.config.ts
+COPY --from=build /app/packages/backend/src/database/migrations ./packages/backend/src/database/migrations
+
+# Expose the port the app runs on
+EXPOSE 4000
+EXPOSE 3000
+
+# Start the application
+CMD ["pnpm", "docker-start"]

package.json

@@ -4,11 +4,19 @@
     "scripts": {
         "dev": "dotenv -- pnpm --filter \"./packages/*\" --parallel dev",
         "build": "pnpm --filter \"./packages/*\" --parallel build",
-        "start:workers": "dotenv -- concurrently \"pnpm --filter @open-archiver/backend start:ingestion-worker\" \"pnpm --filter @open-archiver/backend start:indexing-worker\""
+        "start": "dotenv -- pnpm --filter \"./packages/*\" --parallel start",
+        "start:workers": "dotenv -- concurrently \"pnpm --filter @open-archiver/backend start:ingestion-worker\" \"pnpm --filter @open-archiver/backend start:indexing-worker\" \"pnpm --filter @open-archiver/backend start:sync-scheduler\"",
+        "start:workers:dev": "dotenv -- concurrently \"pnpm --filter @open-archiver/backend start:ingestion-worker:dev\" \"pnpm --filter @open-archiver/backend start:indexing-worker:dev\" \"pnpm --filter @open-archiver/backend start:sync-scheduler:dev\"",
+        "db:generate": "dotenv -- pnpm --filter @open-archiver/backend db:generate",
+        "db:migrate": "dotenv -- pnpm --filter @open-archiver/backend db:migrate",
+        "db:migrate:dev": "dotenv -- pnpm --filter @open-archiver/backend db:migrate:dev",
+        "docker-start": "pnpm db:migrate && concurrently \"pnpm start:workers\" \"pnpm start\""
+    },
+    "dependencies": {
+        "concurrently": "^9.2.0",
+        "dotenv-cli": "8.0.0"
     },
     "devDependencies": {
-        "concurrently": "^9.2.0",
-        "dotenv-cli": "8.0.0",
         "typescript": "5.8.3"
     },
     "packageManager": "pnpm@10.13.1",

packages/backend/drizzle.config.ts

@@ -1,7 +1,7 @@
 import { defineConfig } from 'drizzle-kit';
 import { config } from 'dotenv';
 
-config({ path: '../../.env' });
+config();
 
 if (!process.env.DATABASE_URL) {
     throw new Error('DATABASE_URL is not set in the .env file');

packages/backend/package.json

@@ -6,22 +6,30 @@
     "scripts": {
         "dev": "ts-node-dev --respawn --transpile-only src/index.ts ",
         "build": "tsc",
-        "prestart": "npm run build",
         "start": "node dist/index.js",
-        "start:ingestion-worker": "ts-node-dev --respawn --transpile-only src/workers/ingestion.worker.ts",
-        "start:indexing-worker": "ts-node-dev --respawn --transpile-only src/workers/indexing.worker.ts",
+        "start:ingestion-worker": "node dist/workers/ingestion.worker.js",
+        "start:indexing-worker": "node dist/workers/indexing.worker.js",
+        "start:sync-scheduler": "node dist/jobs/schedulers/sync-scheduler.js",
+        "start:ingestion-worker:dev": "ts-node-dev --respawn --transpile-only src/workers/ingestion.worker.ts",
+        "start:indexing-worker:dev": "ts-node-dev --respawn --transpile-only src/workers/indexing.worker.ts",
+        "start:sync-scheduler:dev": "ts-node-dev --respawn --transpile-only src/jobs/schedulers/sync-scheduler.ts",
         "db:generate": "drizzle-kit generate --config=drizzle.config.ts",
         "db:push": "drizzle-kit push --config=drizzle.config.ts",
-        "db:migrate": "ts-node-dev src/database/migrate.ts"
+        "db:migrate": "node dist/database/migrate.js",
+        "db:migrate:dev": "ts-node-dev src/database/migrate.ts"
     },
     "dependencies": {
+        "drizzle-kit": "^0.31.4",
         "@aws-sdk/client-s3": "^3.844.0",
         "@aws-sdk/lib-storage": "^3.844.0",
         "@azure/msal-node": "^3.6.3",
+        "@microsoft/microsoft-graph-client": "^3.0.7",
         "@open-archiver/types": "workspace:*",
         "axios": "^1.10.0",
         "bcryptjs": "^3.0.2",
         "bullmq": "^5.56.3",
+        "cross-fetch": "^4.1.0",
+        "deepmerge-ts": "^7.1.5",
         "dotenv": "^17.2.0",
         "drizzle-orm": "^0.44.2",
         "express": "^5.1.0",
@@ -48,9 +56,9 @@
         "@bull-board/express": "^6.11.0",
         "@types/express": "^5.0.3",
         "@types/mailparser": "^3.4.6",
+        "@types/microsoft-graph": "^2.40.1",
         "@types/node": "^24.0.12",
         "bull-board": "^2.1.3",
-        "drizzle-kit": "^0.31.4",
         "ts-node-dev": "^2.0.0",
         "typescript": "^5.8.3"
     }

packages/backend/src/api/controllers/dashboard.controller.ts

@@ -0,0 +1,31 @@
+import { Request, Response } from 'express';
+import { dashboardService } from '../../services/DashboardService';
+
+class DashboardController {
+    public async getStats(req: Request, res: Response) {
+        const stats = await dashboardService.getStats();
+        res.json(stats);
+    }
+
+    public async getIngestionHistory(req: Request, res: Response) {
+        const history = await dashboardService.getIngestionHistory();
+        res.json(history);
+    }
+
+    public async getIngestionSources(req: Request, res: Response) {
+        const sources = await dashboardService.getIngestionSources();
+        res.json(sources);
+    }
+
+    public async getRecentSyncs(req: Request, res: Response) {
+        const syncs = await dashboardService.getRecentSyncs();
+        res.json(syncs);
+    }
+
+    public async getIndexedInsights(req: Request, res: Response) {
+        const insights = await dashboardService.getIndexedInsights();
+        res.json(insights);
+    }
+}
+
+export const dashboardController = new DashboardController();

packages/backend/src/api/controllers/ingestion.controller.ts

@@ -80,4 +80,18 @@ export class IngestionController {
             return res.status(500).json({ message: 'An internal server error occurred' });
         }
     };
+
+    public pause = async (req: Request, res: Response): Promise<Response> => {
+        try {
+            const { id } = req.params;
+            const updatedSource = await IngestionService.update(id, { status: 'paused' });
+            return res.status(200).json(updatedSource);
+        } catch (error) {
+            console.error(`Pause ingestion source ${req.params.id} error:`, error);
+            if (error instanceof Error && error.message === 'Ingestion source not found') {
+                return res.status(404).json({ message: error.message });
+            }
+            return res.status(500).json({ message: 'An internal server error occurred' });
+        }
+    };
 }

packages/backend/src/api/routes/dashboard.routes.ts

@@ -0,0 +1,18 @@
+import { Router } from 'express';
+import { dashboardController } from '../controllers/dashboard.controller';
+import { requireAuth } from '../middleware/requireAuth';
+import { IAuthService } from '../../services/AuthService';
+
+export const createDashboardRouter = (authService: IAuthService): Router => {
+    const router = Router();
+
+    router.use(requireAuth(authService));
+
+    router.get('/stats', dashboardController.getStats);
+    router.get('/ingestion-history', dashboardController.getIngestionHistory);
+    router.get('/ingestion-sources', dashboardController.getIngestionSources);
+    router.get('/recent-syncs', dashboardController.getRecentSyncs);
+    router.get('/indexed-insights', dashboardController.getIndexedInsights);
+
+    return router;
+};

packages/backend/src/api/routes/ingestion.routes.ts

@@ -24,5 +24,7 @@ export const createIngestionRouter = (
 
     router.post('/:id/sync', ingestionController.triggerInitialImport);
 
+    router.post('/:id/pause', ingestionController.pause);
+
     return router;
 };

packages/backend/src/database/migrate.ts

@@ -3,7 +3,7 @@ import { drizzle } from 'drizzle-orm/postgres-js';
 import postgres from 'postgres';
 import { config } from 'dotenv';
 
-config({ path: '../../.env' });
+config();
 
 const runMigrate = async () => {
     if (!process.env.DATABASE_URL) {

packages/backend/src/database/migrations/0006_majestic_caretaker.sql

@@ -0,0 +1 @@
+ALTER TABLE "ingestion_sources" ADD COLUMN "sync_state" jsonb;

packages/backend/src/database/migrations/0007_handy_archangel.sql

@@ -0,0 +1 @@
+ALTER TABLE "ingestion_sources" ALTER COLUMN "credentials" SET DATA TYPE text;

packages/backend/src/database/migrations/0008_eminent_the_spike.sql

@@ -0,0 +1 @@
+ALTER TABLE "archived_emails" ADD COLUMN "user_email" text NOT NULL;

packages/backend/src/database/migrations/meta/0006_snapshot.json

@@ -0,0 +1,826 @@
+{
+  "id": "bdc9d789-04c7-4d9f-b4ed-00366b0d3603",
+  "prevId": "4fa75649-1e65-4c61-8cc5-95add8269925",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {
+    "public.archived_emails": {
+      "name": "archived_emails",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "ingestion_source_id": {
+          "name": "ingestion_source_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "message_id_header": {
+          "name": "message_id_header",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sent_at": {
+          "name": "sent_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "subject": {
+          "name": "subject",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_name": {
+          "name": "sender_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_email": {
+          "name": "sender_email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "recipients": {
+          "name": "recipients",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_hash_sha256": {
+          "name": "storage_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_indexed": {
+          "name": "is_indexed",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "has_attachments": {
+          "name": "has_attachments",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "is_on_legal_hold": {
+          "name": "is_on_legal_hold",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "archived_at": {
+          "name": "archived_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "archived_emails_ingestion_source_id_ingestion_sources_id_fk": {
+          "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
+          "tableFrom": "archived_emails",
+          "tableTo": "ingestion_sources",
+          "columnsFrom": [
+            "ingestion_source_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.attachments": {
+      "name": "attachments",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "filename": {
+          "name": "filename",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "mime_type": {
+          "name": "mime_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "content_hash_sha256": {
+          "name": "content_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "attachments_content_hash_sha256_unique": {
+          "name": "attachments_content_hash_sha256_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "content_hash_sha256"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.email_attachments": {
+      "name": "email_attachments",
+      "schema": "",
+      "columns": {
+        "email_id": {
+          "name": "email_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "attachment_id": {
+          "name": "attachment_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "email_attachments_email_id_archived_emails_id_fk": {
+          "name": "email_attachments_email_id_archived_emails_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "archived_emails",
+          "columnsFrom": [
+            "email_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "email_attachments_attachment_id_attachments_id_fk": {
+          "name": "email_attachments_attachment_id_attachments_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "attachments",
+          "columnsFrom": [
+            "attachment_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "restrict",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "email_attachments_email_id_attachment_id_pk": {
+          "name": "email_attachments_email_id_attachment_id_pk",
+          "columns": [
+            "email_id",
+            "attachment_id"
+          ]
+        }
+      },
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.audit_logs": {
+      "name": "audit_logs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "bigserial",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "timestamp": {
+          "name": "timestamp",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "actor_identifier": {
+          "name": "actor_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action": {
+          "name": "action",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "target_type": {
+          "name": "target_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "target_id": {
+          "name": "target_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "details": {
+          "name": "details",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "is_tamper_evident": {
+          "name": "is_tamper_evident",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": false,
+          "default": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ediscovery_cases": {
+      "name": "ediscovery_cases",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'open'"
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "ediscovery_cases_name_unique": {
+          "name": "ediscovery_cases_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.export_jobs": {
+      "name": "export_jobs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "format": {
+          "name": "format",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending'"
+        },
+        "query": {
+          "name": "query",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "file_path": {
+          "name": "file_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "completed_at": {
+          "name": "completed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "export_jobs_case_id_ediscovery_cases_id_fk": {
+          "name": "export_jobs_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "export_jobs",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "set null",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.legal_holds": {
+      "name": "legal_holds",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "custodian_id": {
+          "name": "custodian_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "hold_criteria": {
+          "name": "hold_criteria",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "reason": {
+          "name": "reason",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "applied_by_identifier": {
+          "name": "applied_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "applied_at": {
+          "name": "applied_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "removed_at": {
+          "name": "removed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "legal_holds_case_id_ediscovery_cases_id_fk": {
+          "name": "legal_holds_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "legal_holds_custodian_id_custodians_id_fk": {
+          "name": "legal_holds_custodian_id_custodians_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "custodians",
+          "columnsFrom": [
+            "custodian_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.retention_policies": {
+      "name": "retention_policies",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "priority": {
+          "name": "priority",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "retention_period_days": {
+          "name": "retention_period_days",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action_on_expiry": {
+          "name": "action_on_expiry",
+          "type": "retention_action",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_enabled": {
+          "name": "is_enabled",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": true
+        },
+        "conditions": {
+          "name": "conditions",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "retention_policies_name_unique": {
+          "name": "retention_policies_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.custodians": {
+      "name": "custodians",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "email": {
+          "name": "email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "display_name": {
+          "name": "display_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "source_type": {
+          "name": "source_type",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "custodians_email_unique": {
+          "name": "custodians_email_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "email"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ingestion_sources": {
+      "name": "ingestion_sources",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "provider": {
+          "name": "provider",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "credentials": {
+          "name": "credentials",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "ingestion_status",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending_auth'"
+        },
+        "last_sync_started_at": {
+          "name": "last_sync_started_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_finished_at": {
+          "name": "last_sync_finished_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_status_message": {
+          "name": "last_sync_status_message",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sync_state": {
+          "name": "sync_state",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    }
+  },
+  "enums": {
+    "public.retention_action": {
+      "name": "retention_action",
+      "schema": "public",
+      "values": [
+        "delete_permanently",
+        "notify_admin"
+      ]
+    },
+    "public.ingestion_provider": {
+      "name": "ingestion_provider",
+      "schema": "public",
+      "values": [
+        "google_workspace",
+        "microsoft_365",
+        "generic_imap"
+      ]
+    },
+    "public.ingestion_status": {
+      "name": "ingestion_status",
+      "schema": "public",
+      "values": [
+        "active",
+        "paused",
+        "error",
+        "pending_auth",
+        "syncing",
+        "importing",
+        "auth_success"
+      ]
+    }
+  },
+  "schemas": {},
+  "sequences": {},
+  "roles": {},
+  "policies": {},
+  "views": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}

packages/backend/src/database/migrations/meta/0007_snapshot.json

@@ -0,0 +1,826 @@
+{
+  "id": "2a68f80f-b233-43bd-8280-745bee76ca3e",
+  "prevId": "bdc9d789-04c7-4d9f-b4ed-00366b0d3603",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {
+    "public.archived_emails": {
+      "name": "archived_emails",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "ingestion_source_id": {
+          "name": "ingestion_source_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "message_id_header": {
+          "name": "message_id_header",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sent_at": {
+          "name": "sent_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "subject": {
+          "name": "subject",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_name": {
+          "name": "sender_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_email": {
+          "name": "sender_email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "recipients": {
+          "name": "recipients",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_hash_sha256": {
+          "name": "storage_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_indexed": {
+          "name": "is_indexed",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "has_attachments": {
+          "name": "has_attachments",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "is_on_legal_hold": {
+          "name": "is_on_legal_hold",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "archived_at": {
+          "name": "archived_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "archived_emails_ingestion_source_id_ingestion_sources_id_fk": {
+          "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
+          "tableFrom": "archived_emails",
+          "tableTo": "ingestion_sources",
+          "columnsFrom": [
+            "ingestion_source_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.attachments": {
+      "name": "attachments",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "filename": {
+          "name": "filename",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "mime_type": {
+          "name": "mime_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "content_hash_sha256": {
+          "name": "content_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "attachments_content_hash_sha256_unique": {
+          "name": "attachments_content_hash_sha256_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "content_hash_sha256"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.email_attachments": {
+      "name": "email_attachments",
+      "schema": "",
+      "columns": {
+        "email_id": {
+          "name": "email_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "attachment_id": {
+          "name": "attachment_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "email_attachments_email_id_archived_emails_id_fk": {
+          "name": "email_attachments_email_id_archived_emails_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "archived_emails",
+          "columnsFrom": [
+            "email_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "email_attachments_attachment_id_attachments_id_fk": {
+          "name": "email_attachments_attachment_id_attachments_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "attachments",
+          "columnsFrom": [
+            "attachment_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "restrict",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "email_attachments_email_id_attachment_id_pk": {
+          "name": "email_attachments_email_id_attachment_id_pk",
+          "columns": [
+            "email_id",
+            "attachment_id"
+          ]
+        }
+      },
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.audit_logs": {
+      "name": "audit_logs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "bigserial",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "timestamp": {
+          "name": "timestamp",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "actor_identifier": {
+          "name": "actor_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action": {
+          "name": "action",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "target_type": {
+          "name": "target_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "target_id": {
+          "name": "target_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "details": {
+          "name": "details",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "is_tamper_evident": {
+          "name": "is_tamper_evident",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": false,
+          "default": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ediscovery_cases": {
+      "name": "ediscovery_cases",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'open'"
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "ediscovery_cases_name_unique": {
+          "name": "ediscovery_cases_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.export_jobs": {
+      "name": "export_jobs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "format": {
+          "name": "format",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending'"
+        },
+        "query": {
+          "name": "query",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "file_path": {
+          "name": "file_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "completed_at": {
+          "name": "completed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "export_jobs_case_id_ediscovery_cases_id_fk": {
+          "name": "export_jobs_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "export_jobs",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "set null",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.legal_holds": {
+      "name": "legal_holds",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "custodian_id": {
+          "name": "custodian_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "hold_criteria": {
+          "name": "hold_criteria",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "reason": {
+          "name": "reason",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "applied_by_identifier": {
+          "name": "applied_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "applied_at": {
+          "name": "applied_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "removed_at": {
+          "name": "removed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "legal_holds_case_id_ediscovery_cases_id_fk": {
+          "name": "legal_holds_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "legal_holds_custodian_id_custodians_id_fk": {
+          "name": "legal_holds_custodian_id_custodians_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "custodians",
+          "columnsFrom": [
+            "custodian_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.retention_policies": {
+      "name": "retention_policies",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "priority": {
+          "name": "priority",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "retention_period_days": {
+          "name": "retention_period_days",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action_on_expiry": {
+          "name": "action_on_expiry",
+          "type": "retention_action",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_enabled": {
+          "name": "is_enabled",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": true
+        },
+        "conditions": {
+          "name": "conditions",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "retention_policies_name_unique": {
+          "name": "retention_policies_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.custodians": {
+      "name": "custodians",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "email": {
+          "name": "email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "display_name": {
+          "name": "display_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "source_type": {
+          "name": "source_type",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "custodians_email_unique": {
+          "name": "custodians_email_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "email"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ingestion_sources": {
+      "name": "ingestion_sources",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "provider": {
+          "name": "provider",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "credentials": {
+          "name": "credentials",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "ingestion_status",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending_auth'"
+        },
+        "last_sync_started_at": {
+          "name": "last_sync_started_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_finished_at": {
+          "name": "last_sync_finished_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_status_message": {
+          "name": "last_sync_status_message",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sync_state": {
+          "name": "sync_state",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    }
+  },
+  "enums": {
+    "public.retention_action": {
+      "name": "retention_action",
+      "schema": "public",
+      "values": [
+        "delete_permanently",
+        "notify_admin"
+      ]
+    },
+    "public.ingestion_provider": {
+      "name": "ingestion_provider",
+      "schema": "public",
+      "values": [
+        "google_workspace",
+        "microsoft_365",
+        "generic_imap"
+      ]
+    },
+    "public.ingestion_status": {
+      "name": "ingestion_status",
+      "schema": "public",
+      "values": [
+        "active",
+        "paused",
+        "error",
+        "pending_auth",
+        "syncing",
+        "importing",
+        "auth_success"
+      ]
+    }
+  },
+  "schemas": {},
+  "sequences": {},
+  "roles": {},
+  "policies": {},
+  "views": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}

packages/backend/src/database/migrations/meta/0008_snapshot.json

@@ -0,0 +1,832 @@
+{
+  "id": "86b6960e-1936-4543-846f-a2d24d6dc5d1",
+  "prevId": "2a68f80f-b233-43bd-8280-745bee76ca3e",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {
+    "public.archived_emails": {
+      "name": "archived_emails",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "ingestion_source_id": {
+          "name": "ingestion_source_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "user_email": {
+          "name": "user_email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "message_id_header": {
+          "name": "message_id_header",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sent_at": {
+          "name": "sent_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "subject": {
+          "name": "subject",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_name": {
+          "name": "sender_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_email": {
+          "name": "sender_email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "recipients": {
+          "name": "recipients",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_hash_sha256": {
+          "name": "storage_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_indexed": {
+          "name": "is_indexed",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "has_attachments": {
+          "name": "has_attachments",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "is_on_legal_hold": {
+          "name": "is_on_legal_hold",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "archived_at": {
+          "name": "archived_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "archived_emails_ingestion_source_id_ingestion_sources_id_fk": {
+          "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
+          "tableFrom": "archived_emails",
+          "tableTo": "ingestion_sources",
+          "columnsFrom": [
+            "ingestion_source_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.attachments": {
+      "name": "attachments",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "filename": {
+          "name": "filename",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "mime_type": {
+          "name": "mime_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "content_hash_sha256": {
+          "name": "content_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "attachments_content_hash_sha256_unique": {
+          "name": "attachments_content_hash_sha256_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "content_hash_sha256"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.email_attachments": {
+      "name": "email_attachments",
+      "schema": "",
+      "columns": {
+        "email_id": {
+          "name": "email_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "attachment_id": {
+          "name": "attachment_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "email_attachments_email_id_archived_emails_id_fk": {
+          "name": "email_attachments_email_id_archived_emails_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "archived_emails",
+          "columnsFrom": [
+            "email_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "email_attachments_attachment_id_attachments_id_fk": {
+          "name": "email_attachments_attachment_id_attachments_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "attachments",
+          "columnsFrom": [
+            "attachment_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "restrict",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "email_attachments_email_id_attachment_id_pk": {
+          "name": "email_attachments_email_id_attachment_id_pk",
+          "columns": [
+            "email_id",
+            "attachment_id"
+          ]
+        }
+      },
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.audit_logs": {
+      "name": "audit_logs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "bigserial",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "timestamp": {
+          "name": "timestamp",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "actor_identifier": {
+          "name": "actor_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action": {
+          "name": "action",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "target_type": {
+          "name": "target_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "target_id": {
+          "name": "target_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "details": {
+          "name": "details",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "is_tamper_evident": {
+          "name": "is_tamper_evident",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": false,
+          "default": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ediscovery_cases": {
+      "name": "ediscovery_cases",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'open'"
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "ediscovery_cases_name_unique": {
+          "name": "ediscovery_cases_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.export_jobs": {
+      "name": "export_jobs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "format": {
+          "name": "format",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending'"
+        },
+        "query": {
+          "name": "query",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "file_path": {
+          "name": "file_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "completed_at": {
+          "name": "completed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "export_jobs_case_id_ediscovery_cases_id_fk": {
+          "name": "export_jobs_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "export_jobs",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "set null",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.legal_holds": {
+      "name": "legal_holds",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "custodian_id": {
+          "name": "custodian_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "hold_criteria": {
+          "name": "hold_criteria",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "reason": {
+          "name": "reason",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "applied_by_identifier": {
+          "name": "applied_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "applied_at": {
+          "name": "applied_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "removed_at": {
+          "name": "removed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "legal_holds_case_id_ediscovery_cases_id_fk": {
+          "name": "legal_holds_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "legal_holds_custodian_id_custodians_id_fk": {
+          "name": "legal_holds_custodian_id_custodians_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "custodians",
+          "columnsFrom": [
+            "custodian_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.retention_policies": {
+      "name": "retention_policies",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "priority": {
+          "name": "priority",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "retention_period_days": {
+          "name": "retention_period_days",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action_on_expiry": {
+          "name": "action_on_expiry",
+          "type": "retention_action",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_enabled": {
+          "name": "is_enabled",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": true
+        },
+        "conditions": {
+          "name": "conditions",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "retention_policies_name_unique": {
+          "name": "retention_policies_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.custodians": {
+      "name": "custodians",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "email": {
+          "name": "email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "display_name": {
+          "name": "display_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "source_type": {
+          "name": "source_type",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "custodians_email_unique": {
+          "name": "custodians_email_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "email"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ingestion_sources": {
+      "name": "ingestion_sources",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "provider": {
+          "name": "provider",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "credentials": {
+          "name": "credentials",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "ingestion_status",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending_auth'"
+        },
+        "last_sync_started_at": {
+          "name": "last_sync_started_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_finished_at": {
+          "name": "last_sync_finished_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_status_message": {
+          "name": "last_sync_status_message",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sync_state": {
+          "name": "sync_state",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    }
+  },
+  "enums": {
+    "public.retention_action": {
+      "name": "retention_action",
+      "schema": "public",
+      "values": [
+        "delete_permanently",
+        "notify_admin"
+      ]
+    },
+    "public.ingestion_provider": {
+      "name": "ingestion_provider",
+      "schema": "public",
+      "values": [
+        "google_workspace",
+        "microsoft_365",
+        "generic_imap"
+      ]
+    },
+    "public.ingestion_status": {
+      "name": "ingestion_status",
+      "schema": "public",
+      "values": [
+        "active",
+        "paused",
+        "error",
+        "pending_auth",
+        "syncing",
+        "importing",
+        "auth_success"
+      ]
+    }
+  },
+  "schemas": {},
+  "sequences": {},
+  "roles": {},
+  "policies": {},
+  "views": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}

packages/backend/src/database/migrations/meta/_journal.json

@@ -43,6 +43,27 @@
       "when": 1752606327253,
       "tag": "0005_chunky_sue_storm",
       "breakpoints": true
+    },
+    {
+      "idx": 6,
+      "version": "7",
+      "when": 1753112018514,
+      "tag": "0006_majestic_caretaker",
+      "breakpoints": true
+    },
+    {
+      "idx": 7,
+      "version": "7",
+      "when": 1753190159356,
+      "tag": "0007_handy_archangel",
+      "breakpoints": true
+    },
+    {
+      "idx": 8,
+      "version": "7",
+      "when": 1753370737317,
+      "tag": "0008_eminent_the_spike",
+      "breakpoints": true
     }
   ]
 }

packages/backend/src/database/schema/archived-emails.ts

@@ -7,6 +7,7 @@ export const archivedEmails = pgTable('archived_emails', {
     ingestionSourceId: uuid('ingestion_source_id')
         .notNull()
         .references(() => ingestionSources.id, { onDelete: 'cascade' }),
+    userEmail: text('user_email').notNull(),
     messageIdHeader: text('message_id_header'),
     sentAt: timestamp('sent_at', { withTimezone: true }).notNull(),
     subject: text('subject'),

packages/backend/src/database/schema/ingestion-sources.ts

@@ -20,11 +20,12 @@ export const ingestionSources = pgTable('ingestion_sources', {
     id: uuid('id').primaryKey().defaultRandom(),
     name: text('name').notNull(),
     provider: ingestionProviderEnum('provider').notNull(),
-    credentials: jsonb('credentials'),
+    credentials: text('credentials'),
     status: ingestionStatusEnum('status').notNull().default('pending_auth'),
     lastSyncStartedAt: timestamp('last_sync_started_at', { withTimezone: true }),
     lastSyncFinishedAt: timestamp('last_sync_finished_at', { withTimezone: true }),
     lastSyncStatusMessage: text('last_sync_status_message'),
+    syncState: jsonb('sync_state'),
     createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
     updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow()
 });

packages/backend/src/index.ts

@@ -11,6 +11,7 @@ import { createIngestionRouter } from './api/routes/ingestion.routes';
 import { createArchivedEmailRouter } from './api/routes/archived-email.routes';
 import { createStorageRouter } from './api/routes/storage.routes';
 import { createSearchRouter } from './api/routes/search.routes';
+import { createDashboardRouter } from './api/routes/dashboard.routes';
 import testRouter from './api/routes/test.routes';
 import { AuthService } from './services/AuthService';
 import { AdminUserService } from './services/UserService';
@@ -58,11 +59,13 @@ const ingestionRouter = createIngestionRouter(ingestionController, authService);
 const archivedEmailRouter = createArchivedEmailRouter(archivedEmailController, authService);
 const storageRouter = createStorageRouter(storageController, authService);
 const searchRouter = createSearchRouter(searchController, authService);
+const dashboardRouter = createDashboardRouter(authService);
 app.use('/v1/auth', authRouter);
 app.use('/v1/ingestion-sources', ingestionRouter);
 app.use('/v1/archived-emails', archivedEmailRouter);
 app.use('/v1/storage', storageRouter);
 app.use('/v1/search', searchRouter);
+app.use('/v1/dashboard', dashboardRouter);
 app.use('/v1/test', testRouter);
 
 // Example of a protected route

packages/backend/src/jobs/processors/continuous-sync.processor.ts

@@ -1,11 +1,82 @@
 import { Job } from 'bullmq';
 import { IngestionService } from '../../services/IngestionService';
-import { IInitialImportJob } from '@open-archiver/types';
+import { IContinuousSyncJob } from '@open-archiver/types';
+import { EmailProviderFactory } from '../../services/EmailProviderFactory';
+import { flowProducer } from '../queues';
+import { logger } from '../../config/logger';
 
-const ingestionService = new IngestionService();
+export default async (job: Job<IContinuousSyncJob>) => {
+    const { ingestionSourceId } = job.data;
+    logger.info({ ingestionSourceId }, 'Starting continuous sync job.');
 
-export default async (job: Job<IInitialImportJob>) => {
-    console.log(`Processing continuous sync for ingestion source: ${job.data.ingestionSourceId}`);
-    // This would be similar to performBulkImport, but would likely use the `since` parameter.
-    // For now, we'll just log a message.
+    const source = await IngestionService.findById(ingestionSourceId);
+    if (!source || source.status !== 'active') {
+        logger.warn({ ingestionSourceId, status: source?.status }, 'Skipping continuous sync for non-active source.');
+        return;
+    }
+
+    await IngestionService.update(ingestionSourceId, {
+        status: 'syncing',
+        lastSyncStartedAt: new Date(),
+    });
+
+    const connector = EmailProviderFactory.createConnector(source);
+
+    try {
+        const jobs = [];
+        // if (!connector.listAllUsers) {
+        //     // This is for single-mailbox providers like Generic IMAP
+        //     let userEmail = 'Default';
+        //     if (connector instanceof ImapConnector) {
+        //         userEmail = connector.returnImapUserEmail();
+        //     }
+        //     jobs.push({
+        //         name: 'process-mailbox',
+        //         queueName: 'ingestion',
+        //         data: {
+        //             ingestionSourceId: source.id,
+        //             userEmail: userEmail
+        //         }
+        //     });
+        // } else {
+        // For multi-mailbox providers like Google Workspace and M365
+        for await (const user of connector.listAllUsers()) {
+            if (user.primaryEmail) {
+                jobs.push({
+                    name: 'process-mailbox',
+                    queueName: 'ingestion',
+                    data: {
+                        ingestionSourceId: source.id,
+                        userEmail: user.primaryEmail
+                    }
+                });
+            }
+        }
+        // }
+
+        if (jobs.length > 0) {
+            await flowProducer.add({
+                name: 'sync-cycle-finished',
+                queueName: 'ingestion',
+                data: {
+                    ingestionSourceId,
+                    isInitialImport: false
+                },
+                children: jobs
+            });
+        }
+
+        // The status will be set back to 'active' by the 'sync-cycle-finished' job
+        // once all the mailboxes have been processed.
+        logger.info({ ingestionSourceId }, 'Continuous sync job finished dispatching mailbox jobs.');
+
+    } catch (error) {
+        logger.error({ err: error, ingestionSourceId }, 'Continuous sync job failed.');
+        await IngestionService.update(ingestionSourceId, {
+            status: 'error',
+            lastSyncFinishedAt: new Date(),
+            lastSyncStatusMessage: error instanceof Error ? error.message : 'An unknown error occurred during sync.',
+        });
+        throw error;
+    }
 };

packages/backend/src/jobs/processors/initial-import.processor.ts

@@ -2,8 +2,7 @@ import { Job } from 'bullmq';
 import { IngestionService } from '../../services/IngestionService';
 import { IInitialImportJob } from '@open-archiver/types';
 import { EmailProviderFactory } from '../../services/EmailProviderFactory';
-import { GoogleWorkspaceConnector } from '../../services/ingestion-connectors/GoogleWorkspaceConnector';
-import { ingestionQueue } from '../queues';
+import { flowProducer } from '../queues';
 import { logger } from '../../config/logger';
 
 export default async (job: Job<IInitialImportJob>) => {
@@ -16,28 +15,70 @@ export default async (job: Job<IInitialImportJob>) => {
             throw new Error(`Ingestion source with ID ${ingestionSourceId} not found`);
         }
 
+        await IngestionService.update(ingestionSourceId, {
+            status: 'importing',
+            lastSyncStatusMessage: 'Starting initial import...'
+        });
+
         const connector = EmailProviderFactory.createConnector(source);
 
-        if (connector instanceof GoogleWorkspaceConnector) {
-            let userCount = 0;
-            for await (const user of connector.listAllUsers()) {
-                if (user.primaryEmail) {
-                    await ingestionQueue.add('process-mailbox', {
+        // if (connector instanceof GoogleWorkspaceConnector || connector instanceof MicrosoftConnector) {
+        const jobs = [];
+        let userCount = 0;
+        for await (const user of connector.listAllUsers()) {
+            if (user.primaryEmail) {
+                jobs.push({
+                    name: 'process-mailbox',
+                    queueName: 'ingestion',
+                    data: {
                         ingestionSourceId,
-                        userEmail: user.primaryEmail
-                    });
-                    userCount++;
-                }
+                        userEmail: user.primaryEmail,
+                    }
+                });
+                userCount++;
             }
-            logger.info({ ingestionSourceId, userCount }, `Enqueued mailbox processing jobs for all users`);
-        } else {
-            // For other providers, we might trigger a simpler bulk import directly
-            await new IngestionService().performBulkImport(job.data);
         }
 
+        if (jobs.length > 0) {
+            await flowProducer.add({
+                name: 'sync-cycle-finished',
+                queueName: 'ingestion',
+                data: {
+                    ingestionSourceId,
+                    userCount,
+                    isInitialImport: true
+                },
+                children: jobs
+            });
+        } else {
+            // If there are no users, we can consider the import finished and set to active
+            await IngestionService.update(ingestionSourceId, {
+                status: 'active',
+                lastSyncFinishedAt: new Date(),
+                lastSyncStatusMessage: 'Initial import complete. No users found.'
+            });
+        }
+        // } else {
+        //     // For other providers, we might trigger a simpler bulk import directly
+        //     await new IngestionService().performBulkImport(job.data);
+        //     await flowProducer.add({
+        //         name: 'sync-cycle-finished',
+        //         queueName: 'ingestion',
+        //         data: {
+        //             ingestionSourceId,
+        //             userCount: 1,
+        //             isInitialImport: true
+        //         }
+        //     });
+        // }
+
         logger.info({ ingestionSourceId }, 'Finished initial import master job');
     } catch (error) {
         logger.error({ err: error, ingestionSourceId }, 'Error in initial import master job');
+        await IngestionService.update(ingestionSourceId, {
+            status: 'error',
+            lastSyncStatusMessage: `Initial import failed: ${error instanceof Error ? error.message : 'Unknown error'}`
+        });
         throw error;
     }
 };

packages/backend/src/jobs/processors/process-mailbox.processor.ts

@@ -1,11 +1,11 @@
 import { Job } from 'bullmq';
-import { IProcessMailboxJob } from '@open-archiver/types';
+import { IProcessMailboxJob, SyncState } from '@open-archiver/types';
 import { IngestionService } from '../../services/IngestionService';
 import { logger } from '../../config/logger';
 import { EmailProviderFactory } from '../../services/EmailProviderFactory';
 import { StorageService } from '../../services/StorageService';
 
-export const processMailboxProcessor = async (job: Job<IProcessMailboxJob, any, string>) => {
+export const processMailboxProcessor = async (job: Job<IProcessMailboxJob, SyncState, string>) => {
     const { ingestionSourceId, userEmail } = job.data;
 
     logger.info({ ingestionSourceId, userEmail }, `Processing mailbox for user`);
@@ -20,10 +20,20 @@ export const processMailboxProcessor = async (job: Job<IProcessMailboxJob, any,
         const ingestionService = new IngestionService();
         const storageService = new StorageService();
 
-        for await (const email of connector.fetchEmails(userEmail)) {
-            await ingestionService.processEmail(email, source, storageService);
+        // Pass the sync state for the entire source, the connector will handle per-user logic if necessary
+        for await (const email of connector.fetchEmails(userEmail, source.syncState)) {
+            if (email) {
+                await ingestionService.processEmail(email, source, storageService, userEmail);
+            }
         }
+
+        const newSyncState = connector.getUpdatedSyncState(userEmail);
+        console.log('newSyncState, ', newSyncState);
+
         logger.info({ ingestionSourceId, userEmail }, `Finished processing mailbox for user`);
+
+        // Return the new sync state to be aggregated by the parent flow
+        return newSyncState;
     } catch (error) {
         logger.error({ err: error, ingestionSourceId, userEmail }, 'Error processing mailbox');
         throw error;

packages/backend/src/jobs/processors/schedule-continuous-sync.processor.ts

@@ -0,0 +1,20 @@
+import { Job } from 'bullmq';
+import { db } from '../../database';
+import { ingestionSources } from '../../database/schema';
+import { eq } from 'drizzle-orm';
+import { ingestionQueue } from '../queues';
+
+export default async (job: Job) => {
+    console.log(
+        'Scheduler running: Looking for active ingestion sources to sync.'
+    );
+    const activeSources = await db
+        .select({ id: ingestionSources.id })
+        .from(ingestionSources)
+        .where(eq(ingestionSources.status, 'active'));
+
+    for (const source of activeSources) {
+        // The status field on the ingestion source is used to prevent duplicate syncs.
+        await ingestionQueue.add('continuous-sync', { ingestionSourceId: source.id });
+    }
+};

packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts

@@ -0,0 +1,53 @@
+import { Job, FlowJob } from 'bullmq';
+import { IngestionService } from '../../services/IngestionService';
+import { logger } from '../../config/logger';
+import { SyncState } from '@open-archiver/types';
+import { db } from '../../database';
+import { ingestionSources } from '../../database/schema';
+import { eq } from 'drizzle-orm';
+import { deepmerge } from 'deepmerge-ts';
+
+interface ISyncCycleFinishedJob {
+    ingestionSourceId: string;
+    userCount?: number; // Optional, as it's only relevant for the initial import
+    isInitialImport: boolean;
+}
+
+export default async (job: Job<ISyncCycleFinishedJob, any, string>) => {
+    const { ingestionSourceId, userCount, isInitialImport } = job.data;
+    logger.info({ ingestionSourceId }, 'Sync cycle finished, processing results...');
+
+    try {
+        const childrenJobs = await job.getChildrenValues<SyncState>();
+        const allSyncStates = Object.values(childrenJobs);
+
+        // Merge all sync states from children jobs into one
+        const finalSyncState = deepmerge(...allSyncStates.filter(s => s && Object.keys(s).length > 0));
+
+        let message = 'Continuous sync cycle finished successfully.';
+        if (isInitialImport) {
+            message = `Initial import finished for ${userCount} mailboxes.`;
+        }
+
+        // Update the database with the final aggregated sync state
+        await db
+            .update(ingestionSources)
+            .set({
+                status: 'active',
+                lastSyncFinishedAt: new Date(),
+                lastSyncStatusMessage: message,
+                syncState: finalSyncState
+            })
+            .where(eq(ingestionSources.id, ingestionSourceId));
+
+        logger.info({ ingestionSourceId }, 'Successfully updated status and final sync state.');
+    } catch (error) {
+        logger.error({ err: error, ingestionSourceId }, 'Failed to process finished sync cycle.');
+        // If this fails, we should probably set the status to 'error' to indicate a problem.
+        await IngestionService.update(ingestionSourceId, {
+            status: 'error',
+            lastSyncFinishedAt: new Date(),
+            lastSyncStatusMessage: 'Failed to finalize sync cycle and update sync state.'
+        });
+    }
+};

packages/backend/src/jobs/queues.ts

@@ -1,6 +1,8 @@
-import { Queue } from 'bullmq';
+import { Queue, FlowProducer } from 'bullmq';
 import { connection } from '../config/redis';
 
+export const flowProducer = new FlowProducer({ connection });
+
 // Default job options
 const defaultJobOptions = {
     attempts: 5,

packages/backend/src/jobs/schedulers/sync-scheduler.ts

@@ -0,0 +1,18 @@
+import { ingestionQueue } from '../queues';
+
+const scheduleContinuousSync = async () => {
+    // This job will run every 15 minutes
+    await ingestionQueue.add(
+        'schedule-continuous-sync',
+        {},
+        {
+            repeat: {
+                pattern: '* * * * *', // Every 15 minutes
+            },
+        }
+    );
+};
+
+scheduleContinuousSync().then(() => {
+    console.log('Continuous sync scheduler started.');
+});

packages/backend/src/services/DashboardService.ts

@@ -0,0 +1,89 @@
+import { and, count, eq, gte, sql } from 'drizzle-orm';
+import type { IndexedInsights } from '@open-archiver/types';
+
+import { archivedEmails, ingestionSources } from '../database/schema';
+import { DatabaseService } from './DatabaseService';
+import { SearchService } from './SearchService';
+
+class DashboardService {
+    #db;
+    #searchService;
+
+    constructor(databaseService: DatabaseService, searchService: SearchService) {
+        this.#db = databaseService.db;
+        this.#searchService = searchService;
+    }
+
+    public async getStats() {
+        const totalEmailsArchived = await this.#db.select({ count: count() }).from(archivedEmails);
+        const totalStorageUsed = await this.#db
+            .select({ sum: sql<number>`sum(${archivedEmails.sizeBytes})` })
+            .from(archivedEmails);
+
+        const sevenDaysAgo = new Date();
+        sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7);
+
+        const failedIngestionsLast7Days = await this.#db
+            .select({ count: count() })
+            .from(ingestionSources)
+            .where(
+                and(
+                    eq(ingestionSources.status, 'error'),
+                    gte(ingestionSources.updatedAt, sevenDaysAgo)
+                )
+            );
+
+        return {
+            totalEmailsArchived: totalEmailsArchived[0].count,
+            totalStorageUsed: totalStorageUsed[0].sum || 0,
+            failedIngestionsLast7Days: failedIngestionsLast7Days[0].count
+        };
+    }
+
+    public async getIngestionHistory() {
+        const thirtyDaysAgo = new Date();
+        thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
+
+        const history = await this.#db
+            .select({
+                date: sql<string>`date_trunc('day', ${archivedEmails.archivedAt})`,
+                count: count()
+            })
+            .from(archivedEmails)
+            .where(gte(archivedEmails.archivedAt, thirtyDaysAgo))
+            .groupBy(sql`date_trunc('day', ${archivedEmails.archivedAt})`)
+            .orderBy(sql`date_trunc('day', ${archivedEmails.archivedAt})`);
+
+        return { history };
+    }
+
+    public async getIngestionSources() {
+        const sources = await this.#db
+            .select({
+                id: ingestionSources.id,
+                name: ingestionSources.name,
+                provider: ingestionSources.provider,
+                status: ingestionSources.status,
+                storageUsed: sql<number>`sum(${archivedEmails.sizeBytes})`.mapWith(Number)
+            })
+            .from(ingestionSources)
+            .leftJoin(archivedEmails, eq(ingestionSources.id, archivedEmails.ingestionSourceId))
+            .groupBy(ingestionSources.id);
+
+        return sources;
+    }
+
+    public async getRecentSyncs() {
+        // This is a placeholder as we don't have a sync job table yet.
+        return Promise.resolve([]);
+    }
+
+    public async getIndexedInsights(): Promise<IndexedInsights> {
+        const topSenders = await this.#searchService.getTopSenders(10);
+        return {
+            topSenders
+        };
+    }
+}
+
+export const dashboardService = new DashboardService(new DatabaseService(), new SearchService());

packages/backend/src/services/EmailProviderFactory.ts

@@ -3,7 +3,9 @@ import type {
     GoogleWorkspaceCredentials,
     Microsoft365Credentials,
     GenericImapCredentials,
-    EmailObject
+    EmailObject,
+    SyncState,
+    MailboxUser
 } from '@open-archiver/types';
 import { GoogleWorkspaceConnector } from './ingestion-connectors/GoogleWorkspaceConnector';
 import { MicrosoftConnector } from './ingestion-connectors/MicrosoftConnector';
@@ -12,8 +14,10 @@ import { ImapConnector } from './ingestion-connectors/ImapConnector';
 // Define a common interface for all connectors
 export interface IEmailConnector {
     testConnection(): Promise<boolean>;
-    fetchEmails(userEmail?: string, since?: Date): AsyncGenerator<EmailObject>;
-    listAllUsers?(): AsyncGenerator<any>;
+    fetchEmails(userEmail: string, syncState?: SyncState | null): AsyncGenerator<EmailObject | null>;
+    getUpdatedSyncState(userEmail?: string): SyncState;
+    listAllUsers(): AsyncGenerator<MailboxUser>;
+    returnImapUserEmail?(): string;
 }
 
 export class EmailProviderFactory {

packages/backend/src/services/IndexingService.ts

@@ -73,7 +73,7 @@ export class IndexingService {
     /**
      * Indexes an email object directly, creates a search document, and indexes it.
      */
-    public async indexByEmail(email: EmailObject, ingestionSourceId: string): Promise<void> {
+    public async indexByEmail(email: EmailObject, ingestionSourceId: string, archivedEmailId: string): Promise<void> {
         const attachments: AttachmentsType = [];
         if (email.attachments && email.attachments.length > 0) {
             for (const attachment of email.attachments) {
@@ -84,7 +84,7 @@ export class IndexingService {
                 });
             }
         }
-        const document = await this.createEmailDocumentFromRaw(email, attachments, ingestionSourceId);
+        const document = await this.createEmailDocumentFromRaw(email, attachments, ingestionSourceId, archivedEmailId);
         await this.searchService.addDocuments('emails', [document], 'id');
     }
 
@@ -94,7 +94,8 @@ export class IndexingService {
     private async createEmailDocumentFromRaw(
         email: EmailObject,
         attachments: AttachmentsType,
-        ingestionSourceId: string
+        ingestionSourceId: string,
+        archivedEmailId: string
     ): Promise<EmailDocument> {
         const extractedAttachments = [];
         for (const attachment of attachments) {
@@ -116,7 +117,7 @@ export class IndexingService {
             }
         }
         return {
-            id: email.id,
+            id: archivedEmailId,
             from: email.from[0]?.address,
             to: email.to.map((i) => i.address) || [],
             cc: email.cc?.map((i) => i.address) || [],

packages/backend/src/services/IngestionService.ts

@@ -25,7 +25,7 @@ export class IngestionService {
         const decryptedCredentials = CryptoService.decryptObject<IngestionCredentials>(
             source.credentials as string
         );
-        return { ...source, credentials: decryptedCredentials };
+        return { ...source, credentials: decryptedCredentials } as IngestionSource;
     }
 
     public static async create(dto: CreateIngestionSourceDto): Promise<IngestionSource> {
@@ -106,13 +106,30 @@ export class IngestionService {
     }
 
     public static async delete(id: string): Promise<IngestionSource> {
+        const source = await this.findById(id);
+        if (!source) {
+            throw new Error('Ingestion source not found');
+        }
+
+        // Delete all emails and attachments from storage
+        const storage = new StorageService();
+        const emailPath = `open-archiver/${source.name.replaceAll(' ', '-')}-${source.id}/`;
+        await storage.delete(emailPath);
+
+
+        // Delete all emails from the database
+        // NOTE: This is done by database CASADE, change when CASADE relation no longer exists.
+        // await db.delete(archivedEmails).where(eq(archivedEmails.ingestionSourceId, id));
+
+        // Delete all documents from Meilisearch
+        const searchService = new SearchService();
+        await searchService.deleteDocumentsByFilter('emails', `ingestionSourceId = ${id}`);
+
         const [deletedSource] = await db
             .delete(ingestionSources)
             .where(eq(ingestionSources.id, id))
             .returning();
-        if (!deletedSource) {
-            throw new Error('Ingestion source not found');
-        }
+
         return this.decryptSource(deletedSource);
     }
 
@@ -139,19 +156,35 @@ export class IngestionService {
         });
 
         const connector = EmailProviderFactory.createConnector(source);
-        const storage = new StorageService();
 
         try {
-            for await (const email of connector.fetchEmails()) {
-                await this.processEmail(email, source, storage);
+            if (connector.listAllUsers) {
+                // For multi-mailbox providers, dispatch a job for each user
+                for await (const user of connector.listAllUsers()) {
+                    const userEmail = (user as any).primaryEmail;
+                    if (userEmail) {
+                        await ingestionQueue.add('process-mailbox', {
+                            ingestionSourceId: source.id,
+                            userEmail: userEmail,
+                        });
+                    }
+                }
+            } else {
+                // For single-mailbox providers, dispatch a single job
+                // console.log('source.credentials ', source.credentials);
+                await ingestionQueue.add('process-mailbox', {
+                    ingestionSourceId: source.id,
+                    userEmail: source.credentials.type === 'generic_imap' ? source.credentials.username : 'Default'
+                });
             }
 
-            await IngestionService.update(ingestionSourceId, {
-                status: 'active',
-                lastSyncFinishedAt: new Date(),
-                lastSyncStatusMessage: 'Successfully completed bulk import.'
-            });
-            console.log(`Bulk import finished for source: ${source.name} (${source.id})`);
+
+            // await IngestionService.update(ingestionSourceId, {
+            //     status: 'active',
+            //     lastSyncFinishedAt: new Date(),
+            //     lastSyncStatusMessage: 'Successfully initiated bulk import for all mailboxes.'
+            // });
+            // console.log(`Bulk import job dispatch finished for source: ${source.name} (${source.id})`);
         } catch (error) {
             console.error(`Bulk import failed for source: ${source.name} (${source.id})`, error);
             await IngestionService.update(ingestionSourceId, {
@@ -166,19 +199,21 @@ export class IngestionService {
     public async processEmail(
         email: EmailObject,
         source: IngestionSource,
-        storage: StorageService
+        storage: StorageService,
+        userEmail: string
     ): Promise<void> {
         try {
             console.log('processing email, ', email.id, email.subject);
             const emlBuffer = email.eml ?? Buffer.from(email.body, 'utf-8');
             const emailHash = createHash('sha256').update(emlBuffer).digest('hex');
-            const emailPath = `email-archive/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${email.id}.eml`;
+            const emailPath = `open-archiver/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${email.id}.eml`;
             await storage.put(emailPath, emlBuffer);
 
             const [archivedEmail] = await db
                 .insert(archivedEmails)
                 .values({
                     ingestionSourceId: source.id,
+                    userEmail,
                     messageIdHeader:
                         (email.headers['message-id'] as string) ??
                         `generated-${emailHash}-${source.id}-${email.id}`,
@@ -202,7 +237,7 @@ export class IngestionService {
                 for (const attachment of email.attachments) {
                     const attachmentBuffer = attachment.content;
                     const attachmentHash = createHash('sha256').update(attachmentBuffer).digest('hex');
-                    const attachmentPath = `email-archive/${source.name.replaceAll(' ', '-')}-${source.id}/attachments/${attachment.filename}`;
+                    const attachmentPath = `open-archiver/${source.name.replaceAll(' ', '-')}-${source.id}/attachments/${attachment.filename}`;
                     await storage.put(attachmentPath, attachmentBuffer);
 
                     const [newAttachment] = await db
@@ -236,7 +271,7 @@ export class IngestionService {
             const storageService = new StorageService();
             const databaseService = new DatabaseService();
             const indexingService = new IndexingService(databaseService, searchService, storageService);
-            await indexingService.indexByEmail(email, source.id);
+            await indexingService.indexByEmail(email, source.id, archivedEmail.id);
         } catch (error) {
             logger.error({
                 message: `Failed to process email ${email.id} for source ${source.id}`,

packages/backend/src/services/SearchService.ts

@@ -1,6 +1,6 @@
 import { Index, MeiliSearch, SearchParams } from 'meilisearch';
 import { config } from '../config';
-import type { SearchQuery, SearchResult, EmailDocument } from '@open-archiver/types';
+import type { SearchQuery, SearchResult, EmailDocument, TopSender } from '@open-archiver/types';
 
 export class SearchService {
     private client: MeiliSearch;
@@ -33,6 +33,11 @@ export class SearchService {
         return index.search(query, options);
     }
 
+    public async deleteDocumentsByFilter(indexName: string, filter: string | string[]) {
+        const index = await this.getIndex(indexName);
+        return index.deleteDocuments({ filter });
+    }
+
     public async searchEmails(dto: SearchQuery): Promise<SearchResult> {
         const { query, filters, page = 1, limit = 10, matchingStrategy = 'last' } = dto;
         const index = await this.getIndex<EmailDocument>('emails');
@@ -68,6 +73,26 @@ export class SearchService {
         };
     }
 
+    public async getTopSenders(limit = 10): Promise<TopSender[]> {
+        const index = await this.getIndex<EmailDocument>('emails');
+        const searchResults = await index.search('', {
+            facets: ['from'],
+            limit: 0
+        });
+
+        if (!searchResults.facetDistribution?.from) {
+            return [];
+        }
+
+        // Sort and take top N
+        const sortedSenders = Object.entries(searchResults.facetDistribution.from)
+            .sort(([, countA], [, countB]) => countB - countA)
+            .slice(0, limit)
+            .map(([sender, count]) => ({ sender, count }));
+
+        return sortedSenders;
+    }
+
     public async configureEmailIndex() {
         const index = await this.getIndex('emails');
         await index.updateSettings({

packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts

@@ -3,7 +3,9 @@ import type { admin_directory_v1, gmail_v1, Common } from 'googleapis';
 import type {
     GoogleWorkspaceCredentials,
     EmailObject,
-    EmailAddress
+    EmailAddress,
+    SyncState,
+    MailboxUser
 } from '@open-archiver/types';
 import type { IEmailConnector } from '../EmailProviderFactory';
 import { logger } from '../../config/logger';
@@ -16,6 +18,7 @@ import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser'
 export class GoogleWorkspaceConnector implements IEmailConnector {
     private credentials: GoogleWorkspaceCredentials;
     private serviceAccountCreds: { client_email: string; private_key: string; };
+    private newHistoryId: string | undefined;
 
     constructor(credentials: GoogleWorkspaceCredentials) {
         this.credentials = credentials;
@@ -86,7 +89,7 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
      * This method handles pagination to retrieve the complete list of users.
      * @returns An async generator that yields each user object.
      */
-    public async *listAllUsers(): AsyncGenerator<admin_directory_v1.Schema$User> {
+    public async *listAllUsers(): AsyncGenerator<MailboxUser> {
         const authClient = this.getAuthClient(this.credentials.impersonatedAdminEmail, [
             'https://www.googleapis.com/auth/admin.directory.user.readonly'
         ]);
@@ -105,7 +108,13 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
             const users = res.data.users;
             if (users) {
                 for (const user of users) {
-                    yield user;
+                    if (user.id && user.primaryEmail && user.name?.fullName) {
+                        yield {
+                            id: user.id,
+                            primaryEmail: user.primaryEmail,
+                            displayName: user.name.fullName
+                        };
+                    }
                 }
             }
             pageToken = res.data.nextPageToken ?? undefined;
@@ -113,32 +122,105 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
     }
 
     /**
-     * Fetches emails for a single user, starting from a specific point in time.
+     * Fetches emails for a single user, starting from a specific history ID.
      * This is ideal for continuous synchronization jobs.
      * @param userEmail The email of the user whose mailbox will be read.
-     * @param since Optional date to fetch emails newer than this timestamp.
+     * @param syncState Optional state containing the startHistoryId.
      * @returns An async generator that yields each raw email object.
      */
     public async *fetchEmails(
         userEmail: string,
-        since?: Date
+        syncState?: SyncState | null
     ): AsyncGenerator<EmailObject> {
         const authClient = this.getAuthClient(userEmail, [
             'https://www.googleapis.com/auth/gmail.readonly'
         ]);
-
         const gmail = google.gmail({ version: 'v1', auth: authClient });
         let pageToken: string | undefined = undefined;
 
-        const query = since ? `after:${Math.floor(since.getTime() / 1000)}` : '';
+        const startHistoryId = syncState?.google?.[userEmail]?.historyId;
+
+        // If no sync state is provided for this user, this is an initial import. Get all messages.
+        if (!startHistoryId) {
+            yield* this.fetchAllMessagesForUser(gmail, userEmail);
+            return;
+        }
+
+        this.newHistoryId = startHistoryId;
 
         do {
-            const listResponse: Common.GaxiosResponseWithHTTP2<gmail_v1.Schema$ListMessagesResponse> =
-                await gmail.users.messages.list({
-                    userId: 'me', // 'me' refers to the impersonated user
-                    q: query,
-                    pageToken: pageToken
-                });
+            const historyResponse: Common.GaxiosResponseWithHTTP2<gmail_v1.Schema$ListHistoryResponse> = await gmail.users.history.list({
+                userId: 'me',
+                startHistoryId: this.newHistoryId,
+                pageToken: pageToken,
+                historyTypes: ['messageAdded']
+            });
+
+            const histories = historyResponse.data.history;
+            if (!histories || histories.length === 0) {
+                return;
+            }
+
+            for (const historyRecord of histories) {
+                if (historyRecord.messagesAdded) {
+                    for (const messageAdded of historyRecord.messagesAdded) {
+                        if (messageAdded.message?.id) {
+                            const msgResponse = await gmail.users.messages.get({
+                                userId: 'me',
+                                id: messageAdded.message.id,
+                                format: 'RAW'
+                            });
+
+                            if (msgResponse.data.raw) {
+                                const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
+                                const parsedEmail: ParsedMail = await simpleParser(rawEmail);
+                                const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
+                                    filename: attachment.filename || 'untitled',
+                                    contentType: attachment.contentType,
+                                    size: attachment.size,
+                                    content: attachment.content as Buffer
+                                }));
+                                const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
+                                    if (!addresses) return [];
+                                    const addressArray = Array.isArray(addresses) ? addresses : [addresses];
+                                    return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
+                                };
+                                yield {
+                                    id: msgResponse.data.id!,
+                                    userEmail: userEmail,
+                                    eml: rawEmail,
+                                    from: mapAddresses(parsedEmail.from),
+                                    to: mapAddresses(parsedEmail.to),
+                                    cc: mapAddresses(parsedEmail.cc),
+                                    bcc: mapAddresses(parsedEmail.bcc),
+                                    subject: parsedEmail.subject || '',
+                                    body: parsedEmail.text || '',
+                                    html: parsedEmail.html || '',
+                                    headers: parsedEmail.headers as any,
+                                    attachments,
+                                    receivedAt: parsedEmail.date || new Date(),
+                                };
+                            }
+                        }
+                    }
+                }
+            }
+
+            pageToken = historyResponse.data.nextPageToken ?? undefined;
+            if (historyResponse.data.historyId) {
+                this.newHistoryId = historyResponse.data.historyId;
+            }
+
+        } while (pageToken);
+    }
+
+    private async *fetchAllMessagesForUser(gmail: gmail_v1.Gmail, userEmail: string): AsyncGenerator<EmailObject> {
+        let pageToken: string | undefined = undefined;
+        do {
+            const listResponse: Common.GaxiosResponseWithHTTP2<gmail_v1.Schema$ListMessagesResponse> = await gmail.users.messages.list({
+                userId: 'me',
+                pageToken: pageToken
+            });
 
             const messages = listResponse.data.messages;
             if (!messages || messages.length === 0) {
@@ -150,27 +232,23 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
                     const msgResponse = await gmail.users.messages.get({
                         userId: 'me',
                         id: message.id,
-                        format: 'RAW' // We want the full, raw .eml content
+                        format: 'RAW'
                     });
 
                     if (msgResponse.data.raw) {
-                        // The raw data is base64url encoded, so we need to decode it.
                         const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
                         const parsedEmail: ParsedMail = await simpleParser(rawEmail);
-
                         const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
                             filename: attachment.filename || 'untitled',
                             contentType: attachment.contentType,
                             size: attachment.size,
                             content: attachment.content as Buffer
                         }));
-
                         const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
                             if (!addresses) return [];
                             const addressArray = Array.isArray(addresses) ? addresses : [addresses];
                             return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
                         };
-
                         yield {
                             id: msgResponse.data.id!,
                             userEmail: userEmail,
@@ -189,8 +267,26 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
                     }
                 }
             }
-
             pageToken = listResponse.data.nextPageToken ?? undefined;
         } while (pageToken);
+
+        // After fetching all messages, get the latest history ID to use as the starting point for the next sync.
+        const profileResponse = await gmail.users.getProfile({ userId: 'me' });
+        if (profileResponse.data.historyId) {
+            this.newHistoryId = profileResponse.data.historyId;
+        }
+    }
+
+    public getUpdatedSyncState(userEmail: string): SyncState {
+        if (!this.newHistoryId) {
+            return {};
+        }
+        return {
+            google: {
+                [userEmail]: {
+                    historyId: this.newHistoryId
+                }
+            }
+        };
     }
 }

packages/backend/src/services/ingestion-connectors/ImapConnector.ts

@@ -1,10 +1,11 @@
-import type { GenericImapCredentials, EmailObject, EmailAddress } from '@open-archiver/types';
+import type { GenericImapCredentials, EmailObject, EmailAddress, SyncState, MailboxUser } from '@open-archiver/types';
 import type { IEmailConnector } from '../EmailProviderFactory';
 import { ImapFlow } from 'imapflow';
 import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
 
 export class ImapConnector implements IEmailConnector {
     private client: ImapFlow;
+    private newMaxUid: number = 0;
 
     constructor(private credentials: GenericImapCredentials) {
         this.client = new ImapFlow({
@@ -30,47 +31,100 @@ export class ImapConnector implements IEmailConnector {
         }
     }
 
-    public async *fetchEmails(userEmail?: string, since?: Date): AsyncGenerator<EmailObject> {
+    /**
+    *  We understand that for IMAP inboxes, there is only one user, but we want the IMAP connector to be compatible with other connectors, we return the single user here.
+    * @returns An async generator that yields each user object.
+    */
+    public async *listAllUsers(): AsyncGenerator<MailboxUser> {
+        const emails: string[] = [this.returnImapUserEmail()];
+        for (const [index, email] of emails.entries()) {
+            yield {
+                id: String(index),
+                primaryEmail: email,
+                displayName: email
+            };
+
+        }
+    }
+
+    public returnImapUserEmail(): string {
+        return this.credentials.username;
+    }
+    public async *fetchEmails(userEmail: string, syncState?: SyncState | null): AsyncGenerator<EmailObject | null> {
         await this.client.connect();
         try {
-            await this.client.mailboxOpen('INBOX');
+            const mailbox = await this.client.mailboxOpen('INBOX');
+            const lastUid = syncState?.imap?.maxUid;
 
-            const searchCriteria = since ? { since } : { all: true };
+            // For continuous sync, we start with the last known UID.
+            // For initial sync, we start at 0 and find the highest UID.
+            this.newMaxUid = lastUid || 0;
+
+            // If it's an initial sync, we need to determine the highest UID in the mailbox
+            // to correctly set the state, even if we don't fetch anything.
+            if (!lastUid && mailbox.exists > 0) {
+                const lastMessage = await this.client.fetchOne(String(mailbox.exists), { uid: true });
+                if (lastMessage && lastMessage.uid > this.newMaxUid) {
+                    this.newMaxUid = lastMessage.uid;
+                }
+            }
+
+            const searchCriteria = lastUid ? { uid: `${lastUid + 1}:*` } : { all: true };
+
+            for await (const msg of this.client.fetch(searchCriteria, { envelope: true, source: true, bodyStructure: true, uid: true })) {
+                if (lastUid && msg.uid <= lastUid) {
+                    continue; //in case imapflow returns one email even if it should return no email
+                }
+
+                if (msg.uid > this.newMaxUid) {
+                    this.newMaxUid = msg.uid;
+                }
 
-            for await (const msg of this.client.fetch(searchCriteria, { envelope: true, source: true, bodyStructure: true })) {
                 if (msg.envelope && msg.source) {
-                    const parsedEmail: ParsedMail = await simpleParser(msg.source);
-                    const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
-                        filename: attachment.filename || 'untitled',
-                        contentType: attachment.contentType,
-                        size: attachment.size,
-                        content: attachment.content as Buffer
-                    }));
-
-                    const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
-                        if (!addresses) return [];
-                        const addressArray = Array.isArray(addresses) ? addresses : [addresses];
-                        return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
-                    };
-
-                    yield {
-                        id: msg.uid.toString(),
-                        from: mapAddresses(parsedEmail.from),
-                        to: mapAddresses(parsedEmail.to),
-                        cc: mapAddresses(parsedEmail.cc),
-                        bcc: mapAddresses(parsedEmail.bcc),
-                        subject: parsedEmail.subject || '',
-                        body: parsedEmail.text || '',
-                        html: parsedEmail.html || '',
-                        headers: parsedEmail.headers as any,
-                        attachments,
-                        receivedAt: parsedEmail.date || new Date(),
-                        eml: msg.source
-                    };
+                    yield await this.parseMessage(msg);
                 }
             }
         } finally {
-            await this.client.logout();
+            if (this.client.usable) await this.client.logout();
         }
     }
+
+    private async parseMessage(msg: any): Promise<EmailObject> {
+        const parsedEmail: ParsedMail = await simpleParser(msg.source);
+        const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
+            filename: attachment.filename || 'untitled',
+            contentType: attachment.contentType,
+            size: attachment.size,
+            content: attachment.content as Buffer
+        }));
+
+        const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
+            if (!addresses) return [];
+            const addressArray = Array.isArray(addresses) ? addresses : [addresses];
+            return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
+        };
+
+        return {
+            id: msg.uid.toString(),
+            from: mapAddresses(parsedEmail.from),
+            to: mapAddresses(parsedEmail.to),
+            cc: mapAddresses(parsedEmail.cc),
+            bcc: mapAddresses(parsedEmail.bcc),
+            subject: parsedEmail.subject || '',
+            body: parsedEmail.text || '',
+            html: parsedEmail.html || '',
+            headers: parsedEmail.headers as any,
+            attachments,
+            receivedAt: parsedEmail.date || new Date(),
+            eml: msg.source
+        };
+    }
+
+    public getUpdatedSyncState(): SyncState {
+        return {
+            imap: {
+                maxUid: this.newMaxUid
+            }
+        };
+    }
 }

packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts

@@ -1,99 +1,284 @@
-import type { Microsoft365Credentials, EmailObject, EmailAddress } from '@open-archiver/types';
+import 'cross-fetch/polyfill';
+import type {
+    Microsoft365Credentials,
+    EmailObject,
+    EmailAddress,
+    SyncState,
+    MailboxUser
+} from '@open-archiver/types';
 import type { IEmailConnector } from '../EmailProviderFactory';
-import { ConfidentialClientApplication } from '@azure/msal-node';
+import { logger } from '../../config/logger';
 import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
-import axios from 'axios';
-
-const GRAPH_API_ENDPOINT = 'https://graph.microsoft.com/v1.0';
+import { ConfidentialClientApplication, Configuration, LogLevel } from '@azure/msal-node';
+import { Client } from '@microsoft/microsoft-graph-client';
+import type { User, MailFolder } from 'microsoft-graph';
+import type { AuthProvider } from '@microsoft/microsoft-graph-client';
 
+/**
+ * A connector for Microsoft 365 that uses the Microsoft Graph API with client credentials (app-only)
+ * to access data on behalf of the organization.
+ */
 export class MicrosoftConnector implements IEmailConnector {
-    private cca: ConfidentialClientApplication;
+    private credentials: Microsoft365Credentials;
+    private graphClient: Client;
+    // Store delta tokens for each folder during a sync operation.
+    private newDeltaTokens: { [folderId: string]: string; };
 
-    constructor(private credentials: Microsoft365Credentials) {
-        this.cca = new ConfidentialClientApplication({
+    constructor(credentials: Microsoft365Credentials) {
+        this.credentials = credentials;
+        this.newDeltaTokens = {}; // Initialize as an empty object
+
+        const msalConfig: Configuration = {
             auth: {
                 clientId: this.credentials.clientId,
+                authority: `https://login.microsoftonline.com/${this.credentials.tenantId}`,
                 clientSecret: this.credentials.clientSecret,
             },
-        });
-    }
-
-    private async getAccessToken(): Promise<string> {
-        const result = await this.cca.acquireTokenByClientCredential({
-            scopes: ['https://graph.microsoft.com/.default'],
-        });
-        if (!result?.accessToken) {
-            throw new Error('Failed to acquire access token');
-        }
-        return result.accessToken;
+            system: {
+                loggerOptions: {
+                    loggerCallback(loglevel, message, containsPii) {
+                        if (containsPii) return;
+                        switch (loglevel) {
+                            case LogLevel.Error:
+                                logger.error(message);
+                                return;
+                            case LogLevel.Warning:
+                                logger.warn(message);
+                                return;
+                            case LogLevel.Info:
+                                logger.info(message);
+                                return;
+                            case LogLevel.Verbose:
+                                logger.debug(message);
+                                return;
+                        }
+                    },
+                    piiLoggingEnabled: false,
+                    logLevel: LogLevel.Warning,
+                }
+            }
+        };
+
+        const msalClient = new ConfidentialClientApplication(msalConfig);
+
+        const authProvider: AuthProvider = async (done) => {
+            try {
+                const response = await msalClient.acquireTokenByClientCredential({
+                    scopes: ['https://graph.microsoft.com/.default'],
+                });
+                if (!response?.accessToken) {
+                    throw new Error('Failed to acquire access token.');
+                }
+                done(null, response.accessToken);
+            } catch (error) {
+                logger.error({ err: error }, 'Failed to acquire Microsoft Graph access token');
+                done(error, null);
+            }
+        };
+
+        this.graphClient = Client.init({ authProvider });
     }
 
+    /**
+     * Tests the connection and authentication by attempting to list the first user
+     * from the directory.
+     */
     public async testConnection(): Promise<boolean> {
         try {
-            await this.getAccessToken();
+            await this.graphClient.api('/users').top(1).get();
+            logger.info('Microsoft 365 connection test successful.');
             return true;
         } catch (error) {
-            console.error('Failed to verify Microsoft 365 connection:', error);
+            logger.error({ err: error }, 'Failed to verify Microsoft 365 connection');
             return false;
         }
     }
 
-    public async *fetchEmails(userEmail?: string, since?: Date): AsyncGenerator<EmailObject> {
-        const accessToken = await this.getAccessToken();
-        const headers = { Authorization: `Bearer ${accessToken}` };
+    /**
+     * Lists all users in the Microsoft 365 tenant.
+     * This method handles pagination to retrieve the complete list of users.
+     * @returns An async generator that yields each user object.
+     */
+    public async *listAllUsers(): AsyncGenerator<MailboxUser> {
+        let request = this.graphClient.api('/users').select('id,userPrincipalName,displayName');
 
-        let nextLink: string | undefined = `${GRAPH_API_ENDPOINT}/users/me/messages`;
-        if (since) {
-            nextLink += `?$filter=receivedDateTime ge ${since.toISOString()}`;
-        }
+        try {
+            let response = await request.get();
+            while (response) {
+                for (const user of response.value as User[]) {
+                    if (user.id && user.userPrincipalName && user.displayName) {
+                        yield {
+                            id: user.id,
+                            primaryEmail: user.userPrincipalName,
+                            displayName: user.displayName
+                        };
+                    }
+                }
 
-        while (nextLink) {
-            const res: { data: { value: any[]; '@odata.nextLink'?: string; }; } = await axios.get(
-                nextLink,
-                { headers }
-            );
-            const messages = res.data.value;
-
-            for (const message of messages) {
-                const rawContentRes = await axios.get(
-                    `${GRAPH_API_ENDPOINT}/users/me/messages/${message.id}/$value`,
-                    { headers }
-                );
-                const emlBuffer = Buffer.from(rawContentRes.data, 'utf-8');
-                const parsedEmail: ParsedMail = await simpleParser(emlBuffer);
-                const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
-                    filename: attachment.filename || 'untitled',
-                    contentType: attachment.contentType,
-                    size: attachment.size,
-                    content: attachment.content as Buffer
-                }));
-
-                const mapAddresses = (
-                    addresses: AddressObject | AddressObject[] | undefined
-                ): EmailAddress[] => {
-                    if (!addresses) return [];
-                    const addressArray = Array.isArray(addresses) ? addresses : [addresses];
-                    return addressArray.flatMap(a =>
-                        a.value.map(v => ({ name: v.name, address: v.address || '' }))
-                    );
-                };
-
-                yield {
-                    id: message.id,
-                    from: mapAddresses(parsedEmail.from),
-                    to: mapAddresses(parsedEmail.to),
-                    cc: mapAddresses(parsedEmail.cc),
-                    bcc: mapAddresses(parsedEmail.bcc),
-                    subject: parsedEmail.subject || '',
-                    body: parsedEmail.text || '',
-                    html: parsedEmail.html || '',
-                    headers: parsedEmail.headers as any,
-                    attachments,
-                    receivedAt: parsedEmail.date || new Date(),
-                    eml: emlBuffer
-                };
+                if (response['@odata.nextLink']) {
+                    response = await this.graphClient.api(response['@odata.nextLink']).get();
+                } else {
+                    break;
+                }
             }
-            nextLink = res.data['@odata.nextLink'];
+        } catch (error) {
+            logger.error({ err: error }, 'Failed to list all users from Microsoft 365');
+            throw error;
         }
     }
+
+    /**
+     * Fetches emails for a single user by iterating through all mail folders and
+     * performing a delta query on each.
+     * @param userEmail The user principal name or ID of the user.
+     * @param syncState Optional state containing the deltaTokens for each folder.
+     * @returns An async generator that yields each raw email object.
+     */
+    public async *fetchEmails(
+        userEmail: string,
+        syncState?: SyncState | null
+    ): AsyncGenerator<EmailObject> {
+        this.newDeltaTokens = syncState?.microsoft?.[userEmail]?.deltaTokens || {};
+
+        try {
+            const folders = this.listAllFolders(userEmail);
+            for await (const folder of folders) {
+                if (folder.id) {
+                    logger.info({ userEmail, folderId: folder.id, folderName: folder.displayName }, 'Syncing folder');
+                    yield* this.syncFolder(userEmail, folder.id, this.newDeltaTokens[folder.id]);
+                }
+            }
+        } catch (error) {
+            logger.error({ err: error, userEmail }, 'Failed to fetch emails from Microsoft 365');
+            throw error;
+        }
+    }
+
+    /**
+     * Lists all mail folders for a given user.
+     * @param userEmail The user principal name or ID.
+     * @returns An async generator that yields each mail folder.
+     */
+    private async *listAllFolders(userEmail: string): AsyncGenerator<MailFolder> {
+        let requestUrl: string | undefined = `/users/${userEmail}/mailFolders`;
+
+        while (requestUrl) {
+            try {
+                const response = await this.graphClient.api(requestUrl).get();
+                for (const folder of response.value as MailFolder[]) {
+                    yield folder;
+                }
+                requestUrl = response['@odata.nextLink'];
+            } catch (error) {
+                logger.error({ err: error, userEmail }, 'Failed to list mail folders');
+                throw error; // Stop if we can't list folders
+            }
+        }
+    }
+
+    /**
+     * Performs a delta sync on a single mail folder.
+     * @param userEmail The user's email.
+     * @param folderId The ID of the folder to sync.
+     * @param deltaToken The existing delta token for this folder, if any.
+     * @returns An async generator that yields email objects.
+     */
+    private async *syncFolder(
+        userEmail: string,
+        folderId: string,
+        deltaToken?: string
+    ): AsyncGenerator<EmailObject> {
+        let requestUrl: string | undefined;
+
+        if (deltaToken) {
+            // Continuous sync
+            requestUrl = deltaToken;
+        } else {
+            // Initial sync
+            requestUrl = `/users/${userEmail}/mailFolders/${folderId}/messages/delta`;
+        }
+
+        while (requestUrl) {
+            try {
+                const response = await this.graphClient.api(requestUrl).get();
+
+                for (const message of response.value) {
+                    if (message.id && !(message as any)['@removed']) {
+                        const rawEmail = await this.getRawEmail(userEmail, message.id);
+                        if (rawEmail) {
+                            yield await this.parseEmail(rawEmail, message.id, userEmail);
+                        }
+                    }
+                }
+
+                if (response['@odata.deltaLink']) {
+                    this.newDeltaTokens[folderId] = response['@odata.deltaLink'];
+                }
+
+                requestUrl = response['@odata.nextLink'];
+            } catch (error) {
+                logger.error({ err: error, userEmail, folderId }, 'Failed to sync mail folder');
+                // Continue to the next folder if one fails
+                return;
+            }
+        }
+    }
+
+    private async getRawEmail(userEmail: string, messageId: string): Promise<Buffer | null> {
+        try {
+            const response = await this.graphClient.api(`/users/${userEmail}/messages/${messageId}/$value`).getStream();
+            const chunks: any[] = [];
+            for await (const chunk of response) {
+                chunks.push(chunk);
+            }
+            return Buffer.concat(chunks);
+        } catch (error) {
+            logger.error({ err: error, userEmail, messageId }, 'Failed to fetch raw email content.');
+            return null;
+        }
+    }
+
+    private async parseEmail(rawEmail: Buffer, messageId: string, userEmail: string): Promise<EmailObject> {
+        const parsedEmail: ParsedMail = await simpleParser(rawEmail);
+        const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
+            filename: attachment.filename || 'untitled',
+            contentType: attachment.contentType,
+            size: attachment.size,
+            content: attachment.content as Buffer
+        }));
+        const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
+            if (!addresses) return [];
+            const addressArray = Array.isArray(addresses) ? addresses : [addresses];
+            return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
+        };
+
+        return {
+            id: messageId,
+            userEmail: userEmail,
+            eml: rawEmail,
+            from: mapAddresses(parsedEmail.from),
+            to: mapAddresses(parsedEmail.to),
+            cc: mapAddresses(parsedEmail.cc),
+            bcc: mapAddresses(parsedEmail.bcc),
+            subject: parsedEmail.subject || '',
+            body: parsedEmail.text || '',
+            html: parsedEmail.html || '',
+            headers: parsedEmail.headers as any,
+            attachments,
+            receivedAt: parsedEmail.date || new Date(),
+        };
+    }
+
+    public getUpdatedSyncState(userEmail: string): SyncState {
+        if (Object.keys(this.newDeltaTokens).length === 0) {
+            return {};
+        }
+        return {
+            microsoft: {
+                [userEmail]: {
+                    deltaTokens: this.newDeltaTokens
+                }
+            }
+        };
+    }
 }

packages/backend/src/services/storage/LocalFileSystemProvider.ts

@@ -37,8 +37,9 @@ export class LocalFileSystemProvider implements IStorageProvider {
     async delete(filePath: string): Promise<void> {
         const fullPath = path.join(this.rootPath, filePath);
         try {
-            await fs.unlink(fullPath);
+            await fs.rm(fullPath, { recursive: true, force: true });
         } catch (error: any) {
+            // Even with force: true, other errors might occur (e.g., permissions)
             if (error.code !== 'ENOENT') {
                 throw error;
             }

packages/backend/src/services/storage/S3StorageProvider.ts

@@ -5,6 +5,8 @@ import {
     DeleteObjectCommand,
     HeadObjectCommand,
     NotFound,
+    ListObjectsV2Command,
+    DeleteObjectsCommand,
 } from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
 import { Readable } from 'stream';
@@ -60,11 +62,28 @@ export class S3StorageProvider implements IStorageProvider {
     }
 
     async delete(path: string): Promise<void> {
-        const command = new DeleteObjectCommand({
+        // List all objects with the given prefix
+        const listCommand = new ListObjectsV2Command({
             Bucket: this.bucket,
-            Key: path,
+            Prefix: path,
         });
-        await this.client.send(command);
+        const listedObjects = await this.client.send(listCommand);
+
+        if (!listedObjects.Contents || listedObjects.Contents.length === 0) {
+            return;
+        }
+
+        // Create a list of objects to delete
+        const deleteParams = {
+            Bucket: this.bucket,
+            Delete: {
+                Objects: listedObjects.Contents.map(({ Key }) => ({ Key })),
+            },
+        };
+
+        // Delete the objects
+        const deleteCommand = new DeleteObjectsCommand(deleteParams);
+        await this.client.send(deleteCommand);
     }
 
     async exists(path: string): Promise<boolean> {

packages/backend/src/workers/ingestion.worker.ts

@@ -2,14 +2,20 @@ import { Worker } from 'bullmq';
 import { connection } from '../config/redis';
 import initialImportProcessor from '../jobs/processors/initial-import.processor';
 import continuousSyncProcessor from '../jobs/processors/continuous-sync.processor';
+import scheduleContinuousSyncProcessor from '../jobs/processors/schedule-continuous-sync.processor';
 import { processMailboxProcessor } from '../jobs/processors/process-mailbox.processor';
+import syncCycleFinishedProcessor from '../jobs/processors/sync-cycle-finished.processor';
 
 const processor = async (job: any) => {
     switch (job.name) {
         case 'initial-import':
             return initialImportProcessor(job);
+        case 'sync-cycle-finished':
+            return syncCycleFinishedProcessor(job);
         case 'continuous-sync':
             return continuousSyncProcessor(job);
+        case 'schedule-continuous-sync':
+            return scheduleContinuousSyncProcessor(job);
         case 'process-mailbox':
             return processMailboxProcessor(job);
         default:

packages/frontend/package.json

@@ -6,6 +6,7 @@
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
+		"start": "node build/index.js",
 		"preview": "vite preview",
 		"prepare": "svelte-kit sync || echo ''",
 		"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
@@ -15,28 +16,32 @@
 	},
 	"dependencies": {
 		"@open-archiver/types": "workspace:*",
+		"d3-shape": "^3.2.0",
 		"jose": "^6.0.1",
 		"lucide-svelte": "^0.525.0",
 		"postal-mime": "^2.4.4",
-		"svelte-persisted-store": "^0.12.0"
+		"svelte-persisted-store": "^0.12.0",
+		"@sveltejs/kit": "^2.16.0",
+		"bits-ui": "^2.8.10",
+		"clsx": "^2.1.1",
+		"tailwind-merge": "^3.3.1",
+		"tailwind-variants": "^1.0.0"
 	},
 	"devDependencies": {
 		"@internationalized/date": "^3.8.2",
 		"@lucide/svelte": "^0.515.0",
 		"@sveltejs/adapter-auto": "^6.0.0",
-		"@sveltejs/kit": "^2.16.0",
+		"@sveltejs/adapter-node": "^5.2.13",
 		"@sveltejs/vite-plugin-svelte": "^5.0.0",
 		"@tailwindcss/vite": "^4.0.0",
-		"bits-ui": "^2.8.10",
-		"clsx": "^2.1.1",
+		"@types/d3-shape": "^3.1.7",
 		"dotenv": "^17.2.0",
+		"layerchart": "2.0.0-next.27",
 		"prettier": "^3.4.2",
 		"prettier-plugin-svelte": "^3.3.3",
 		"prettier-plugin-tailwindcss": "^0.6.11",
 		"svelte": "^5.0.0",
 		"svelte-check": "^4.0.0",
-		"tailwind-merge": "^3.3.1",
-		"tailwind-variants": "^1.0.0",
 		"tailwindcss": "^4.0.0",
 		"tw-animate-css": "^1.3.5",
 		"typescript": "^5.0.0",

packages/frontend/src/app.css

@@ -1,121 +1,121 @@
-@import "tailwindcss";
+@import 'tailwindcss';
 
-@import "tw-animate-css";
+@import 'tw-animate-css';
 
 @custom-variant dark (&:is(.dark *));
 
 :root {
-  --radius: 0.625rem;
-  --background: oklch(1 0 0);
-  --foreground: oklch(0.129 0.042 264.695);
-  --card: oklch(1 0 0);
-  --card-foreground: oklch(0.129 0.042 264.695);
-  --popover: oklch(1 0 0);
-  --popover-foreground: oklch(0.129 0.042 264.695);
-  --primary: oklch(0.208 0.042 265.755);
-  --primary-foreground: oklch(0.984 0.003 247.858);
-  --secondary: oklch(0.968 0.007 247.896);
-  --secondary-foreground: oklch(0.208 0.042 265.755);
-  --muted: oklch(0.968 0.007 247.896);
-  --muted-foreground: oklch(0.554 0.046 257.417);
-  --accent: oklch(0.968 0.007 247.896);
-  --accent-foreground: oklch(0.208 0.042 265.755);
-  --destructive: oklch(0.577 0.245 27.325);
-  --border: oklch(0.929 0.013 255.508);
-  --input: oklch(0.929 0.013 255.508);
-  --ring: oklch(0.704 0.04 256.788);
-  --chart-1: oklch(0.646 0.222 41.116);
-  --chart-2: oklch(0.6 0.118 184.704);
-  --chart-3: oklch(0.398 0.07 227.392);
-  --chart-4: oklch(0.828 0.189 84.429);
-  --chart-5: oklch(0.769 0.188 70.08);
-  --sidebar: oklch(0.984 0.003 247.858);
-  --sidebar-foreground: oklch(0.129 0.042 264.695);
-  --sidebar-primary: oklch(0.208 0.042 265.755);
-  --sidebar-primary-foreground: oklch(0.984 0.003 247.858);
-  --sidebar-accent: oklch(0.968 0.007 247.896);
-  --sidebar-accent-foreground: oklch(0.208 0.042 265.755);
-  --sidebar-border: oklch(0.929 0.013 255.508);
-  --sidebar-ring: oklch(0.704 0.04 256.788);
+	--radius: 0.65rem;
+	--background: oklch(1 0 0);
+	--foreground: oklch(0.141 0.005 285.823);
+	--card: oklch(1 0 0);
+	--card-foreground: oklch(0.141 0.005 285.823);
+	--popover: oklch(1 0 0);
+	--popover-foreground: oklch(0.141 0.005 285.823);
+	--primary: oklch(0.705 0.213 47.604);
+	--primary-foreground: oklch(0.98 0.016 73.684);
+	--secondary: oklch(0.967 0.001 286.375);
+	--secondary-foreground: oklch(0.21 0.006 285.885);
+	--muted: oklch(0.967 0.001 286.375);
+	--muted-foreground: oklch(0.552 0.016 285.938);
+	--accent: oklch(0.967 0.001 286.375);
+	--accent-foreground: oklch(0.21 0.006 285.885);
+	--destructive: oklch(0.577 0.245 27.325);
+	--border: oklch(0.92 0.004 286.32);
+	--input: oklch(0.92 0.004 286.32);
+	--ring: oklch(0.705 0.213 47.604);
+	--chart-1: oklch(0.646 0.222 41.116);
+	--chart-2: oklch(0.705 0.213 47.604);
+	--chart-3: oklch(0.837 0.128 66.29);
+	--chart-4: oklch(0.553 0.195 38.402);
+	--chart-5: oklch(0.47 0.157 37.304);
+	--sidebar: oklch(0.985 0 0);
+	--sidebar-foreground: oklch(0.141 0.005 285.823);
+	--sidebar-primary: oklch(0.705 0.213 47.604);
+	--sidebar-primary-foreground: oklch(0.98 0.016 73.684);
+	--sidebar-accent: oklch(0.967 0.001 286.375);
+	--sidebar-accent-foreground: oklch(0.21 0.006 285.885);
+	--sidebar-border: oklch(0.92 0.004 286.32);
+	--sidebar-ring: oklch(0.705 0.213 47.604);
 }
 
 .dark {
-  --background: oklch(0.129 0.042 264.695);
-  --foreground: oklch(0.984 0.003 247.858);
-  --card: oklch(0.208 0.042 265.755);
-  --card-foreground: oklch(0.984 0.003 247.858);
-  --popover: oklch(0.208 0.042 265.755);
-  --popover-foreground: oklch(0.984 0.003 247.858);
-  --primary: oklch(0.929 0.013 255.508);
-  --primary-foreground: oklch(0.208 0.042 265.755);
-  --secondary: oklch(0.279 0.041 260.031);
-  --secondary-foreground: oklch(0.984 0.003 247.858);
-  --muted: oklch(0.279 0.041 260.031);
-  --muted-foreground: oklch(0.704 0.04 256.788);
-  --accent: oklch(0.279 0.041 260.031);
-  --accent-foreground: oklch(0.984 0.003 247.858);
-  --destructive: oklch(0.704 0.191 22.216);
-  --border: oklch(1 0 0 / 10%);
-  --input: oklch(1 0 0 / 15%);
-  --ring: oklch(0.551 0.027 264.364);
-  --chart-1: oklch(0.488 0.243 264.376);
-  --chart-2: oklch(0.696 0.17 162.48);
-  --chart-3: oklch(0.769 0.188 70.08);
-  --chart-4: oklch(0.627 0.265 303.9);
-  --chart-5: oklch(0.645 0.246 16.439);
-  --sidebar: oklch(0.208 0.042 265.755);
-  --sidebar-foreground: oklch(0.984 0.003 247.858);
-  --sidebar-primary: oklch(0.488 0.243 264.376);
-  --sidebar-primary-foreground: oklch(0.984 0.003 247.858);
-  --sidebar-accent: oklch(0.279 0.041 260.031);
-  --sidebar-accent-foreground: oklch(0.984 0.003 247.858);
-  --sidebar-border: oklch(1 0 0 / 10%);
-  --sidebar-ring: oklch(0.551 0.027 264.364);
+	--background: oklch(0.141 0.005 285.823);
+	--foreground: oklch(0.985 0 0);
+	--card: oklch(0.21 0.006 285.885);
+	--card-foreground: oklch(0.985 0 0);
+	--popover: oklch(0.21 0.006 285.885);
+	--popover-foreground: oklch(0.985 0 0);
+	--primary: oklch(0.646 0.222 41.116);
+	--primary-foreground: oklch(0.98 0.016 73.684);
+	--secondary: oklch(0.274 0.006 286.033);
+	--secondary-foreground: oklch(0.985 0 0);
+	--muted: oklch(0.274 0.006 286.033);
+	--muted-foreground: oklch(0.705 0.015 286.067);
+	--accent: oklch(0.274 0.006 286.033);
+	--accent-foreground: oklch(0.985 0 0);
+	--destructive: oklch(0.704 0.191 22.216);
+	--border: oklch(1 0 0 / 10%);
+	--input: oklch(1 0 0 / 15%);
+	--ring: oklch(0.646 0.222 41.116);
+	--chart-1: oklch(0.646 0.222 41.116);
+	--chart-2: oklch(0.705 0.213 47.604);
+	--chart-3: oklch(0.837 0.128 66.29);
+	--chart-4: oklch(0.553 0.195 38.402);
+	--chart-5: oklch(0.47 0.157 37.304);
+	--sidebar: oklch(0.21 0.006 285.885);
+	--sidebar-foreground: oklch(0.985 0 0);
+	--sidebar-primary: oklch(0.646 0.222 41.116);
+	--sidebar-primary-foreground: oklch(0.98 0.016 73.684);
+	--sidebar-accent: oklch(0.274 0.006 286.033);
+	--sidebar-accent-foreground: oklch(0.985 0 0);
+	--sidebar-border: oklch(1 0 0 / 10%);
+	--sidebar-ring: oklch(0.646 0.222 41.116);
 }
 
 @theme inline {
-  --radius-sm: calc(var(--radius) - 4px);
-  --radius-md: calc(var(--radius) - 2px);
-  --radius-lg: var(--radius);
-  --radius-xl: calc(var(--radius) + 4px);
-  --color-background: var(--background);
-  --color-foreground: var(--foreground);
-  --color-card: var(--card);
-  --color-card-foreground: var(--card-foreground);
-  --color-popover: var(--popover);
-  --color-popover-foreground: var(--popover-foreground);
-  --color-primary: var(--primary);
-  --color-primary-foreground: var(--primary-foreground);
-  --color-secondary: var(--secondary);
-  --color-secondary-foreground: var(--secondary-foreground);
-  --color-muted: var(--muted);
-  --color-muted-foreground: var(--muted-foreground);
-  --color-accent: var(--accent);
-  --color-accent-foreground: var(--accent-foreground);
-  --color-destructive: var(--destructive);
-  --color-border: var(--border);
-  --color-input: var(--input);
-  --color-ring: var(--ring);
-  --color-chart-1: var(--chart-1);
-  --color-chart-2: var(--chart-2);
-  --color-chart-3: var(--chart-3);
-  --color-chart-4: var(--chart-4);
-  --color-chart-5: var(--chart-5);
-  --color-sidebar: var(--sidebar);
-  --color-sidebar-foreground: var(--sidebar-foreground);
-  --color-sidebar-primary: var(--sidebar-primary);
-  --color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
-  --color-sidebar-accent: var(--sidebar-accent);
-  --color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
-  --color-sidebar-border: var(--sidebar-border);
-  --color-sidebar-ring: var(--sidebar-ring);
+	--radius-sm: calc(var(--radius) - 4px);
+	--radius-md: calc(var(--radius) - 2px);
+	--radius-lg: var(--radius);
+	--radius-xl: calc(var(--radius) + 4px);
+	--color-background: var(--background);
+	--color-foreground: var(--foreground);
+	--color-card: var(--card);
+	--color-card-foreground: var(--card-foreground);
+	--color-popover: var(--popover);
+	--color-popover-foreground: var(--popover-foreground);
+	--color-primary: var(--primary);
+	--color-primary-foreground: var(--primary-foreground);
+	--color-secondary: var(--secondary);
+	--color-secondary-foreground: var(--secondary-foreground);
+	--color-muted: var(--muted);
+	--color-muted-foreground: var(--muted-foreground);
+	--color-accent: var(--accent);
+	--color-accent-foreground: var(--accent-foreground);
+	--color-destructive: var(--destructive);
+	--color-border: var(--border);
+	--color-input: var(--input);
+	--color-ring: var(--ring);
+	--color-chart-1: var(--chart-1);
+	--color-chart-2: var(--chart-2);
+	--color-chart-3: var(--chart-3);
+	--color-chart-4: var(--chart-4);
+	--color-chart-5: var(--chart-5);
+	--color-sidebar: var(--sidebar);
+	--color-sidebar-foreground: var(--sidebar-foreground);
+	--color-sidebar-primary: var(--sidebar-primary);
+	--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
+	--color-sidebar-accent: var(--sidebar-accent);
+	--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
+	--color-sidebar-border: var(--sidebar-border);
+	--color-sidebar-ring: var(--sidebar-ring);
 }
 
 @layer base {
-  * {
-    @apply border-border outline-ring/50;
-  }
-  body {
-    @apply bg-background text-foreground;
-  }
-}
+	* {
+		@apply border-border outline-ring/50;
+	}
+	body {
+		@apply bg-background text-foreground;
+	}
+}

packages/frontend/src/lib/api.client.ts

@@ -1,5 +1,4 @@
 import { authStore } from '$lib/stores/auth.store';
-import type { User } from '@open-archiver/types';
 import { get } from 'svelte/store';
 
 const BASE_URL = '/api/v1'; // Using a relative URL for proxying

packages/frontend/src/lib/components/custom/EmailPreview.svelte

@@ -6,15 +6,16 @@
 		raw,
 		rawHtml
 	}: { raw?: Buffer | { type: 'Buffer'; data: number[] } | undefined; rawHtml?: string } = $props();
-
 	let parsedEmail: Email | null = $state(null);
 	let isLoading = $state(true);
 
 	// By adding a <base> tag, all relative and absolute links in the HTML document
 	// will open in a new tab by default.
 	let emailHtml = $derived(() => {
-		if (parsedEmail && parsedEmail?.html) {
+		if (parsedEmail && parsedEmail.html) {
 			return `<base target="_blank" />${parsedEmail.html}`;
+		} else if (parsedEmail && parsedEmail.text) {
+			return `<base target="_blank" />${parsedEmail.text}`;
 		} else if (rawHtml) {
 			return `<base target="_blank" />${rawHtml}`;
 		}

packages/frontend/src/lib/components/custom/EmptyState.svelte

@@ -0,0 +1,51 @@
+<script lang="ts">
+	import { Button, buttonVariants } from '$lib/components/ui/button/index.js';
+	import * as Dialog from '$lib/components/ui/dialog/index.js';
+	import { type Snippet } from 'svelte';
+
+	let {
+		header,
+		text,
+		buttonText,
+		click
+	}: {
+		header: string;
+		text: string;
+		buttonText?: string;
+		click: () => void;
+	} = $props();
+</script>
+
+<div class="space-y-4 rounded-lg border-2 border-dashed border-gray-300 p-6 text-center">
+	<div>
+		<svg
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			aria-hidden="true"
+			class="mx-auto size-12 text-gray-400"
+		>
+			<path
+				d="M9 13h6m-3-3v6m-9 1V7a2 2 0 012-2h6l2 2h6a2 2 0 012 2v8a2 2 0 01-2 2H5a2 2 0 01-2-2z"
+				stroke-width="2"
+				vector-effect="non-scaling-stroke"
+				stroke-linecap="round"
+				stroke-linejoin="round"
+			/>
+		</svg>
+	</div>
+
+	<h3 class="mt-2 text-sm font-semibold text-gray-900">{header}</h3>
+	<p class="mt-1 text-sm text-gray-500">{text}</p>
+	<div>
+		<Button
+			variant="outline"
+			class="cursor-pointer"
+			onclick={() => {
+				click();
+			}}
+		>
+			{buttonText}
+		</Button>
+	</div>
+</div>

packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte

@@ -12,7 +12,7 @@
 		onSubmit
 	}: {
 		source?: IngestionSource | null;
-		onSubmit: (data: CreateIngestionSourceDto) => void;
+		onSubmit: (data: CreateIngestionSourceDto) => Promise<void>;
 	} = $props();
 
 	const providerOptions = [
@@ -29,13 +29,25 @@
 		}
 	});
 
+	$effect(() => {
+		formData.providerConfig.type = formData.provider;
+		console.log(formData);
+	});
+
 	const triggerContent = $derived(
 		providerOptions.find((p) => p.value === formData.provider)?.label ?? 'Select a provider'
 	);
 
-	const handleSubmit = (event: Event) => {
+	let isSubmitting = $state(false);
+
+	const handleSubmit = async (event: Event) => {
 		event.preventDefault();
-		onSubmit(formData);
+		isSubmitting = true;
+		try {
+			await onSubmit(formData);
+		} finally {
+			isSubmitting = false;
+		}
 	};
 </script>
 
@@ -78,17 +90,23 @@
 		</div>
 	{:else if formData.provider === 'microsoft_365'}
 		<div class="grid grid-cols-4 items-center gap-4">
-			<Label for="clientId" class="text-right">Client ID</Label>
+			<Label for="clientId" class="text-right">Application (Client) ID</Label>
 			<Input id="clientId" bind:value={formData.providerConfig.clientId} class="col-span-3" />
 		</div>
 		<div class="grid grid-cols-4 items-center gap-4">
-			<Label for="clientSecret" class="text-right">Client Secret</Label>
+			<Label for="clientSecret" class="text-right">Client Secret Value</Label>
 			<Input
 				id="clientSecret"
+				type="password"
+				placeholder="Enter the secret Value, not the Secret ID"
 				bind:value={formData.providerConfig.clientSecret}
 				class="col-span-3"
 			/>
 		</div>
+		<div class="grid grid-cols-4 items-center gap-4">
+			<Label for="tenantId" class="text-right">Directory (Tenant) ID</Label>
+			<Input id="tenantId" bind:value={formData.providerConfig.tenantId} class="col-span-3" />
+		</div>
 	{:else if formData.provider === 'generic_imap'}
 		<div class="grid grid-cols-4 items-center gap-4">
 			<Label for="host" class="text-right">Host</Label>
@@ -113,6 +131,12 @@
 		</div>
 	{/if}
 	<Dialog.Footer>
-		<Button type="submit">Save changes</Button>
+		<Button type="submit" disabled={isSubmitting}>
+			{#if isSubmitting}
+				Submitting...
+			{:else}
+				Save changes
+			{/if}
+		</Button>
 	</Dialog.Footer>
 </form>

packages/frontend/src/lib/components/custom/charts/IngestionHistoryChart.svelte

@@ -0,0 +1,54 @@
+<script lang="ts">
+	import * as Chart from '$lib/components/ui/chart/index.js';
+	import { AreaChart } from 'layerchart';
+	import { curveCatmullRom } from 'd3-shape';
+	import type { ChartConfig } from '$lib/components/ui/chart';
+
+	export let data: { date: Date; count: number }[];
+
+	const chartConfig = {
+		count: {
+			label: 'Emails Ingested',
+			color: 'var(--chart-1)'
+		}
+	} satisfies ChartConfig;
+</script>
+
+<Chart.Container config={chartConfig} class="min-h-[300px] w-full">
+	<AreaChart
+		{data}
+		x="date"
+		y="count"
+		yDomain={[0, Math.max(...data.map((d) => d.count)) * 1.1]}
+		axis
+		legend={false}
+		series={[
+			{
+				key: 'count',
+				...chartConfig.count
+			}
+		]}
+		cRange={[
+			'var(--color-chart-1)',
+			'var(--color-chart-2)',
+			'var(--color-chart-3)',
+			'var(--color-chart-4)',
+			'var(--color-chart-5)'
+		]}
+		labels={{}}
+		props={{
+			xAxis: {
+				format: (d) =>
+					new Date(d).toLocaleDateString('en-US', {
+						month: 'short',
+						day: 'numeric'
+					})
+			},
+			area: { curve: curveCatmullRom }
+		}}
+	>
+		{#snippet tooltip()}
+			<Chart.Tooltip />
+		{/snippet}
+	</AreaChart>
+</Chart.Container>

packages/frontend/src/lib/components/custom/charts/StorageBySourceChart.svelte

@@ -0,0 +1,35 @@
+<script lang="ts">
+	import * as Chart from '$lib/components/ui/chart/index.js';
+	import { PieChart } from 'layerchart';
+	import type { IngestionSourceStats } from '@open-archiver/types';
+	import type { ChartConfig } from '$lib/components/ui/chart';
+
+	export let data: IngestionSourceStats[];
+
+	const chartConfig = {
+		storageUsed: {
+			label: 'Storage Used'
+		}
+	} satisfies ChartConfig;
+</script>
+
+<Chart.Container config={chartConfig} class="h-full min-h-[300px] w-full">
+	<PieChart
+		{data}
+		key="name"
+		value="storageUsed"
+		label="name"
+		legend={{}}
+		cRange={[
+			'var(--color-chart-1)',
+			'var(--color-chart-2)',
+			'var(--color-chart-3)',
+			'var(--color-chart-4)',
+			'var(--color-chart-5)'
+		]}
+	>
+		{#snippet tooltip()}
+			<Chart.Tooltip></Chart.Tooltip>
+		{/snippet}
+	</PieChart>
+</Chart.Container>

packages/frontend/src/lib/components/custom/charts/TopSendersChart.svelte

@@ -0,0 +1,44 @@
+<script lang="ts">
+	import * as Chart from '$lib/components/ui/chart/index.js';
+	import { BarChart } from 'layerchart';
+	import type { TopSender } from '@open-archiver/types';
+	import type { ChartConfig } from '$lib/components/ui/chart';
+
+	export let data: TopSender[];
+
+	const chartConfig = {
+		count: {
+			label: 'Emails'
+		}
+	} satisfies ChartConfig;
+</script>
+
+<Chart.Container config={chartConfig} class="min-h-[300px] w-full">
+	<BarChart
+		{data}
+		x="count"
+		y="sender"
+		orientation="horizontal"
+		xDomain={[0, Math.max(...data.map((d) => d.count)) * 1.1]}
+		axis={'x'}
+		legend={false}
+		series={[
+			{
+				key: 'count',
+				...chartConfig.count
+			}
+		]}
+		cRange={[
+			'var(--color-chart-1)',
+			'var(--color-chart-2)',
+			'var(--color-chart-3)',
+			'var(--color-chart-4)',
+			'var(--color-chart-5)'
+		]}
+		labels={{}}
+	>
+		{#snippet tooltip()}
+			<Chart.Tooltip />
+		{/snippet}
+	</BarChart>
+</Chart.Container>

packages/frontend/src/lib/components/ui/badge/badge.svelte

@@ -0,0 +1,49 @@
+<script lang="ts" module>
+	import { type VariantProps, tv } from 'tailwind-variants';
+
+	export const badgeVariants = tv({
+		base: 'focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive inline-flex w-fit shrink-0 items-center justify-center gap-1 overflow-hidden whitespace-nowrap rounded-md border px-2 py-0.5 text-xs font-medium transition-[color,box-shadow] focus-visible:ring-[3px] [&>svg]:pointer-events-none [&>svg]:size-3',
+		variants: {
+			variant: {
+				default: 'bg-primary text-primary-foreground [a&]:hover:bg-primary/90 border-transparent',
+				secondary:
+					'bg-secondary text-secondary-foreground [a&]:hover:bg-secondary/90 border-transparent',
+				destructive:
+					'bg-destructive [a&]:hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/70 border-transparent text-white',
+				outline: 'text-foreground [a&]:hover:bg-accent [a&]:hover:text-accent-foreground'
+			}
+		},
+		defaultVariants: {
+			variant: 'default'
+		}
+	});
+
+	export type BadgeVariant = VariantProps<typeof badgeVariants>['variant'];
+</script>
+
+<script lang="ts">
+	import type { HTMLAnchorAttributes } from 'svelte/elements';
+	import { cn, type WithElementRef } from '$lib/utils.js';
+
+	let {
+		ref = $bindable(null),
+		href,
+		class: className,
+		variant = 'default',
+		children,
+		...restProps
+	}: WithElementRef<HTMLAnchorAttributes> & {
+		variant?: BadgeVariant;
+	} = $props();
+</script>
+
+<svelte:element
+	this={href ? 'a' : 'span'}
+	bind:this={ref}
+	data-slot="badge"
+	{href}
+	class={cn(badgeVariants({ variant }), className)}
+	{...restProps}
+>
+	{@render children?.()}
+</svelte:element>

packages/frontend/src/lib/components/ui/badge/index.ts

@@ -0,0 +1,2 @@
+export { default as Badge } from "./badge.svelte";
+export { badgeVariants, type BadgeVariant } from "./badge.svelte";

packages/frontend/src/lib/components/ui/chart/chart-container.svelte

@@ -0,0 +1,80 @@
+<script lang="ts">
+	import { cn, type WithElementRef } from "$lib/utils.js";
+	import type { HTMLAttributes } from "svelte/elements";
+	import ChartStyle from "./chart-style.svelte";
+	import { setChartContext, type ChartConfig } from "./chart-utils.js";
+
+	const uid = $props.id();
+
+	let {
+		ref = $bindable(null),
+		id = uid,
+		class: className,
+		children,
+		config,
+		...restProps
+	}: WithElementRef<HTMLAttributes<HTMLElement>> & {
+		config: ChartConfig;
+	} = $props();
+
+	const chartId = `chart-${id || uid.replace(/:/g, "")}`;
+
+	setChartContext({
+		get config() {
+			return config;
+		},
+	});
+</script>
+
+<div
+	bind:this={ref}
+	data-chart={chartId}
+	data-slot="chart"
+	class={cn(
+		"flex aspect-video justify-center overflow-visible text-xs",
+		// Overrides
+		//
+		// Stroke around dots/marks when hovering
+		"[&_.stroke-white]:stroke-transparent",
+		// override the default stroke color of lines
+		"[&_.lc-line]:stroke-border/50",
+
+		// by default, layerchart shows a line intersecting the point when hovering, this hides that
+		"[&_.lc-highlight-line]:stroke-0",
+
+		// by default, when you hover a point on a stacked series chart, it will drop the opacity
+		// of the other series, this overrides that
+		"[&_.lc-area-path]:opacity-100 [&_.lc-highlight-line]:opacity-100 [&_.lc-highlight-point]:opacity-100 [&_.lc-spline-path]:opacity-100 [&_.lc-text-svg]:overflow-visible [&_.lc-text]:text-xs",
+
+		// We don't want the little tick lines between the axis labels and the chart, so we remove
+		// the stroke. The alternative is to manually disable `tickMarks` on the x/y axis of every
+		// chart.
+		"[&_.lc-axis-tick]:stroke-0",
+
+		// We don't want to display the rule on the x/y axis, as there is already going to be
+		// a grid line there and rule ends up overlapping the marks because it is rendered after
+		// the marks
+		"[&_.lc-rule-x-line:not(.lc-grid-x-rule)]:stroke-0 [&_.lc-rule-y-line:not(.lc-grid-y-rule)]:stroke-0",
+		"[&_.lc-grid-x-radial-line]:stroke-border [&_.lc-grid-x-radial-circle]:stroke-border",
+		"[&_.lc-grid-y-radial-line]:stroke-border [&_.lc-grid-y-radial-circle]:stroke-border",
+
+		// Legend adjustments
+		"[&_.lc-legend-swatch-button]:items-center [&_.lc-legend-swatch-button]:gap-1.5",
+		"[&_.lc-legend-swatch-group]:items-center [&_.lc-legend-swatch-group]:gap-4",
+		"[&_.lc-legend-swatch]:size-2.5 [&_.lc-legend-swatch]:rounded-[2px]",
+
+		// Labels
+		"[&_.lc-labels-text:not([fill])]:fill-foreground [&_text]:stroke-transparent",
+
+		// Tick labels on th x/y axes
+		"[&_.lc-axis-tick-label]:fill-muted-foreground [&_.lc-axis-tick-label]:font-normal",
+		"[&_.lc-tooltip-rects-g]:fill-transparent",
+		"[&_.lc-layout-svg-g]:fill-transparent",
+		"[&_.lc-root-container]:w-full",
+		className
+	)}
+	{...restProps}
+>
+	<ChartStyle id={chartId} {config} />
+	{@render children?.()}
+</div>

packages/frontend/src/lib/components/ui/chart/chart-style.svelte

@@ -0,0 +1,36 @@
+<script lang="ts">
+	import { THEMES, type ChartConfig } from "./chart-utils.js";
+
+	let { id, config }: { id: string; config: ChartConfig } = $props();
+
+	const colorConfig = $derived(
+		config ? Object.entries(config).filter(([, config]) => config.theme || config.color) : null
+	);
+
+	const themeContents = $derived.by(() => {
+		if (!colorConfig || !colorConfig.length) return;
+
+		const themeContents = [];
+		for (let [_theme, prefix] of Object.entries(THEMES)) {
+			let content = `${prefix} [data-chart=${id}] {\n`;
+			const color = colorConfig.map(([key, itemConfig]) => {
+				const theme = _theme as keyof typeof itemConfig.theme;
+				const color = itemConfig.theme?.[theme] || itemConfig.color;
+				return color ? `\t--color-${key}: ${color};` : null;
+			});
+
+			content += color.join("\n") + "\n}";
+
+			themeContents.push(content);
+		}
+
+		return themeContents.join("\n");
+	});
+</script>
+
+{#if themeContents}
+	{#key id}
+		<!-- eslint-disable-next-line svelte/no-at-html-tags -->
+		{@html `<style>${themeContents}</style>`}
+	{/key}
+{/if}

packages/frontend/src/lib/components/ui/chart/chart-tooltip.svelte

@@ -0,0 +1,159 @@
+<script lang="ts">
+	import { cn, type WithElementRef, type WithoutChildren } from "$lib/utils.js";
+	import type { HTMLAttributes } from "svelte/elements";
+	import { getPayloadConfigFromPayload, useChart, type TooltipPayload } from "./chart-utils.js";
+	import { getTooltipContext, Tooltip as TooltipPrimitive } from "layerchart";
+	import type { Snippet } from "svelte";
+
+	// eslint-disable-next-line @typescript-eslint/no-explicit-any
+	function defaultFormatter(value: any, _payload: TooltipPayload[]) {
+		return `${value}`;
+	}
+
+	let {
+		ref = $bindable(null),
+		class: className,
+		hideLabel = false,
+		indicator = "dot",
+		hideIndicator = false,
+		labelKey,
+		label,
+		labelFormatter = defaultFormatter,
+		labelClassName,
+		formatter,
+		nameKey,
+		color,
+		...restProps
+	}: WithoutChildren<WithElementRef<HTMLAttributes<HTMLDivElement>>> & {
+		hideLabel?: boolean;
+		label?: string;
+		indicator?: "line" | "dot" | "dashed";
+		nameKey?: string;
+		labelKey?: string;
+		hideIndicator?: boolean;
+		labelClassName?: string;
+		labelFormatter?: // eslint-disable-next-line @typescript-eslint/no-explicit-any
+		((value: any, payload: TooltipPayload[]) => string | number | Snippet) | null;
+		formatter?: Snippet<
+			[
+				{
+					value: unknown;
+					name: string;
+					item: TooltipPayload;
+					index: number;
+					payload: TooltipPayload[];
+				},
+			]
+		>;
+	} = $props();
+
+	const chart = useChart();
+	const tooltipCtx = getTooltipContext();
+
+	const formattedLabel = $derived.by(() => {
+		if (hideLabel || !tooltipCtx.payload?.length) return null;
+
+		const [item] = tooltipCtx.payload;
+		const key = labelKey ?? item?.label ?? item?.name ?? "value";
+
+		const itemConfig = getPayloadConfigFromPayload(chart.config, item, key);
+
+		const value =
+			!labelKey && typeof label === "string"
+				? (chart.config[label as keyof typeof chart.config]?.label ?? label)
+				: (itemConfig?.label ?? item.label);
+
+		if (value === undefined) return null;
+		if (!labelFormatter) return value;
+		return labelFormatter(value, tooltipCtx.payload);
+	});
+
+	const nestLabel = $derived(tooltipCtx.payload.length === 1 && indicator !== "dot");
+</script>
+
+{#snippet TooltipLabel()}
+	{#if formattedLabel}
+		<div class={cn("font-medium", labelClassName)}>
+			{#if typeof formattedLabel === "function"}
+				{@render formattedLabel()}
+			{:else}
+				{formattedLabel}
+			{/if}
+		</div>
+	{/if}
+{/snippet}
+
+<TooltipPrimitive.Root variant="none">
+	<div
+		class={cn(
+			"border-border/50 bg-background grid min-w-[9rem] items-start gap-1.5 rounded-lg border px-2.5 py-1.5 text-xs shadow-xl",
+			className
+		)}
+		{...restProps}
+	>
+		{#if !nestLabel}
+			{@render TooltipLabel()}
+		{/if}
+		<div class="grid gap-1.5">
+			{#each tooltipCtx.payload as item, i (item.key + i)}
+				{@const key = `${nameKey || item.key || item.name || "value"}`}
+				{@const itemConfig = getPayloadConfigFromPayload(chart.config, item, key)}
+				{@const indicatorColor = color || item.payload?.color || item.color}
+				<div
+					class={cn(
+						"[&>svg]:text-muted-foreground flex w-full flex-wrap items-stretch gap-2 [&>svg]:size-2.5",
+						indicator === "dot" && "items-center"
+					)}
+				>
+					{#if formatter && item.value !== undefined && item.name}
+						{@render formatter({
+							value: item.value,
+							name: item.name,
+							item,
+							index: i,
+							payload: tooltipCtx.payload,
+						})}
+					{:else}
+						{#if itemConfig?.icon}
+							<itemConfig.icon />
+						{:else if !hideIndicator}
+							<div
+								style="--color-bg: {indicatorColor}; --color-border: {indicatorColor};"
+								class={cn(
+									"border-(--color-border) bg-(--color-bg) shrink-0 rounded-[2px]",
+									{
+										"size-2.5": indicator === "dot",
+										"h-full w-1": indicator === "line",
+										"w-0 border-[1.5px] border-dashed bg-transparent":
+											indicator === "dashed",
+										"my-0.5": nestLabel && indicator === "dashed",
+									}
+								)}
+							></div>
+						{/if}
+						<div
+							class={cn(
+								"flex flex-1 shrink-0 justify-between leading-none",
+								nestLabel ? "items-end" : "items-center"
+							)}
+						>
+							<div class="grid gap-1.5">
+								{#if nestLabel}
+									{@render TooltipLabel()}
+								{/if}
+								<span class="text-muted-foreground">
+									{itemConfig?.label || item.name}
+								</span>
+							</div>
+							{#if item.value !== undefined}
+								<span class="text-foreground font-mono font-medium tabular-nums">
+									{item.value.toLocaleString()}
+								</span>
+							{/if}
+						</div>
+					{/if}
+				</div>
+			{/each}
+		</div>
+	</div>
+</TooltipPrimitive.Root>

packages/frontend/src/lib/components/ui/chart/chart-utils.ts

@@ -0,0 +1,66 @@
+import type { Tooltip } from "layerchart";
+import { getContext, setContext, type Component, type ComponentProps, type Snippet } from "svelte";
+
+export const THEMES = { light: "", dark: ".dark" } as const;
+
+export type ChartConfig = {
+	[k in string]: {
+		label?: string;
+		icon?: Component;
+	} & (
+		| { color?: string; theme?: never }
+		| { color?: never; theme: Record<keyof typeof THEMES, string> }
+	);
+};
+
+export type ExtractSnippetParams<T> = T extends Snippet<[infer P]> ? P : never;
+
+export type TooltipPayload = ExtractSnippetParams<
+	ComponentProps<typeof Tooltip.Root>["children"]
+>["payload"][number];
+
+// Helper to extract item config from a payload.
+export function getPayloadConfigFromPayload(
+	config: ChartConfig,
+	payload: TooltipPayload,
+	key: string
+) {
+	if (typeof payload !== "object" || payload === null) return undefined;
+
+	const payloadPayload =
+		"payload" in payload && typeof payload.payload === "object" && payload.payload !== null
+			? payload.payload
+			: undefined;
+
+	let configLabelKey: string = key;
+
+	if (payload.key === key) {
+		configLabelKey = payload.key;
+	} else if (payload.name === key) {
+		configLabelKey = payload.name;
+	} else if (key in payload && typeof payload[key as keyof typeof payload] === "string") {
+		configLabelKey = payload[key as keyof typeof payload] as string;
+	} else if (
+		payloadPayload !== undefined &&
+		key in payloadPayload &&
+		typeof payloadPayload[key as keyof typeof payloadPayload] === "string"
+	) {
+		configLabelKey = payloadPayload[key as keyof typeof payloadPayload] as string;
+	}
+
+	return configLabelKey in config ? config[configLabelKey] : config[key as keyof typeof config];
+}
+
+type ChartContextValue = {
+	config: ChartConfig;
+};
+
+const chartContextKey = Symbol("chart-context");
+
+export function setChartContext(value: ChartContextValue) {
+	return setContext(chartContextKey, value);
+}
+
+export function useChart() {
+	return getContext<ChartContextValue>(chartContextKey);
+}

packages/frontend/src/lib/components/ui/chart/index.ts

@@ -0,0 +1,6 @@
+import ChartContainer from "./chart-container.svelte";
+import ChartTooltip from "./chart-tooltip.svelte";
+
+export { getPayloadConfigFromPayload, type ChartConfig } from "./chart-utils.js";
+
+export { ChartContainer, ChartTooltip, ChartContainer as Container, ChartTooltip as Tooltip };

packages/frontend/src/lib/components/ui/switch/index.ts

@@ -0,0 +1,7 @@
+import Root from "./switch.svelte";
+
+export {
+	Root,
+	//
+	Root as Switch,
+};

packages/frontend/src/lib/components/ui/switch/switch.svelte

@@ -0,0 +1,29 @@
+<script lang="ts">
+	import { Switch as SwitchPrimitive } from "bits-ui";
+	import { cn, type WithoutChildrenOrChild } from "$lib/utils.js";
+
+	let {
+		ref = $bindable(null),
+		class: className,
+		checked = $bindable(false),
+		...restProps
+	}: WithoutChildrenOrChild<SwitchPrimitive.RootProps> = $props();
+</script>
+
+<SwitchPrimitive.Root
+	bind:ref
+	bind:checked
+	data-slot="switch"
+	class={cn(
+		"data-[state=checked]:bg-primary data-[state=unchecked]:bg-input focus-visible:border-ring focus-visible:ring-ring/50 dark:data-[state=unchecked]:bg-input/80 shadow-xs peer inline-flex h-[1.15rem] w-8 shrink-0 items-center rounded-full border border-transparent outline-none transition-all focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50",
+		className
+	)}
+	{...restProps}
+>
+	<SwitchPrimitive.Thumb
+		data-slot="switch-thumb"
+		class={cn(
+			"bg-background dark:data-[state=unchecked]:bg-foreground dark:data-[state=checked]:bg-primary-foreground pointer-events-none block size-4 rounded-full ring-0 transition-transform data-[state=checked]:translate-x-[calc(100%-2px)] data-[state=unchecked]:translate-x-0"
+		)}
+	/>
+</SwitchPrimitive.Root>

packages/frontend/src/lib/server/api.ts

@@ -2,6 +2,7 @@ import type { RequestEvent } from '@sveltejs/kit';
 
 const BASE_URL = '/api/v1'; // Using a relative URL for proxying
 
+
 /**
  * A custom fetch wrapper for the server-side to automatically handle authentication headers.
  * @param url The URL to fetch, relative to the API base.

packages/frontend/src/lib/utils.ts

@@ -5,9 +5,21 @@ export function cn(...inputs: ClassValue[]) {
 	return twMerge(clsx(inputs));
 }
 
+export function formatBytes(bytes: number, decimals = 2) {
+	if (bytes === 0) return '0 Bytes';
+
+	const k = 1024;
+	const dm = decimals < 0 ? 0 : decimals;
+	const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+
+	const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+	return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
+}
+
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export type WithoutChild<T> = T extends { child?: any } ? Omit<T, "child"> : T;
+export type WithoutChild<T> = T extends { child?: any; } ? Omit<T, "child"> : T;
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export type WithoutChildren<T> = T extends { children?: any } ? Omit<T, "children"> : T;
+export type WithoutChildren<T> = T extends { children?: any; } ? Omit<T, "children"> : T;
 export type WithoutChildrenOrChild<T> = WithoutChildren<WithoutChild<T>>;
-export type WithElementRef<T, U extends HTMLElement = HTMLElement> = T & { ref?: U | null };
+export type WithElementRef<T, U extends HTMLElement = HTMLElement> = T & { ref?: U | null; };

packages/frontend/src/routes/api/[...slug]/+server.ts

@@ -0,0 +1,30 @@
+import { env } from '$env/dynamic/private';
+import type { RequestHandler } from '@sveltejs/kit';
+
+const BACKEND_URL = `http://localhost:${env.PORT_BACKEND || 4000}`;
+
+const handleRequest: RequestHandler = async ({ request, params }) => {
+    const url = new URL(request.url);
+    const slug = params.slug || '';
+    const targetUrl = `${BACKEND_URL}/${slug}${url.search}`;
+
+    // Create a new request with the same method, headers, and body
+    const proxyRequest = new Request(targetUrl, {
+        method: request.method,
+        headers: request.headers,
+        body: request.body,
+        duplex: 'half' // Required for streaming request bodies
+    } as RequestInit);
+
+    // Forward the request to the backend
+    const response = await fetch(proxyRequest);
+
+    // Return the response from the backend
+    return response;
+};
+
+export const GET = handleRequest;
+export const POST = handleRequest;
+export const PUT = handleRequest;
+export const PATCH = handleRequest;
+export const DELETE = handleRequest;

packages/frontend/src/routes/dashboard/+layout.svelte

@@ -20,7 +20,7 @@
 
 <header class="bg-background sticky top-0 z-40 border-b">
 	<div class="container mx-auto flex h-16 flex-row items-center justify-between">
-		<a href="/dashboard" class="text-primary flex flex-row items-center gap-2 font-bold">
+		<a href="/dashboard" class="flex flex-row items-center gap-2 font-bold">
 			<img src="/logos/logo-sq.svg" alt="OpenArchiver Logo" class="h-8 w-8" />
 			<span>Open Archiver</span>
 		</a>

packages/frontend/src/routes/dashboard/+page.server.ts

@@ -0,0 +1,83 @@
+import type { PageServerLoad } from './$types';
+import { api } from '$lib/server/api';
+import type {
+    DashboardStats,
+    IngestionHistory,
+    IngestionSourceStats,
+    RecentSync,
+    IndexedInsights
+} from '@open-archiver/types';
+
+export const load: PageServerLoad = async (event) => {
+    const fetchStats = async (): Promise<DashboardStats | null> => {
+        try {
+            const response = await api('/dashboard/stats', event);
+            if (!response.ok) throw new Error('Failed to fetch stats');
+            return await response.json();
+        } catch (error) {
+            console.error('Dashboard Stats Error:', error);
+            return null;
+        }
+    };
+
+    const fetchIngestionHistory = async (): Promise<IngestionHistory | null> => {
+        try {
+            const response = await api('/dashboard/ingestion-history', event);
+            if (!response.ok) throw new Error('Failed to fetch ingestion history');
+            return await response.json();
+        } catch (error) {
+            console.error('Ingestion History Error:', error);
+            return null;
+        }
+    };
+
+    const fetchIngestionSources = async (): Promise<IngestionSourceStats[] | null> => {
+        try {
+            const response = await api('/dashboard/ingestion-sources', event);
+            if (!response.ok) throw new Error('Failed to fetch ingestion sources');
+            return await response.json();
+        } catch (error) {
+            console.error('Ingestion Sources Error:', error);
+            return null;
+        }
+    };
+
+    const fetchRecentSyncs = async (): Promise<RecentSync[] | null> => {
+        try {
+            const response = await api('/dashboard/recent-syncs', event);
+            if (!response.ok) throw new Error('Failed to fetch recent syncs');
+            return await response.json();
+        } catch (error) {
+            console.error('Recent Syncs Error:', error);
+            return null;
+        }
+    };
+
+    const fetchIndexedInsights = async (): Promise<IndexedInsights | null> => {
+        try {
+            const response = await api('/dashboard/indexed-insights', event);
+            if (!response.ok) throw new Error('Failed to fetch indexed insights');
+            return await response.json();
+        } catch (error) {
+            console.error('Indexed Insights Error:', error);
+            return null;
+        }
+    };
+
+    const [stats, ingestionHistory, ingestionSources, recentSyncs, indexedInsights] =
+        await Promise.all([
+            fetchStats(),
+            fetchIngestionHistory(),
+            fetchIngestionSources(),
+            fetchRecentSyncs(),
+            fetchIndexedInsights()
+        ]);
+
+    return {
+        stats,
+        ingestionHistory,
+        ingestionSources,
+        recentSyncs,
+        indexedInsights
+    };
+};

packages/frontend/src/routes/dashboard/+page.svelte

@@ -1,15 +1,132 @@
 <script lang="ts">
-	import { authStore } from '$lib/stores/auth.store';
+	import type { PageData } from './$types';
+	import * as Card from '$lib/components/ui/card/index.js';
+	import { formatBytes } from '$lib/utils';
+	import EmptyState from '$lib/components/custom/EmptyState.svelte';
+	import { goto } from '$app/navigation';
+	import { Archive, CircleAlert, HardDrive } from 'lucide-svelte';
+	import TopSendersChart from '$lib/components/custom/charts/TopSendersChart.svelte';
+	import IngestionHistoryChart from '$lib/components/custom/charts/IngestionHistoryChart.svelte';
+	import StorageBySourceChart from '$lib/components/custom/charts/StorageBySourceChart.svelte';
+	let { data }: { data: PageData } = $props();
+
+	const transformedHistory = $derived(
+		data.ingestionHistory?.history.map((item) => ({
+			...item,
+			date: new Date(item.date)
+		})) ?? []
+	);
 </script>
 
 <svelte:head>
-	<title>Dashboard - Open Archiver</title>
+	<title>Dashboard - OpenArchiver</title>
+	<meta name="description" content="Overview of your email archive." />
 </svelte:head>
 
-<div class="">
-	<div class="flex items-center justify-between">
-		<h1 class="text-2xl font-bold">Dashboard</h1>
+<div class="flex-1 space-y-4">
+	<div class="flex items-center justify-between space-y-2">
+		<h2 class="text-3xl font-bold tracking-tight">Dashboard</h2>
 	</div>
-	<p class="mt-4">Welcome, {$authStore.user?.email}!</p>
-	<p>You are logged in.</p>
+	{#if !data.ingestionSources || data.ingestionSources?.length === 0}
+		<div>
+			<EmptyState
+				buttonText="Create an ingestion"
+				header="You don't have any ingestion source set up."
+				text="Add an ingestion source to start archiving your inboxes."
+				click={() => {
+					goto('/dashboard/ingestions');
+				}}
+			></EmptyState>
+		</div>
+	{:else}
+		<!-- show data -->
+		<div class="space-y-4">
+			{#if data.stats}
+				<div class="grid gap-4 md:grid-cols-2 lg:grid-cols-3">
+					<Card.Root>
+						<Card.Header class="flex flex-row items-center justify-between space-y-0 pb-2">
+							<Card.Title class="text-sm font-medium">Total Emails Archived</Card.Title>
+							<Archive class="text-muted-foreground h-4 w-4" />
+						</Card.Header>
+						<Card.Content>
+							<div class="text-primary text-2xl font-bold">{data.stats.totalEmailsArchived}</div>
+						</Card.Content>
+					</Card.Root>
+					<Card.Root>
+						<Card.Header class="flex flex-row items-center justify-between space-y-0 pb-2">
+							<Card.Title class="text-sm font-medium">Total Storage Used</Card.Title>
+							<HardDrive class="text-muted-foreground h-4 w-4" />
+						</Card.Header>
+						<Card.Content>
+							<div class="text-primary text-2xl font-bold">
+								{formatBytes(data.stats.totalStorageUsed)}
+							</div>
+						</Card.Content>
+					</Card.Root>
+					<Card.Root class="">
+						<Card.Header class="flex flex-row items-center justify-between space-y-0 pb-2">
+							<Card.Title class="text-sm font-medium">Failed Ingestions (Last 7 Days)</Card.Title>
+							<CircleAlert class=" text-muted-foreground h-4 w-4" />
+						</Card.Header>
+						<Card.Content>
+							<div
+								class="  text-2xl font-bold text-green-500"
+								class:text-destructive={data.stats.failedIngestionsLast7Days > 0}
+							>
+								{data.stats.failedIngestionsLast7Days}
+							</div>
+						</Card.Content>
+					</Card.Root>
+				</div>
+			{/if}
+
+			<div class="grid grid-cols-1 gap-4 lg:grid-cols-3">
+				<div class=" lg:col-span-2">
+					<Card.Root>
+						<Card.Header>
+							<Card.Title>Ingestion History</Card.Title>
+						</Card.Header>
+						<Card.Content class=" pl-4">
+							{#if transformedHistory.length > 0}
+								<IngestionHistoryChart data={transformedHistory} />
+							{:else}
+								<p>No ingestion history available.</p>
+							{/if}
+						</Card.Content>
+					</Card.Root>
+				</div>
+				<div class=" lg:col-span-1">
+					<Card.Root class="h-full">
+						<Card.Header>
+							<Card.Title>Storage by Ingestion Source (Bytes)</Card.Title>
+						</Card.Header>
+						<Card.Content class="h-full">
+							{#if data.ingestionSources && data.ingestionSources.length > 0}
+								<StorageBySourceChart data={data.ingestionSources} />
+							{:else}
+								<p>No ingestion sources available.</p>
+							{/if}
+						</Card.Content>
+					</Card.Root>
+				</div>
+			</div>
+			<div>
+				<h1 class="text-xl font-semibold leading-6">Indexed insights</h1>
+			</div>
+			<div class="grid grid-cols-1">
+				<Card.Root>
+					<Card.Header>
+						<Card.Title>Top 10 Senders</Card.Title>
+					</Card.Header>
+					<Card.Content>
+						{#if data.indexedInsights && data.indexedInsights.topSenders.length > 0}
+							<TopSendersChart data={data.indexedInsights.topSenders} />
+						{:else}
+							<p>No indexed insights available.</p>
+						{/if}
+					</Card.Content>
+				</Card.Root>
+			</div>
+		</div>
+	{/if}
 </div>

packages/frontend/src/routes/dashboard/archived-emails/+page.svelte

@@ -95,6 +95,7 @@
 		<Table.Header>
 			<Table.Row>
 				<Table.Head>Date</Table.Head>
+				<Table.Head>Inbox</Table.Head>
 				<Table.Head>Subject</Table.Head>
 				<Table.Head>Sender</Table.Head>
 				<Table.Head>Attachments</Table.Head>
@@ -106,10 +107,13 @@
 				{#each archivedEmails.items as email (email.id)}
 					<Table.Row>
 						<Table.Cell>{new Date(email.sentAt).toLocaleString()}</Table.Cell>
+						<Table.Cell>{email.userEmail}</Table.Cell>
 						<Table.Cell>
-							<a href={`/dashboard/archived-emails/${email.id}`}>
-								{email.subject}
-							</a>
+							<div class="max-w-100 truncate">
+								<a href={`/dashboard/archived-emails/${email.id}`}>
+									{email.subject}
+								</a>
+							</div>
 						</Table.Cell>
 						<Table.Cell>{email.senderEmail}</Table.Cell>
 						<Table.Cell>{email.hasAttachments ? 'Yes' : 'No'}</Table.Cell>

packages/frontend/src/routes/dashboard/archived-emails/[id]/+page.svelte

@@ -8,7 +8,6 @@
 
 	let { data }: { data: PageData } = $props();
 	const { email } = data;
-	console.log(email);
 	async function download(path: string, filename: string) {
 		if (!browser) return;
 
@@ -37,7 +36,7 @@
 
 {#if email}
 	<div class="grid grid-cols-3 gap-6">
-		<div class="col-span-2">
+		<div class="col-span-3 md:col-span-2">
 			<Card.Root>
 				<Card.Header>
 					<Card.Title>{email.subject || 'No Subject'}</Card.Title>
@@ -80,7 +79,7 @@
 				</Card.Content>
 			</Card.Root>
 		</div>
-		<div class="col-span-1">
+		<div class="col-span-3 md:col-span-1">
 			<Card.Root>
 				<Card.Header>
 					<Card.Title>Actions</Card.Title>

packages/frontend/src/routes/dashboard/ingestions/+page.svelte

@@ -5,15 +5,21 @@
 	import * as DropdownMenu from '$lib/components/ui/dropdown-menu';
 	import { MoreHorizontal } from 'lucide-svelte';
 	import * as Dialog from '$lib/components/ui/dialog';
+	import { Switch } from '$lib/components/ui/switch';
 	import IngestionSourceForm from '$lib/components/custom/IngestionSourceForm.svelte';
 	import { api } from '$lib/api.client';
 	import type { IngestionSource, CreateIngestionSourceDto } from '@open-archiver/types';
+	import Badge from '$lib/components/ui/badge/badge.svelte';
+	import type { BadgeVariant } from '$lib/components/ui/badge/badge.svelte';
 
 	let { data }: { data: PageData } = $props();
 
 	let ingestionSources = $state(data.ingestionSources);
 	let isDialogOpen = $state(false);
+	let isDeleteDialogOpen = $state(false);
 	let selectedSource = $state<IngestionSource | null>(null);
+	let sourceToDelete = $state<IngestionSource | null>(null);
+	let isDeleting = $state(false);
 
 	const openCreateDialog = () => {
 		selectedSource = null;
@@ -25,12 +31,22 @@
 		isDialogOpen = true;
 	};
 
-	const handleDelete = async (id: string) => {
-		if (!confirm('Are you sure you want to delete this ingestion source?')) {
-			return;
+	const openDeleteDialog = (source: IngestionSource) => {
+		sourceToDelete = source;
+		isDeleteDialogOpen = true;
+	};
+
+	const confirmDelete = async () => {
+		if (!sourceToDelete) return;
+		isDeleting = true;
+		try {
+			await api(`/ingestion-sources/${sourceToDelete.id}`, { method: 'DELETE' });
+			ingestionSources = ingestionSources.filter((s) => s.id !== sourceToDelete!.id);
+			isDeleteDialogOpen = false;
+			sourceToDelete = null;
+		} finally {
+			isDeleting = false;
 		}
-		await api(`/ingestion-sources/${id}`, { method: 'DELETE' });
-		ingestionSources = ingestionSources.filter((s) => s.id !== id);
 	};
 
 	const handleSync = async (id: string) => {
@@ -44,6 +60,27 @@
 		ingestionSources = updatedSources;
 	};
 
+	const handleToggle = async (source: IngestionSource) => {
+		const isPaused = source.status === 'paused';
+		const newStatus = isPaused ? 'active' : 'paused';
+
+		if (newStatus === 'paused') {
+			await api(`/ingestion-sources/${source.id}/pause`, { method: 'POST' });
+		} else {
+			await api(`/ingestion-sources/${source.id}`, {
+				method: 'PUT',
+				body: JSON.stringify({ status: 'active' })
+			});
+		}
+
+		ingestionSources = ingestionSources.map((s) => {
+			if (s.id === source.id) {
+				return { ...s, status: newStatus };
+			}
+			return s;
+		});
+	};
+
 	const handleFormSubmit = async (formData: CreateIngestionSourceDto) => {
 		if (selectedSource) {
 			// Update
@@ -66,6 +103,27 @@
 		}
 		isDialogOpen = false;
 	};
+
+	function getStatusClasses(status: IngestionSource['status']): string {
+		switch (status) {
+			case 'active':
+				return 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300';
+			case 'paused':
+				return 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-300';
+			case 'error':
+				return 'bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-300';
+			case 'syncing':
+				return 'bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-300';
+			case 'importing':
+				return 'bg-purple-100 text-purple-800 dark:bg-purple-900 dark:text-purple-300';
+			case 'pending_auth':
+				return 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-300';
+			case 'auth_success':
+				return 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-300';
+			default:
+				return 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-300';
+		}
+	}
 </script>
 
 <div class="">
@@ -81,6 +139,7 @@
 					<Table.Head>Name</Table.Head>
 					<Table.Head>Provider</Table.Head>
 					<Table.Head>Status</Table.Head>
+					<Table.Head>Active</Table.Head>
 					<Table.Head>Created At</Table.Head>
 					<Table.Head class="text-right">Actions</Table.Head>
 				</Table.Row>
@@ -89,9 +148,24 @@
 				{#if ingestionSources.length > 0}
 					{#each ingestionSources as source (source.id)}
 						<Table.Row>
-							<Table.Cell>{source.name}</Table.Cell>
-							<Table.Cell>{source.provider}</Table.Cell>
-							<Table.Cell>{source.status}</Table.Cell>
+							<Table.Cell>
+								<a href="/dashboard/archived-emails?ingestionSourceId={source.id}">{source.name}</a>
+							</Table.Cell>
+							<Table.Cell class="capitalize">{source.provider.split('_').join(' ')}</Table.Cell>
+							<Table.Cell class="min-w-24">
+								<Badge class="{getStatusClasses(source.status)} capitalize">
+									{source.status.split('_').join(' ')}
+								</Badge>
+							</Table.Cell>
+							<Table.Cell>
+								<Switch
+									id={`active-switch-${source.id}`}
+									class="cursor-pointer"
+									checked={source.status !== 'paused'}
+									onCheckedChange={() => handleToggle(source)}
+									disabled={source.status !== 'active' && source.status !== 'paused'}
+								/>
+							</Table.Cell>
 							<Table.Cell>{new Date(source.createdAt).toLocaleDateString()}</Table.Cell>
 							<Table.Cell class="text-right">
 								<DropdownMenu.Root>
@@ -109,7 +183,7 @@
 										<DropdownMenu.Item onclick={() => handleSync(source.id)}>Sync</DropdownMenu.Item
 										>
 										<DropdownMenu.Separator />
-										<DropdownMenu.Item class="text-red-600" onclick={() => handleDelete(source.id)}
+										<DropdownMenu.Item class="text-red-600" onclick={() => openDeleteDialog(source)}
 											>Delete</DropdownMenu.Item
 										>
 									</DropdownMenu.Content>
@@ -140,3 +214,23 @@
 		<IngestionSourceForm source={selectedSource} onSubmit={handleFormSubmit} />
 	</Dialog.Content>
 </Dialog.Root>
+
+<Dialog.Root bind:open={isDeleteDialogOpen}>
+	<Dialog.Content class="sm:max-w-lg">
+		<Dialog.Header>
+			<Dialog.Title>Are you sure you want to delete this ingestion?</Dialog.Title>
+			<Dialog.Description>
+				This will delete all archived emails, attachments, indexing, and files associated with this
+				ingestion. If you only want to stop syncing new emails, you can pause the ingestion instead.
+			</Dialog.Description>
+		</Dialog.Header>
+		<Dialog.Footer class="sm:justify-start">
+			<Button type="button" variant="destructive" onclick={confirmDelete} disabled={isDeleting}
+				>{#if isDeleting}Deleting...{:else}Confirm{/if}</Button
+			>
+			<Dialog.Close>
+				<Button type="button" variant="secondary">Cancel</Button>
+			</Dialog.Close>
+		</Dialog.Footer>
+	</Dialog.Content>
+</Dialog.Root>

packages/frontend/svelte.config.js

@@ -1,4 +1,4 @@
-import adapter from '@sveltejs/adapter-auto';
+import adapter from '@sveltejs/adapter-node';
 import { vitePreprocess } from '@sveltejs/vite-plugin-svelte';
 
 /** @type {import('@sveltejs/kit').Config} */

packages/frontend/vite.config.ts

@@ -16,5 +16,8 @@ export default defineConfig({
 				rewrite: (path) => path.replace(/^\/api/, '')
 			}
 		}
+	},
+	ssr: {
+		noExternal: ['layerchart']
 	}
 });

packages/types/src/archived-emails.types.ts

@@ -23,6 +23,7 @@ export interface Attachment {
 export interface ArchivedEmail {
     id: string;
     ingestionSourceId: string;
+    userEmail: string;
     messageIdHeader: string | null;
     sentAt: Date;
     subject: string | null;

packages/types/src/dashboard.types.ts

@@ -0,0 +1,38 @@
+export interface DashboardStats {
+    totalEmailsArchived: number;
+    totalStorageUsed: number;
+    failedIngestionsLast7Days: number;
+}
+
+export interface IngestionHistory {
+    history: {
+        date: string;
+        count: number;
+    }[];
+}
+
+export interface IngestionSourceStats {
+    id: string;
+    name: string;
+    provider: string;
+    status: string;
+    storageUsed: number;
+}
+
+export interface RecentSync {
+    id: string;
+    sourceName: string;
+    startTime: string;
+    duration: number;
+    emailsProcessed: number;
+    status: string;
+}
+
+export interface TopSender {
+    sender: string;
+    count: number;
+}
+
+export interface IndexedInsights {
+    topSenders: TopSender[];
+}

packages/types/src/index.ts

@@ -5,3 +5,4 @@ export * from './storage.types';
 export * from './email.types';
 export * from './archived-emails.types';
 export * from './search.types';
+export * from './dashboard.types';

packages/types/src/ingestion.types.ts

@@ -1,3 +1,20 @@
+export type SyncState = {
+    google?: {
+        [userEmail: string]: {
+            historyId: string;
+        };
+    };
+    microsoft?: {
+        [userEmail: string]: {
+            deltaTokens: { [folderId: string]: string; };
+        };
+    };
+    imap?: {
+        maxUid: number;
+    };
+    lastSyncTimestamp?: string;
+};
+
 export type IngestionProvider = 'google_workspace' | 'microsoft_365' | 'generic_imap';
 
 export type IngestionStatus =
@@ -9,7 +26,11 @@ export type IngestionStatus =
     | 'importing'
     | 'auth_success';
 
-export interface GenericImapCredentials {
+export interface BaseIngestionCredentials {
+    type: IngestionProvider;
+}
+
+export interface GenericImapCredentials extends BaseIngestionCredentials {
     type: 'generic_imap';
     host: string;
     port: number;
@@ -18,7 +39,7 @@ export interface GenericImapCredentials {
     password?: string;
 }
 
-export interface GoogleWorkspaceCredentials {
+export interface GoogleWorkspaceCredentials extends BaseIngestionCredentials {
     type: 'google_workspace';
     /**
      * The full JSON content of the Google Service Account key.
@@ -31,10 +52,11 @@ export interface GoogleWorkspaceCredentials {
     impersonatedAdminEmail: string;
 }
 
-export interface Microsoft365Credentials {
+export interface Microsoft365Credentials extends BaseIngestionCredentials {
     type: 'microsoft_365';
     clientId: string;
     clientSecret: string;
+    tenantId: string;
 }
 
 // Discriminated union for all possible credential types
@@ -51,6 +73,10 @@ export interface IngestionSource {
     createdAt: Date;
     updatedAt: Date;
     credentials: IngestionCredentials;
+    lastSyncStartedAt?: Date | null;
+    lastSyncFinishedAt?: Date | null;
+    lastSyncStatusMessage?: string | null;
+    syncState?: SyncState | null;
 }
 
 export interface CreateIngestionSourceDto {
@@ -67,6 +93,11 @@ export interface UpdateIngestionSourceDto {
     lastSyncStartedAt?: Date;
     lastSyncFinishedAt?: Date;
     lastSyncStatusMessage?: string;
+    syncState?: SyncState;
+}
+
+export interface IContinuousSyncJob {
+    ingestionSourceId: string;
 }
 
 export interface IInitialImportJob {
@@ -77,3 +108,9 @@ export interface IProcessMailboxJob {
     ingestionSourceId: string;
     userEmail: string;
 }
+
+export type MailboxUser = {
+    id: string;
+    primaryEmail: string;
+    displayName: string;
+};