Merge pull request #15 from LogicLabs-OU/dev

Thread discovery

.dockerignore

@@ -0,0 +1,46 @@
+# Git
+.git
+.gitignore
+
+# Node
+node_modules
+.pnpm-store
+
+# Env
+.env
+.env.*
+!/.env.example
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# IDEs
+.idea
+.vscode
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
+
+# Docker
+docker-compose.yml
+.dockerignore
+Dockerfile
+
+# Local data
+meili_data

.env.example

@@ -1,25 +1,40 @@
-# Application
+
+# --- Application Settings ---
+# Set to 'production' for production environments
 NODE_ENV=development
 PORT_BACKEND=4000
 PORT_FRONTEND=3000
 
-# PostgreSQL
-DATABASE_URL="postgresql://admin:password@postgres:5432/open_archive?schema=public"
+# --- Docker Compose Service Configuration ---
+# These variables are used by docker-compose.yml to configure the services. Leave them unchanged if you use Docker services for Postgresql, Valkey (Redis) and Meilisearch. If you decide to use your own instances of these services, you can substitute them with your own connection credentials.
 
-# Redis
-REDIS_HOST=redis
-REDIS_PORT=6379
+# PostgreSQL
+POSTGRES_DB=open_archive
+POSTGRES_USER=admin
+POSTGRES_PASSWORD=password
+DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}"
 
 # Meilisearch
 MEILI_MASTER_KEY=aSampleMasterKey
 MEILI_HOST=http://meilisearch:7700
 
-# Storage
+
+
+# Valkey (Redis compatible)
+REDIS_HOST=valkey
+REDIS_PORT=6379
+REDIS_PASSWORD=defaultredispassword
+# If you run Valkey service from Docker Compose, set the REDIS_TLS_ENABLED variable to false.
+REDIS_TLS_ENABLED=false
+
+
+# --- Storage Settings ---
 # Choose your storage backend. Valid options are 'local' or 's3'.
 STORAGE_TYPE=local
 
 # --- Local Storage Settings ---
-# The absolute path on the server where files will be stored.
+# The path inside the container where files will be stored.
+# This is mapped to a Docker volume for persistence.
 # This is only used if STORAGE_TYPE is 'local'.
 STORAGE_LOCAL_ROOT_PATH=/var/data/open-archiver
 
@@ -33,18 +48,20 @@ STORAGE_S3_REGION=
 # Set to 'true' for MinIO and other non-AWS S3 services
 STORAGE_S3_FORCE_PATH_STYLE=false
 
+# --- Security & Authentication ---
+
 # JWT
-JWT_SECRET="a-very-secret-key"
+# IMPORTANT: Change this to a long, random, and secret string in your .env file
+JWT_SECRET=a-very-secret-key-that-you-should-change
 JWT_EXPIRES_IN="7d"
 
-
-
-# Admin users
-ADMIN_EMAIL=admin@local.com 
-ADMIN_PASSWORD=a_strong_pass
+# Admin User
+# Set the credentials for the initial admin user.
+ADMIN_EMAIL=admin@local.com
+ADMIN_PASSWORD=a_strong_password_that_you_should_change
 SUPER_API_KEY=
 
-# Master Encryption Key for sensitive data
+# Master Encryption Key for sensitive data (Such as Ingestion source credentials and passwords)
 # IMPORTANT: Generate a secure, random 32-byte hex string for this
-# Use `openssl rand -hex 32` to generate a key
+# You can use `openssl rand -hex 32` to generate a key.
 ENCRYPTION_KEY=

.github/CLA.md

@@ -0,0 +1,25 @@
+# Contributor License Agreement (CLA)
+
+This license is for your protection as a Contributor as well as the protection of the maintainers of the Open Archiver software; it does not change your rights to use your own Contributions for any other purpose. In the following, the maintainers of Open Archiver are referred to as "Open Archiver".
+
+You accept and agree to the following terms and conditions for Your present and future Contributions submitted to "Open Archiver". Except for the license granted herein to Open Archiver and recipients of software distributed by "Open Archiver", You reserve all right, title, and interest in and to Your Contributions.
+
+1. Definitions.
+
+    "You" (or "Your") shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with "Open Archiver". For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor.
+
+    "Contribution" shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to "Open Archiver" for inclusion in, or documentation of, any of the products owned or managed by "Open Archiver" (the "Work"). For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to "Open Archiver" or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, "Open Archiver" for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution."
+
+2. Grant of Copyright License. Subject to the terms and conditions of this Agreement, You grant to "Open Archiver" and to recipients of software distributed by "Open Archiver" a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works.
+
+3. Grant of Patent License. Subject to the terms and conditions of this Agreement, You grant to "Open Archiver" and to recipients of software distributed by "Open Archiver" a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed.
+
+4. You represent that you are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes your Contributions, you represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to "Open Archiver", or that your employer has executed a separate Contributor License Agreement with "Open Archiver".
+
+5. You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware and which are associated with any part of Your Contributions.
+
+6. You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE.
+
+7. Should You wish to submit work that is not Your original creation, You may submit it to "Open Archiver" separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as "Submitted on behalf of a third-party: [named here]".
+
+8. You agree to notify "Open Archiver" of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect.

.github/workflows/cla.yml

@@ -0,0 +1,32 @@
+name: 'CLA Assistant'
+on:
+    issue_comment:
+        types: [created]
+    pull_request_target:
+        types: [opened, closed, synchronize]
+
+# explicitly configure permissions, in case your GITHUB_TOKEN workflow permissions are set to read-only in repository settings
+permissions:
+    actions: write
+    contents: write # this can be 'read' if the signatures are in remote repository
+    pull-requests: write
+    statuses: write
+
+jobs:
+    CLAAssistant:
+        runs-on: ubuntu-latest
+        steps:
+            - name: 'CLA Assistant'
+              if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
+              uses: contributor-assistant/github-action@v2.6.1
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+              with:
+                  path-to-signatures: 'signatures/version1/cla.json'
+                  path-to-document: 'https://github.com/LogicLabs-OU/OpenArchiver/tree/main/.github/CLA.md'
+                  branch: 'main'
+                  allowlist: 'wayneshn'
+
+                  remote-organization-name: 'LogicLabs-OU'
+                  remote-repository-name: 'cla-db'

.github/workflows/deploy-docs.yml

@@ -0,0 +1,43 @@
+name: Deploy Docs to GitHub Pages
+
+on:
+    push:
+        branches:
+            - main
+        paths:
+            - 'docs/**'
+
+jobs:
+    build-and-deploy:
+        runs-on: ubuntu-latest
+        permissions:
+            contents: write
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+
+            - name: Setup pnpm
+              uses: pnpm/action-setup@v2
+              with:
+                  version: 10.13.1
+
+            - name: Setup Node.js
+              uses: actions/setup-node@v4
+              with:
+                  node-version: '22'
+                  cache: 'pnpm'
+
+            - name: Install dependencies
+              run: pnpm install --frozen-lockfile
+
+            - name: Build documentation site
+              run: pnpm docs:build
+
+            - name: Create CNAME file
+              run: echo "docs.openarchiver.com" > docs/.vitepress/dist/CNAME
+
+            - name: Deploy to GitHub Pages
+              uses: peaceiris/actions-gh-pages@v3
+              with:
+                  github_token: ${{ secrets.GITHUB_TOKEN }}
+                  publish_dir: docs/.vitepress/dist

.github/workflows/docker-deployment.yml

@@ -0,0 +1,41 @@
+name: docker-deployment
+
+on:
+    push:
+        branches:
+            - main
+        paths-ignore:
+            - '**.md'
+            - 'docs/**'
+
+jobs:
+    docker:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+
+            - name: Login to Docker Hub
+              uses: docker/login-action@v3
+              with:
+                  username: ${{ vars.DOCKERHUB_USERNAME }}
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v3
+
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+
+            - name: Extract short SHA
+              id: sha
+              run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+            - name: Build and push
+              uses: docker/build-push-action@v6
+              with:
+                  context: .
+                  file: ./docker/Dockerfile
+                  platforms: linux/amd64,linux/arm64
+                  push: true
+                  tags: logiclabshq/open-archiver:${{ steps.sha.outputs.sha }}

.gitignore

@@ -9,7 +9,6 @@ dist
 **/meili_data/
 
 # PNPM
-pnpm-lock.yaml
 pnpm-debug.log
 
 # IDE
@@ -21,3 +20,7 @@ pnpm-debug.log
 
 # Dev
 .dev
+
+# Vitepress
+docs/.vitepress/dist
+docs/.vitepress/cache

CONTRIBUTING.md

@@ -29,6 +29,10 @@ If you have an idea for an enhancement, please open an issue to discuss it. This
 6.  **Update the documentation** if your changes require it.
 7.  **Submit a pull request** to the `main` branch of the main repository. Please provide a clear description of the problem and solution. Include the relevant issue number if applicable.
 
+## Contributor License Agreement
+
+By submitting a pull request to this repository, you agree to the terms and conditions of our [Contributor License Agreement](./.github/CLA.md)
+
 ## Code of Conduct
 
 This project and everyone participating in it is governed by the [Open Archiver Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior.

README.md

@@ -1,27 +1,48 @@
 # Open Archiver
 
-> [!WARNING]
-> This project is currently in active development and has not yet reached a stable General Availability (GA) release. It is not recommended for use in production environments. We welcome contributions from the community to help us accelerate development and improve the platform. Please see our [Contributing](#contributing) section for more details.
+![Docker Compose](https://img.shields.io/badge/Docker%20Compose-up-4A4A4A?style=for-the-badge&logo=docker)
+![PostgreSQL](https://img.shields.io/badge/PostgreSQL-6B6B6B?style=for-the-badge&logo=postgresql)
+![Meilisearch](https://img.shields.io/badge/Meilisearch-2F2F2F?style=for-the-badge&logo=meilisearch)
 
 **A secure, sovereign, and affordable open-source platform for email archiving and eDiscovery.**
 
-Open Archiver enables individuals and organizations to take control of their digital communication history. It provides a robust, self-hosted solution for ingesting, storing, indexing, and searching emails from major platforms, ensuring a permanent, tamper-proof record of your most critical data, free from vendor lock-in.
+Open Archiver provides a robust, self-hosted solution for archiving, storing, indexing, and searching emails from major platforms, including Google Workspace (Gmail), Microsoft 365, as well as generic IMAP-enabled email inboxes. Use Open Archiver to keep a permanent, tamper-proof record of your communication history, free from vendor lock-in.
 
-![Open Archiver Preview](assets/screenshots/email.png)
+## Screenshots
 
-## Vision
+![Open Archiver Preview](assets/screenshots/dashboard-1.png)
+_Dashboard_
 
-To provide individuals and organizations with a secure, sovereign, and affordable platform to preserve and access their digital communication history.
+![Open Archiver Preview](assets/screenshots/archived-emails.png)
+_Archived emails_
+
+![Open Archiver Preview](assets/screenshots/search.png)
+_Full-text search across all your emails and attachments_
+
+## Community
+
+We are committed to build an engaging community around Open Archiver, and we are inviting all of you to join our community on Discord to get real-time support and connect with the team.
+
+[![Discord](https://img.shields.io/badge/Join%20our%20Discord-7289DA?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/MTtD7BhuTQ)
+
+[![Bluesky](https://img.shields.io/badge/Follow%20us%20on%20Bluesky-0265D4?style=for-the-badge&logo=bluesky&logoColor=white)](https://bsky.app/profile/openarchiver.bsky.social)
+
+## Live demo
+
+Check out the live demo here: https://demo.openarchiver.com
+
+Username: admin@local.com
+
+Password: openarchiver_demo
 
 ## Key Features
 
 -   **Universal Ingestion**: Connect to Google Workspace, Microsoft 365, and standard IMAP servers to perform initial bulk imports and maintain continuous, real-time synchronization.
 -   **Secure & Efficient Storage**: Emails are stored in the standard `.eml` format. The system uses deduplication and compression to minimize storage costs. All data is encrypted at rest.
--   **Pluggable Storage Backends**: Start with local filesystem storage and scale to S3-compatible object storage (like AWS S3 or MinIO) as your needs grow.
--   **Powerful Search & eDiscovery**: A high-performance search engine indexes the full text of emails and attachments (PDF, DOCX, etc.). The intuitive UI supports advanced search operators, filtering, and case management.
--   **Compliance & Retention**: Define granular retention policies to automatically manage the lifecycle of your data. Place legal holds on communications to prevent deletion during litigation.
--   **Comprehensive Auditing**: An immutable audit trail logs all system activities, ensuring you have a clear record of who accessed what and when.
--   **Role-Based Access Control (RBAC)**: Enforce the principle of least privilege with pre-defined roles for Administrators, Auditors, and End Users.
+-   **Pluggable Storage Backends**: Support both local filesystem storage and S3-compatible object storage (like AWS S3 or MinIO).
+-   **Powerful Search & eDiscovery**: A high-performance search engine indexes the full text of emails and attachments (PDF, DOCX, etc.).
+-   **Compliance & Retention**: Define granular retention policies to automatically manage the lifecycle of your data. Place legal holds on communications to prevent deletion during litigation (TBD).
+-   **Comprehensive Auditing**: An immutable audit trail logs all system activities, ensuring you have a clear record of who accessed what and when (TBD).
 
 ## Tech Stack
 
@@ -29,17 +50,17 @@ Open Archiver is built on a modern, scalable, and maintainable technology stack:
 
 -   **Frontend**: SvelteKit with Svelte 5
 -   **Backend**: Node.js with Express.js & TypeScript
--   **Job Queue**: BullMQ on Redis for robust, asynchronous processing
+-   **Job Queue**: BullMQ on Redis for robust, asynchronous processing. (We use Valkey as the Redis service in the Docker Compose deployment mode, but you can use Redis as well.)
 -   **Search Engine**: Meilisearch for blazingly fast and resource-efficient search
 -   **Database**: PostgreSQL for metadata, user management, and audit logs
--   **Deployment**: Docker Compose for easy, one-command deployment
+-   **Deployment**: Docker Compose deployment
 
-## Getting Started
+## Deployment
 
 ### Prerequisites
 
 -   [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/)
--   A server or local machine with at least 2GB of RAM.
+-   A server or local machine with at least 4GB of RAM (2GB of RAM if you use external Postgres, Redis (Valkey) and Meilisearch instances).
 
 ### Installation
 
@@ -57,7 +78,7 @@ Open Archiver is built on a modern, scalable, and maintainable technology stack:
     cp .env.example .env
     ```
 
-    You will need to edit the `.env` file to set your database passwords, secret keys, and other essential configuration.
+    You will need to edit the `.env` file to set your admin passwords, secret keys, and other essential configuration. Read the .env.example for how to set up.
 
 3.  **Run the application:**
 
@@ -65,14 +86,22 @@ Open Archiver is built on a modern, scalable, and maintainable technology stack:
     docker compose up -d
     ```
 
-    This command will build the necessary Docker images and start all the services (frontend, backend, database, etc.) in the background.
+    This command will pull the pre-built Docker images and start all the services (frontend, backend, database, etc.) in the background.
 
 4.  **Access the application:**
     Once the services are running, you can access the Open Archiver web interface by navigating to `http://localhost:3000` in your web browser.
 
+## Data Source Configuration
+
+After deploying the application, you will need to configure one or more ingestion sources to begin archiving emails. Follow our detailed guides to connect to your email provider:
+
+-   [Connecting to Google Workspace](https://docs.openarchiver.com/user-guides/email-providers/google-workspace.html)
+-   [Connecting to Microsoft 365](https://docs.openarchiver.com/user-guides/email-providers/imap.html)
+-   [Connecting to a Generic IMAP Server](https://docs.openarchiver.com/user-guides/email-providers/imap.html)
+
 ## Contributing
 
-We welcome contributions from the community! Whether you're a developer, a designer, or just an enthusiast, there are many ways to get involved.
+We welcome contributions from the community!
 
 -   **Reporting Bugs**: If you find a bug, please open an issue on our GitHub repository.
 -   **Suggesting Enhancements**: Have an idea for a new feature? We'd love to hear it. Open an issue to start the discussion.
@@ -80,6 +109,4 @@ We welcome contributions from the community! Whether you're a developer, a desig
 
 Please read our `CONTRIBUTING.md` file for more details on our code of conduct and the process for submitting pull requests.
 
-## License
-
-This project is licensed under the AGPL-3.0 License.
+## Star History [![Star History Chart](https://api.star-history.com/svg?repos=LogicLabs-OU/OpenArchiver&type=Date)](https://www.star-history.com/#LogicLabs-OU/OpenArchiver&Date)

docker-compose.yml

@@ -1,77 +1,74 @@
 version: '3.8'
 
 services:
-    frontend:
-        build:
-            context: ./packages/frontend
-            dockerfile: Dockerfile
+    open-archiver:
+        image: logiclabshq/open-archiver:latest
+        container_name: open-archiver
+        restart: unless-stopped
         ports:
-            - '3000:3000'
-        depends_on:
-            - backend-api
+            - '4000:4000' # Backend
+            - '3000:3000' # Frontend
         env_file:
-            - ./.env
-
-    backend-api:
-        build:
-            context: ./packages/backend
-            dockerfile: Dockerfile
-        ports:
-            - '4000:4000'
+            - .env
+        volumes:
+            - archiver-data:/var/data/open-archiver
         depends_on:
             - postgres
-            - redis
-        env_file:
-            - ./.env
-
-    ingestion-worker:
-        build:
-            context: ./packages/backend
-            dockerfile: Dockerfile
-        command: 'pnpm ts-node-dev --respawn --transpile-only src/workers/ingestion.worker.ts'
-        depends_on:
-            - postgres
-            - redis
-        env_file:
-            - ./.env
-
-    indexing-worker:
-        build:
-            context: ./packages/backend
-            dockerfile: Dockerfile
-        command: 'pnpm ts-node-dev --respawn --transpile-only src/workers/indexing.worker.ts'
-        depends_on:
-            - postgres
-            - redis
-        env_file:
-            - ./.env
+            - valkey
+            - meilisearch
+        networks:
+            - open-archiver-net
 
     postgres:
-        image: postgres:15
+        image: postgres:17-alpine
+        container_name: postgres
+        restart: unless-stopped
+        environment:
+            POSTGRES_DB: ${POSTGRES_DB:-open_archive}
+            POSTGRES_USER: ${POSTGRES_USER:-admin}
+            POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password}
+        volumes:
+            - pgdata:/var/lib/postgresql/data
         ports:
             - '5432:5432'
-        volumes:
-            - postgres_data:/var/lib/postgresql/data
-        env_file:
-            - ./.env
+        networks:
+            - open-archiver-net
 
-    redis:
-        image: redis:7
+    valkey:
+        image: valkey/valkey:8-alpine
+        container_name: valkey
+        restart: unless-stopped
+        command: valkey-server --requirepass ${REDIS_PASSWORD}
         ports:
             - '6379:6379'
         volumes:
-            - redis_data:/data
+            - valkeydata:/data
+        networks:
+            - open-archiver-net
 
     meilisearch:
-        image: getmeili/meilisearch:v1.3
+        image: getmeili/meilisearch:v1.15
+        container_name: meilisearch
+        restart: unless-stopped
+        environment:
+            MEILI_MASTER_KEY: ${MEILI_MASTER_KEY:-aSampleMasterKey}
         ports:
             - '7700:7700'
         volumes:
-            - meili_data:/meili_data
-        env_file:
-            - ./.env
+            - meilidata:/meili_data
+        networks:
+            - open-archiver-net
 
 volumes:
-    postgres_data:
-    redis_data:
-    meili_data:
+    pgdata:
+        driver: local
+    valkeydata:
+        driver: local
+    meilidata:
+        driver: local
+    archiver-data:
+        driver: local
+
+networks:
+    open-archiver-net:
+        driver: bridge

docker/Dockerfile

@@ -0,0 +1,61 @@
+# Dockerfile for Open Archiver
+
+# 1. Build Stage: Install all dependencies and build the project
+FROM node:22-alpine AS build
+WORKDIR /app
+
+# Install pnpm
+RUN npm install -g pnpm
+
+# Copy manifests and lockfile
+COPY package.json pnpm-workspace.yaml pnpm-lock.yaml* ./
+COPY packages/backend/package.json ./packages/backend/
+COPY packages/frontend/package.json ./packages/frontend/
+COPY packages/types/package.json ./packages/types/
+COPY packages/frontend/svelte.config.js ./packages/frontend/
+
+# Install all dependencies. Use --shamefully-hoist to create a flat node_modules structure
+RUN pnpm install --shamefully-hoist --frozen-lockfile --prod=false
+
+# Copy the rest of the source code
+COPY . .
+
+# Build all packages.
+RUN pnpm build
+
+# 2. Production Stage: Install only production dependencies and copy built artifacts
+FROM node:22-alpine AS production
+WORKDIR /app
+
+# Install pnpm
+RUN npm install -g pnpm
+
+# Copy manifests and lockfile
+COPY package.json pnpm-workspace.yaml pnpm-lock.yaml* ./
+COPY packages/backend/package.json ./packages/backend/
+COPY packages/frontend/package.json ./packages/frontend/
+COPY packages/types/package.json ./packages/types/
+
+# Install production dependencies
+# RUN pnpm install --shamefully-hoist --frozen-lockfile --prod=true
+
+# Copy built application from build stage
+COPY --from=build /app/packages/backend/dist ./packages/backend/dist
+COPY --from=build /app/packages/frontend/build ./packages/frontend/build
+COPY --from=build /app/packages/types/dist ./packages/types/dist
+COPY --from=build /app/packages/backend/drizzle.config.ts ./packages/backend/drizzle.config.ts
+COPY --from=build /app/packages/backend/src/database/migrations ./packages/backend/src/database/migrations
+
+# Copy the entrypoint script and make it executable
+COPY docker/docker-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh
+
+# Expose the port the app runs on
+EXPOSE 4000
+EXPOSE 3000
+
+# Set the entrypoint
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+# Start the application
+CMD ["pnpm", "docker-start"]

docker/docker-entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# Exit immediately if a command exits with a non-zero status
+set -e
+
+# Run pnpm install to ensure all dependencies, including native addons,
+# are built for the container's architecture. This is crucial for
+# multi-platform Docker images, as it prevents "exec format error"
+# when running on a different architecture than the one used for building.
+pnpm install --frozen-lockfile --prod
+
+# Run database migrations before starting the application to prevent
+# race conditions where the app starts before the database is ready.
+pnpm db:migrate
+
+# Execute the main container command
+exec "$@"

docs/.vitepress/config.mts

@@ -0,0 +1,69 @@
+import { defineConfig } from 'vitepress';
+
+export default defineConfig({
+    head: [
+        [
+            'script',
+            {
+                defer: '',
+                src: 'https://analytics.zenceipt.com/script.js',
+                'data-website-id': '2c8b452e-eab5-4f82-8ead-902d8f8b976f'
+            }
+        ]
+    ],
+    title: 'Open Archiver',
+    description: 'Official documentation for the Open Archiver project.',
+    themeConfig: {
+        search: {
+            provider: 'local'
+        },
+        logo: {
+            src: '/logo-sq.svg'
+        },
+        nav: [
+            { text: 'Home', link: '/' },
+            { text: 'Github', link: 'https://github.com/LogicLabs-OU/OpenArchiver' },
+            { text: "Website", link: 'https://openarchiver.com/' },
+            { text: "Discord", link: 'https://discord.gg/MTtD7BhuTQ' }
+        ],
+        sidebar: [
+            {
+                text: 'User Guides',
+                items: [
+                    { text: 'Get Started', link: '/' },
+                    { text: 'Installation', link: '/user-guides/installation' },
+                    {
+                        text: 'Email Providers',
+                        link: '/user-guides/email-providers/',
+                        collapsed: true,
+                        items: [
+                            { text: 'Google Workspace', link: '/user-guides/email-providers/google-workspace' },
+                            { text: 'Generic IMAP Server', link: '/user-guides/email-providers/imap' },
+                            { text: 'Microsoft 365', link: '/user-guides/email-providers/microsoft-365' }
+                        ]
+                    }
+                ]
+            },
+            {
+                text: 'API Reference',
+                items: [
+                    { text: 'Overview', link: '/api/' },
+                    { text: 'Authentication', link: '/api/authentication' },
+                    { text: 'Auth', link: '/api/auth' },
+                    { text: 'Archived Email', link: '/api/archived-email' },
+                    { text: 'Dashboard', link: '/api/dashboard' },
+                    { text: 'Ingestion', link: '/api/ingestion' },
+                    { text: 'Search', link: '/api/search' },
+                    { text: 'Storage', link: '/api/storage' }
+                ]
+            },
+            {
+                text: 'Services',
+                items: [
+                    { text: 'Overview', link: '/services/' },
+                    { text: 'Storage Service', link: '/services/storage-service' }
+                ]
+            }
+        ]
+    }
+});

docs/SUMMARY.md

@@ -0,0 +1,17 @@
+# Table of contents
+
+## User guides
+
+-   [Get started](index.md)
+-   [Installation](user-guides/installation.md)
+-   [email-providers](user-guides/email-providers/index.md)
+    -   [Connecting to Google Workspace](user-guides/email-providers/google-workspace.md)
+    -   [Connecting to a Generic IMAP Server](user-guides/email-providers/imap.md)
+    -   [Connecting to Microsoft 365](user-guides/email-providers/microsoft-365.md)
+
+---
+
+-   [api](api/index.md)
+    -   [Ingestion Sources API Documentation](api/ingestion.md)
+-   [services](services/index.md)
+    -   [Pluggable Storage Service (StorageService)](services/storage-service.md)

docs/api/archived-email.md

@@ -0,0 +1,111 @@
+# Archived Email Service API
+
+The Archived Email Service is responsible for retrieving archived emails and their details from the database and storage.
+
+## Endpoints
+
+All endpoints in this service require authentication.
+
+### GET /api/v1/archived-emails/ingestion-source/:ingestionSourceId
+
+Retrieves a paginated list of archived emails for a specific ingestion source.
+
+**Access:** Authenticated
+
+#### URL Parameters
+
+| Parameter           | Type   | Description                                       |
+| :------------------ | :----- | :------------------------------------------------ |
+| `ingestionSourceId` | string | The ID of the ingestion source to get emails for. |
+
+#### Query Parameters
+
+| Parameter | Type   | Description                     | Default |
+| :-------- | :----- | :------------------------------ | :------ |
+| `page`    | number | The page number for pagination. | 1       |
+| `limit`   | number | The number of items per page.   | 10      |
+
+#### Responses
+
+-   **200 OK:** A paginated list of archived emails.
+
+    ```json
+    {
+        "items": [
+            {
+                "id": "email-id",
+                "subject": "Test Email",
+                "from": "sender@example.com",
+                "sentAt": "2023-10-27T10:00:00.000Z",
+                "hasAttachments": true,
+                "recipients": [
+                    { "name": "Recipient 1", "email": "recipient1@example.com" }
+                ]
+            }
+        ],
+        "total": 100,
+        "page": 1,
+        "limit": 10
+    }
+    ```
+
+-   **500 Internal Server Error:** An unexpected error occurred.
+
+### GET /api/v1/archived-emails/:id
+
+Retrieves a single archived email by its ID, including its raw content and attachments.
+
+**Access:** Authenticated
+
+#### URL Parameters
+
+| Parameter | Type   | Description                   |
+| :-------- | :----- | :---------------------------- |
+| `id`      | string | The ID of the archived email. |
+
+#### Responses
+
+-   **200 OK:** The archived email details.
+
+    ```json
+    {
+        "id": "email-id",
+        "subject": "Test Email",
+        "from": "sender@example.com",
+        "sentAt": "2023-10-27T10:00:00.000Z",
+        "hasAttachments": true,
+        "recipients": [
+            { "name": "Recipient 1", "email": "recipient1@example.com" }
+        ],
+        "raw": "...",
+        "attachments": [
+            {
+                "id": "attachment-id",
+                "filename": "document.pdf",
+                "mimeType": "application/pdf",
+                "sizeBytes": 12345
+            }
+        ]
+    }
+    ```
+
+-   **404 Not Found:** The archived email with the specified ID was not found.
+-   **500 Internal Server Error:** An unexpected error occurred.
+
+## Service Methods
+
+### `getArchivedEmails(ingestionSourceId: string, page: number, limit: number): Promise<PaginatedArchivedEmails>`
+
+Retrieves a paginated list of archived emails from the database for a given ingestion source.
+
+-   **ingestionSourceId:** The ID of the ingestion source.
+-   **page:** The page number for pagination.
+-   **limit:** The number of items per page.
+-   **Returns:** A promise that resolves to a `PaginatedArchivedEmails` object.
+
+### `getArchivedEmailById(emailId: string): Promise<ArchivedEmail | null>`
+
+Retrieves a single archived email by its ID, including its raw content and attachments.
+
+-   **emailId:** The ID of the archived email.
+-   **Returns:** A promise that resolves to an `ArchivedEmail` object or `null` if not found.

docs/api/auth.md

@@ -0,0 +1,84 @@
+# Auth Service API
+
+The Auth Service is responsible for handling user authentication, including login and token verification.
+
+## Endpoints
+
+### POST /api/v1/auth/login
+
+Authenticates a user and returns a JWT if the credentials are valid.
+
+**Access:** Public
+
+**Rate Limiting:** This endpoint is rate-limited to prevent brute-force attacks.
+
+#### Request Body
+
+| Field      | Type   | Description               |
+| :--------- | :----- | :------------------------ |
+| `email`    | string | The user's email address. |
+| `password` | string | The user's password.      |
+
+#### Responses
+
+-   **200 OK:** Authentication successful.
+
+    ```json
+    {
+        "accessToken": "your.jwt.token",
+        "user": {
+            "id": "user-id",
+            "email": "user@example.com",
+            "role": "user"
+        }
+    }
+    ```
+
+-   **400 Bad Request:** Email or password not provided.
+
+    ```json
+    {
+        "message": "Email and password are required"
+    }
+    ```
+
+-   **401 Unauthorized:** Invalid credentials.
+
+    ```json
+    {
+        "message": "Invalid credentials"
+    }
+    ```
+
+-   **500 Internal Server Error:** An unexpected error occurred.
+
+    ```json
+    {
+        "message": "An internal server error occurred"
+    }
+    ```
+
+## Service Methods
+
+### `verifyPassword(password: string, hash: string): Promise<boolean>`
+
+Compares a plain-text password with a hashed password to verify its correctness.
+
+-   **password:** The plain-text password.
+-   **hash:** The hashed password to compare against.
+-   **Returns:** A promise that resolves to `true` if the password is valid, otherwise `false`.
+
+### `login(email: string, password: string): Promise<LoginResponse | null>`
+
+Handles the user login process. It finds the user by email, verifies the password, and generates a JWT upon successful authentication.
+
+-   **email:** The user's email.
+-   **password:** The user's password.
+-   **Returns:** A promise that resolves to a `LoginResponse` object containing the `accessToken` and `user` details, or `null` if authentication fails.
+
+### `verifyToken(token: string): Promise<AuthTokenPayload | null>`
+
+Verifies the authenticity and expiration of a JWT.
+
+-   **token:** The JWT string to verify.
+-   **Returns:** A promise that resolves to the token's `AuthTokenPayload` if valid, otherwise `null`.

docs/api/authentication.md

@@ -0,0 +1,60 @@
+# API Authentication
+
+To access protected API endpoints, you need to include a JSON Web Token (JWT) in the `Authorization` header of your requests.
+
+## Obtaining a JWT
+
+First, you need to authenticate with the `/api/v1/auth/login` endpoint by providing your email and password. If the credentials are correct, the API will return an `accessToken`.
+
+**Request:**
+
+```http
+POST /api/v1/auth/login
+Content-Type: application/json
+
+{
+  "email": "user@example.com",
+  "password": "your-password"
+}
+```
+
+**Successful Response:**
+
+```json
+{
+    "accessToken": "your.jwt.token",
+    "user": {
+        "id": "user-id",
+        "email": "user@example.com",
+        "role": "user"
+    }
+}
+```
+
+## Making Authenticated Requests
+
+Once you have the `accessToken`, you must include it in the `Authorization` header of all subsequent requests to protected endpoints, using the `Bearer` scheme.
+
+**Example:**
+
+```http
+GET /api/v1/dashboard/stats
+Authorization: Bearer your.jwt.token
+```
+
+If the token is missing, expired, or invalid, the API will respond with a `401 Unauthorized` status code.
+
+## Using a Super API Key
+
+Alternatively, for server-to-server communication or scripts, you can use a super API key. This key provides unrestricted access to the API and should be kept secret.
+
+You can set the `SUPER_API_KEY` in your `.env` file.
+
+To authenticate using the super API key, include it in the `Authorization` header as a Bearer token.
+
+**Example:**
+
+```http
+GET /api/v1/dashboard/stats
+Authorization: Bearer your-super-secret-api-key
+```

docs/api/dashboard.md

@@ -0,0 +1,114 @@
+# Dashboard Service API
+
+The Dashboard Service provides endpoints for retrieving statistics and data for the main dashboard.
+
+## Endpoints
+
+All endpoints in this service require authentication.
+
+### GET /api/v1/dashboard/stats
+
+Retrieves overall statistics, including the total number of archived emails, total storage used, and the number of failed ingestions in the last 7 days.
+
+**Access:** Authenticated
+
+#### Responses
+
+-   **200 OK:** An object containing the dashboard statistics.
+
+    ```json
+    {
+        "totalEmailsArchived": 12345,
+        "totalStorageUsed": 54321098,
+        "failedIngestionsLast7Days": 3
+    }
+    ```
+
+### GET /api/v1/dashboard/ingestion-history
+
+Retrieves the email ingestion history for the last 30 days, grouped by day.
+
+**Access:** Authenticated
+
+#### Responses
+
+-   **200 OK:** An object containing the ingestion history.
+
+    ```json
+    {
+        "history": [
+            {
+                "date": "2023-09-27T00:00:00.000Z",
+                "count": 150
+            },
+            {
+                "date": "2023-09-28T00:00:00.000Z",
+                "count": 200
+            }
+        ]
+    }
+    ```
+
+### GET /api/v1/dashboard/ingestion-sources
+
+Retrieves a list of all ingestion sources along with their status and storage usage.
+
+**Access:** Authenticated
+
+#### Responses
+
+-   **200 OK:** An array of ingestion source objects.
+
+    ```json
+    [
+        {
+            "id": "source-id-1",
+            "name": "Google Workspace",
+            "provider": "google",
+            "status": "active",
+            "storageUsed": 12345678
+        },
+        {
+            "id": "source-id-2",
+            "name": "Microsoft 365",
+            "provider": "microsoft",
+            "status": "error",
+            "storageUsed": 87654321
+        }
+    ]
+    ```
+
+### GET /api/v1/dashboard/recent-syncs
+
+Retrieves a list of recent synchronization jobs. (Note: This is currently a placeholder and will return an empty array).
+
+**Access:** Authenticated
+
+#### Responses
+
+-   **200 OK:** An empty array.
+
+    ```json
+    []
+    ```
+
+### GET /api/v1/dashboard/indexed-insights
+
+Retrieves insights from the indexed email data, such as the top senders.
+
+**Access:** Authenticated
+
+#### Responses
+
+-   **200 OK:** An object containing indexed insights.
+
+    ```json
+    {
+        "topSenders": [
+            {
+                "sender": "user@example.com",
+                "count": 42
+            }
+        ]
+    }
+    ```

docs/api/index.md

@@ -0,0 +1,18 @@
+# API Overview
+
+Welcome to the Open Archiver API documentation. This section provides detailed information about the available API endpoints.
+
+All API endpoints are prefixed with `/api/v1`.
+
+## Authentication
+
+Before making requests to protected endpoints, you must authenticate with the API. See the [Authentication Guide](./authentication.md) for details on how to obtain and use API tokens.
+
+## API Services
+
+-   [**Auth Service**](./auth.md): Handles user authentication.
+-   [**Archived Email Service**](./archived-email.md): Manages archived emails.
+-   [**Dashboard Service**](./dashboard.md): Provides data for the main dashboard.
+-   [**Ingestion Service**](./ingestion.md): Manages email ingestion sources.
+-   [**Search Service**](./search.md): Handles email search functionality.
+-   [**Storage Service**](./storage.md): Manages file storage and downloads.

docs/api/ingestion.md

@@ -1,206 +1,168 @@
-# Ingestion Sources API Documentation
+# Ingestion Service API
 
-A comprehensive guide to using the Ingestion Sources API.
+The Ingestion Service manages ingestion sources, which are configurations for connecting to email providers and importing emails.
 
-**Base Path:** `/v1/ingestion-sources`
+## Endpoints
 
----
+All endpoints in this service require authentication.
 
-## Authentication
+### POST /api/v1/ingestion
 
-All endpoints in this API are protected and require authentication. Requests must include an `Authorization` header containing a valid Bearer token. This can be a JWT obtained from the login endpoint or a `SUPER_API_KEY` for administrative tasks.
+Creates a new ingestion source.
 
-**Header Example:**
-`Authorization: Bearer <YOUR_JWT_OR_SUPER_API_KEY>`
+**Access:** Authenticated
 
----
+#### Request Body
 
-## Core Concepts
+The request body should be a `CreateIngestionSourceDto` object.
 
-### Ingestion Providers
-
-The `provider` field determines the type of email source. Each provider requires a different configuration object, for example:
-
--   `google_workspace`: For connecting to Google Workspace accounts via OAuth 2.0.
--   `microsoft_365`: For connecting to Microsoft 365 accounts via OAuth 2.0.
--   `generic_imap`: For connecting to any email server that supports IMAP.
-
-### Ingestion Status
-
-The `status` field tracks the state of the ingestion source.
-
--   `pending_auth`: The source has been created but requires user authorization (OAuth flow).
--   `active`: The source is authenticated and ready to sync.
--   `syncing`: An import job is currently in progress.
--   `importing`: initial syncing in progress
--   `paused`: The source is temporarily disabled.
--   `error`: An error occurred during the last sync.
-
----
-
-## 1. Create Ingestion Source
-
--   **Method:** `POST`
--   **Path:** `/`
--   **Description:** Registers a new source for email ingestion. The `providerConfig` will vary based on the selected `provider`.
-
-#### Request Body (`CreateIngestionSourceDto`)
-
--   `name` (string, required): A user-friendly name for the source (e.g., "Marketing Department G-Suite").
--   `provider` (string, required): One of `google_workspace`, `microsoft_365`, or `generic_imap`.
--   `providerConfig` (object, required): Configuration specific to the provider.
-
-##### `providerConfig` for `google_workspace` / `microsoft_365`
-
-```json
-{
-    "name": "Corporate Google Workspace",
-    "provider": "google_workspace",
-    "providerConfig": {
-        "clientId": "your-oauth-client-id.apps.googleusercontent.com",
-        "clientSecret": "your-super-secret-client-secret",
-        "redirectUri": "https://yourapp.com/oauth/google/callback"
-    }
-}
-```
-
-##### `providerConfig` for `generic_imap`
-
-```json
-{
-    "name": "Legacy IMAP Server",
-    "provider": "generic_imap",
-    "providerConfig": {
-        "host": "imap.example.com",
-        "port": 993,
-        "secure": true,
-        "username": "archive-user",
-        "password": "imap-password"
-    }
+```typescript
+interface CreateIngestionSourceDto {
+    name: string;
+    provider: 'google' | 'microsoft' | 'generic_imap';
+    providerConfig: IngestionCredentials;
 }
 ```
 
 #### Responses
 
--   **Success (`201 Created`):** Returns the full `IngestionSource` object, which now includes a system-generated `id` and default status.
+-   **201 Created:** The newly created ingestion source.
+-   **500 Internal Server Error:** An unexpected error occurred.
 
-    ```json
-    {
-        "id": "a1b2c3d4-e5f6-7890-1234-567890abcdef",
-        "name": "Corporate Google Workspace",
-        "provider": "google_workspace",
-        "status": "pending_auth",
-        "createdAt": "2025-07-11T12:00:00.000Z",
-        "updatedAt": "2025-07-11T12:00:00.000Z",
-        "providerConfig": { ... }
-    }
-    ```
+### GET /api/v1/ingestion
 
--   **Error (`500 Internal Server Error`):** Indicates a server-side problem during creation.
+Retrieves all ingestion sources.
 
----
-
-## 2. Get All Ingestion Sources
-
--   **Method:** `GET`
--   **Path:** `/`
--   **Description:** Retrieves a list of all configured ingestion sources for the organization.
+**Access:** Authenticated
 
 #### Responses
 
--   **Success (`200 OK`):** Returns an array of `IngestionSource` objects.
+-   **200 OK:** An array of ingestion source objects.
+-   **500 Internal Server Error:** An unexpected error occurred.
 
--   **Error (`500 Internal Server Error`):** Indicates a server-side problem.
+### GET /api/v1/ingestion/:id
 
----
+Retrieves a single ingestion source by its ID.
 
-## 3. Get Ingestion Source by ID
-
--   **Method:** `GET`
--   **Path:** `/:id`
--   **Description:** Fetches the details of a specific ingestion source.
+**Access:** Authenticated
 
 #### URL Parameters
 
--   `id` (string, required): The UUID of the ingestion source.
+| Parameter | Type   | Description                     |
+| :-------- | :----- | :------------------------------ |
+| `id`      | string | The ID of the ingestion source. |
 
 #### Responses
 
--   **Success (`200 OK`):** Returns the corresponding `IngestionSource` object.
+-   **200 OK:** The ingestion source object.
+-   **404 Not Found:** Ingestion source not found.
+-   **500 Internal Server Error:** An unexpected error occurred.
 
--   **Error (`404 Not Found`):** Returned if no source with the given ID exists.
--   **Error (`500 Internal Server Error`):** Indicates a server-side problem.
+### PUT /api/v1/ingestion/:id
 
----
+Updates an existing ingestion source.
 
-## 4. Update Ingestion Source
-
--   **Method:** `PUT`
--   **Path:** `/:id`
--   **Description:** Modifies an existing ingestion source. This is useful for changing the name, pausing a source, or updating its configuration.
+**Access:** Authenticated
 
 #### URL Parameters
 
--   `id` (string, required): The UUID of the ingestion source to update.
+| Parameter | Type   | Description                     |
+| :-------- | :----- | :------------------------------ |
+| `id`      | string | The ID of the ingestion source. |
 
-#### Request Body (`UpdateIngestionSourceDto`)
+#### Request Body
 
-All fields are optional. Only include the fields you want to change.
+The request body should be an `UpdateIngestionSourceDto` object.
 
-```json
-{
-    "name": "Marketing Dept G-Suite (Paused)",
-    "status": "paused"
+```typescript
+interface UpdateIngestionSourceDto {
+    name?: string;
+    provider?: 'google' | 'microsoft' | 'generic_imap';
+    providerConfig?: IngestionCredentials;
+    status?:
+        | 'pending_auth'
+        | 'auth_success'
+        | 'importing'
+        | 'active'
+        | 'paused'
+        | 'error';
 }
 ```
 
 #### Responses
 
--   **Success (`200 OK`):** Returns the complete, updated `IngestionSource` object.
+-   **200 OK:** The updated ingestion source object.
+-   **404 Not Found:** Ingestion source not found.
+-   **500 Internal Server Error:** An unexpected error occurred.
 
--   **Error (`404 Not Found`):** Returned if no source with the given ID exists.
--   **Error (`500 Internal Server Error`):** Indicates a server-side problem.
+### DELETE /api/v1/ingestion/:id
 
----
+Deletes an ingestion source and all associated data.
 
-## 5. Delete Ingestion Source
-
--   **Method:** `DELETE`
--   **Path:** `/:id`
--   **Description:** Permanently removes an ingestion source. This action cannot be undone.
+**Access:** Authenticated
 
 #### URL Parameters
 
--   `id` (string, required): The UUID of the ingestion source to delete.
+| Parameter | Type   | Description                     |
+| :-------- | :----- | :------------------------------ |
+| `id`      | string | The ID of the ingestion source. |
 
 #### Responses
 
--   **Success (`204 No Content`):** Indicates successful deletion with no body content.
+-   **204 No Content:** The ingestion source was deleted successfully.
+-   **404 Not Found:** Ingestion source not found.
+-   **500 Internal Server Error:** An unexpected error occurred.
 
--   **Error (`404 Not Found`):** Returned if no source with the given ID exists.
--   **Error (`500 Internal Server Error`):** Indicates a server-side problem.
+### POST /api/v1/ingestion/:id/import
 
----
+Triggers the initial import process for an ingestion source.
 
-## 6. Trigger Initial Import
-
--   **Method:** `POST`
--   **Path:** `/:id/sync`
--   **Description:** Initiates the email import process for a given source. This is an asynchronous operation that enqueues a background job and immediately returns a response. The status of the source will be updated to `importing`.
+**Access:** Authenticated
 
 #### URL Parameters
 
--   `id` (string, required): The UUID of the ingestion source to sync.
+| Parameter | Type   | Description                     |
+| :-------- | :----- | :------------------------------ |
+| `id`      | string | The ID of the ingestion source. |
 
 #### Responses
 
--   **Success (`202 Accepted`):** Confirms that the sync request has been accepted for processing.
+-   **202 Accepted:** The initial import was triggered successfully.
+-   **404 Not Found:** Ingestion source not found.
+-   **500 Internal Server Error:** An unexpected error occurred.
 
-    ```json
-    {
-        "message": "Initial import triggered successfully."
-    }
-    ```
+### POST /api/v1/ingestion/:id/pause
 
--   **Error (`404 Not Found`):** Returned if no source with the given ID exists.
--   **Error (`500 Internal Server Error`):** Indicates a server-side problem.
+Pauses an active ingestion source.
+
+**Access:** Authenticated
+
+#### URL Parameters
+
+| Parameter | Type   | Description                     |
+| :-------- | :----- | :------------------------------ |
+| `id`      | string | The ID of the ingestion source. |
+
+#### Responses
+
+-   **200 OK:** The updated ingestion source object with a `paused` status.
+-   **404 Not Found:** Ingestion source not found.
+-   **500 Internal Server Error:** An unexpected error occurred.
+
+### POST /api/v1/ingestion/:id/sync
+
+Triggers a forced synchronization for an ingestion source.
+
+**Access:** Authenticated
+
+#### URL Parameters
+
+| Parameter | Type   | Description                     |
+| :-------- | :----- | :------------------------------ |
+| `id`      | string | The ID of the ingestion source. |
+
+#### Responses
+
+-   **202 Accepted:** The force sync was triggered successfully.
+-   **404 Not Found:** Ingestion source not found.
+-   **500 Internal Server Error:** An unexpected error occurred.

docs/api/search.md

@@ -0,0 +1,50 @@
+# Search Service API
+
+The Search Service provides an endpoint for searching indexed emails.
+
+## Endpoints
+
+All endpoints in this service require authentication.
+
+### GET /api/v1/search
+
+Performs a search query against the indexed emails.
+
+**Access:** Authenticated
+
+#### Query Parameters
+
+| Parameter          | Type   | Description                                                            | Default |
+| :----------------- | :----- | :--------------------------------------------------------------------- | :------ |
+| `keywords`         | string | The search query.                                                      |         |
+| `page`             | number | The page number for pagination.                                        | 1       |
+| `limit`            | number | The number of items per page.                                          | 10      |
+| `matchingStrategy` | string | The matching strategy to use (`all` or `last`).                        | `last`  |
+| `filters`          | object | Key-value pairs for filtering results (e.g., `from=user@example.com`). |         |
+
+#### Responses
+
+-   **200 OK:** A search result object.
+
+    ```json
+    {
+        "hits": [
+            {
+                "id": "email-id",
+                "subject": "Test Email",
+                "from": "sender@example.com",
+                "_formatted": {
+                    "subject": "<em>Test</em> Email"
+                }
+            }
+        ],
+        "total": 1,
+        "page": 1,
+        "limit": 10,
+        "totalPages": 1,
+        "processingTimeMs": 5
+    }
+    ```
+
+-   **400 Bad Request:** Keywords are required.
+-   **500 Internal Server Error:** An unexpected error occurred.

docs/api/storage.md

@@ -0,0 +1,26 @@
+# Storage Service API
+
+The Storage Service provides an endpoint for downloading files from the configured storage provider.
+
+## Endpoints
+
+All endpoints in this service require authentication.
+
+### GET /api/v1/storage/download
+
+Downloads a file from the storage.
+
+**Access:** Authenticated
+
+#### Query Parameters
+
+| Parameter | Type   | Description                                       |
+| :-------- | :----- | :------------------------------------------------ |
+| `path`    | string | The path to the file within the storage provider. |
+
+#### Responses
+
+-   **200 OK:** The file stream.
+-   **400 Bad Request:** File path is required or invalid.
+-   **404 Not Found:** File not found.
+-   **500 Internal Server Error:** An unexpected error occurred.

docs/index.md

@@ -0,0 +1,42 @@
+# Get Started 👋
+
+Welcome to Open Archiver! This guide will help you get started with setting up and using the platform.
+
+## What is Open Archiver? 🛡️
+
+**A secure, sovereign, and affordable open-source platform for email archiving and eDiscovery.**
+
+Open Archiver provides a robust, self-hosted solution for archiving, storing, indexing, and searching emails from major platforms, including Google Workspace (Gmail), Microsoft 365, as well as generic IMAP-enabled email inboxes. Use Open Archiver to keep a permanent, tamper-proof record of your communication history, free from vendor lock-in.
+
+## Key Features ✨
+
+-   **Universal Ingestion**: Connect to Google Workspace, Microsoft 365, and standard IMAP servers to perform initial bulk imports and maintain continuous, real-time synchronization.
+-   **Secure & Efficient Storage**: Emails are stored in the standard `.eml` format. The system uses deduplication and compression to minimize storage costs. All data is encrypted at rest.
+-   **Pluggable Storage Backends**: Support both local filesystem storage and S3-compatible object storage (like AWS S3 or MinIO).
+-   **Powerful Search & eDiscovery**: A high-performance search engine indexes the full text of emails and attachments (PDF, DOCX, etc.).
+-   **Compliance & Retention**: Define granular retention policies to automatically manage the lifecycle of your data. Place legal holds on communications to prevent deletion during litigation (TBD).
+-   **Comprehensive Auditing**: An immutable audit trail logs all system activities, ensuring you have a clear record of who accessed what and when (TBD).
+
+## Installation 🚀
+
+To get your own instance of Open Archiver running, follow our detailed installation guide:
+
+-   [Installation Guide](./user-guides/installation.md)
+
+## Data Source Configuration 🔌
+
+After deploying the application, you will need to configure one or more ingestion sources to begin archiving emails. Follow our detailed guides to connect to your email provider:
+
+-   [Connecting to Google Workspace](./user-guides/email-providers/google-workspace.md)
+-   [Connecting to Microsoft 365](./user-guides/email-providers/microsoft-365.md)
+-   [Connecting to a Generic IMAP Server](./user-guides/email-providers/imap.md)
+
+## Contributing ❤️
+
+We welcome contributions from the community!
+
+-   **Reporting Bugs**: If you find a bug, please open an issue on our GitHub repository.
+-   **Suggesting Enhancements**: Have an idea for a new feature? We'd love to hear it. Open an issue to start the discussion.
+-   **Code Contributions**: If you'd like to contribute code, please fork the repository and submit a pull request.
+
+Please read our `CONTRIBUTING.md` file for more details on our code of conduct and the process for submitting pull requests.

docs/services/index.md

@@ -0,0 +1,2 @@
+# services
+

docs/user-guides/email-providers/google-workspace.md

@@ -0,0 +1,130 @@
+# Connecting to Google Workspace
+
+This guide provides instructions for Google Workspace administrators to set up a connection that allows the archiving of all user mailboxes within their organization.
+
+The connection uses a **Google Cloud Service Account** with **Domain-Wide Delegation**. This is a secure method that grants the archiving service permission to access user data on behalf of the administrator, without requiring individual user passwords or consent.
+
+## Prerequisites
+
+-   You must have **Super Administrator** privileges in your Google Workspace account.
+-   You must have access to the **Google Cloud Console** associated with your organization.
+
+## Setup Overview
+
+The setup process involves three main parts:
+
+1.  Configuring the necessary permissions in the Google Cloud Console.
+2.  Authorizing the service account in the Google Workspace Admin Console.
+3.  Entering the generated credentials into the OpenArchiver application.
+
+---
+
+### Part 1: Google Cloud Console Setup
+
+In this part, you will create a service account and enable the APIs it needs to function.
+
+1.  **Create a Google Cloud Project:**
+
+    -   Go to the [Google Cloud Console](https://console.cloud.google.com/).
+    -   If you don't already have one, create a new project for the archiving service (e.g., "Email Archiver").
+
+2.  **Enable Required APIs:**
+
+    -   In your selected project, navigate to the **"APIs & Services" > "Library"** section.
+    -   Search for and enable the following two APIs:
+        -   **Gmail API**
+        -   **Admin SDK API**
+
+3.  **Create a Service Account:**
+
+    -   Navigate to **"IAM & Admin" > "Service Accounts"**.
+    -   Click **"Create Service Account"**.
+    -   Give the service account a name (e.g., `email-archiver-service`) and a description.
+    -   Click **"Create and Continue"**. You do not need to grant this service account any roles on the project. Click **"Done"**.
+
+4.  **Generate a JSON Key:**
+    -   Find the service account you just created in the list.
+    -   Click the three-dot menu under **"Actions"** and select **"Manage keys"**.
+    -   Click **"Add Key"** > **"Create new key"**.
+    -   Select **JSON** as the key type and click **"Create"**.
+    -   A JSON file will be downloaded to your computer. **Keep this file secure, as it contains private credentials.** You will need the contents of this file in Part 3.
+
+### Troubleshooting
+
+#### Error: "iam.disableServiceAccountKeyCreation"
+
+If you receive an error message stating `The organization policy constraint 'iam.disableServiceAccountKeyCreation' is enforced` when trying to create a JSON key, it means your Google Cloud organization has a policy preventing the creation of new service account keys.
+
+To resolve this, you must have **Organization Administrator** permissions.
+
+1.  **Navigate to your Organization:** In the Google Cloud Console, use the project selector at the top of the page to select your organization node (it usually has a building icon).
+2.  **Go to IAM:** From the navigation menu, select **"IAM & Admin" > "IAM"**.
+3.  **Edit Your Permissions:** Find your user account in the list and click the pencil icon to edit roles. Add the following two roles:
+    -   `Organization Policy Administrator`
+    -   `Organization Administrator`
+        _Note: These roles are only available at the organization level, not the project level._
+4.  **Modify the Policy:**
+    -   Navigate to **"IAM & Admin" > "Organization Policies"**.
+    -   In the filter box, search for the policy **"iam.disableServiceAccountKeyCreation"**.
+    -   Click on the policy to edit it.
+    -   You can either disable the policy entirely (if your security rules permit) or add a rule to exclude the specific project you are using for the archiver from this policy.
+5.  **Retry Key Creation:** Once the policy is updated, return to your project and you should be able to generate the JSON key as described in Part 1.
+
+---
+
+### Part 2: Grant Domain-Wide Delegation
+
+Now, you will authorize the service account you created to access data from your Google Workspace.
+
+1.  **Get the Service Account's Client ID:**
+
+    -   Go back to the list of service accounts in the Google Cloud Console.
+    -   Click on the service account you created.
+    -   Under the **"Details"** tab, find and copy the **Unique ID** (this is the Client ID).
+
+2.  **Authorize the Client in Google Workspace:**
+
+    -   Go to your **Google Workspace Admin Console** at [admin.google.com](https://admin.google.com).
+    -   Navigate to **Security > Access and data control > API controls**.
+    -   Under the "Domain-wide Delegation" section, click **"Manage Domain-wide Delegation"**.
+    -   Click **"Add new"**.
+
+3.  **Enter Client Details and Scopes:**
+    -   In the **Client ID** field, paste the **Unique ID** you copied from the service account.
+    -   In the **OAuth scopes** field, paste the following two scopes exactly as they appear, separated by a comma:
+        ```
+        https://www.googleapis.com/auth/admin.directory.user.readonly,https://www.googleapis.com/auth/gmail.readonly
+        ```
+    -   Click **"Authorize"**.
+
+The service account is now permitted to list users and read their email data across your domain.
+
+---
+
+### Part 3: Connecting in OpenArchiver
+
+Finally, you will provide the generated credentials to the application.
+
+1.  **Navigate to Ingestion Sources:**
+    From the main dashboard, go to the **Ingestion Sources** page.
+
+2.  **Create a New Source:**
+    Click the **"Create New"** button.
+
+3.  **Fill in the Configuration Details:**
+
+    -   **Name:** Give the source a name (e.g., "Google Workspace Archive").
+    -   **Provider:** Select **"Google Workspace"** from the dropdown.
+    -   **Service Account Key (JSON):** Open the JSON file you downloaded in Part 1. Copy the entire content of the file and paste it into this text area.
+    -   **Impersonated Admin Email:** Enter the email address of a Super Administrator in your Google Workspace (e.g., `admin@your-domain.com`). The service will use this user's authority to discover all other users.
+
+4.  **Save Changes:**
+    Click **"Save changes"**.
+
+## What Happens Next?
+
+Once the connection is saved and verified, the system will begin the archiving process:
+
+1.  **User Discovery:** The service will first connect to the Admin SDK to get a list of all active users in your Google Workspace.
+2.  **Initial Import:** The system will then start a background job to import the mailboxes of all discovered users. The status will show as **"Importing"**. This can take a significant amount of time depending on the number of users and the size of their mailboxes.
+3.  **Continuous Sync:** After the initial import is complete, the status will change to **"Active"**. The system will then periodically check each user's mailbox for new emails and archive them automatically.

docs/user-guides/email-providers/imap.md

@@ -0,0 +1,67 @@
+# Connecting to a Generic IMAP Server
+
+This guide will walk you through connecting a standard IMAP email account as an ingestion source. This allows you to archive emails from any provider that supports the IMAP protocol, which is common for many self-hosted or traditional email services.
+
+## Step-by-Step Guide
+
+1.  **Navigate to Ingestion Sources:**
+    From the main dashboard, go to the **Ingestions** page.
+
+2.  **Create a New Source:**
+    Click the **"Create New"** button to open the ingestion source configuration dialog.
+
+3.  **Fill in the Configuration Details:**
+    You will see a form with several fields. Here is how to fill them out for an IMAP connection:
+
+    -   **Name:** Give your ingestion source a descriptive name that you will easily recognize, such as "Work Email (IMAP)" or "Personal Gmail".
+
+    -   **Provider:** From the dropdown menu, select **"Generic IMAP"**. This will reveal the specific fields required for an IMAP connection.
+
+    -   **Host:** Enter the server address for your email provider's IMAP service. This often looks like `imap.your-provider.com` or `mail.your-domain.com`.
+
+    -   **Port:** Enter the port number for the IMAP server. For a secure connection (which is strongly recommended), this is typically `993`.
+
+    -   **Username:** Enter the full email address or username you use to log in to your email account.
+
+    -   **Password:** Enter the password for your email account.
+
+4.  **Save Changes:**
+    Once you have filled in all the details, click the **"Save changes"** button.
+
+## Security Recommendation: Use an App Password
+
+For enhanced security, we strongly recommend using an **"app password"** (sometimes called an "app-specific password") instead of your main account password.
+
+Many email providers (like Gmail, Outlook, and Fastmail) allow you to generate a unique password that grants access only to a specific application (in this case, the archiving service). If you ever need to revoke access, you can simply delete the app password without affecting your main account login.
+
+Please consult your email provider's documentation to see if they support app passwords and how to create one.
+
+### How to Obtain an App Password for Gmail
+
+1.  **Enable 2-Step Verification:** You must have 2-Step Verification turned on for your Google Account.
+2.  **Go to App Passwords:** Visit [myaccount.google.com/apppasswords](https://myaccount.google.com/apppasswords). You may be asked to sign in again.
+3.  **Create the Password:**
+    -   At the bottom, click **"Select app"** and choose **"Other (Custom name)"**.
+    -   Give it a name you'll recognize, like "OpenArchiver".
+    -   Click **"Generate"**.
+4.  **Use the Password:** A 16-digit password will be displayed. Copy this password and paste it into the **Password** field in the OpenArchiver ingestion source form.
+
+### How to Obtain an App Password for Outlook/Microsoft Accounts
+
+1.  **Enable Two-Step Verification:** You must have two-step verification enabled for your Microsoft account.
+2.  **Go to Security Options:** Sign in to your Microsoft account and navigate to the [Advanced security options](https://account.live.com/proofs/manage/additional).
+3.  **Create a New App Password:**
+    -   Scroll down to the **"App passwords"** section.
+    -   Click **"Create a new app password"**.
+4.  **Use the Password:** A new password will be generated. Use this password in the **Password** field in the OpenArchiver ingestion source form.
+
+## What Happens Next?
+
+After you save the connection, the system will attempt to connect to the IMAP server. The status of the ingestion source will update to reflect its current state:
+
+-   **Importing:** The system is performing the initial, one-time import of all emails from your `INBOX`. This may take a while depending on the size of your mailbox.
+-   **Active:** The initial import is complete, and the system will now periodically check for and archive new emails.
+-   **Paused:** The connection is valid, but the system will not check for new emails until you resume it.
+-   **Error:** The system was unable to connect using the provided credentials. Please double-check your Host, Port, Username, and Password and try again.
+
+You can view, edit, pause, or manually sync any of your ingestion sources from the main table on the **Ingestions** page.

docs/user-guides/email-providers/index.md

@@ -0,0 +1,9 @@
+# Connecting Email Providers
+
+Open Archiver can connect to a variety of email sources to ingest and archive your emails. This section provides guides for connecting to popular email providers.
+
+Choose your provider from the list below to get started:
+
+-   [Google Workspace](./google-workspace.md)
+-   [Microsoft 365](./microsoft-365.md)
+-   [Generic IMAP Server](./imap.md)

docs/user-guides/email-providers/microsoft-365.md

@@ -0,0 +1,94 @@
+# Connecting to Microsoft 365
+
+This guide provides instructions for Microsoft 365 administrators to set up a connection that allows the archiving of all user mailboxes within their organization.
+
+The connection uses the **Microsoft Graph API** and an **App Registration** in Microsoft Entra ID. This is a secure, standard method that grants the archiving service permission to read email data on your behalf without ever needing to handle user passwords.
+
+## Prerequisites
+
+-   You must have one of the following administrator roles in your Microsoft 365 tenant: **Global Administrator**, **Application Administrator**, or **Cloud Application Administrator**.
+
+## Setup Overview
+
+The setup process involves four main parts, all performed within the Microsoft Entra admin center and the OpenArchiver application:
+
+1.  Registering a new application identity for the archiver in Entra ID.
+2.  Granting the application the specific permissions it needs to read mail.
+3.  Creating a secure password (a client secret) for the application.
+4.  Entering the generated credentials into the OpenArchiver application.
+
+---
+
+### Part 1: Register a New Application in Microsoft Entra ID
+
+First, you will create an "App registration," which acts as an identity for the archiving service within your Microsoft 365 ecosystem.
+
+1.  Sign in to the [Microsoft Entra admin center](https://entra.microsoft.com).
+2.  In the left-hand navigation pane, go to **Identity > Applications > App registrations**.
+3.  Click the **+ New registration** button at the top of the page.
+4.  On the "Register an application" screen:
+    -   **Name:** Give the application a descriptive name you will recognize, such as `OpenArchiver Service`.
+    -   **Supported account types:** Select **"Accounts in this organizational directory only (Default Directory only - Single tenant)"**. This is the most secure option.
+    -   **Redirect URI (optional):** You can leave this blank.
+5.  Click the **Register** button. You will be taken to the application's main "Overview" page.
+
+---
+
+### Part 2: Grant API Permissions
+
+Next, you must grant the application the specific permissions required to read user profiles and their mailboxes.
+
+1.  From your new application's page, select **API permissions** from the left-hand menu.
+2.  Click the **+ Add a permission** button.
+3.  In the "Request API permissions" pane, select **Microsoft Graph**.
+4.  Select **Application permissions**. This is critical as it allows the service to run in the background without a user being signed in.
+5.  In the "Select permissions" search box, find and check the boxes for the following two permissions:
+    -   `Mail.Read`
+    -   `User.Read.All`
+6.  Click the **Add permissions** button at the bottom.
+7.  **Crucial Final Step:** You will now see the permissions in your list with a warning status. You must grant consent on behalf of your organization. Click the **"Grant admin consent for [Your Organization's Name]"** button located above the permissions table. Click **Yes** in the confirmation dialog. The status for both permissions should now show a green checkmark.
+
+---
+
+### Part 3: Create a Client Secret
+
+The client secret is a password that the archiving service will use to authenticate. Treat this with the same level of security as an administrator's password.
+
+1.  In your application's menu, navigate to **Certificates & secrets**.
+2.  Select the **Client secrets** tab and click **+ New client secret**.
+3.  In the pane that appears:
+    -   **Description:** Enter a clear description, such as `OpenArchiver Key`.
+    -   **Expires:** Select an expiry duration. We recommend **12 or 24 months**. Set a calendar reminder to renew it before it expires to prevent service interruption.
+4.  Click **Add**.
+5.  **IMMEDIATELY COPY THE SECRET:** The secret is now visible in the **"Value"** column. This is the only time it will be fully displayed. Copy this value now and store it in a secure password manager before navigating away. If you lose it, you must create a new one.
+
+---
+
+### Part 4: Connecting in OpenArchiver
+
+You now have the three pieces of information required to configure the connection.
+
+1.  **Navigate to Ingestion Sources:**
+    In the OpenArchiver application, go to the **Ingestion Sources** page.
+
+2.  **Create a New Source:**
+    Click the **"Create New"** button.
+
+3.  **Fill in the Configuration Details:**
+
+    -   **Name:** Give the source a name (e.g., "Microsoft 365 Archive").
+    -   **Provider:** Select **"Microsoft 365"** from the dropdown.
+    -   **Application (Client) ID:** Go to the **Overview** page of your app registration in the Entra admin center and copy this value.
+    -   **Directory (Tenant) ID:** This value is also on the **Overview** page.
+    -   **Client Secret Value:** Paste the secret **Value** (not the Secret ID) that you copied and saved in the previous step.
+
+4.  **Save Changes:**
+    Click **"Save changes"**.
+
+## What Happens Next?
+
+Once the connection is saved, the system will begin the archiving process:
+
+1.  **User Discovery:** The service will connect to the Microsoft Graph API to get a list of all users in your organization.
+2.  **Initial Import:** The system will begin a background job to import the mailboxes of all discovered users, folder by folder. The status will show as **"Importing"**. This can take a significant amount of time.
+3.  **Continuous Sync:** After the initial import, the status will change to **"Active"**. The system will use Microsoft Graph's delta query feature to efficiently fetch only new or changed emails, ensuring the archive stays up-to-date.

docs/user-guides/installation.md

@@ -0,0 +1,163 @@
+# Installation Guide
+
+This guide will walk you through setting up Open Archiver using Docker Compose. This is the recommended method for deploying the application.
+
+## Prerequisites
+
+-   [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) installed on your server or local machine.
+-   A server or local machine with at least 4GB of RAM (2GB of RAM if you use external Postgres, Redis (Valkey) and Meilisearch instances).
+-   [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) installed on your server or local machine.
+
+## 1. Clone the Repository
+
+First, clone the Open Archiver repository to your machine:
+
+```bash
+git clone https://github.com/LogicLabs-OU/OpenArchiver.git
+cd OpenArchiver
+```
+
+## 2. Configure Your Environment
+
+The application is configured using environment variables. You'll need to create a `.env` file to store your configuration.
+
+Copy the example environment file for Docker:
+
+```bash
+cp .env.example.docker .env
+```
+
+Now, open the `.env` file in a text editor and customize the settings.
+
+### Important Configuration
+
+You must change the following placeholder values to secure your instance:
+
+-   `POSTGRES_PASSWORD`: A strong, unique password for the database.
+-   `REDIS_PASSWORD`: A strong, unique password for the Valkey/Redis service.
+-   `MEILI_MASTER_KEY`: A complex key for Meilisearch.
+-   `JWT_SECRET`: A long, random string for signing authentication tokens.
+-   `ADMIN_PASSWORD`: A strong password for the initial admin user.
+-   `ENCRYPTION_KEY`: A 32-byte hex string for encrypting sensitive data in the database. You can generate one with the following command:
+    ```bash
+    openssl rand -hex 32
+    ```
+
+### Storage Configuration
+
+By default, the Docker Compose setup uses local filesystem storage, which is persisted using a Docker volume named `archiver-data`. This is suitable for most use cases.
+
+If you want to use S3-compatible object storage, change the `STORAGE_TYPE` to `s3` and fill in your S3 credentials (`STORAGE_S3_*` variables). When `STORAGE_TYPE` is set to `local`, the S3-related variables are not required.
+
+### Using External Services
+
+For convenience, the `docker-compose.yml` file includes services for PostgreSQL, Valkey (Redis), and Meilisearch. However, you can use your own external or managed instances for these services.
+
+To do so:
+
+1.  **Update your `.env` file**: Change the host, port, and credential variables to point to your external service instances. For example, you would update `DATABASE_URL`, `REDIS_HOST`, and `MEILI_HOST`.
+2.  **Modify `docker-compose.yml`**: Remove or comment out the service definitions for `postgres`, `valkey`, and `meilisearch` from your `docker-compose.yml` file.
+
+This will configure the Open Archiver application to connect to your services instead of starting the default ones.
+
+### Environment Variable Reference
+
+Here is a complete list of environment variables available for configuration:
+
+#### Application Settings
+
+| Variable        | Description                        | Default Value |
+| --------------- | ---------------------------------- | ------------- |
+| `NODE_ENV`      | The application environment.       | `development` |
+| `PORT_BACKEND`  | The port for the backend service.  | `4000`        |
+| `PORT_FRONTEND` | The port for the frontend service. | `3000`        |
+
+#### Docker Compose Service Configuration
+
+These variables are used by `docker-compose.yml` to configure the services.
+
+| Variable            | Description                                     | Default Value                                            |
+| ------------------- | ----------------------------------------------- | -------------------------------------------------------- |
+| `POSTGRES_DB`       | The name of the PostgreSQL database.            | `open_archive`                                           |
+| `POSTGRES_USER`     | The username for the PostgreSQL database.       | `admin`                                                  |
+| `POSTGRES_PASSWORD` | The password for the PostgreSQL database.       | `password`                                               |
+| `DATABASE_URL`      | The connection URL for the PostgreSQL database. | `postgresql://admin:password@postgres:5432/open_archive` |
+| `MEILI_MASTER_KEY`  | The master key for Meilisearch.                 | `aSampleMasterKey`                                       |
+| `MEILI_HOST`        | The host for the Meilisearch service.           | `http://meilisearch:7700`                                |
+| `REDIS_HOST`        | The host for the Valkey (Redis) service.        | `valkey`                                                 |
+| `REDIS_PORT`        | The port for the Valkey (Redis) service.        | `6379`                                                   |
+| `REDIS_PASSWORD`    | The password for the Valkey (Redis) service.    | `defaultredispassword`                                   |
+| `REDIS_TLS_ENABLED` | Enable or disable TLS for Redis.                | `false`                                                  |
+
+#### Storage Settings
+
+| Variable                       | Description                                                                           | Default Value             |
+| ------------------------------ | ------------------------------------------------------------------------------------- | ------------------------- |
+| `STORAGE_TYPE`                 | The storage backend to use (`local` or `s3`).                                         | `local`                   |
+| `STORAGE_LOCAL_ROOT_PATH`      | The root path for local file storage.                                                 | `/var/data/open-archiver` |
+| `STORAGE_S3_ENDPOINT`          | The endpoint for S3-compatible storage (required if `STORAGE_TYPE` is `s3`).          |                           |
+| `STORAGE_S3_BUCKET`            | The bucket name for S3-compatible storage (required if `STORAGE_TYPE` is `s3`).       |                           |
+| `STORAGE_S3_ACCESS_KEY_ID`     | The access key ID for S3-compatible storage (required if `STORAGE_TYPE` is `s3`).     |                           |
+| `STORAGE_S3_SECRET_ACCESS_KEY` | The secret access key for S3-compatible storage (required if `STORAGE_TYPE` is `s3`). |                           |
+| `STORAGE_S3_REGION`            | The region for S3-compatible storage (required if `STORAGE_TYPE` is `s3`).            |                           |
+| `STORAGE_S3_FORCE_PATH_STYLE`  | Force path-style addressing for S3 (optional).                                        | `false`                   |
+
+#### Security & Authentication
+
+| Variable         | Description                                         | Default Value                              |
+| ---------------- | --------------------------------------------------- | ------------------------------------------ |
+| `JWT_SECRET`     | A secret key for signing JWT tokens.                | `a-very-secret-key-that-you-should-change` |
+| `JWT_EXPIRES_IN` | The expiration time for JWT tokens.                 | `7d`                                       |
+| `ADMIN_EMAIL`    | The email for the initial admin user.               | `admin@local.com`                          |
+| `ADMIN_PASSWORD` | The password for the initial admin user.            | `a_strong_password_that_you_should_change` |
+| `SUPER_API_KEY`  | An API key with super admin privileges.             |                                            |
+| `ENCRYPTION_KEY` | A 32-byte hex string for encrypting sensitive data. |                                            |
+
+## 3. Run the Application
+
+Once you have configured your `.env` file, you can start all the services using Docker Compose:
+
+```bash
+docker compose up -d
+```
+
+This command will:
+
+-   Pull the required Docker images for the frontend, backend, database, and other services.
+-   Create and start the containers in the background (`-d` flag).
+-   Create the persistent volumes for your data.
+
+You can check the status of the running containers with:
+
+```bash
+docker compose ps
+```
+
+## 4. Access the Application
+
+Once the services are running, you can access the Open Archiver web interface by navigating to `http://localhost:3000` in your web browser.
+
+You can log in with the `ADMIN_EMAIL` and `ADMIN_PASSWORD` you configured in your `.env` file.
+
+## 5. Next Steps
+
+After successfully deploying and logging into Open Archiver, the next step is to configure your ingestion sources to start archiving emails.
+
+-   [Connecting to Google Workspace](./email-providers/google-workspace.md)
+-   [Connecting to Microsoft 365](./email-providers/microsoft-365.md)
+-   [Connecting to a Generic IMAP Server](./email-providers/imap.md)
+
+## Updating Your Installation
+
+To update your Open Archiver instance to the latest version, run the following commands:
+
+```bash
+# Pull the latest changes from the repository
+git pull
+
+# Pull the latest Docker images
+docker compose pull
+
+# Restart the services with the new images
+docker compose up -d
+```

package.json

@@ -3,13 +3,25 @@
     "private": true,
     "scripts": {
         "dev": "dotenv -- pnpm --filter \"./packages/*\" --parallel dev",
-        "build": "pnpm --filter \"./packages/*\" --parallel build",
-        "start:workers": "dotenv -- concurrently \"pnpm --filter @open-archiver/backend start:ingestion-worker\" \"pnpm --filter @open-archiver/backend start:indexing-worker\""
+        "build": "pnpm --filter \"./packages/*\" build",
+        "start": "dotenv -- pnpm --filter \"./packages/*\" --parallel start",
+        "start:workers": "dotenv -- concurrently \"pnpm --filter @open-archiver/backend start:ingestion-worker\" \"pnpm --filter @open-archiver/backend start:indexing-worker\" \"pnpm --filter @open-archiver/backend start:sync-scheduler\"",
+        "start:workers:dev": "dotenv -- concurrently \"pnpm --filter @open-archiver/backend start:ingestion-worker:dev\" \"pnpm --filter @open-archiver/backend start:indexing-worker:dev\" \"pnpm --filter @open-archiver/backend start:sync-scheduler:dev\"",
+        "db:generate": "dotenv -- pnpm --filter @open-archiver/backend db:generate",
+        "db:migrate": "dotenv -- pnpm --filter @open-archiver/backend db:migrate",
+        "db:migrate:dev": "dotenv -- pnpm --filter @open-archiver/backend db:migrate:dev",
+        "docker-start": "concurrently \"pnpm start:workers\" \"pnpm start\"",
+        "docs:dev": "vitepress dev docs --port 3009",
+        "docs:build": "vitepress build docs",
+        "docs:preview": "vitepress preview docs"
+    },
+    "dependencies": {
+        "concurrently": "^9.2.0",
+        "dotenv-cli": "8.0.0"
     },
     "devDependencies": {
-        "concurrently": "^9.2.0",
-        "dotenv-cli": "8.0.0",
-        "typescript": "5.8.3"
+        "typescript": "5.8.3",
+        "vitepress": "^1.6.3"
     },
     "packageManager": "pnpm@10.13.1",
     "engines": {

packages/backend/drizzle.config.ts

@@ -1,7 +1,7 @@
 import { defineConfig } from 'drizzle-kit';
 import { config } from 'dotenv';
 
-config({ path: '../../.env' });
+config();
 
 if (!process.env.DATABASE_URL) {
     throw new Error('DATABASE_URL is not set in the .env file');

packages/backend/package.json

@@ -6,25 +6,34 @@
     "scripts": {
         "dev": "ts-node-dev --respawn --transpile-only src/index.ts ",
         "build": "tsc",
-        "prestart": "npm run build",
         "start": "node dist/index.js",
-        "start:ingestion-worker": "ts-node-dev --respawn --transpile-only src/workers/ingestion.worker.ts",
-        "start:indexing-worker": "ts-node-dev --respawn --transpile-only src/workers/indexing.worker.ts",
+        "start:ingestion-worker": "node dist/workers/ingestion.worker.js",
+        "start:indexing-worker": "node dist/workers/indexing.worker.js",
+        "start:sync-scheduler": "node dist/jobs/schedulers/sync-scheduler.js",
+        "start:ingestion-worker:dev": "ts-node-dev --respawn --transpile-only src/workers/ingestion.worker.ts",
+        "start:indexing-worker:dev": "ts-node-dev --respawn --transpile-only src/workers/indexing.worker.ts",
+        "start:sync-scheduler:dev": "ts-node-dev --respawn --transpile-only src/jobs/schedulers/sync-scheduler.ts",
         "db:generate": "drizzle-kit generate --config=drizzle.config.ts",
         "db:push": "drizzle-kit push --config=drizzle.config.ts",
-        "db:migrate": "ts-node-dev src/database/migrate.ts"
+        "db:migrate": "node dist/database/migrate.js",
+        "db:migrate:dev": "ts-node-dev src/database/migrate.ts"
     },
     "dependencies": {
         "@aws-sdk/client-s3": "^3.844.0",
         "@aws-sdk/lib-storage": "^3.844.0",
         "@azure/msal-node": "^3.6.3",
+        "@microsoft/microsoft-graph-client": "^3.0.7",
         "@open-archiver/types": "workspace:*",
         "axios": "^1.10.0",
         "bcryptjs": "^3.0.2",
         "bullmq": "^5.56.3",
+        "cross-fetch": "^4.1.0",
+        "deepmerge-ts": "^7.1.5",
         "dotenv": "^17.2.0",
+        "drizzle-kit": "^0.31.4",
         "drizzle-orm": "^0.44.2",
         "express": "^5.1.0",
+        "express-rate-limit": "^8.0.1",
         "express-validator": "^7.2.1",
         "google-auth-library": "^10.1.0",
         "googleapis": "^152.0.0",
@@ -48,9 +57,9 @@
         "@bull-board/express": "^6.11.0",
         "@types/express": "^5.0.3",
         "@types/mailparser": "^3.4.6",
+        "@types/microsoft-graph": "^2.40.1",
         "@types/node": "^24.0.12",
         "bull-board": "^2.1.3",
-        "drizzle-kit": "^0.31.4",
         "ts-node-dev": "^2.0.0",
         "typescript": "^5.8.3"
     }

packages/backend/src/api/controllers/auth.controller.ts

@@ -24,7 +24,6 @@ export class AuthController {
 
             return res.status(200).json(result);
         } catch (error) {
-            // In a real application, you'd want to log this error.
             console.error('Login error:', error);
             return res.status(500).json({ message: 'An internal server error occurred' });
         }

packages/backend/src/api/controllers/dashboard.controller.ts

@@ -0,0 +1,31 @@
+import { Request, Response } from 'express';
+import { dashboardService } from '../../services/DashboardService';
+
+class DashboardController {
+    public async getStats(req: Request, res: Response) {
+        const stats = await dashboardService.getStats();
+        res.json(stats);
+    }
+
+    public async getIngestionHistory(req: Request, res: Response) {
+        const history = await dashboardService.getIngestionHistory();
+        res.json(history);
+    }
+
+    public async getIngestionSources(req: Request, res: Response) {
+        const sources = await dashboardService.getIngestionSources();
+        res.json(sources);
+    }
+
+    public async getRecentSyncs(req: Request, res: Response) {
+        const syncs = await dashboardService.getRecentSyncs();
+        res.json(syncs);
+    }
+
+    public async getIndexedInsights(req: Request, res: Response) {
+        const insights = await dashboardService.getIndexedInsights();
+        res.json(insights);
+    }
+}
+
+export const dashboardController = new DashboardController();

packages/backend/src/api/controllers/ingestion.controller.ts

@@ -1,23 +1,47 @@
 import { Request, Response } from 'express';
 import { IngestionService } from '../../services/IngestionService';
-import { CreateIngestionSourceDto, UpdateIngestionSourceDto } from '@open-archiver/types';
+import {
+    CreateIngestionSourceDto,
+    UpdateIngestionSourceDto,
+    IngestionSource,
+    SafeIngestionSource
+} from '@open-archiver/types';
+import { logger } from '../../config/logger';
+import { config } from '../../config';
 
 export class IngestionController {
+    /**
+     * Converts an IngestionSource object to a safe version for client-side consumption
+     * by removing the credentials.
+     * @param source The full IngestionSource object.
+     * @returns An object conforming to the SafeIngestionSource type.
+     */
+    private toSafeIngestionSource(source: IngestionSource): SafeIngestionSource {
+        const { credentials, ...safeSource } = source;
+        return safeSource;
+    }
+
     public create = async (req: Request, res: Response): Promise<Response> => {
+        if (config.app.isDemo) {
+            return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
+        }
         try {
             const dto: CreateIngestionSourceDto = req.body;
             const newSource = await IngestionService.create(dto);
-            return res.status(201).json(newSource);
-        } catch (error) {
-            console.error('Create ingestion source error:', error);
-            return res.status(500).json({ message: 'An internal server error occurred' });
+            const safeSource = this.toSafeIngestionSource(newSource);
+            return res.status(201).json(safeSource);
+        } catch (error: any) {
+            logger.error({ err: error }, 'Create ingestion source error');
+            // Return a 400 Bad Request for connection errors
+            return res.status(400).json({ message: error.message || 'Failed to create ingestion source due to a connection error.' });
         }
     };
 
     public findAll = async (req: Request, res: Response): Promise<Response> => {
         try {
             const sources = await IngestionService.findAll();
-            return res.status(200).json(sources);
+            const safeSources = sources.map(this.toSafeIngestionSource);
+            return res.status(200).json(safeSources);
         } catch (error) {
             console.error('Find all ingestion sources error:', error);
             return res.status(500).json({ message: 'An internal server error occurred' });
@@ -28,7 +52,8 @@ export class IngestionController {
         try {
             const { id } = req.params;
             const source = await IngestionService.findById(id);
-            return res.status(200).json(source);
+            const safeSource = this.toSafeIngestionSource(source);
+            return res.status(200).json(safeSource);
         } catch (error) {
             console.error(`Find ingestion source by id ${req.params.id} error:`, error);
             if (error instanceof Error && error.message === 'Ingestion source not found') {
@@ -39,11 +64,15 @@ export class IngestionController {
     };
 
     public update = async (req: Request, res: Response): Promise<Response> => {
+        if (config.app.isDemo) {
+            return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
+        }
         try {
             const { id } = req.params;
             const dto: UpdateIngestionSourceDto = req.body;
             const updatedSource = await IngestionService.update(id, dto);
-            return res.status(200).json(updatedSource);
+            const safeSource = this.toSafeIngestionSource(updatedSource);
+            return res.status(200).json(safeSource);
         } catch (error) {
             console.error(`Update ingestion source ${req.params.id} error:`, error);
             if (error instanceof Error && error.message === 'Ingestion source not found') {
@@ -54,6 +83,9 @@ export class IngestionController {
     };
 
     public delete = async (req: Request, res: Response): Promise<Response> => {
+        if (config.app.isDemo) {
+            return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
+        }
         try {
             const { id } = req.params;
             await IngestionService.delete(id);
@@ -68,6 +100,9 @@ export class IngestionController {
     };
 
     public triggerInitialImport = async (req: Request, res: Response): Promise<Response> => {
+        if (config.app.isDemo) {
+            return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
+        }
         try {
             const { id } = req.params;
             await IngestionService.triggerInitialImport(id);
@@ -80,4 +115,39 @@ export class IngestionController {
             return res.status(500).json({ message: 'An internal server error occurred' });
         }
     };
+
+    public pause = async (req: Request, res: Response): Promise<Response> => {
+        if (config.app.isDemo) {
+            return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
+        }
+        try {
+            const { id } = req.params;
+            const updatedSource = await IngestionService.update(id, { status: 'paused' });
+            const safeSource = this.toSafeIngestionSource(updatedSource);
+            return res.status(200).json(safeSource);
+        } catch (error) {
+            console.error(`Pause ingestion source ${req.params.id} error:`, error);
+            if (error instanceof Error && error.message === 'Ingestion source not found') {
+                return res.status(404).json({ message: error.message });
+            }
+            return res.status(500).json({ message: 'An internal server error occurred' });
+        }
+    };
+
+    public triggerForceSync = async (req: Request, res: Response): Promise<Response> => {
+        if (config.app.isDemo) {
+            return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
+        }
+        try {
+            const { id } = req.params;
+            await IngestionService.triggerForceSync(id);
+            return res.status(202).json({ message: 'Force sync triggered successfully.' });
+        } catch (error) {
+            console.error(`Trigger force sync for ${req.params.id} error:`, error);
+            if (error instanceof Error && error.message === 'Ingestion source not found') {
+                return res.status(404).json({ message: error.message });
+            }
+            return res.status(500).json({ message: 'An internal server error occurred' });
+        }
+    };
 }

packages/backend/src/api/controllers/storage.controller.ts

@@ -1,28 +1,46 @@
 import { Request, Response } from 'express';
 import { StorageService } from '../../services/StorageService';
+import * as path from 'path';
+import { storage as storageConfig } from '../../config/storage';
 
 export class StorageController {
     constructor(private storageService: StorageService) { }
 
     public downloadFile = async (req: Request, res: Response): Promise<void> => {
-        const filePath = req.query.path as string;
+        const unsafePath = req.query.path as string;
 
-        if (!filePath) {
+        if (!unsafePath) {
             res.status(400).send('File path is required');
             return;
         }
 
+        // Normalize the path to prevent directory traversal
+        const normalizedPath = path.normalize(unsafePath).replace(/^(\.\.(\/|\\|$))+/, '');
+
+        // Determine the base path from storage configuration
+        const basePath = storageConfig.type === 'local' ? storageConfig.rootPath : '/';
+
+        // Resolve the full path and ensure it's within the storage directory
+        const fullPath = path.join(basePath, normalizedPath);
+
+        if (!fullPath.startsWith(basePath)) {
+            res.status(400).send('Invalid file path');
+            return;
+        }
+
+        // Use the sanitized, relative path for storage service operations
+        const safePath = path.relative(basePath, fullPath);
+
         try {
-            const fileExists = await this.storageService.exists(filePath);
+            const fileExists = await this.storageService.exists(safePath);
             if (!fileExists) {
-                console.log(filePath);
                 res.status(404).send('File not found');
                 return;
             }
 
-            const fileStream = await this.storageService.get(filePath);
-            const fileName = filePath.split('/').pop();
-            res.setHeader('Content-Disposition', `attachment; filename=${fileName}`);
+            const fileStream = await this.storageService.get(safePath);
+            const fileName = path.basename(safePath);
+            res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`);
             fileStream.pipe(res);
         } catch (error) {
             console.error('Error downloading file:', error);

packages/backend/src/api/middleware/rateLimiter.ts

@@ -0,0 +1,10 @@
+import rateLimit from 'express-rate-limit';
+
+// Rate limiter to prevent brute-force attacks on the login endpoint
+export const loginRateLimiter = rateLimit({
+    windowMs: 15 * 60 * 1000, // 15 minutes
+    max: 10, // Limit each IP to 10 login requests per windowMs
+    message: 'Too many login attempts from this IP, please try again after 15 minutes',
+    standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
+    legacyHeaders: false, // Disable the `X-RateLimit-*` headers
+});

packages/backend/src/api/middleware/requireAuth.ts

@@ -20,8 +20,8 @@ export const requireAuth = (authService: IAuthService) => {
         }
         const token = authHeader.split(' ')[1];
         try {
-            // use a SUPER_API_KEY for all authentications.
-            if (token === process.env.SUPER_API_KEY) {
+            // use a SUPER_API_KEY for all authentications. add process.env.SUPER_API_KEY conditional check in case user didn't set a SUPER_API_KEY.
+            if (process.env.SUPER_API_KEY && token === process.env.SUPER_API_KEY) {
                 next();
                 return;
             }

packages/backend/src/api/routes/auth.routes.ts

@@ -1,4 +1,5 @@
 import { Router } from 'express';
+import { loginRateLimiter } from '../middleware/rateLimiter';
 import type { AuthController } from '../controllers/auth.controller';
 
 export const createAuthRouter = (authController: AuthController): Router => {
@@ -9,7 +10,7 @@ export const createAuthRouter = (authController: AuthController): Router => {
      * @description Authenticates a user and returns a JWT.
      * @access Public
      */
-    router.post('/login', authController.login);
+    router.post('/login', loginRateLimiter, authController.login);
 
     return router;
 };

packages/backend/src/api/routes/dashboard.routes.ts

@@ -0,0 +1,18 @@
+import { Router } from 'express';
+import { dashboardController } from '../controllers/dashboard.controller';
+import { requireAuth } from '../middleware/requireAuth';
+import { IAuthService } from '../../services/AuthService';
+
+export const createDashboardRouter = (authService: IAuthService): Router => {
+    const router = Router();
+
+    router.use(requireAuth(authService));
+
+    router.get('/stats', dashboardController.getStats);
+    router.get('/ingestion-history', dashboardController.getIngestionHistory);
+    router.get('/ingestion-sources', dashboardController.getIngestionSources);
+    router.get('/recent-syncs', dashboardController.getRecentSyncs);
+    router.get('/indexed-insights', dashboardController.getIndexedInsights);
+
+    return router;
+};

packages/backend/src/api/routes/ingestion.routes.ts

@@ -22,7 +22,11 @@ export const createIngestionRouter = (
 
     router.delete('/:id', ingestionController.delete);
 
-    router.post('/:id/sync', ingestionController.triggerInitialImport);
+    router.post('/:id/import', ingestionController.triggerInitialImport);
+
+    router.post('/:id/pause', ingestionController.pause);
+
+    router.post('/:id/sync', ingestionController.triggerForceSync);
 
     return router;
 };

packages/backend/src/api/routes/test.routes.ts

@@ -3,16 +3,7 @@ import { ingestionQueue } from '../../jobs/queues';
 
 const router: Router = Router();
 
-router.post('/trigger-job', async (req, res) => {
-    try {
-        const job = await ingestionQueue.add('initial-import', {
-            ingestionSourceId: 'test-source-id-test-2345'
-        });
-        res.status(202).json({ message: 'Test job triggered successfully', jobId: job.id });
-    } catch (error) {
-        console.error('Failed to trigger test job', error);
-        res.status(500).json({ message: 'Failed to trigger test job' });
-    }
-});
+
+
 
 export default router;

packages/backend/src/config/app.ts

@@ -4,4 +4,5 @@ export const app = {
     nodeEnv: process.env.NODE_ENV || 'development',
     port: process.env.PORT_BACKEND ? parseInt(process.env.PORT_BACKEND, 10) : 4000,
     encryptionKey: process.env.ENCRYPTION_KEY,
+    isDemo: process.env.IS_DEMO === 'true',
 };

packages/backend/src/config/redis.ts

@@ -3,12 +3,17 @@ import 'dotenv/config';
 /**
  * @see https://github.com/taskforcesh/bullmq/blob/master/docs/gitbook/guide/connections.md
  */
-export const connection = {
+const connectionOptions: any = {
     host: process.env.REDIS_HOST || 'localhost',
     port: (process.env.REDIS_PORT && parseInt(process.env.REDIS_PORT, 10)) || 6379,
     password: process.env.REDIS_PASSWORD,
-    maxRetriesPerRequest: null,
-    tls: {
-        rejectUnauthorized: false
-    }
+    enableReadyCheck: true,
 };
+
+if (process.env.REDIS_TLS_ENABLED === 'true') {
+    connectionOptions.tls = {
+        rejectUnauthorized: false
+    };
+}
+
+export const connection = connectionOptions;

packages/backend/src/database/migrate.ts

@@ -3,7 +3,7 @@ import { drizzle } from 'drizzle-orm/postgres-js';
 import postgres from 'postgres';
 import { config } from 'dotenv';
 
-config({ path: '../../.env' });
+config();
 
 const runMigrate = async () => {
     if (!process.env.DATABASE_URL) {

packages/backend/src/database/migrations/0006_majestic_caretaker.sql

@@ -0,0 +1 @@
+ALTER TABLE "ingestion_sources" ADD COLUMN "sync_state" jsonb;

packages/backend/src/database/migrations/0007_handy_archangel.sql

@@ -0,0 +1 @@
+ALTER TABLE "ingestion_sources" ALTER COLUMN "credentials" SET DATA TYPE text;

packages/backend/src/database/migrations/0008_eminent_the_spike.sql

@@ -0,0 +1 @@
+ALTER TABLE "archived_emails" ADD COLUMN "user_email" text NOT NULL;

packages/backend/src/database/migrations/0009_late_lenny_balinger.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "archived_emails" ADD COLUMN "thread_id" text;--> statement-breakpoint
+CREATE INDEX "thread_id_idx" ON "archived_emails" USING btree ("thread_id");

packages/backend/src/database/migrations/meta/0006_snapshot.json

@@ -0,0 +1,826 @@
+{
+  "id": "bdc9d789-04c7-4d9f-b4ed-00366b0d3603",
+  "prevId": "4fa75649-1e65-4c61-8cc5-95add8269925",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {
+    "public.archived_emails": {
+      "name": "archived_emails",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "ingestion_source_id": {
+          "name": "ingestion_source_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "message_id_header": {
+          "name": "message_id_header",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sent_at": {
+          "name": "sent_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "subject": {
+          "name": "subject",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_name": {
+          "name": "sender_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_email": {
+          "name": "sender_email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "recipients": {
+          "name": "recipients",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_hash_sha256": {
+          "name": "storage_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_indexed": {
+          "name": "is_indexed",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "has_attachments": {
+          "name": "has_attachments",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "is_on_legal_hold": {
+          "name": "is_on_legal_hold",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "archived_at": {
+          "name": "archived_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "archived_emails_ingestion_source_id_ingestion_sources_id_fk": {
+          "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
+          "tableFrom": "archived_emails",
+          "tableTo": "ingestion_sources",
+          "columnsFrom": [
+            "ingestion_source_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.attachments": {
+      "name": "attachments",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "filename": {
+          "name": "filename",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "mime_type": {
+          "name": "mime_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "content_hash_sha256": {
+          "name": "content_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "attachments_content_hash_sha256_unique": {
+          "name": "attachments_content_hash_sha256_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "content_hash_sha256"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.email_attachments": {
+      "name": "email_attachments",
+      "schema": "",
+      "columns": {
+        "email_id": {
+          "name": "email_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "attachment_id": {
+          "name": "attachment_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "email_attachments_email_id_archived_emails_id_fk": {
+          "name": "email_attachments_email_id_archived_emails_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "archived_emails",
+          "columnsFrom": [
+            "email_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "email_attachments_attachment_id_attachments_id_fk": {
+          "name": "email_attachments_attachment_id_attachments_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "attachments",
+          "columnsFrom": [
+            "attachment_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "restrict",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "email_attachments_email_id_attachment_id_pk": {
+          "name": "email_attachments_email_id_attachment_id_pk",
+          "columns": [
+            "email_id",
+            "attachment_id"
+          ]
+        }
+      },
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.audit_logs": {
+      "name": "audit_logs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "bigserial",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "timestamp": {
+          "name": "timestamp",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "actor_identifier": {
+          "name": "actor_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action": {
+          "name": "action",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "target_type": {
+          "name": "target_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "target_id": {
+          "name": "target_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "details": {
+          "name": "details",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "is_tamper_evident": {
+          "name": "is_tamper_evident",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": false,
+          "default": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ediscovery_cases": {
+      "name": "ediscovery_cases",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'open'"
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "ediscovery_cases_name_unique": {
+          "name": "ediscovery_cases_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.export_jobs": {
+      "name": "export_jobs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "format": {
+          "name": "format",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending'"
+        },
+        "query": {
+          "name": "query",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "file_path": {
+          "name": "file_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "completed_at": {
+          "name": "completed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "export_jobs_case_id_ediscovery_cases_id_fk": {
+          "name": "export_jobs_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "export_jobs",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "set null",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.legal_holds": {
+      "name": "legal_holds",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "custodian_id": {
+          "name": "custodian_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "hold_criteria": {
+          "name": "hold_criteria",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "reason": {
+          "name": "reason",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "applied_by_identifier": {
+          "name": "applied_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "applied_at": {
+          "name": "applied_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "removed_at": {
+          "name": "removed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "legal_holds_case_id_ediscovery_cases_id_fk": {
+          "name": "legal_holds_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "legal_holds_custodian_id_custodians_id_fk": {
+          "name": "legal_holds_custodian_id_custodians_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "custodians",
+          "columnsFrom": [
+            "custodian_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.retention_policies": {
+      "name": "retention_policies",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "priority": {
+          "name": "priority",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "retention_period_days": {
+          "name": "retention_period_days",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action_on_expiry": {
+          "name": "action_on_expiry",
+          "type": "retention_action",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_enabled": {
+          "name": "is_enabled",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": true
+        },
+        "conditions": {
+          "name": "conditions",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "retention_policies_name_unique": {
+          "name": "retention_policies_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.custodians": {
+      "name": "custodians",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "email": {
+          "name": "email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "display_name": {
+          "name": "display_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "source_type": {
+          "name": "source_type",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "custodians_email_unique": {
+          "name": "custodians_email_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "email"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ingestion_sources": {
+      "name": "ingestion_sources",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "provider": {
+          "name": "provider",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "credentials": {
+          "name": "credentials",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "ingestion_status",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending_auth'"
+        },
+        "last_sync_started_at": {
+          "name": "last_sync_started_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_finished_at": {
+          "name": "last_sync_finished_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_status_message": {
+          "name": "last_sync_status_message",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sync_state": {
+          "name": "sync_state",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    }
+  },
+  "enums": {
+    "public.retention_action": {
+      "name": "retention_action",
+      "schema": "public",
+      "values": [
+        "delete_permanently",
+        "notify_admin"
+      ]
+    },
+    "public.ingestion_provider": {
+      "name": "ingestion_provider",
+      "schema": "public",
+      "values": [
+        "google_workspace",
+        "microsoft_365",
+        "generic_imap"
+      ]
+    },
+    "public.ingestion_status": {
+      "name": "ingestion_status",
+      "schema": "public",
+      "values": [
+        "active",
+        "paused",
+        "error",
+        "pending_auth",
+        "syncing",
+        "importing",
+        "auth_success"
+      ]
+    }
+  },
+  "schemas": {},
+  "sequences": {},
+  "roles": {},
+  "policies": {},
+  "views": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}

packages/backend/src/database/migrations/meta/0007_snapshot.json

@@ -0,0 +1,826 @@
+{
+  "id": "2a68f80f-b233-43bd-8280-745bee76ca3e",
+  "prevId": "bdc9d789-04c7-4d9f-b4ed-00366b0d3603",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {
+    "public.archived_emails": {
+      "name": "archived_emails",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "ingestion_source_id": {
+          "name": "ingestion_source_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "message_id_header": {
+          "name": "message_id_header",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sent_at": {
+          "name": "sent_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "subject": {
+          "name": "subject",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_name": {
+          "name": "sender_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_email": {
+          "name": "sender_email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "recipients": {
+          "name": "recipients",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_hash_sha256": {
+          "name": "storage_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_indexed": {
+          "name": "is_indexed",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "has_attachments": {
+          "name": "has_attachments",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "is_on_legal_hold": {
+          "name": "is_on_legal_hold",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "archived_at": {
+          "name": "archived_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "archived_emails_ingestion_source_id_ingestion_sources_id_fk": {
+          "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
+          "tableFrom": "archived_emails",
+          "tableTo": "ingestion_sources",
+          "columnsFrom": [
+            "ingestion_source_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.attachments": {
+      "name": "attachments",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "filename": {
+          "name": "filename",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "mime_type": {
+          "name": "mime_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "content_hash_sha256": {
+          "name": "content_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "attachments_content_hash_sha256_unique": {
+          "name": "attachments_content_hash_sha256_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "content_hash_sha256"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.email_attachments": {
+      "name": "email_attachments",
+      "schema": "",
+      "columns": {
+        "email_id": {
+          "name": "email_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "attachment_id": {
+          "name": "attachment_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "email_attachments_email_id_archived_emails_id_fk": {
+          "name": "email_attachments_email_id_archived_emails_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "archived_emails",
+          "columnsFrom": [
+            "email_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "email_attachments_attachment_id_attachments_id_fk": {
+          "name": "email_attachments_attachment_id_attachments_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "attachments",
+          "columnsFrom": [
+            "attachment_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "restrict",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "email_attachments_email_id_attachment_id_pk": {
+          "name": "email_attachments_email_id_attachment_id_pk",
+          "columns": [
+            "email_id",
+            "attachment_id"
+          ]
+        }
+      },
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.audit_logs": {
+      "name": "audit_logs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "bigserial",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "timestamp": {
+          "name": "timestamp",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "actor_identifier": {
+          "name": "actor_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action": {
+          "name": "action",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "target_type": {
+          "name": "target_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "target_id": {
+          "name": "target_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "details": {
+          "name": "details",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "is_tamper_evident": {
+          "name": "is_tamper_evident",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": false,
+          "default": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ediscovery_cases": {
+      "name": "ediscovery_cases",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'open'"
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "ediscovery_cases_name_unique": {
+          "name": "ediscovery_cases_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.export_jobs": {
+      "name": "export_jobs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "format": {
+          "name": "format",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending'"
+        },
+        "query": {
+          "name": "query",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "file_path": {
+          "name": "file_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "completed_at": {
+          "name": "completed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "export_jobs_case_id_ediscovery_cases_id_fk": {
+          "name": "export_jobs_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "export_jobs",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "set null",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.legal_holds": {
+      "name": "legal_holds",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "custodian_id": {
+          "name": "custodian_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "hold_criteria": {
+          "name": "hold_criteria",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "reason": {
+          "name": "reason",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "applied_by_identifier": {
+          "name": "applied_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "applied_at": {
+          "name": "applied_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "removed_at": {
+          "name": "removed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "legal_holds_case_id_ediscovery_cases_id_fk": {
+          "name": "legal_holds_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "legal_holds_custodian_id_custodians_id_fk": {
+          "name": "legal_holds_custodian_id_custodians_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "custodians",
+          "columnsFrom": [
+            "custodian_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.retention_policies": {
+      "name": "retention_policies",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "priority": {
+          "name": "priority",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "retention_period_days": {
+          "name": "retention_period_days",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action_on_expiry": {
+          "name": "action_on_expiry",
+          "type": "retention_action",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_enabled": {
+          "name": "is_enabled",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": true
+        },
+        "conditions": {
+          "name": "conditions",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "retention_policies_name_unique": {
+          "name": "retention_policies_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.custodians": {
+      "name": "custodians",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "email": {
+          "name": "email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "display_name": {
+          "name": "display_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "source_type": {
+          "name": "source_type",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "custodians_email_unique": {
+          "name": "custodians_email_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "email"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ingestion_sources": {
+      "name": "ingestion_sources",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "provider": {
+          "name": "provider",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "credentials": {
+          "name": "credentials",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "ingestion_status",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending_auth'"
+        },
+        "last_sync_started_at": {
+          "name": "last_sync_started_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_finished_at": {
+          "name": "last_sync_finished_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_status_message": {
+          "name": "last_sync_status_message",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sync_state": {
+          "name": "sync_state",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    }
+  },
+  "enums": {
+    "public.retention_action": {
+      "name": "retention_action",
+      "schema": "public",
+      "values": [
+        "delete_permanently",
+        "notify_admin"
+      ]
+    },
+    "public.ingestion_provider": {
+      "name": "ingestion_provider",
+      "schema": "public",
+      "values": [
+        "google_workspace",
+        "microsoft_365",
+        "generic_imap"
+      ]
+    },
+    "public.ingestion_status": {
+      "name": "ingestion_status",
+      "schema": "public",
+      "values": [
+        "active",
+        "paused",
+        "error",
+        "pending_auth",
+        "syncing",
+        "importing",
+        "auth_success"
+      ]
+    }
+  },
+  "schemas": {},
+  "sequences": {},
+  "roles": {},
+  "policies": {},
+  "views": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}

packages/backend/src/database/migrations/meta/0008_snapshot.json

@@ -0,0 +1,832 @@
+{
+  "id": "86b6960e-1936-4543-846f-a2d24d6dc5d1",
+  "prevId": "2a68f80f-b233-43bd-8280-745bee76ca3e",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {
+    "public.archived_emails": {
+      "name": "archived_emails",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "ingestion_source_id": {
+          "name": "ingestion_source_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "user_email": {
+          "name": "user_email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "message_id_header": {
+          "name": "message_id_header",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sent_at": {
+          "name": "sent_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "subject": {
+          "name": "subject",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_name": {
+          "name": "sender_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_email": {
+          "name": "sender_email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "recipients": {
+          "name": "recipients",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_hash_sha256": {
+          "name": "storage_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_indexed": {
+          "name": "is_indexed",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "has_attachments": {
+          "name": "has_attachments",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "is_on_legal_hold": {
+          "name": "is_on_legal_hold",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "archived_at": {
+          "name": "archived_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "archived_emails_ingestion_source_id_ingestion_sources_id_fk": {
+          "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
+          "tableFrom": "archived_emails",
+          "tableTo": "ingestion_sources",
+          "columnsFrom": [
+            "ingestion_source_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.attachments": {
+      "name": "attachments",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "filename": {
+          "name": "filename",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "mime_type": {
+          "name": "mime_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "content_hash_sha256": {
+          "name": "content_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "attachments_content_hash_sha256_unique": {
+          "name": "attachments_content_hash_sha256_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "content_hash_sha256"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.email_attachments": {
+      "name": "email_attachments",
+      "schema": "",
+      "columns": {
+        "email_id": {
+          "name": "email_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "attachment_id": {
+          "name": "attachment_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "email_attachments_email_id_archived_emails_id_fk": {
+          "name": "email_attachments_email_id_archived_emails_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "archived_emails",
+          "columnsFrom": [
+            "email_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "email_attachments_attachment_id_attachments_id_fk": {
+          "name": "email_attachments_attachment_id_attachments_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "attachments",
+          "columnsFrom": [
+            "attachment_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "restrict",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "email_attachments_email_id_attachment_id_pk": {
+          "name": "email_attachments_email_id_attachment_id_pk",
+          "columns": [
+            "email_id",
+            "attachment_id"
+          ]
+        }
+      },
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.audit_logs": {
+      "name": "audit_logs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "bigserial",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "timestamp": {
+          "name": "timestamp",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "actor_identifier": {
+          "name": "actor_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action": {
+          "name": "action",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "target_type": {
+          "name": "target_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "target_id": {
+          "name": "target_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "details": {
+          "name": "details",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "is_tamper_evident": {
+          "name": "is_tamper_evident",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": false,
+          "default": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ediscovery_cases": {
+      "name": "ediscovery_cases",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'open'"
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "ediscovery_cases_name_unique": {
+          "name": "ediscovery_cases_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.export_jobs": {
+      "name": "export_jobs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "format": {
+          "name": "format",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending'"
+        },
+        "query": {
+          "name": "query",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "file_path": {
+          "name": "file_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "completed_at": {
+          "name": "completed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "export_jobs_case_id_ediscovery_cases_id_fk": {
+          "name": "export_jobs_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "export_jobs",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "set null",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.legal_holds": {
+      "name": "legal_holds",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "custodian_id": {
+          "name": "custodian_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "hold_criteria": {
+          "name": "hold_criteria",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "reason": {
+          "name": "reason",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "applied_by_identifier": {
+          "name": "applied_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "applied_at": {
+          "name": "applied_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "removed_at": {
+          "name": "removed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "legal_holds_case_id_ediscovery_cases_id_fk": {
+          "name": "legal_holds_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "legal_holds_custodian_id_custodians_id_fk": {
+          "name": "legal_holds_custodian_id_custodians_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "custodians",
+          "columnsFrom": [
+            "custodian_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.retention_policies": {
+      "name": "retention_policies",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "priority": {
+          "name": "priority",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "retention_period_days": {
+          "name": "retention_period_days",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action_on_expiry": {
+          "name": "action_on_expiry",
+          "type": "retention_action",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_enabled": {
+          "name": "is_enabled",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": true
+        },
+        "conditions": {
+          "name": "conditions",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "retention_policies_name_unique": {
+          "name": "retention_policies_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.custodians": {
+      "name": "custodians",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "email": {
+          "name": "email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "display_name": {
+          "name": "display_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "source_type": {
+          "name": "source_type",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "custodians_email_unique": {
+          "name": "custodians_email_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "email"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ingestion_sources": {
+      "name": "ingestion_sources",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "provider": {
+          "name": "provider",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "credentials": {
+          "name": "credentials",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "ingestion_status",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending_auth'"
+        },
+        "last_sync_started_at": {
+          "name": "last_sync_started_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_finished_at": {
+          "name": "last_sync_finished_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_status_message": {
+          "name": "last_sync_status_message",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sync_state": {
+          "name": "sync_state",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    }
+  },
+  "enums": {
+    "public.retention_action": {
+      "name": "retention_action",
+      "schema": "public",
+      "values": [
+        "delete_permanently",
+        "notify_admin"
+      ]
+    },
+    "public.ingestion_provider": {
+      "name": "ingestion_provider",
+      "schema": "public",
+      "values": [
+        "google_workspace",
+        "microsoft_365",
+        "generic_imap"
+      ]
+    },
+    "public.ingestion_status": {
+      "name": "ingestion_status",
+      "schema": "public",
+      "values": [
+        "active",
+        "paused",
+        "error",
+        "pending_auth",
+        "syncing",
+        "importing",
+        "auth_success"
+      ]
+    }
+  },
+  "schemas": {},
+  "sequences": {},
+  "roles": {},
+  "policies": {},
+  "views": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}

packages/backend/src/database/migrations/meta/0009_snapshot.json

@@ -0,0 +1,854 @@
+{
+  "id": "701eda75-451a-4a6d-87e3-b6658fca65da",
+  "prevId": "86b6960e-1936-4543-846f-a2d24d6dc5d1",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {
+    "public.archived_emails": {
+      "name": "archived_emails",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "thread_id": {
+          "name": "thread_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "ingestion_source_id": {
+          "name": "ingestion_source_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "user_email": {
+          "name": "user_email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "message_id_header": {
+          "name": "message_id_header",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sent_at": {
+          "name": "sent_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "subject": {
+          "name": "subject",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_name": {
+          "name": "sender_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sender_email": {
+          "name": "sender_email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "recipients": {
+          "name": "recipients",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_hash_sha256": {
+          "name": "storage_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_indexed": {
+          "name": "is_indexed",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "has_attachments": {
+          "name": "has_attachments",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "is_on_legal_hold": {
+          "name": "is_on_legal_hold",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": false
+        },
+        "archived_at": {
+          "name": "archived_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {
+        "thread_id_idx": {
+          "name": "thread_id_idx",
+          "columns": [
+            {
+              "expression": "thread_id",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": false,
+          "concurrently": false,
+          "method": "btree",
+          "with": {}
+        }
+      },
+      "foreignKeys": {
+        "archived_emails_ingestion_source_id_ingestion_sources_id_fk": {
+          "name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
+          "tableFrom": "archived_emails",
+          "tableTo": "ingestion_sources",
+          "columnsFrom": [
+            "ingestion_source_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.attachments": {
+      "name": "attachments",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "filename": {
+          "name": "filename",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "mime_type": {
+          "name": "mime_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "size_bytes": {
+          "name": "size_bytes",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "content_hash_sha256": {
+          "name": "content_hash_sha256",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "storage_path": {
+          "name": "storage_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "attachments_content_hash_sha256_unique": {
+          "name": "attachments_content_hash_sha256_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "content_hash_sha256"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.email_attachments": {
+      "name": "email_attachments",
+      "schema": "",
+      "columns": {
+        "email_id": {
+          "name": "email_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "attachment_id": {
+          "name": "attachment_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "email_attachments_email_id_archived_emails_id_fk": {
+          "name": "email_attachments_email_id_archived_emails_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "archived_emails",
+          "columnsFrom": [
+            "email_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "email_attachments_attachment_id_attachments_id_fk": {
+          "name": "email_attachments_attachment_id_attachments_id_fk",
+          "tableFrom": "email_attachments",
+          "tableTo": "attachments",
+          "columnsFrom": [
+            "attachment_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "restrict",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "email_attachments_email_id_attachment_id_pk": {
+          "name": "email_attachments_email_id_attachment_id_pk",
+          "columns": [
+            "email_id",
+            "attachment_id"
+          ]
+        }
+      },
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.audit_logs": {
+      "name": "audit_logs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "bigserial",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "timestamp": {
+          "name": "timestamp",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "actor_identifier": {
+          "name": "actor_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action": {
+          "name": "action",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "target_type": {
+          "name": "target_type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "target_id": {
+          "name": "target_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "details": {
+          "name": "details",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "is_tamper_evident": {
+          "name": "is_tamper_evident",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": false,
+          "default": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ediscovery_cases": {
+      "name": "ediscovery_cases",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'open'"
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "ediscovery_cases_name_unique": {
+          "name": "ediscovery_cases_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.export_jobs": {
+      "name": "export_jobs",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "format": {
+          "name": "format",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending'"
+        },
+        "query": {
+          "name": "query",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "file_path": {
+          "name": "file_path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_by_identifier": {
+          "name": "created_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "completed_at": {
+          "name": "completed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "export_jobs_case_id_ediscovery_cases_id_fk": {
+          "name": "export_jobs_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "export_jobs",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "set null",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.legal_holds": {
+      "name": "legal_holds",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "case_id": {
+          "name": "case_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "custodian_id": {
+          "name": "custodian_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "hold_criteria": {
+          "name": "hold_criteria",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "reason": {
+          "name": "reason",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "applied_by_identifier": {
+          "name": "applied_by_identifier",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "applied_at": {
+          "name": "applied_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "removed_at": {
+          "name": "removed_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "legal_holds_case_id_ediscovery_cases_id_fk": {
+          "name": "legal_holds_case_id_ediscovery_cases_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "ediscovery_cases",
+          "columnsFrom": [
+            "case_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        },
+        "legal_holds_custodian_id_custodians_id_fk": {
+          "name": "legal_holds_custodian_id_custodians_id_fk",
+          "tableFrom": "legal_holds",
+          "tableTo": "custodians",
+          "columnsFrom": [
+            "custodian_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "cascade",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.retention_policies": {
+      "name": "retention_policies",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "priority": {
+          "name": "priority",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "retention_period_days": {
+          "name": "retention_period_days",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "action_on_expiry": {
+          "name": "action_on_expiry",
+          "type": "retention_action",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "is_enabled": {
+          "name": "is_enabled",
+          "type": "boolean",
+          "primaryKey": false,
+          "notNull": true,
+          "default": true
+        },
+        "conditions": {
+          "name": "conditions",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "retention_policies_name_unique": {
+          "name": "retention_policies_name_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "name"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.custodians": {
+      "name": "custodians",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "email": {
+          "name": "email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "display_name": {
+          "name": "display_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "source_type": {
+          "name": "source_type",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {
+        "custodians_email_unique": {
+          "name": "custodians_email_unique",
+          "nullsNotDistinct": false,
+          "columns": [
+            "email"
+          ]
+        }
+      },
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
+    "public.ingestion_sources": {
+      "name": "ingestion_sources",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "gen_random_uuid()"
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "provider": {
+          "name": "provider",
+          "type": "ingestion_provider",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "credentials": {
+          "name": "credentials",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "status": {
+          "name": "status",
+          "type": "ingestion_status",
+          "typeSchema": "public",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "'pending_auth'"
+        },
+        "last_sync_started_at": {
+          "name": "last_sync_started_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_finished_at": {
+          "name": "last_sync_finished_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "last_sync_status_message": {
+          "name": "last_sync_status_message",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "sync_state": {
+          "name": "sync_state",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    }
+  },
+  "enums": {
+    "public.retention_action": {
+      "name": "retention_action",
+      "schema": "public",
+      "values": [
+        "delete_permanently",
+        "notify_admin"
+      ]
+    },
+    "public.ingestion_provider": {
+      "name": "ingestion_provider",
+      "schema": "public",
+      "values": [
+        "google_workspace",
+        "microsoft_365",
+        "generic_imap"
+      ]
+    },
+    "public.ingestion_status": {
+      "name": "ingestion_status",
+      "schema": "public",
+      "values": [
+        "active",
+        "paused",
+        "error",
+        "pending_auth",
+        "syncing",
+        "importing",
+        "auth_success"
+      ]
+    }
+  },
+  "schemas": {},
+  "sequences": {},
+  "roles": {},
+  "policies": {},
+  "views": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}

packages/backend/src/database/migrations/meta/_journal.json

@@ -43,6 +43,34 @@
       "when": 1752606327253,
       "tag": "0005_chunky_sue_storm",
       "breakpoints": true
+    },
+    {
+      "idx": 6,
+      "version": "7",
+      "when": 1753112018514,
+      "tag": "0006_majestic_caretaker",
+      "breakpoints": true
+    },
+    {
+      "idx": 7,
+      "version": "7",
+      "when": 1753190159356,
+      "tag": "0007_handy_archangel",
+      "breakpoints": true
+    },
+    {
+      "idx": 8,
+      "version": "7",
+      "when": 1753370737317,
+      "tag": "0008_eminent_the_spike",
+      "breakpoints": true
+    },
+    {
+      "idx": 9,
+      "version": "7",
+      "when": 1754337938241,
+      "tag": "0009_late_lenny_balinger",
+      "breakpoints": true
     }
   ]
 }

packages/backend/src/database/schema/archived-emails.ts

@@ -1,26 +1,36 @@
 import { relations } from 'drizzle-orm';
-import { boolean, jsonb, pgTable, text, timestamp, uuid, bigint } from 'drizzle-orm/pg-core';
+import { boolean, jsonb, pgTable, text, timestamp, uuid, bigint, index } from 'drizzle-orm/pg-core';
 import { ingestionSources } from './ingestion-sources';
 
-export const archivedEmails = pgTable('archived_emails', {
-    id: uuid('id').primaryKey().defaultRandom(),
-    ingestionSourceId: uuid('ingestion_source_id')
-        .notNull()
-        .references(() => ingestionSources.id, { onDelete: 'cascade' }),
-    messageIdHeader: text('message_id_header'),
-    sentAt: timestamp('sent_at', { withTimezone: true }).notNull(),
-    subject: text('subject'),
-    senderName: text('sender_name'),
-    senderEmail: text('sender_email').notNull(),
-    recipients: jsonb('recipients'),
-    storagePath: text('storage_path').notNull(),
-    storageHashSha256: text('storage_hash_sha256').notNull(),
-    sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
-    isIndexed: boolean('is_indexed').notNull().default(false),
-    hasAttachments: boolean('has_attachments').notNull().default(false),
-    isOnLegalHold: boolean('is_on_legal_hold').notNull().default(false),
-    archivedAt: timestamp('archived_at', { withTimezone: true }).notNull().defaultNow(),
-});
+export const archivedEmails = pgTable(
+    'archived_emails',
+    {
+        id: uuid('id').primaryKey().defaultRandom(),
+        threadId: text('thread_id'),
+        ingestionSourceId: uuid('ingestion_source_id')
+            .notNull()
+            .references(() => ingestionSources.id, { onDelete: 'cascade' }),
+        userEmail: text('user_email').notNull(),
+        messageIdHeader: text('message_id_header'),
+        sentAt: timestamp('sent_at', { withTimezone: true }).notNull(),
+        subject: text('subject'),
+        senderName: text('sender_name'),
+        senderEmail: text('sender_email').notNull(),
+        recipients: jsonb('recipients'),
+        storagePath: text('storage_path').notNull(),
+        storageHashSha256: text('storage_hash_sha256').notNull(),
+        sizeBytes: bigint('size_bytes', { mode: 'number' }).notNull(),
+        isIndexed: boolean('is_indexed').notNull().default(false),
+        hasAttachments: boolean('has_attachments').notNull().default(false),
+        isOnLegalHold: boolean('is_on_legal_hold').notNull().default(false),
+        archivedAt: timestamp('archived_at', { withTimezone: true }).notNull().defaultNow(),
+    },
+    (table) => {
+        return {
+            threadIdIdx: index('thread_id_idx').on(table.threadId)
+        };
+    }
+);
 
 export const archivedEmailsRelations = relations(archivedEmails, ({ one }) => ({
     ingestionSource: one(ingestionSources, {

packages/backend/src/database/schema/ingestion-sources.ts

@@ -20,11 +20,12 @@ export const ingestionSources = pgTable('ingestion_sources', {
     id: uuid('id').primaryKey().defaultRandom(),
     name: text('name').notNull(),
     provider: ingestionProviderEnum('provider').notNull(),
-    credentials: jsonb('credentials'),
+    credentials: text('credentials'),
     status: ingestionStatusEnum('status').notNull().default('pending_auth'),
     lastSyncStartedAt: timestamp('last_sync_started_at', { withTimezone: true }),
     lastSyncFinishedAt: timestamp('last_sync_finished_at', { withTimezone: true }),
     lastSyncStatusMessage: text('last_sync_status_message'),
+    syncState: jsonb('sync_state'),
     createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
     updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow()
 });

packages/backend/src/index.ts

@@ -11,6 +11,7 @@ import { createIngestionRouter } from './api/routes/ingestion.routes';
 import { createArchivedEmailRouter } from './api/routes/archived-email.routes';
 import { createStorageRouter } from './api/routes/storage.routes';
 import { createSearchRouter } from './api/routes/search.routes';
+import { createDashboardRouter } from './api/routes/dashboard.routes';
 import testRouter from './api/routes/test.routes';
 import { AuthService } from './services/AuthService';
 import { AdminUserService } from './services/UserService';
@@ -58,11 +59,13 @@ const ingestionRouter = createIngestionRouter(ingestionController, authService);
 const archivedEmailRouter = createArchivedEmailRouter(archivedEmailController, authService);
 const storageRouter = createStorageRouter(storageController, authService);
 const searchRouter = createSearchRouter(searchController, authService);
+const dashboardRouter = createDashboardRouter(authService);
 app.use('/v1/auth', authRouter);
 app.use('/v1/ingestion-sources', ingestionRouter);
 app.use('/v1/archived-emails', archivedEmailRouter);
 app.use('/v1/storage', storageRouter);
 app.use('/v1/search', searchRouter);
+app.use('/v1/dashboard', dashboardRouter);
 app.use('/v1/test', testRouter);
 
 // Example of a protected route

packages/backend/src/jobs/processors/continuous-sync.processor.ts

@@ -1,11 +1,79 @@
 import { Job } from 'bullmq';
 import { IngestionService } from '../../services/IngestionService';
-import { IInitialImportJob } from '@open-archiver/types';
+import { IContinuousSyncJob } from '@open-archiver/types';
+import { EmailProviderFactory } from '../../services/EmailProviderFactory';
+import { flowProducer } from '../queues';
+import { logger } from '../../config/logger';
 
-const ingestionService = new IngestionService();
+export default async (job: Job<IContinuousSyncJob>) => {
+    const { ingestionSourceId } = job.data;
+    logger.info({ ingestionSourceId }, 'Starting continuous sync job.');
 
-export default async (job: Job<IInitialImportJob>) => {
-    console.log(`Processing continuous sync for ingestion source: ${job.data.ingestionSourceId}`);
-    // This would be similar to performBulkImport, but would likely use the `since` parameter.
-    // For now, we'll just log a message.
+    const source = await IngestionService.findById(ingestionSourceId);
+    if (!source || !['error', 'active'].includes(source.status)) {
+        logger.warn({ ingestionSourceId, status: source?.status }, 'Skipping continuous sync for non-active or non-error source.');
+        return;
+    }
+
+    await IngestionService.update(ingestionSourceId, {
+        status: 'syncing',
+        lastSyncStartedAt: new Date(),
+    });
+
+    const connector = EmailProviderFactory.createConnector(source);
+
+    try {
+        const jobs = [];
+        for await (const user of connector.listAllUsers()) {
+            if (user.primaryEmail) {
+                jobs.push({
+                    name: 'process-mailbox',
+                    queueName: 'ingestion',
+                    data: {
+                        ingestionSourceId: source.id,
+                        userEmail: user.primaryEmail
+                    },
+                    opts: {
+                        removeOnComplete: {
+                            age: 60 * 10 // 10 minutes
+                        },
+                        removeOnFail: {
+                            age: 60 * 30 // 30 minutes
+                        },
+                        timeout: 1000 * 60 * 30 // 30 minutes
+                    }
+                });
+            }
+        }
+        // }
+
+        if (jobs.length > 0) {
+            await flowProducer.add({
+                name: 'sync-cycle-finished',
+                queueName: 'ingestion',
+                data: {
+                    ingestionSourceId,
+                    isInitialImport: false
+                },
+                children: jobs,
+                opts: {
+                    removeOnComplete: true,
+                    removeOnFail: true
+                }
+            });
+        }
+
+        // The status will be set back to 'active' by the 'sync-cycle-finished' job
+        // once all the mailboxes have been processed.
+        logger.info({ ingestionSourceId }, 'Continuous sync job finished dispatching mailbox jobs.');
+
+    } catch (error) {
+        logger.error({ err: error, ingestionSourceId }, 'Continuous sync job failed.');
+        await IngestionService.update(ingestionSourceId, {
+            status: 'error',
+            lastSyncFinishedAt: new Date(),
+            lastSyncStatusMessage: error instanceof Error ? error.message : 'An unknown error occurred during sync.',
+        });
+        throw error;
+    }
 };

packages/backend/src/jobs/processors/initial-import.processor.ts

@@ -1,11 +1,11 @@
-import { Job } from 'bullmq';
+import { Job, FlowChildJob } from 'bullmq';
 import { IngestionService } from '../../services/IngestionService';
 import { IInitialImportJob } from '@open-archiver/types';
 import { EmailProviderFactory } from '../../services/EmailProviderFactory';
-import { GoogleWorkspaceConnector } from '../../services/ingestion-connectors/GoogleWorkspaceConnector';
-import { ingestionQueue } from '../queues';
+import { flowProducer } from '../queues';
 import { logger } from '../../config/logger';
 
+
 export default async (job: Job<IInitialImportJob>) => {
     const { ingestionSourceId } = job.data;
     logger.info({ ingestionSourceId }, 'Starting initial import master job');
@@ -16,28 +16,85 @@ export default async (job: Job<IInitialImportJob>) => {
             throw new Error(`Ingestion source with ID ${ingestionSourceId} not found`);
         }
 
+        await IngestionService.update(ingestionSourceId, {
+            status: 'importing',
+            lastSyncStatusMessage: 'Starting initial import...'
+        });
+
         const connector = EmailProviderFactory.createConnector(source);
 
-        if (connector instanceof GoogleWorkspaceConnector) {
-            let userCount = 0;
-            for await (const user of connector.listAllUsers()) {
-                if (user.primaryEmail) {
-                    await ingestionQueue.add('process-mailbox', {
+        // if (connector instanceof GoogleWorkspaceConnector || connector instanceof MicrosoftConnector) {
+        const jobs: FlowChildJob[] = [];
+        let userCount = 0;
+        for await (const user of connector.listAllUsers()) {
+            if (user.primaryEmail) {
+                jobs.push({
+                    name: 'process-mailbox',
+                    queueName: 'ingestion',
+                    data: {
                         ingestionSourceId,
-                        userEmail: user.primaryEmail
-                    });
-                    userCount++;
-                }
+                        userEmail: user.primaryEmail,
+                    },
+                    opts: {
+                        removeOnComplete: {
+                            age: 60 * 10 // 10 minutes
+                        },
+                        removeOnFail: {
+                            age: 60 * 30 // 30 minutes
+                        },
+                        attempts: 1,
+                        // failParentOnFailure: true
+                    }
+                });
+                userCount++;
             }
-            logger.info({ ingestionSourceId, userCount }, `Enqueued mailbox processing jobs for all users`);
-        } else {
-            // For other providers, we might trigger a simpler bulk import directly
-            await new IngestionService().performBulkImport(job.data);
         }
 
+        if (jobs.length > 0) {
+            logger.info({ ingestionSourceId, userCount }, 'Adding sync-cycle-finished job to the queue');
+            await flowProducer.add({
+                name: 'sync-cycle-finished',
+                queueName: 'ingestion',
+                data: {
+                    ingestionSourceId,
+                    userCount,
+                    isInitialImport: true
+                },
+                children: jobs,
+                opts: {
+                    removeOnComplete: true,
+                    removeOnFail: true
+                }
+            });
+        } else {
+            // If there are no users, we can consider the import finished and set to active
+            await IngestionService.update(ingestionSourceId, {
+                status: 'active',
+                lastSyncFinishedAt: new Date(),
+                lastSyncStatusMessage: 'Initial import complete. No users found.'
+            });
+        }
+        // } else {
+        //     // For other providers, we might trigger a simpler bulk import directly
+        //     await new IngestionService().performBulkImport(job.data);
+        //     await flowProducer.add({
+        //         name: 'sync-cycle-finished',
+        //         queueName: 'ingestion',
+        //         data: {
+        //             ingestionSourceId,
+        //             userCount: 1,
+        //             isInitialImport: true
+        //         }
+        //     });
+        // }
+
         logger.info({ ingestionSourceId }, 'Finished initial import master job');
     } catch (error) {
         logger.error({ err: error, ingestionSourceId }, 'Error in initial import master job');
+        await IngestionService.update(ingestionSourceId, {
+            status: 'error',
+            lastSyncStatusMessage: `Initial import failed: ${error instanceof Error ? error.message : 'Unknown error'}`
+        });
         throw error;
     }
 };

packages/backend/src/jobs/processors/process-mailbox.processor.ts

@@ -1,11 +1,19 @@
 import { Job } from 'bullmq';
-import { IProcessMailboxJob } from '@open-archiver/types';
+import { IProcessMailboxJob, SyncState, ProcessMailboxError } from '@open-archiver/types';
 import { IngestionService } from '../../services/IngestionService';
 import { logger } from '../../config/logger';
 import { EmailProviderFactory } from '../../services/EmailProviderFactory';
 import { StorageService } from '../../services/StorageService';
 
-export const processMailboxProcessor = async (job: Job<IProcessMailboxJob, any, string>) => {
+/**
+ * This processor handles the ingestion of emails for a single user's mailbox.
+ * If an error occurs during processing (e.g., an API failure),
+ * it catches the exception and returns a structured error object instead of throwing.
+ * This prevents a single failed mailbox from halting the entire sync cycle for all users.
+ * The parent 'sync-cycle-finished' job is responsible for inspecting the results of all
+ * 'process-mailbox' jobs, aggregating successes, and reporting detailed failures.
+ */
+export const processMailboxProcessor = async (job: Job<IProcessMailboxJob, SyncState, string>) => {
     const { ingestionSourceId, userEmail } = job.data;
 
     logger.info({ ingestionSourceId, userEmail }, `Processing mailbox for user`);
@@ -20,12 +28,26 @@ export const processMailboxProcessor = async (job: Job<IProcessMailboxJob, any,
         const ingestionService = new IngestionService();
         const storageService = new StorageService();
 
-        for await (const email of connector.fetchEmails(userEmail)) {
-            await ingestionService.processEmail(email, source, storageService);
+        // Pass the sync state for the entire source, the connector will handle per-user logic if necessary
+        for await (const email of connector.fetchEmails(userEmail, source.syncState)) {
+            if (email) {
+                await ingestionService.processEmail(email, source, storageService, userEmail);
+            }
         }
+
+        const newSyncState = connector.getUpdatedSyncState(userEmail);
+
         logger.info({ ingestionSourceId, userEmail }, `Finished processing mailbox for user`);
+
+        // Return the new sync state to be aggregated by the parent flow
+        return newSyncState;
     } catch (error) {
         logger.error({ err: error, ingestionSourceId, userEmail }, 'Error processing mailbox');
-        throw error;
+        const errorMessage = error instanceof Error ? error.message : 'An unknown error occurred';
+        const processMailboxError: ProcessMailboxError = {
+            error: true,
+            message: `Failed to process mailbox for ${userEmail}: ${errorMessage}`
+        };
+        return processMailboxError;
     }
 };

packages/backend/src/jobs/processors/schedule-continuous-sync.processor.ts

@@ -0,0 +1,26 @@
+import { Job } from 'bullmq';
+import { db } from '../../database';
+import { ingestionSources } from '../../database/schema';
+import { or, eq } from 'drizzle-orm';
+import { ingestionQueue } from '../queues';
+
+export default async (job: Job) => {
+    console.log(
+        'Scheduler running: Looking for active or error ingestion sources to sync.'
+    );
+    // find all sources that have the status of active or error for continuous syncing.
+    const sourcesToSync = await db
+        .select({ id: ingestionSources.id })
+        .from(ingestionSources)
+        .where(
+            or(
+                eq(ingestionSources.status, 'active'),
+                eq(ingestionSources.status, 'error')
+            )
+        );
+
+    for (const source of sourcesToSync) {
+        // The status field on the ingestion source is used to prevent duplicate syncs.
+        await ingestionQueue.add('continuous-sync', { ingestionSourceId: source.id });
+    }
+};

packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts

@@ -0,0 +1,77 @@
+import { Job } from 'bullmq';
+import { IngestionService } from '../../services/IngestionService';
+import { logger } from '../../config/logger';
+import { SyncState, ProcessMailboxError } from '@open-archiver/types';
+import { db } from '../../database';
+import { ingestionSources } from '../../database/schema';
+import { eq } from 'drizzle-orm';
+import { deepmerge } from 'deepmerge-ts';
+
+interface ISyncCycleFinishedJob {
+    ingestionSourceId: string;
+    userCount?: number; // Optional, as it's only relevant for the initial import
+    isInitialImport: boolean;
+}
+
+/**
+ * This processor runs after all 'process-mailbox' jobs for a sync cycle have completed.
+ * It is responsible for aggregating the results and finalizing the sync status.
+ * It inspects the return values of all child jobs to identify successes and failures.
+ *
+ * If any child jobs returned an error object, this processor will:
+ * 1. Mark the overall ingestion status as 'error'.
+ * 2. Aggregate the detailed error messages from all failed jobs.
+ * 3. Save the sync state from any jobs that *did* succeed, preserving partial progress.
+ *
+ * If all child jobs succeeded, it marks the ingestion as 'active' and saves the final
+ * aggregated sync state from all children.
+ *
+ */
+export default async (job: Job<ISyncCycleFinishedJob, any, string>) => {
+    const { ingestionSourceId, userCount, isInitialImport } = job.data;
+    logger.info({ ingestionSourceId, userCount, isInitialImport }, 'Sync cycle finished job started');
+
+    try {
+        const childrenValues = await job.getChildrenValues<SyncState | ProcessMailboxError>();
+        const allChildJobs = Object.values(childrenValues);
+        // if data has error property, it is a failed job
+        const failedJobs = allChildJobs.filter(v => v && (v as any).error) as ProcessMailboxError[];
+        // if data doesn't have error property, it is a successful job with SyncState
+        const successfulJobs = allChildJobs.filter(v => !v || !(v as any).error) as SyncState[];
+
+        const finalSyncState = deepmerge(...successfulJobs.filter(s => s && Object.keys(s).length > 0));
+
+        let status: 'active' | 'error' = 'active';
+        let message: string;
+
+        if (failedJobs.length > 0) {
+            status = 'error';
+            const errorMessages = failedJobs.map(j => j.message).join('\n');
+            message = `Sync cycle completed with ${failedJobs.length} error(s):\n${errorMessages}`;
+            logger.error({ ingestionSourceId, errors: errorMessages }, 'Sync cycle finished with errors.');
+        } else {
+            message = 'Continuous sync cycle finished successfully.';
+            if (isInitialImport) {
+                message = `Initial import finished for ${userCount} mailboxes.`;
+            }
+            logger.info({ ingestionSourceId }, 'Successfully updated status and final sync state.');
+        }
+
+        await db
+            .update(ingestionSources)
+            .set({
+                status,
+                lastSyncFinishedAt: new Date(),
+                lastSyncStatusMessage: message,
+                syncState: finalSyncState
+            })
+            .where(eq(ingestionSources.id, ingestionSourceId));
+    } catch (error) {
+        logger.error({ err: error, ingestionSourceId }, 'An unexpected error occurred while finalizing the sync cycle.');
+        await IngestionService.update(ingestionSourceId, {
+            status: 'error',
+            lastSyncFinishedAt: new Date(),
+            lastSyncStatusMessage: 'An unexpected error occurred while finalizing the sync cycle.'
+        });
+    }
+};

packages/backend/src/jobs/queues.ts

@@ -1,6 +1,8 @@
-import { Queue } from 'bullmq';
+import { Queue, FlowProducer } from 'bullmq';
 import { connection } from '../config/redis';
 
+export const flowProducer = new FlowProducer({ connection });
+
 // Default job options
 const defaultJobOptions = {
     attempts: 5,

packages/backend/src/jobs/schedulers/sync-scheduler.ts

@@ -0,0 +1,18 @@
+import { ingestionQueue } from '../queues';
+
+const scheduleContinuousSync = async () => {
+    // This job will run every 15 minutes
+    await ingestionQueue.add(
+        'schedule-continuous-sync',
+        {},
+        {
+            repeat: {
+                pattern: '* * * * *', // Every 1 minute
+            },
+        }
+    );
+};
+
+scheduleContinuousSync().then(() => {
+    console.log('Continuous sync scheduler started.');
+});

packages/backend/src/services/ArchivedEmailService.ts

@@ -1,7 +1,7 @@
-import { count, desc, eq } from 'drizzle-orm';
+import { count, desc, eq, asc } from 'drizzle-orm';
 import { db } from '../database';
 import { archivedEmails, attachments, emailAttachments } from '../database/schema';
-import type { PaginatedArchivedEmails, ArchivedEmail, Recipient } from '@open-archiver/types';
+import type { PaginatedArchivedEmails, ArchivedEmail, Recipient, ThreadEmail } from '@open-archiver/types';
 import { StorageService } from './StorageService';
 import type { Readable } from 'stream';
 
@@ -11,6 +11,8 @@ interface DbRecipients {
     bcc: { name: string; address: string; }[];
 }
 
+
+
 async function streamToBuffer(stream: Readable): Promise<Buffer> {
     return new Promise((resolve, reject) => {
         const chunks: Buffer[] = [];
@@ -75,6 +77,21 @@ export class ArchivedEmailService {
             return null;
         }
 
+        let threadEmails: ThreadEmail[] = [];
+
+        if (email.threadId) {
+            threadEmails = await db.query.archivedEmails.findMany({
+                where: eq(archivedEmails.threadId, email.threadId),
+                orderBy: [asc(archivedEmails.sentAt)],
+                columns: {
+                    id: true,
+                    subject: true,
+                    sentAt: true,
+                    senderEmail: true,
+                },
+            });
+        }
+
         const storage = new StorageService();
         const rawStream = await storage.get(email.storagePath);
         const raw = await streamToBuffer(rawStream as Readable);
@@ -82,7 +99,8 @@ export class ArchivedEmailService {
         const mappedEmail = {
             ...email,
             recipients: this.mapRecipients(email.recipients),
-            raw
+            raw,
+            thread: threadEmails
         };
 
         if (email.hasAttachments) {

packages/backend/src/services/CryptoService.ts

@@ -29,20 +29,25 @@ export class CryptoService {
         return Buffer.concat([salt, iv, tag, encrypted]).toString('hex');
     }
 
-    public static decrypt(encrypted: string): string {
-        const data = Buffer.from(encrypted, 'hex');
-        const salt = data.subarray(0, SALT_LENGTH);
-        const iv = data.subarray(SALT_LENGTH, SALT_LENGTH + IV_LENGTH);
-        const tag = data.subarray(SALT_LENGTH + IV_LENGTH, SALT_LENGTH + IV_LENGTH + TAG_LENGTH);
-        const encryptedValue = data.subarray(SALT_LENGTH + IV_LENGTH + TAG_LENGTH);
+    public static decrypt(encrypted: string): string | null {
+        try {
+            const data = Buffer.from(encrypted, 'hex');
+            const salt = data.subarray(0, SALT_LENGTH);
+            const iv = data.subarray(SALT_LENGTH, SALT_LENGTH + IV_LENGTH);
+            const tag = data.subarray(SALT_LENGTH + IV_LENGTH, SALT_LENGTH + IV_LENGTH + TAG_LENGTH);
+            const encryptedValue = data.subarray(SALT_LENGTH + IV_LENGTH + TAG_LENGTH);
 
-        const key = getKey(salt);
-        const decipher = createDecipheriv(ALGORITHM, key, iv);
-        decipher.setAuthTag(tag);
+            const key = getKey(salt);
+            const decipher = createDecipheriv(ALGORITHM, key, iv);
+            decipher.setAuthTag(tag);
 
-        const decrypted = Buffer.concat([decipher.update(encryptedValue), decipher.final()]);
+            const decrypted = Buffer.concat([decipher.update(encryptedValue), decipher.final()]);
 
-        return decrypted.toString('utf8');
+            return decrypted.toString('utf8');
+        } catch (error) {
+            console.error('Decryption failed:', error);
+            return null;
+        }
     }
 
     public static encryptObject<T extends object>(obj: T): string {
@@ -50,8 +55,16 @@ export class CryptoService {
         return this.encrypt(jsonString);
     }
 
-    public static decryptObject<T extends object>(encrypted: string): T {
+    public static decryptObject<T extends object>(encrypted: string): T | null {
         const decryptedString = this.decrypt(encrypted);
-        return JSON.parse(decryptedString) as T;
+        if (!decryptedString) {
+            return null;
+        }
+        try {
+            return JSON.parse(decryptedString) as T;
+        } catch (error) {
+            console.error('Failed to parse decrypted JSON:', error);
+            return null;
+        }
     }
 }

packages/backend/src/services/DashboardService.ts

@@ -0,0 +1,89 @@
+import { and, count, eq, gte, sql } from 'drizzle-orm';
+import type { IndexedInsights } from '@open-archiver/types';
+
+import { archivedEmails, ingestionSources } from '../database/schema';
+import { DatabaseService } from './DatabaseService';
+import { SearchService } from './SearchService';
+
+class DashboardService {
+    #db;
+    #searchService;
+
+    constructor(databaseService: DatabaseService, searchService: SearchService) {
+        this.#db = databaseService.db;
+        this.#searchService = searchService;
+    }
+
+    public async getStats() {
+        const totalEmailsArchived = await this.#db.select({ count: count() }).from(archivedEmails);
+        const totalStorageUsed = await this.#db
+            .select({ sum: sql<number>`sum(${archivedEmails.sizeBytes})` })
+            .from(archivedEmails);
+
+        const sevenDaysAgo = new Date();
+        sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7);
+
+        const failedIngestionsLast7Days = await this.#db
+            .select({ count: count() })
+            .from(ingestionSources)
+            .where(
+                and(
+                    eq(ingestionSources.status, 'error'),
+                    gte(ingestionSources.updatedAt, sevenDaysAgo)
+                )
+            );
+
+        return {
+            totalEmailsArchived: totalEmailsArchived[0].count,
+            totalStorageUsed: totalStorageUsed[0].sum || 0,
+            failedIngestionsLast7Days: failedIngestionsLast7Days[0].count
+        };
+    }
+
+    public async getIngestionHistory() {
+        const thirtyDaysAgo = new Date();
+        thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
+
+        const history = await this.#db
+            .select({
+                date: sql<string>`date_trunc('day', ${archivedEmails.archivedAt})`,
+                count: count()
+            })
+            .from(archivedEmails)
+            .where(gte(archivedEmails.archivedAt, thirtyDaysAgo))
+            .groupBy(sql`date_trunc('day', ${archivedEmails.archivedAt})`)
+            .orderBy(sql`date_trunc('day', ${archivedEmails.archivedAt})`);
+
+        return { history };
+    }
+
+    public async getIngestionSources() {
+        const sources = await this.#db
+            .select({
+                id: ingestionSources.id,
+                name: ingestionSources.name,
+                provider: ingestionSources.provider,
+                status: ingestionSources.status,
+                storageUsed: sql<number>`sum(${archivedEmails.sizeBytes})`.mapWith(Number)
+            })
+            .from(ingestionSources)
+            .leftJoin(archivedEmails, eq(ingestionSources.id, archivedEmails.ingestionSourceId))
+            .groupBy(ingestionSources.id);
+
+        return sources;
+    }
+
+    public async getRecentSyncs() {
+        // This is a placeholder as we don't have a sync job table yet.
+        return Promise.resolve([]);
+    }
+
+    public async getIndexedInsights(): Promise<IndexedInsights> {
+        const topSenders = await this.#searchService.getTopSenders(10);
+        return {
+            topSenders
+        };
+    }
+}
+
+export const dashboardService = new DashboardService(new DatabaseService(), new SearchService());

packages/backend/src/services/EmailProviderFactory.ts

@@ -3,7 +3,9 @@ import type {
     GoogleWorkspaceCredentials,
     Microsoft365Credentials,
     GenericImapCredentials,
-    EmailObject
+    EmailObject,
+    SyncState,
+    MailboxUser
 } from '@open-archiver/types';
 import { GoogleWorkspaceConnector } from './ingestion-connectors/GoogleWorkspaceConnector';
 import { MicrosoftConnector } from './ingestion-connectors/MicrosoftConnector';
@@ -12,8 +14,10 @@ import { ImapConnector } from './ingestion-connectors/ImapConnector';
 // Define a common interface for all connectors
 export interface IEmailConnector {
     testConnection(): Promise<boolean>;
-    fetchEmails(userEmail?: string, since?: Date): AsyncGenerator<EmailObject>;
-    listAllUsers?(): AsyncGenerator<any>;
+    fetchEmails(userEmail: string, syncState?: SyncState | null): AsyncGenerator<EmailObject | null>;
+    getUpdatedSyncState(userEmail?: string): SyncState;
+    listAllUsers(): AsyncGenerator<MailboxUser>;
+    returnImapUserEmail?(): string;
 }
 
 export class EmailProviderFactory {

packages/backend/src/services/IndexingService.ts

@@ -1,4 +1,4 @@
-import { Attachment, EmailDocument, EmailObject } from '@open-archiver/types';
+import { Attachment, EmailAddress, EmailDocument, EmailObject } from '@open-archiver/types';
 import { SearchService } from './SearchService';
 import { StorageService } from './StorageService';
 import { extractText } from '../helpers/textExtractor';
@@ -73,7 +73,7 @@ export class IndexingService {
     /**
      * Indexes an email object directly, creates a search document, and indexes it.
      */
-    public async indexByEmail(email: EmailObject, ingestionSourceId: string): Promise<void> {
+    public async indexByEmail(email: EmailObject, ingestionSourceId: string, archivedEmailId: string): Promise<void> {
         const attachments: AttachmentsType = [];
         if (email.attachments && email.attachments.length > 0) {
             for (const attachment of email.attachments) {
@@ -84,7 +84,7 @@ export class IndexingService {
                 });
             }
         }
-        const document = await this.createEmailDocumentFromRaw(email, attachments, ingestionSourceId);
+        const document = await this.createEmailDocumentFromRaw(email, attachments, ingestionSourceId, archivedEmailId);
         await this.searchService.addDocuments('emails', [document], 'id');
     }
 
@@ -94,7 +94,8 @@ export class IndexingService {
     private async createEmailDocumentFromRaw(
         email: EmailObject,
         attachments: AttachmentsType,
-        ingestionSourceId: string
+        ingestionSourceId: string,
+        archivedEmailId: string
     ): Promise<EmailDocument> {
         const extractedAttachments = [];
         for (const attachment of attachments) {
@@ -116,11 +117,11 @@ export class IndexingService {
             }
         }
         return {
-            id: email.id,
+            id: archivedEmailId,
             from: email.from[0]?.address,
-            to: email.to.map((i) => i.address) || [],
-            cc: email.cc?.map((i) => i.address) || [],
-            bcc: email.bcc?.map((i) => i.address) || [],
+            to: email.to.map((i: EmailAddress) => i.address) || [],
+            cc: email.cc?.map((i: EmailAddress) => i.address) || [],
+            bcc: email.bcc?.map((i: EmailAddress) => i.address) || [],
             subject: email.subject || '',
             body: email.body || email.html || '',
             attachments: extractedAttachments,

packages/backend/src/services/IngestionService.ts

@@ -6,10 +6,11 @@ import type {
     IngestionSource,
     IngestionCredentials
 } from '@open-archiver/types';
-import { eq } from 'drizzle-orm';
+import { and, desc, eq } from 'drizzle-orm';
 import { CryptoService } from './CryptoService';
 import { EmailProviderFactory } from './EmailProviderFactory';
 import { ingestionQueue } from '../jobs/queues';
+import type { JobType } from 'bullmq';
 import { StorageService } from './StorageService';
 import type { IInitialImportJob, EmailObject } from '@open-archiver/types';
 import { archivedEmails, attachments as attachmentsSchema, emailAttachments } from '../database/schema';
@@ -21,11 +22,17 @@ import { DatabaseService } from './DatabaseService';
 
 
 export class IngestionService {
-    private static decryptSource(source: typeof ingestionSources.$inferSelect): IngestionSource {
+    private static decryptSource(source: typeof ingestionSources.$inferSelect): IngestionSource | null {
         const decryptedCredentials = CryptoService.decryptObject<IngestionCredentials>(
             source.credentials as string
         );
-        return { ...source, credentials: decryptedCredentials };
+
+        if (!decryptedCredentials) {
+            logger.error({ sourceId: source.id }, 'Failed to decrypt ingestion source credentials.');
+            return null;
+        }
+
+        return { ...source, credentials: decryptedCredentials } as IngestionSource;
     }
 
     public static async create(dto: CreateIngestionSourceDto): Promise<IngestionSource> {
@@ -42,21 +49,29 @@ export class IngestionService {
         const [newSource] = await db.insert(ingestionSources).values(valuesToInsert).returning();
 
         const decryptedSource = this.decryptSource(newSource);
-
-        // Test the connection
-        const connector = EmailProviderFactory.createConnector(decryptedSource);
-        const isConnected = await connector.testConnection();
-
-        if (isConnected) {
-            return await this.update(decryptedSource.id, { status: 'auth_success' });
+        if (!decryptedSource) {
+            await this.delete(newSource.id);
+            throw new Error('Failed to process newly created ingestion source due to a decryption error.');
         }
+        const connector = EmailProviderFactory.createConnector(decryptedSource);
 
-        return decryptedSource;
+        try {
+            await connector.testConnection();
+            // If connection succeeds, update status to auth_success, which triggers the initial import.
+            return await this.update(decryptedSource.id, { status: 'auth_success' });
+        } catch (error) {
+            // If connection fails, delete the newly created source and throw the error.
+            await this.delete(decryptedSource.id);
+            throw error;
+        }
     }
 
     public static async findAll(): Promise<IngestionSource[]> {
-        const sources = await db.select().from(ingestionSources);
-        return sources.map(this.decryptSource);
+        const sources = await db.select().from(ingestionSources).orderBy(desc(ingestionSources.createdAt));
+        return sources.flatMap(source => {
+            const decrypted = this.decryptSource(source);
+            return decrypted ? [decrypted] : [];
+        });
     }
 
     public static async findById(id: string): Promise<IngestionSource> {
@@ -64,7 +79,11 @@ export class IngestionService {
         if (!source) {
             throw new Error('Ingestion source not found');
         }
-        return this.decryptSource(source);
+        const decryptedSource = this.decryptSource(source);
+        if (!decryptedSource) {
+            throw new Error('Failed to decrypt ingestion source credentials.');
+        }
+        return decryptedSource;
     }
 
     public static async update(
@@ -94,6 +113,10 @@ export class IngestionService {
 
         const decryptedSource = this.decryptSource(updatedSource);
 
+        if (!decryptedSource) {
+            throw new Error('Failed to process updated ingestion source due to a decryption error.');
+        }
+
         // If the status has changed to auth_success, trigger the initial import
         if (
             originalSource.status !== 'auth_success' &&
@@ -106,22 +129,74 @@ export class IngestionService {
     }
 
     public static async delete(id: string): Promise<IngestionSource> {
+        const source = await this.findById(id);
+        if (!source) {
+            throw new Error('Ingestion source not found');
+        }
+
+        // Delete all emails and attachments from storage
+        const storage = new StorageService();
+        const emailPath = `open-archiver/${source.name.replaceAll(' ', '-')}-${source.id}/`;
+        await storage.delete(emailPath);
+
+
+        // Delete all emails from the database
+        // NOTE: This is done by database CASADE, change when CASADE relation no longer exists.
+        // await db.delete(archivedEmails).where(eq(archivedEmails.ingestionSourceId, id));
+
+        // Delete all documents from Meilisearch
+        const searchService = new SearchService();
+        await searchService.deleteDocumentsByFilter('emails', `ingestionSourceId = ${id}`);
+
         const [deletedSource] = await db
             .delete(ingestionSources)
             .where(eq(ingestionSources.id, id))
             .returning();
-        if (!deletedSource) {
-            throw new Error('Ingestion source not found');
+
+        const decryptedSource = this.decryptSource(deletedSource);
+        if (!decryptedSource) {
+            // Even if decryption fails, we should confirm deletion.
+            // We might return a simpler object or just a success message.
+            // For now, we'll indicate the issue but still confirm deletion happened.
+            logger.warn({ sourceId: deletedSource.id }, 'Could not decrypt credentials of deleted source, but deletion was successful.');
+            return { ...deletedSource, credentials: null } as unknown as IngestionSource;
         }
-        return this.decryptSource(deletedSource);
+        return decryptedSource;
     }
 
-    public static async triggerInitialImport(id: string): Promise<IngestionSource> {
+    public static async triggerInitialImport(id: string): Promise<void> {
         const source = await this.findById(id);
 
         await ingestionQueue.add('initial-import', { ingestionSourceId: source.id });
 
-        return await this.update(id, { status: 'importing' });
+    }
+
+    public static async triggerForceSync(id: string): Promise<void> {
+        const source = await this.findById(id);
+        logger.info({ ingestionSourceId: id }, 'Force syncing started.');
+        if (!source) {
+            throw new Error('Ingestion source not found');
+        }
+
+        // Clean up existing jobs for this source to break any stuck flows
+        const jobTypes: JobType[] = ['active', 'waiting', 'failed', 'delayed', 'paused'];
+        const jobs = await ingestionQueue.getJobs(jobTypes);
+        for (const job of jobs) {
+            if (job.data.ingestionSourceId === id) {
+                try {
+                    await job.remove();
+                    logger.info({ jobId: job.id, ingestionSourceId: id }, 'Removed stale job during force sync.');
+                } catch (error) {
+                    logger.error({ err: error, jobId: job.id }, 'Failed to remove stale job.');
+                }
+            }
+        }
+
+        // Reset status to 'active'
+        await this.update(id, { status: 'active', lastSyncStatusMessage: 'Force sync triggered by user.' });
+
+
+        await ingestionQueue.add('continuous-sync', { ingestionSourceId: source.id });
     }
 
     public async performBulkImport(job: IInitialImportJob): Promise<void> {
@@ -139,19 +214,35 @@ export class IngestionService {
         });
 
         const connector = EmailProviderFactory.createConnector(source);
-        const storage = new StorageService();
 
         try {
-            for await (const email of connector.fetchEmails()) {
-                await this.processEmail(email, source, storage);
+            if (connector.listAllUsers) {
+                // For multi-mailbox providers, dispatch a job for each user
+                for await (const user of connector.listAllUsers()) {
+                    const userEmail = user.primaryEmail;
+                    if (userEmail) {
+                        await ingestionQueue.add('process-mailbox', {
+                            ingestionSourceId: source.id,
+                            userEmail: userEmail,
+                        });
+                    }
+                }
+            } else {
+                // For single-mailbox providers, dispatch a single job
+                // console.log('source.credentials ', source.credentials);
+                await ingestionQueue.add('process-mailbox', {
+                    ingestionSourceId: source.id,
+                    userEmail: source.credentials.type === 'generic_imap' ? source.credentials.username : 'Default'
+                });
             }
 
-            await IngestionService.update(ingestionSourceId, {
-                status: 'active',
-                lastSyncFinishedAt: new Date(),
-                lastSyncStatusMessage: 'Successfully completed bulk import.'
-            });
-            console.log(`Bulk import finished for source: ${source.name} (${source.id})`);
+
+            // await IngestionService.update(ingestionSourceId, {
+            //     status: 'active',
+            //     lastSyncFinishedAt: new Date(),
+            //     lastSyncStatusMessage: 'Successfully initiated bulk import for all mailboxes.'
+            // });
+            // console.log(`Bulk import job dispatch finished for source: ${source.name} (${source.id})`);
         } catch (error) {
             console.error(`Bulk import failed for source: ${source.name} (${source.id})`, error);
             await IngestionService.update(ingestionSourceId, {
@@ -166,22 +257,48 @@ export class IngestionService {
     public async processEmail(
         email: EmailObject,
         source: IngestionSource,
-        storage: StorageService
+        storage: StorageService,
+        userEmail: string
     ): Promise<void> {
         try {
+            // Generate a unique message ID for the email. If the email already has a message-id header, use that.
+            // Otherwise, generate a new one based on the email's hash, source ID, and email ID.
+            const messageIdHeader = email.headers.get('message-id');
+            let messageId: string | undefined;
+            if (Array.isArray(messageIdHeader)) {
+                messageId = messageIdHeader[0];
+            } else if (typeof messageIdHeader === 'string') {
+                messageId = messageIdHeader;
+            }
+            if (!messageId) {
+                messageId = `generated-${createHash('sha256').update(email.eml ?? Buffer.from(email.body, 'utf-8')).digest('hex')}-${source.id}-${email.id}`;
+            }
+            // Check if an email with the same message ID has already been imported for the current ingestion source. This is to prevent duplicate imports when an email is present in multiple mailboxes (e.g., "Inbox" and "All Mail").
+            const existingEmail = await db.query.archivedEmails.findFirst({
+                where: and(
+                    eq(archivedEmails.messageIdHeader, messageId),
+                    eq(archivedEmails.ingestionSourceId, source.id)
+                )
+            });
+
+            if (existingEmail) {
+                logger.info({ messageId, ingestionSourceId: source.id }, 'Skipping duplicate email');
+                return;
+            }
+
             console.log('processing email, ', email.id, email.subject);
             const emlBuffer = email.eml ?? Buffer.from(email.body, 'utf-8');
             const emailHash = createHash('sha256').update(emlBuffer).digest('hex');
-            const emailPath = `email-archive/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${email.id}.eml`;
+            const emailPath = `open-archiver/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${email.id}.eml`;
             await storage.put(emailPath, emlBuffer);
 
             const [archivedEmail] = await db
                 .insert(archivedEmails)
                 .values({
                     ingestionSourceId: source.id,
-                    messageIdHeader:
-                        (email.headers['message-id'] as string) ??
-                        `generated-${emailHash}-${source.id}-${email.id}`,
+                    userEmail,
+                    threadId: email.threadId,
+                    messageIdHeader: messageId,
                     sentAt: email.receivedAt,
                     subject: email.subject,
                     senderName: email.from[0]?.name,
@@ -202,7 +319,7 @@ export class IngestionService {
                 for (const attachment of email.attachments) {
                     const attachmentBuffer = attachment.content;
                     const attachmentHash = createHash('sha256').update(attachmentBuffer).digest('hex');
-                    const attachmentPath = `email-archive/${source.name.replaceAll(' ', '-')}-${source.id}/attachments/${attachment.filename}`;
+                    const attachmentPath = `open-archiver/${source.name.replaceAll(' ', '-')}-${source.id}/attachments/${attachment.filename}`;
                     await storage.put(attachmentPath, attachmentBuffer);
 
                     const [newAttachment] = await db
@@ -220,10 +337,13 @@ export class IngestionService {
                         })
                         .returning();
 
-                    await db.insert(emailAttachments).values({
-                        emailId: archivedEmail.id,
-                        attachmentId: newAttachment.id
-                    });
+                    await db
+                        .insert(emailAttachments)
+                        .values({
+                            emailId: archivedEmail.id,
+                            attachmentId: newAttachment.id
+                        })
+                        .onConflictDoNothing();
                 }
             }
             // adding to indexing queue
@@ -236,7 +356,7 @@ export class IngestionService {
             const storageService = new StorageService();
             const databaseService = new DatabaseService();
             const indexingService = new IndexingService(databaseService, searchService, storageService);
-            await indexingService.indexByEmail(email, source.id);
+            await indexingService.indexByEmail(email, source.id, archivedEmail.id);
         } catch (error) {
             logger.error({
                 message: `Failed to process email ${email.id} for source ${source.id}`,

packages/backend/src/services/SearchService.ts

@@ -1,6 +1,6 @@
 import { Index, MeiliSearch, SearchParams } from 'meilisearch';
 import { config } from '../config';
-import type { SearchQuery, SearchResult, EmailDocument } from '@open-archiver/types';
+import type { SearchQuery, SearchResult, EmailDocument, TopSender } from '@open-archiver/types';
 
 export class SearchService {
     private client: MeiliSearch;
@@ -33,6 +33,11 @@ export class SearchService {
         return index.search(query, options);
     }
 
+    public async deleteDocumentsByFilter(indexName: string, filter: string | string[]) {
+        const index = await this.getIndex(indexName);
+        return index.deleteDocuments({ filter });
+    }
+
     public async searchEmails(dto: SearchQuery): Promise<SearchResult> {
         const { query, filters, page = 1, limit = 10, matchingStrategy = 'last' } = dto;
         const index = await this.getIndex<EmailDocument>('emails');
@@ -68,6 +73,26 @@ export class SearchService {
         };
     }
 
+    public async getTopSenders(limit = 10): Promise<TopSender[]> {
+        const index = await this.getIndex<EmailDocument>('emails');
+        const searchResults = await index.search('', {
+            facets: ['from'],
+            limit: 0
+        });
+
+        if (!searchResults.facetDistribution?.from) {
+            return [];
+        }
+
+        // Sort and take top N
+        const sortedSenders = Object.entries(searchResults.facetDistribution.from)
+            .sort(([, countA], [, countB]) => countB - countA)
+            .slice(0, limit)
+            .map(([sender, count]) => ({ sender, count }));
+
+        return sortedSenders;
+    }
+
     public async configureEmailIndex() {
         const index = await this.getIndex('emails');
         await index.updateSettings({

packages/backend/src/services/UserService.ts

@@ -3,7 +3,7 @@ import type { User } from '@open-archiver/types';
 import type { IUserService } from './AuthService';
 
 // This is a mock implementation of the IUserService.
-// In a real application, this service would interact with a database.
+// Later on, this service would interact with a database.
 export class AdminUserService implements IUserService {
     #users: User[] = [];
 
@@ -24,7 +24,7 @@ export class AdminUserService implements IUserService {
     }
 
     public async findByEmail(email: string): Promise<User | null> {
-        // In a real implementation, this would be a database query.
+        // once user service is ready, this would be a database query.
         const user = this.#users.find(u => u.email === email);
         return user || null;
     }

packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts

@@ -3,11 +3,14 @@ import type { admin_directory_v1, gmail_v1, Common } from 'googleapis';
 import type {
     GoogleWorkspaceCredentials,
     EmailObject,
-    EmailAddress
+    EmailAddress,
+    SyncState,
+    MailboxUser
 } from '@open-archiver/types';
 import type { IEmailConnector } from '../EmailProviderFactory';
 import { logger } from '../../config/logger';
-import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
+import { simpleParser, ParsedMail, Attachment, AddressObject, Headers } from 'mailparser';
+import { getThreadId } from './utils';
 
 /**
  * A connector for Google Workspace that uses a service account with domain-wide delegation
@@ -16,6 +19,7 @@ import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser'
 export class GoogleWorkspaceConnector implements IEmailConnector {
     private credentials: GoogleWorkspaceCredentials;
     private serviceAccountCreds: { client_email: string; private_key: string; };
+    private newHistoryId: string | undefined;
 
     constructor(credentials: GoogleWorkspaceCredentials) {
         this.credentials = credentials;
@@ -51,6 +55,7 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
         return jwtClient;
     }
 
+
     /**
      * Tests the connection and authentication by attempting to list the first user
      * from the directory, impersonating the admin user.
@@ -77,7 +82,7 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
             return true;
         } catch (error) {
             logger.error({ err: error }, 'Failed to verify Google Workspace connection');
-            return false;
+            throw error;
         }
     }
 
@@ -86,7 +91,7 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
      * This method handles pagination to retrieve the complete list of users.
      * @returns An async generator that yields each user object.
      */
-    public async *listAllUsers(): AsyncGenerator<admin_directory_v1.Schema$User> {
+    public async *listAllUsers(): AsyncGenerator<MailboxUser> {
         const authClient = this.getAuthClient(this.credentials.impersonatedAdminEmail, [
             'https://www.googleapis.com/auth/admin.directory.user.readonly'
         ]);
@@ -105,7 +110,13 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
             const users = res.data.users;
             if (users) {
                 for (const user of users) {
-                    yield user;
+                    if (user.id && user.primaryEmail && user.name?.fullName) {
+                        yield {
+                            id: user.id,
+                            primaryEmail: user.primaryEmail,
+                            displayName: user.name.fullName
+                        };
+                    }
                 }
             }
             pageToken = res.data.nextPageToken ?? undefined;
@@ -113,32 +124,116 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
     }
 
     /**
-     * Fetches emails for a single user, starting from a specific point in time.
+     * Fetches emails for a single user, starting from a specific history ID.
      * This is ideal for continuous synchronization jobs.
      * @param userEmail The email of the user whose mailbox will be read.
-     * @param since Optional date to fetch emails newer than this timestamp.
+     * @param syncState Optional state containing the startHistoryId.
      * @returns An async generator that yields each raw email object.
      */
     public async *fetchEmails(
         userEmail: string,
-        since?: Date
+        syncState?: SyncState | null
     ): AsyncGenerator<EmailObject> {
         const authClient = this.getAuthClient(userEmail, [
             'https://www.googleapis.com/auth/gmail.readonly'
         ]);
-
         const gmail = google.gmail({ version: 'v1', auth: authClient });
         let pageToken: string | undefined = undefined;
 
-        const query = since ? `after:${Math.floor(since.getTime() / 1000)}` : '';
+        const startHistoryId = syncState?.google?.[userEmail]?.historyId;
+
+        // If no sync state is provided for this user, this is an initial import. Get all messages.
+        if (!startHistoryId) {
+            yield* this.fetchAllMessagesForUser(gmail, userEmail);
+            return;
+        }
+
+        this.newHistoryId = startHistoryId;
 
         do {
-            const listResponse: Common.GaxiosResponseWithHTTP2<gmail_v1.Schema$ListMessagesResponse> =
-                await gmail.users.messages.list({
-                    userId: 'me', // 'me' refers to the impersonated user
-                    q: query,
-                    pageToken: pageToken
-                });
+            const historyResponse: Common.GaxiosResponseWithHTTP2<gmail_v1.Schema$ListHistoryResponse> = await gmail.users.history.list({
+                userId: userEmail,
+                startHistoryId: this.newHistoryId,
+                pageToken: pageToken,
+                historyTypes: ['messageAdded']
+            });
+
+            const histories = historyResponse.data.history;
+            if (!histories || histories.length === 0) {
+                return;
+            }
+
+            for (const historyRecord of histories) {
+                if (historyRecord.messagesAdded) {
+                    for (const messageAdded of historyRecord.messagesAdded) {
+                        if (messageAdded.message?.id) {
+                            try {
+                                const msgResponse = await gmail.users.messages.get({
+                                    userId: userEmail,
+                                    id: messageAdded.message.id,
+                                    format: 'RAW'
+                                });
+
+                                if (msgResponse.data.raw) {
+                                    const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
+                                    const parsedEmail: ParsedMail = await simpleParser(rawEmail);
+                                    const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
+                                        filename: attachment.filename || 'untitled',
+                                        contentType: attachment.contentType,
+                                        size: attachment.size,
+                                        content: attachment.content as Buffer
+                                    }));
+                                    const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
+                                        if (!addresses) return [];
+                                        const addressArray = Array.isArray(addresses) ? addresses : [addresses];
+                                        return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
+                                    };
+                                    const threadId = getThreadId(parsedEmail.headers);
+                                    console.log('threadId', threadId);
+                                    yield {
+                                        id: msgResponse.data.id!,
+                                        threadId,
+                                        userEmail: userEmail,
+                                        eml: rawEmail,
+                                        from: mapAddresses(parsedEmail.from),
+                                        to: mapAddresses(parsedEmail.to),
+                                        cc: mapAddresses(parsedEmail.cc),
+                                        bcc: mapAddresses(parsedEmail.bcc),
+                                        subject: parsedEmail.subject || '',
+                                        body: parsedEmail.text || '',
+                                        html: parsedEmail.html || '',
+                                        headers: parsedEmail.headers,
+                                        attachments,
+                                        receivedAt: parsedEmail.date || new Date(),
+                                    };
+                                }
+                            } catch (error: any) {
+                                if (error.code === 404) {
+                                    logger.warn({ messageId: messageAdded.message.id, userEmail }, 'Message not found, skipping.');
+                                } else {
+                                    throw error;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            pageToken = historyResponse.data.nextPageToken ?? undefined;
+            if (historyResponse.data.historyId) {
+                this.newHistoryId = historyResponse.data.historyId;
+            }
+
+        } while (pageToken);
+    }
+
+    private async *fetchAllMessagesForUser(gmail: gmail_v1.Gmail, userEmail: string): AsyncGenerator<EmailObject> {
+        let pageToken: string | undefined = undefined;
+        do {
+            const listResponse: Common.GaxiosResponseWithHTTP2<gmail_v1.Schema$ListMessagesResponse> = await gmail.users.messages.list({
+                userId: userEmail,
+                pageToken: pageToken
+            });
 
             const messages = listResponse.data.messages;
             if (!messages || messages.length === 0) {
@@ -147,50 +242,75 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
 
             for (const message of messages) {
                 if (message.id) {
-                    const msgResponse = await gmail.users.messages.get({
-                        userId: 'me',
-                        id: message.id,
-                        format: 'RAW' // We want the full, raw .eml content
-                    });
+                    try {
+                        const msgResponse = await gmail.users.messages.get({
+                            userId: userEmail,
+                            id: message.id,
+                            format: 'RAW'
+                        });
 
-                    if (msgResponse.data.raw) {
-                        // The raw data is base64url encoded, so we need to decode it.
-                        const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
-                        const parsedEmail: ParsedMail = await simpleParser(rawEmail);
-
-                        const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
-                            filename: attachment.filename || 'untitled',
-                            contentType: attachment.contentType,
-                            size: attachment.size,
-                            content: attachment.content as Buffer
-                        }));
-
-                        const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
-                            if (!addresses) return [];
-                            const addressArray = Array.isArray(addresses) ? addresses : [addresses];
-                            return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
-                        };
-
-                        yield {
-                            id: msgResponse.data.id!,
-                            userEmail: userEmail,
-                            eml: rawEmail,
-                            from: mapAddresses(parsedEmail.from),
-                            to: mapAddresses(parsedEmail.to),
-                            cc: mapAddresses(parsedEmail.cc),
-                            bcc: mapAddresses(parsedEmail.bcc),
-                            subject: parsedEmail.subject || '',
-                            body: parsedEmail.text || '',
-                            html: parsedEmail.html || '',
-                            headers: parsedEmail.headers as any,
-                            attachments,
-                            receivedAt: parsedEmail.date || new Date(),
-                        };
+                        if (msgResponse.data.raw) {
+                            const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
+                            const parsedEmail: ParsedMail = await simpleParser(rawEmail);
+                            const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
+                                filename: attachment.filename || 'untitled',
+                                contentType: attachment.contentType,
+                                size: attachment.size,
+                                content: attachment.content as Buffer
+                            }));
+                            const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
+                                if (!addresses) return [];
+                                const addressArray = Array.isArray(addresses) ? addresses : [addresses];
+                                return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
+                            };
+                            const threadId = getThreadId(parsedEmail.headers);
+                            console.log('threadId', threadId);
+                            yield {
+                                id: msgResponse.data.id!,
+                                threadId,
+                                userEmail: userEmail,
+                                eml: rawEmail,
+                                from: mapAddresses(parsedEmail.from),
+                                to: mapAddresses(parsedEmail.to),
+                                cc: mapAddresses(parsedEmail.cc),
+                                bcc: mapAddresses(parsedEmail.bcc),
+                                subject: parsedEmail.subject || '',
+                                body: parsedEmail.text || '',
+                                html: parsedEmail.html || '',
+                                headers: parsedEmail.headers,
+                                attachments,
+                                receivedAt: parsedEmail.date || new Date(),
+                            };
+                        }
+                    } catch (error: any) {
+                        if (error.code === 404) {
+                            logger.warn({ messageId: message.id, userEmail }, 'Message not found during initial import, skipping.');
+                        } else {
+                            throw error;
+                        }
                     }
                 }
             }
-
             pageToken = listResponse.data.nextPageToken ?? undefined;
         } while (pageToken);
+
+        // After fetching all messages, get the latest history ID to use as the starting point for the next sync.
+        const profileResponse = await gmail.users.getProfile({ userId: userEmail });
+        if (profileResponse.data.historyId) {
+            this.newHistoryId = profileResponse.data.historyId;
+        }
+    }
+
+    public getUpdatedSyncState(userEmail: string): SyncState {
+        if (!this.newHistoryId) {
+            return {};
+        }
+        return {
+            google: {
+                [userEmail]: {
+                    historyId: this.newHistoryId
+                }
+            }
+        };
     }
 }

packages/backend/src/services/ingestion-connectors/ImapConnector.ts

@@ -1,10 +1,14 @@
-import type { GenericImapCredentials, EmailObject, EmailAddress } from '@open-archiver/types';
+import type { GenericImapCredentials, EmailObject, EmailAddress, SyncState, MailboxUser } from '@open-archiver/types';
 import type { IEmailConnector } from '../EmailProviderFactory';
 import { ImapFlow } from 'imapflow';
-import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
+import { simpleParser, ParsedMail, Attachment, AddressObject, Headers } from 'mailparser';
+import { logger } from '../../config/logger';
+import { getThreadId } from './utils';
 
 export class ImapConnector implements IEmailConnector {
     private client: ImapFlow;
+    private newMaxUids: { [mailboxPath: string]: number; } = {};
+    private isConnected = false;
 
     constructor(private credentials: GenericImapCredentials) {
         this.client = new ImapFlow({
@@ -15,62 +19,206 @@ export class ImapConnector implements IEmailConnector {
                 user: this.credentials.username,
                 pass: this.credentials.password,
             },
-            logger: false, // Set to true for verbose logging
+            logger: logger.child({ module: 'ImapFlow' }),
         });
+
+        // Handles client-level errors, like unexpected disconnects, to prevent crashes.
+        this.client.on('error', (err) => {
+            logger.error({ err }, 'IMAP client error');
+            this.isConnected = false;
+        });
+    }
+
+    /**
+     * Establishes a connection to the IMAP server if not already connected.
+     */
+    private async connect(): Promise<void> {
+        if (this.isConnected && this.client.usable) {
+            return;
+        }
+        try {
+            await this.client.connect();
+            this.isConnected = true;
+        } catch (err: any) {
+            this.isConnected = false;
+            logger.error({ err }, 'IMAP connection failed');
+            if (err.responseText) {
+                throw new Error(`IMAP Connection Error: ${err.responseText}`);
+            }
+            throw err;
+        }
+    }
+
+    /**
+     * Disconnects from the IMAP server if the connection is active.
+     */
+    private async disconnect(): Promise<void> {
+        if (this.isConnected && this.client.usable) {
+            await this.client.logout();
+            this.isConnected = false;
+        }
     }
 
     public async testConnection(): Promise<boolean> {
         try {
-            await this.client.connect();
-            await this.client.logout();
+            await this.connect();
+            await this.disconnect();
             return true;
         } catch (error) {
-            console.error('Failed to verify IMAP connection:', error);
-            return false;
+            logger.error({ error }, 'Failed to verify IMAP connection');
+            throw error;
         }
     }
 
-    public async *fetchEmails(userEmail?: string, since?: Date): AsyncGenerator<EmailObject> {
-        await this.client.connect();
+    /**
+    *  We understand that for IMAP inboxes, there is only one user, but we want the IMAP connector to be compatible with other connectors, we return the single user here.
+    * @returns An async generator that yields each user object.
+    */
+    public async *listAllUsers(): AsyncGenerator<MailboxUser> {
         try {
-            await this.client.mailboxOpen('INBOX');
+            const emails: string[] = [this.returnImapUserEmail()];
+            for (const [index, email] of emails.entries()) {
+                yield {
+                    id: String(index),
+                    primaryEmail: email,
+                    displayName: email
+                };
 
-            const searchCriteria = since ? { since } : { all: true };
+            }
+        } finally {
+            await this.disconnect();
+        }
+    }
 
-            for await (const msg of this.client.fetch(searchCriteria, { envelope: true, source: true, bodyStructure: true })) {
-                if (msg.envelope && msg.source) {
-                    const parsedEmail: ParsedMail = await simpleParser(msg.source);
-                    const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
-                        filename: attachment.filename || 'untitled',
-                        contentType: attachment.contentType,
-                        size: attachment.size,
-                        content: attachment.content as Buffer
-                    }));
+    public returnImapUserEmail(): string {
+        return this.credentials.username;
+    }
 
-                    const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
-                        if (!addresses) return [];
-                        const addressArray = Array.isArray(addresses) ? addresses : [addresses];
-                        return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
-                    };
+    /**
+     * Wraps an IMAP operation with a retry mechanism to handle transient network errors.
+     * @param action The async function to execute.
+     * @param maxRetries The maximum number of retries.
+     * @returns The result of the action.
+     */
+    private async withRetry<T>(action: () => Promise<T>, maxRetries = 3): Promise<T> {
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+            try {
+                await this.connect();
+                return await action();
+            } catch (err: any) {
+                logger.error({ err, attempt }, `IMAP operation failed on attempt ${attempt}`);
+                this.isConnected = false; // Force reconnect on next attempt
+                if (attempt === maxRetries) {
+                    logger.error({ err }, 'IMAP operation failed after all retries.');
+                    throw err;
+                }
+                // Wait for a short period before retrying
+                await new Promise(resolve => setTimeout(resolve, 1000 * attempt));
+            }
+        }
+        // This line should be unreachable
+        throw new Error('IMAP operation failed after all retries.');
+    }
 
-                    yield {
-                        id: msg.uid.toString(),
-                        from: mapAddresses(parsedEmail.from),
-                        to: mapAddresses(parsedEmail.to),
-                        cc: mapAddresses(parsedEmail.cc),
-                        bcc: mapAddresses(parsedEmail.bcc),
-                        subject: parsedEmail.subject || '',
-                        body: parsedEmail.text || '',
-                        html: parsedEmail.html || '',
-                        headers: parsedEmail.headers as any,
-                        attachments,
-                        receivedAt: parsedEmail.date || new Date(),
-                        eml: msg.source
-                    };
+    public async *fetchEmails(userEmail: string, syncState?: SyncState | null): AsyncGenerator<EmailObject | null> {
+        try {
+            const mailboxes = await this.withRetry(() => this.client.list());
+            // console.log('fetched mailboxes:', mailboxes);
+            const processableMailboxes = mailboxes.filter(mailbox => {
+                // filter out trash and all mail emails
+                if (mailbox.specialUse) {
+                    const specialUse = mailbox.specialUse.toLowerCase();
+                    if (specialUse === '\\junk' || specialUse === '\\trash' || specialUse === '\\all') {
+                        return false;
+                    }
+                }
+                // Fallback to checking flags
+                if (mailbox.flags.has('\\Noselect') || mailbox.flags.has('\\Trash') || mailbox.flags.has('\\Junk') || mailbox.flags.has('\\All')) {
+                    return false;
+                }
+
+                return true;
+            });
+
+            for (const mailboxInfo of processableMailboxes) {
+                const mailboxPath = mailboxInfo.path;
+                const mailbox = await this.withRetry(() => this.client.mailboxOpen(mailboxPath));
+                const lastUid = syncState?.imap?.[mailboxPath]?.maxUid;
+                let currentMaxUid = lastUid || 0;
+
+                if (!lastUid && mailbox.exists > 0) {
+                    const lastMessage = await this.client.fetchOne(String(mailbox.exists), { uid: true });
+                    if (lastMessage && lastMessage.uid > currentMaxUid) {
+                        currentMaxUid = lastMessage.uid;
+                    }
+                }
+                this.newMaxUids[mailboxPath] = currentMaxUid;
+
+                const searchCriteria = lastUid ? { uid: `${lastUid + 1}:*` } : { all: true };
+
+                // Only fetch if the mailbox has messages, to avoid errors on empty mailboxes with some IMAP servers.
+                if (mailbox.exists > 0) {
+                    for await (const msg of this.client.fetch(searchCriteria, { envelope: true, source: true, bodyStructure: true, uid: true })) {
+                        if (lastUid && msg.uid <= lastUid) {
+                            continue;
+                        }
+
+                        if (msg.uid > this.newMaxUids[mailboxPath]) {
+                            this.newMaxUids[mailboxPath] = msg.uid;
+                        }
+
+                        if (msg.envelope && msg.source) {
+                            yield await this.parseMessage(msg);
+                        }
+                    }
                 }
             }
         } finally {
-            await this.client.logout();
+            await this.disconnect();
         }
     }
+
+    private async parseMessage(msg: any): Promise<EmailObject> {
+        const parsedEmail: ParsedMail = await simpleParser(msg.source);
+        const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
+            filename: attachment.filename || 'untitled',
+            contentType: attachment.contentType,
+            size: attachment.size,
+            content: attachment.content as Buffer
+        }));
+
+        const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
+            if (!addresses) return [];
+            const addressArray = Array.isArray(addresses) ? addresses : [addresses];
+            return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
+        };
+
+        const threadId = getThreadId(parsedEmail.headers);
+
+        return {
+            id: msg.uid.toString(),
+            threadId: threadId,
+            from: mapAddresses(parsedEmail.from),
+            to: mapAddresses(parsedEmail.to),
+            cc: mapAddresses(parsedEmail.cc),
+            bcc: mapAddresses(parsedEmail.bcc),
+            subject: parsedEmail.subject || '',
+            body: parsedEmail.text || '',
+            html: parsedEmail.html || '',
+            headers: parsedEmail.headers,
+            attachments,
+            receivedAt: parsedEmail.date || new Date(),
+            eml: msg.source
+        };
+    }
+
+    public getUpdatedSyncState(): SyncState {
+        const imapSyncState: { [mailboxPath: string]: { maxUid: number; }; } = {};
+        for (const [path, uid] of Object.entries(this.newMaxUids)) {
+            imapSyncState[path] = { maxUid: uid };
+        }
+        return {
+            imap: imapSyncState
+        };
+    }
 }

packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts

@@ -1,99 +1,288 @@
-import type { Microsoft365Credentials, EmailObject, EmailAddress } from '@open-archiver/types';
+import 'cross-fetch/polyfill';
+import type {
+    Microsoft365Credentials,
+    EmailObject,
+    EmailAddress,
+    SyncState,
+    MailboxUser
+} from '@open-archiver/types';
 import type { IEmailConnector } from '../EmailProviderFactory';
-import { ConfidentialClientApplication } from '@azure/msal-node';
+import { logger } from '../../config/logger';
 import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
-import axios from 'axios';
-
-const GRAPH_API_ENDPOINT = 'https://graph.microsoft.com/v1.0';
+import { ConfidentialClientApplication, Configuration, LogLevel } from '@azure/msal-node';
+import { Client } from '@microsoft/microsoft-graph-client';
+import type { User, MailFolder } from 'microsoft-graph';
+import type { AuthProvider } from '@microsoft/microsoft-graph-client';
 
+/**
+ * A connector for Microsoft 365 that uses the Microsoft Graph API with client credentials (app-only)
+ * to access data on behalf of the organization.
+ */
 export class MicrosoftConnector implements IEmailConnector {
-    private cca: ConfidentialClientApplication;
+    private credentials: Microsoft365Credentials;
+    private graphClient: Client;
+    // Store delta tokens for each folder during a sync operation.
+    private newDeltaTokens: { [folderId: string]: string; };
 
-    constructor(private credentials: Microsoft365Credentials) {
-        this.cca = new ConfidentialClientApplication({
+    constructor(credentials: Microsoft365Credentials) {
+        this.credentials = credentials;
+        this.newDeltaTokens = {}; // Initialize as an empty object
+
+        const msalConfig: Configuration = {
             auth: {
                 clientId: this.credentials.clientId,
+                authority: `https://login.microsoftonline.com/${this.credentials.tenantId}`,
                 clientSecret: this.credentials.clientSecret,
             },
-        });
-    }
-
-    private async getAccessToken(): Promise<string> {
-        const result = await this.cca.acquireTokenByClientCredential({
-            scopes: ['https://graph.microsoft.com/.default'],
-        });
-        if (!result?.accessToken) {
-            throw new Error('Failed to acquire access token');
-        }
-        return result.accessToken;
+            system: {
+                loggerOptions: {
+                    loggerCallback(loglevel, message, containsPii) {
+                        if (containsPii) return;
+                        switch (loglevel) {
+                            case LogLevel.Error:
+                                logger.error(message);
+                                return;
+                            case LogLevel.Warning:
+                                logger.warn(message);
+                                return;
+                            case LogLevel.Info:
+                                logger.info(message);
+                                return;
+                            case LogLevel.Verbose:
+                                logger.debug(message);
+                                return;
+                        }
+                    },
+                    piiLoggingEnabled: false,
+                    logLevel: LogLevel.Warning,
+                }
+            }
+        };
+
+        const msalClient = new ConfidentialClientApplication(msalConfig);
+
+        const authProvider: AuthProvider = async (done) => {
+            try {
+                const response = await msalClient.acquireTokenByClientCredential({
+                    scopes: ['https://graph.microsoft.com/.default'],
+                });
+                if (!response?.accessToken) {
+                    throw new Error('Failed to acquire access token.');
+                }
+                done(null, response.accessToken);
+            } catch (error) {
+                logger.error({ err: error }, 'Failed to acquire Microsoft Graph access token');
+                done(error, null);
+            }
+        };
+
+        this.graphClient = Client.init({ authProvider });
     }
 
+    /**
+     * Tests the connection and authentication by attempting to list the first user
+     * from the directory.
+     */
     public async testConnection(): Promise<boolean> {
         try {
-            await this.getAccessToken();
+            await this.graphClient.api('/users').top(1).get();
+            logger.info('Microsoft 365 connection test successful.');
             return true;
         } catch (error) {
-            console.error('Failed to verify Microsoft 365 connection:', error);
-            return false;
+            logger.error({ err: error }, 'Failed to verify Microsoft 365 connection');
+            throw error;
         }
     }
 
-    public async *fetchEmails(userEmail?: string, since?: Date): AsyncGenerator<EmailObject> {
-        const accessToken = await this.getAccessToken();
-        const headers = { Authorization: `Bearer ${accessToken}` };
+    /**
+     * Lists all users in the Microsoft 365 tenant.
+     * This method handles pagination to retrieve the complete list of users.
+     * @returns An async generator that yields each user object.
+     */
+    public async *listAllUsers(): AsyncGenerator<MailboxUser> {
+        let request = this.graphClient.api('/users').select('id,userPrincipalName,displayName');
 
-        let nextLink: string | undefined = `${GRAPH_API_ENDPOINT}/users/me/messages`;
-        if (since) {
-            nextLink += `?$filter=receivedDateTime ge ${since.toISOString()}`;
-        }
+        try {
+            let response = await request.get();
+            while (response) {
+                for (const user of response.value as User[]) {
+                    if (user.id && user.userPrincipalName && user.displayName) {
+                        yield {
+                            id: user.id,
+                            primaryEmail: user.userPrincipalName,
+                            displayName: user.displayName
+                        };
+                    }
+                }
 
-        while (nextLink) {
-            const res: { data: { value: any[]; '@odata.nextLink'?: string; }; } = await axios.get(
-                nextLink,
-                { headers }
-            );
-            const messages = res.data.value;
-
-            for (const message of messages) {
-                const rawContentRes = await axios.get(
-                    `${GRAPH_API_ENDPOINT}/users/me/messages/${message.id}/$value`,
-                    { headers }
-                );
-                const emlBuffer = Buffer.from(rawContentRes.data, 'utf-8');
-                const parsedEmail: ParsedMail = await simpleParser(emlBuffer);
-                const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
-                    filename: attachment.filename || 'untitled',
-                    contentType: attachment.contentType,
-                    size: attachment.size,
-                    content: attachment.content as Buffer
-                }));
-
-                const mapAddresses = (
-                    addresses: AddressObject | AddressObject[] | undefined
-                ): EmailAddress[] => {
-                    if (!addresses) return [];
-                    const addressArray = Array.isArray(addresses) ? addresses : [addresses];
-                    return addressArray.flatMap(a =>
-                        a.value.map(v => ({ name: v.name, address: v.address || '' }))
-                    );
-                };
-
-                yield {
-                    id: message.id,
-                    from: mapAddresses(parsedEmail.from),
-                    to: mapAddresses(parsedEmail.to),
-                    cc: mapAddresses(parsedEmail.cc),
-                    bcc: mapAddresses(parsedEmail.bcc),
-                    subject: parsedEmail.subject || '',
-                    body: parsedEmail.text || '',
-                    html: parsedEmail.html || '',
-                    headers: parsedEmail.headers as any,
-                    attachments,
-                    receivedAt: parsedEmail.date || new Date(),
-                    eml: emlBuffer
-                };
+                if (response['@odata.nextLink']) {
+                    response = await this.graphClient.api(response['@odata.nextLink']).get();
+                } else {
+                    break;
+                }
             }
-            nextLink = res.data['@odata.nextLink'];
+        } catch (error) {
+            logger.error({ err: error }, 'Failed to list all users from Microsoft 365');
+            throw error;
         }
     }
+
+    /**
+     * Fetches emails for a single user by iterating through all mail folders and
+     * performing a delta query on each.
+     * @param userEmail The user principal name or ID of the user.
+     * @param syncState Optional state containing the deltaTokens for each folder.
+     * @returns An async generator that yields each raw email object.
+     */
+    public async *fetchEmails(
+        userEmail: string,
+        syncState?: SyncState | null
+    ): AsyncGenerator<EmailObject> {
+        this.newDeltaTokens = syncState?.microsoft?.[userEmail]?.deltaTokens || {};
+
+        try {
+            const folders = this.listAllFolders(userEmail);
+            for await (const folder of folders) {
+                if (folder.id) {
+                    logger.info({ userEmail, folderId: folder.id, folderName: folder.displayName }, 'Syncing folder');
+                    yield* this.syncFolder(userEmail, folder.id, this.newDeltaTokens[folder.id]);
+                }
+            }
+        } catch (error) {
+            logger.error({ err: error, userEmail }, 'Failed to fetch emails from Microsoft 365');
+            throw error;
+        }
+    }
+
+    /**
+     * Lists all mail folders for a given user.
+     * @param userEmail The user principal name or ID.
+     * @returns An async generator that yields each mail folder.
+     */
+    private async *listAllFolders(userEmail: string): AsyncGenerator<MailFolder> {
+        let requestUrl: string | undefined = `/users/${userEmail}/mailFolders`;
+
+        while (requestUrl) {
+            try {
+                const response = await this.graphClient.api(requestUrl).get();
+                for (const folder of response.value as MailFolder[]) {
+                    yield folder;
+                }
+                requestUrl = response['@odata.nextLink'];
+            } catch (error) {
+                logger.error({ err: error, userEmail }, 'Failed to list mail folders');
+                throw error; // Stop if we can't list folders
+            }
+        }
+    }
+
+    /**
+     * Performs a delta sync on a single mail folder.
+     * @param userEmail The user's email.
+     * @param folderId The ID of the folder to sync.
+     * @param deltaToken The existing delta token for this folder, if any.
+     * @returns An async generator that yields email objects.
+     */
+    private async *syncFolder(
+        userEmail: string,
+        folderId: string,
+        deltaToken?: string
+    ): AsyncGenerator<EmailObject> {
+        let requestUrl: string | undefined;
+
+        if (deltaToken) {
+            // Continuous sync
+            requestUrl = deltaToken;
+        } else {
+            // Initial sync
+            requestUrl = `/users/${userEmail}/mailFolders/${folderId}/messages/delta`;
+        }
+
+        while (requestUrl) {
+            try {
+                const response = await this.graphClient.api(requestUrl)
+                    .select('id,conversationId,@removed')
+                    .get();
+
+                for (const message of response.value) {
+                    if (message.id && !(message)['@removed']) {
+                        const rawEmail = await this.getRawEmail(userEmail, message.id);
+                        if (rawEmail) {
+                            const emailObject = await this.parseEmail(rawEmail, message.id, userEmail);
+                            emailObject.threadId = message.conversationId; // Add conversationId as threadId
+                            yield emailObject;
+                        }
+                    }
+                }
+
+                if (response['@odata.deltaLink']) {
+                    this.newDeltaTokens[folderId] = response['@odata.deltaLink'];
+                }
+
+                requestUrl = response['@odata.nextLink'];
+            } catch (error) {
+                logger.error({ err: error, userEmail, folderId }, 'Failed to sync mail folder');
+                // Continue to the next folder if one fails
+                return;
+            }
+        }
+    }
+
+    private async getRawEmail(userEmail: string, messageId: string): Promise<Buffer | null> {
+        try {
+            const response = await this.graphClient.api(`/users/${userEmail}/messages/${messageId}/$value`).getStream();
+            const chunks: any[] = [];
+            for await (const chunk of response) {
+                chunks.push(chunk);
+            }
+            return Buffer.concat(chunks);
+        } catch (error) {
+            logger.error({ err: error, userEmail, messageId }, 'Failed to fetch raw email content.');
+            return null;
+        }
+    }
+
+    private async parseEmail(rawEmail: Buffer, messageId: string, userEmail: string): Promise<EmailObject> {
+        const parsedEmail: ParsedMail = await simpleParser(rawEmail);
+        const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
+            filename: attachment.filename || 'untitled',
+            contentType: attachment.contentType,
+            size: attachment.size,
+            content: attachment.content as Buffer
+        }));
+        const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
+            if (!addresses) return [];
+            const addressArray = Array.isArray(addresses) ? addresses : [addresses];
+            return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
+        };
+
+        return {
+            id: messageId,
+            userEmail: userEmail,
+            eml: rawEmail,
+            from: mapAddresses(parsedEmail.from),
+            to: mapAddresses(parsedEmail.to),
+            cc: mapAddresses(parsedEmail.cc),
+            bcc: mapAddresses(parsedEmail.bcc),
+            subject: parsedEmail.subject || '',
+            body: parsedEmail.text || '',
+            html: parsedEmail.html || '',
+            headers: parsedEmail.headers,
+            attachments,
+            receivedAt: parsedEmail.date || new Date(),
+        };
+    }
+
+    public getUpdatedSyncState(userEmail: string): SyncState {
+        if (Object.keys(this.newDeltaTokens).length === 0) {
+            return {};
+        }
+        return {
+            microsoft: {
+                [userEmail]: {
+                    deltaTokens: this.newDeltaTokens
+                }
+            }
+        };
+    }
 }

packages/backend/src/services/ingestion-connectors/utils.ts

@@ -0,0 +1,47 @@
+
+import type { Headers } from 'mailparser';
+
+function getHeaderValue(header: any): string | undefined {
+    if (typeof header === 'string') {
+        return header;
+    }
+    if (Array.isArray(header)) {
+        return getHeaderValue(header[0]);
+    }
+    if (typeof header === 'object' && header !== null && 'value' in header) {
+        return getHeaderValue(header.value);
+    }
+    return undefined;
+}
+
+export function getThreadId(headers: Headers): string | undefined {
+
+    const referencesHeader = headers.get('references');
+
+    if (referencesHeader) {
+        const references = getHeaderValue(referencesHeader);
+        if (references) {
+            return references.split(' ')[0].trim();
+        }
+    }
+
+    const inReplyToHeader = headers.get('in-reply-to');
+
+    if (inReplyToHeader) {
+        const inReplyTo = getHeaderValue(inReplyToHeader);
+        if (inReplyTo) {
+            return inReplyTo.trim();
+        }
+    }
+
+    const messageIdHeader = headers.get('message-id');
+
+    if (messageIdHeader) {
+        const messageId = getHeaderValue(messageIdHeader);
+        if (messageId) {
+            return messageId.trim();
+        }
+    }
+    console.warn('No thread ID found, returning undefined');
+    return undefined;
+}

packages/backend/src/services/storage/LocalFileSystemProvider.ts

@@ -37,8 +37,9 @@ export class LocalFileSystemProvider implements IStorageProvider {
     async delete(filePath: string): Promise<void> {
         const fullPath = path.join(this.rootPath, filePath);
         try {
-            await fs.unlink(fullPath);
+            await fs.rm(fullPath, { recursive: true, force: true });
         } catch (error: any) {
+            // Even with force: true, other errors might occur (e.g., permissions)
             if (error.code !== 'ENOENT') {
                 throw error;
             }

packages/backend/src/services/storage/S3StorageProvider.ts

@@ -5,6 +5,8 @@ import {
     DeleteObjectCommand,
     HeadObjectCommand,
     NotFound,
+    ListObjectsV2Command,
+    DeleteObjectsCommand,
 } from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
 import { Readable } from 'stream';
@@ -60,11 +62,28 @@ export class S3StorageProvider implements IStorageProvider {
     }
 
     async delete(path: string): Promise<void> {
-        const command = new DeleteObjectCommand({
+        // List all objects with the given prefix
+        const listCommand = new ListObjectsV2Command({
             Bucket: this.bucket,
-            Key: path,
+            Prefix: path,
         });
-        await this.client.send(command);
+        const listedObjects = await this.client.send(listCommand);
+
+        if (!listedObjects.Contents || listedObjects.Contents.length === 0) {
+            return;
+        }
+
+        // Create a list of objects to delete
+        const deleteParams = {
+            Bucket: this.bucket,
+            Delete: {
+                Objects: listedObjects.Contents.map(({ Key }) => ({ Key })),
+            },
+        };
+
+        // Delete the objects
+        const deleteCommand = new DeleteObjectsCommand(deleteParams);
+        await this.client.send(deleteCommand);
     }
 
     async exists(path: string): Promise<boolean> {

packages/backend/src/workers/ingestion.worker.ts

@@ -2,14 +2,20 @@ import { Worker } from 'bullmq';
 import { connection } from '../config/redis';
 import initialImportProcessor from '../jobs/processors/initial-import.processor';
 import continuousSyncProcessor from '../jobs/processors/continuous-sync.processor';
+import scheduleContinuousSyncProcessor from '../jobs/processors/schedule-continuous-sync.processor';
 import { processMailboxProcessor } from '../jobs/processors/process-mailbox.processor';
+import syncCycleFinishedProcessor from '../jobs/processors/sync-cycle-finished.processor';
 
 const processor = async (job: any) => {
     switch (job.name) {
         case 'initial-import':
             return initialImportProcessor(job);
+        case 'sync-cycle-finished':
+            return syncCycleFinishedProcessor(job);
         case 'continuous-sync':
             return continuousSyncProcessor(job);
+        case 'schedule-continuous-sync':
+            return scheduleContinuousSyncProcessor(job);
         case 'process-mailbox':
             return processMailboxProcessor(job);
         default:
@@ -27,6 +33,7 @@ const worker = new Worker('ingestion', processor, {
     },
 });
 
+
 console.log('Ingestion worker started');
 
 process.on('SIGINT', () => worker.close());

packages/backend/tsconfig.json

@@ -3,7 +3,6 @@
     "compilerOptions": {
         "outDir": "./dist",
         "rootDir": "./src",
-        "composite": true,
         "emitDecoratorMetadata": true,
         "experimentalDecorators": true
     },

packages/frontend/package.json

@@ -1,11 +1,12 @@
 {
-	"name": "frontend",
+	"name": "@open-archiver/frontend",
 	"private": true,
 	"version": "0.0.1",
 	"type": "module",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
+		"start": "node build/index.js",
 		"preview": "vite preview",
 		"prepare": "svelte-kit sync || echo ''",
 		"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
@@ -14,29 +15,36 @@
 		"lint": "prettier --check ."
 	},
 	"dependencies": {
+		"@iconify/svelte": "^5.0.1",
 		"@open-archiver/types": "workspace:*",
+		"@sveltejs/kit": "^2.16.0",
+		"bits-ui": "^2.8.10",
+		"clsx": "^2.1.1",
+		"d3-shape": "^3.2.0",
 		"jose": "^6.0.1",
 		"lucide-svelte": "^0.525.0",
 		"postal-mime": "^2.4.4",
-		"svelte-persisted-store": "^0.12.0"
+		"svelte-persisted-store": "^0.12.0",
+		"tailwind-merge": "^3.3.1",
+		"tailwind-variants": "^1.0.0"
 	},
 	"devDependencies": {
 		"@internationalized/date": "^3.8.2",
 		"@lucide/svelte": "^0.515.0",
 		"@sveltejs/adapter-auto": "^6.0.0",
-		"@sveltejs/kit": "^2.16.0",
+		"@sveltejs/adapter-node": "^5.2.13",
 		"@sveltejs/vite-plugin-svelte": "^5.0.0",
 		"@tailwindcss/vite": "^4.0.0",
-		"bits-ui": "^2.8.10",
-		"clsx": "^2.1.1",
+		"@types/d3-shape": "^3.1.7",
 		"dotenv": "^17.2.0",
+		"layerchart": "2.0.0-next.27",
+		"mode-watcher": "^1.1.0",
 		"prettier": "^3.4.2",
 		"prettier-plugin-svelte": "^3.3.3",
 		"prettier-plugin-tailwindcss": "^0.6.11",
 		"svelte": "^5.0.0",
 		"svelte-check": "^4.0.0",
-		"tailwind-merge": "^3.3.1",
-		"tailwind-variants": "^1.0.0",
+		"svelte-sonner": "^1.0.5",
 		"tailwindcss": "^4.0.0",
 		"tw-animate-css": "^1.3.5",
 		"typescript": "^5.0.0",

packages/frontend/src/app.css

@@ -1,121 +1,121 @@
-@import "tailwindcss";
+@import 'tailwindcss';
 
-@import "tw-animate-css";
+@import 'tw-animate-css';
 
 @custom-variant dark (&:is(.dark *));
 
 :root {
-  --radius: 0.625rem;
-  --background: oklch(1 0 0);
-  --foreground: oklch(0.129 0.042 264.695);
-  --card: oklch(1 0 0);
-  --card-foreground: oklch(0.129 0.042 264.695);
-  --popover: oklch(1 0 0);
-  --popover-foreground: oklch(0.129 0.042 264.695);
-  --primary: oklch(0.208 0.042 265.755);
-  --primary-foreground: oklch(0.984 0.003 247.858);
-  --secondary: oklch(0.968 0.007 247.896);
-  --secondary-foreground: oklch(0.208 0.042 265.755);
-  --muted: oklch(0.968 0.007 247.896);
-  --muted-foreground: oklch(0.554 0.046 257.417);
-  --accent: oklch(0.968 0.007 247.896);
-  --accent-foreground: oklch(0.208 0.042 265.755);
-  --destructive: oklch(0.577 0.245 27.325);
-  --border: oklch(0.929 0.013 255.508);
-  --input: oklch(0.929 0.013 255.508);
-  --ring: oklch(0.704 0.04 256.788);
-  --chart-1: oklch(0.646 0.222 41.116);
-  --chart-2: oklch(0.6 0.118 184.704);
-  --chart-3: oklch(0.398 0.07 227.392);
-  --chart-4: oklch(0.828 0.189 84.429);
-  --chart-5: oklch(0.769 0.188 70.08);
-  --sidebar: oklch(0.984 0.003 247.858);
-  --sidebar-foreground: oklch(0.129 0.042 264.695);
-  --sidebar-primary: oklch(0.208 0.042 265.755);
-  --sidebar-primary-foreground: oklch(0.984 0.003 247.858);
-  --sidebar-accent: oklch(0.968 0.007 247.896);
-  --sidebar-accent-foreground: oklch(0.208 0.042 265.755);
-  --sidebar-border: oklch(0.929 0.013 255.508);
-  --sidebar-ring: oklch(0.704 0.04 256.788);
+	--radius: 0.65rem;
+	--background: oklch(1 0 0);
+	--foreground: oklch(0.141 0.005 285.823);
+	--card: oklch(1 0 0);
+	--card-foreground: oklch(0.141 0.005 285.823);
+	--popover: oklch(1 0 0);
+	--popover-foreground: oklch(0.141 0.005 285.823);
+	--primary: oklch(0.705 0.213 47.604);
+	--primary-foreground: oklch(0.98 0.016 73.684);
+	--secondary: oklch(0.967 0.001 286.375);
+	--secondary-foreground: oklch(0.21 0.006 285.885);
+	--muted: oklch(0.967 0.001 286.375);
+	--muted-foreground: oklch(0.552 0.016 285.938);
+	--accent: oklch(0.967 0.001 286.375);
+	--accent-foreground: oklch(0.21 0.006 285.885);
+	--destructive: oklch(0.577 0.245 27.325);
+	--border: oklch(0.92 0.004 286.32);
+	--input: oklch(0.92 0.004 286.32);
+	--ring: oklch(0.705 0.213 47.604);
+	--chart-1: oklch(0.705 0.213 47.604);
+	--chart-2: oklch(0.5093 0.0758 213.43);
+	--chart-3: oklch(0.9227 0.0517 91.38);
+	--chart-4: oklch(0.7509 0.1563 46.19);
+	--chart-5: oklch(0.7145 0.1478 266.89);
+	--sidebar: oklch(0.985 0 0);
+	--sidebar-foreground: oklch(0.141 0.005 285.823);
+	--sidebar-primary: oklch(0.705 0.213 47.604);
+	--sidebar-primary-foreground: oklch(0.98 0.016 73.684);
+	--sidebar-accent: oklch(0.967 0.001 286.375);
+	--sidebar-accent-foreground: oklch(0.21 0.006 285.885);
+	--sidebar-border: oklch(0.92 0.004 286.32);
+	--sidebar-ring: oklch(0.705 0.213 47.604);
 }
 
 .dark {
-  --background: oklch(0.129 0.042 264.695);
-  --foreground: oklch(0.984 0.003 247.858);
-  --card: oklch(0.208 0.042 265.755);
-  --card-foreground: oklch(0.984 0.003 247.858);
-  --popover: oklch(0.208 0.042 265.755);
-  --popover-foreground: oklch(0.984 0.003 247.858);
-  --primary: oklch(0.929 0.013 255.508);
-  --primary-foreground: oklch(0.208 0.042 265.755);
-  --secondary: oklch(0.279 0.041 260.031);
-  --secondary-foreground: oklch(0.984 0.003 247.858);
-  --muted: oklch(0.279 0.041 260.031);
-  --muted-foreground: oklch(0.704 0.04 256.788);
-  --accent: oklch(0.279 0.041 260.031);
-  --accent-foreground: oklch(0.984 0.003 247.858);
-  --destructive: oklch(0.704 0.191 22.216);
-  --border: oklch(1 0 0 / 10%);
-  --input: oklch(1 0 0 / 15%);
-  --ring: oklch(0.551 0.027 264.364);
-  --chart-1: oklch(0.488 0.243 264.376);
-  --chart-2: oklch(0.696 0.17 162.48);
-  --chart-3: oklch(0.769 0.188 70.08);
-  --chart-4: oklch(0.627 0.265 303.9);
-  --chart-5: oklch(0.645 0.246 16.439);
-  --sidebar: oklch(0.208 0.042 265.755);
-  --sidebar-foreground: oklch(0.984 0.003 247.858);
-  --sidebar-primary: oklch(0.488 0.243 264.376);
-  --sidebar-primary-foreground: oklch(0.984 0.003 247.858);
-  --sidebar-accent: oklch(0.279 0.041 260.031);
-  --sidebar-accent-foreground: oklch(0.984 0.003 247.858);
-  --sidebar-border: oklch(1 0 0 / 10%);
-  --sidebar-ring: oklch(0.551 0.027 264.364);
+	--background: oklch(0.141 0.005 285.823);
+	--foreground: oklch(0.985 0 0);
+	--card: oklch(0.21 0.006 285.885);
+	--card-foreground: oklch(0.985 0 0);
+	--popover: oklch(0.21 0.006 285.885);
+	--popover-foreground: oklch(0.985 0 0);
+	--primary: oklch(0.646 0.222 41.116);
+	--primary-foreground: oklch(0.98 0.016 73.684);
+	--secondary: oklch(0.274 0.006 286.033);
+	--secondary-foreground: oklch(0.985 0 0);
+	--muted: oklch(0.274 0.006 286.033);
+	--muted-foreground: oklch(0.705 0.015 286.067);
+	--accent: oklch(0.274 0.006 286.033);
+	--accent-foreground: oklch(0.985 0 0);
+	--destructive: oklch(0.704 0.191 22.216);
+	--border: oklch(1 0 0 / 10%);
+	--input: oklch(1 0 0 / 15%);
+	--ring: oklch(0.646 0.222 41.116);
+	--chart-1: oklch(0.705 0.213 47.604);
+	--chart-2: oklch(0.5093 0.0758 213.43);
+	--chart-3: oklch(0.9227 0.0517 91.38);
+	--chart-4: oklch(0.7509 0.1563 46.19);
+	--chart-5: oklch(0.7145 0.1478 266.89);
+	--sidebar: oklch(0.21 0.006 285.885);
+	--sidebar-foreground: oklch(0.985 0 0);
+	--sidebar-primary: oklch(0.646 0.222 41.116);
+	--sidebar-primary-foreground: oklch(0.98 0.016 73.684);
+	--sidebar-accent: oklch(0.274 0.006 286.033);
+	--sidebar-accent-foreground: oklch(0.985 0 0);
+	--sidebar-border: oklch(1 0 0 / 10%);
+	--sidebar-ring: oklch(0.646 0.222 41.116);
 }
 
 @theme inline {
-  --radius-sm: calc(var(--radius) - 4px);
-  --radius-md: calc(var(--radius) - 2px);
-  --radius-lg: var(--radius);
-  --radius-xl: calc(var(--radius) + 4px);
-  --color-background: var(--background);
-  --color-foreground: var(--foreground);
-  --color-card: var(--card);
-  --color-card-foreground: var(--card-foreground);
-  --color-popover: var(--popover);
-  --color-popover-foreground: var(--popover-foreground);
-  --color-primary: var(--primary);
-  --color-primary-foreground: var(--primary-foreground);
-  --color-secondary: var(--secondary);
-  --color-secondary-foreground: var(--secondary-foreground);
-  --color-muted: var(--muted);
-  --color-muted-foreground: var(--muted-foreground);
-  --color-accent: var(--accent);
-  --color-accent-foreground: var(--accent-foreground);
-  --color-destructive: var(--destructive);
-  --color-border: var(--border);
-  --color-input: var(--input);
-  --color-ring: var(--ring);
-  --color-chart-1: var(--chart-1);
-  --color-chart-2: var(--chart-2);
-  --color-chart-3: var(--chart-3);
-  --color-chart-4: var(--chart-4);
-  --color-chart-5: var(--chart-5);
-  --color-sidebar: var(--sidebar);
-  --color-sidebar-foreground: var(--sidebar-foreground);
-  --color-sidebar-primary: var(--sidebar-primary);
-  --color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
-  --color-sidebar-accent: var(--sidebar-accent);
-  --color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
-  --color-sidebar-border: var(--sidebar-border);
-  --color-sidebar-ring: var(--sidebar-ring);
+	--radius-sm: calc(var(--radius) - 4px);
+	--radius-md: calc(var(--radius) - 2px);
+	--radius-lg: var(--radius);
+	--radius-xl: calc(var(--radius) + 4px);
+	--color-background: var(--background);
+	--color-foreground: var(--foreground);
+	--color-card: var(--card);
+	--color-card-foreground: var(--card-foreground);
+	--color-popover: var(--popover);
+	--color-popover-foreground: var(--popover-foreground);
+	--color-primary: var(--primary);
+	--color-primary-foreground: var(--primary-foreground);
+	--color-secondary: var(--secondary);
+	--color-secondary-foreground: var(--secondary-foreground);
+	--color-muted: var(--muted);
+	--color-muted-foreground: var(--muted-foreground);
+	--color-accent: var(--accent);
+	--color-accent-foreground: var(--accent-foreground);
+	--color-destructive: var(--destructive);
+	--color-border: var(--border);
+	--color-input: var(--input);
+	--color-ring: var(--ring);
+	--color-chart-1: var(--chart-1);
+	--color-chart-2: var(--chart-2);
+	--color-chart-3: var(--chart-3);
+	--color-chart-4: var(--chart-4);
+	--color-chart-5: var(--chart-5);
+	--color-sidebar: var(--sidebar);
+	--color-sidebar-foreground: var(--sidebar-foreground);
+	--color-sidebar-primary: var(--sidebar-primary);
+	--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
+	--color-sidebar-accent: var(--sidebar-accent);
+	--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
+	--color-sidebar-border: var(--sidebar-border);
+	--color-sidebar-ring: var(--sidebar-ring);
 }
 
 @layer base {
-  * {
-    @apply border-border outline-ring/50;
-  }
-  body {
-    @apply bg-background text-foreground;
-  }
-}
+	* {
+		@apply border-border outline-ring/50;
+	}
+	body {
+		@apply bg-background text-foreground;
+	}
+}

packages/frontend/src/hooks.server.ts

@@ -1,15 +1,18 @@
 import type { Handle } from '@sveltejs/kit';
 import { jwtVerify } from 'jose';
 import type { User } from '@open-archiver/types';
+import 'dotenv/config';
 
-const JWT_SECRET = new TextEncoder().encode('a-very-secret-key');
+
+
+const JWT_SECRET_ENCODED = new TextEncoder().encode(process.env.JWT_SECRET);
 
 export const handle: Handle = async ({ event, resolve }) => {
     const token = event.cookies.get('accessToken');
 
     if (token) {
         try {
-            const { payload } = await jwtVerify(token, JWT_SECRET);
+            const { payload } = await jwtVerify(token, JWT_SECRET_ENCODED);
             event.locals.user = payload as Omit<User, 'passwordHash'>;
             event.locals.accessToken = token;
         } catch (error) {

packages/frontend/src/lib/api.client.ts

@@ -1,5 +1,4 @@
 import { authStore } from '$lib/stores/auth.store';
-import type { User } from '@open-archiver/types';
 import { get } from 'svelte/store';
 
 const BASE_URL = '/api/v1'; // Using a relative URL for proxying

packages/frontend/src/lib/components/custom/EmailPreview.svelte

@@ -6,15 +6,16 @@
 		raw,
 		rawHtml
 	}: { raw?: Buffer | { type: 'Buffer'; data: number[] } | undefined; rawHtml?: string } = $props();
-
 	let parsedEmail: Email | null = $state(null);
 	let isLoading = $state(true);
 
 	// By adding a <base> tag, all relative and absolute links in the HTML document
 	// will open in a new tab by default.
 	let emailHtml = $derived(() => {
-		if (parsedEmail && parsedEmail?.html) {
+		if (parsedEmail && parsedEmail.html) {
 			return `<base target="_blank" />${parsedEmail.html}`;
+		} else if (parsedEmail && parsedEmail.text) {
+			return `<base target="_blank" />${parsedEmail.text}`;
 		} else if (rawHtml) {
 			return `<base target="_blank" />${rawHtml}`;
 		}

packages/frontend/src/lib/components/custom/EmailThread.svelte

@@ -0,0 +1,62 @@
+<script lang="ts">
+	import { goto } from '$app/navigation';
+	import type { ArchivedEmail } from '@open-archiver/types';
+
+	let {
+		thread,
+		currentEmailId
+	}: {
+		thread: ArchivedEmail['thread'];
+		currentEmailId: string;
+	} = $props();
+</script>
+
+<div>
+	<div class="relative border-l-2 border-gray-200 pl-6">
+		{#if thread}
+			{#each thread as item, i (item.id)}
+				<div class="mb-8">
+					<span
+						class="absolute -left-3 flex h-6 w-6 items-center justify-center rounded-full bg-gray-200 ring-8 ring-white"
+					>
+						<svg
+							class="h-3 w-3 text-gray-600"
+							fill="currentColor"
+							viewBox="0 0 20 20"
+							xmlns="http://www.w3.org/2000/svg"
+							><path
+								fill-rule="evenodd"
+								d="M6 2a1 1 0 00-1 1v1H4a2 2 0 00-2 2v10a2 2 0 002 2h12a2 2 0 002-2V6a2 2 0 00-2-2h-1V3a1 1 0 10-2 0v1H7V3a1 1 0 00-1-1zm0 5a1 1 0 000 2h8a1 1 0 100-2H6z"
+								clip-rule="evenodd"
+							></path></svg
+						>
+					</span>
+					<h4
+						class:font-bold={item.id === currentEmailId}
+						class="text-md mb-2 {item.id !== currentEmailId
+							? 'text-blue-500 hover:underline'
+							: 'text-gray-900'}"
+					>
+						{#if item.id !== currentEmailId}
+							<a
+								href="/dashboard/archived-emails/{item.id}"
+								onclick={(e) => {
+									e.preventDefault();
+									goto(`/dashboard/archived-emails/${item.id}`, {
+										invalidateAll: true
+									});
+								}}>{item.subject || 'No Subject'}</a
+							>
+						{:else}
+							{item.subject || 'No Subject'}
+						{/if}
+					</h4>
+					<div class="flex flex-col space-y-2 text-sm font-normal leading-none text-gray-400">
+						<span>From: {item.senderEmail}</span>
+						<time class="">{new Date(item.sentAt).toLocaleString()}</time>
+					</div>
+				</div>
+			{/each}
+		{/if}
+	</div>
+</div>

packages/frontend/src/lib/components/custom/EmptyState.svelte

@@ -0,0 +1,49 @@
+<script lang="ts">
+	import { Button } from '$lib/components/ui/button/index.js';
+
+	let {
+		header,
+		text,
+		buttonText,
+		click
+	}: {
+		header: string;
+		text: string;
+		buttonText?: string;
+		click: () => void;
+	} = $props();
+</script>
+
+<div class="space-y-4 rounded-lg border-2 border-dashed border-gray-300 p-6 text-center">
+	<div>
+		<svg
+			viewBox="0 0 24 24"
+			fill="none"
+			stroke="currentColor"
+			aria-hidden="true"
+			class="mx-auto size-12 text-gray-400"
+		>
+			<path
+				d="M9 13h6m-3-3v6m-9 1V7a2 2 0 012-2h6l2 2h6a2 2 0 012 2v8a2 2 0 01-2 2H5a2 2 0 01-2-2z"
+				stroke-width="2"
+				vector-effect="non-scaling-stroke"
+				stroke-linecap="round"
+				stroke-linejoin="round"
+			/>
+		</svg>
+	</div>
+
+	<h3 class="0 mt-2 text-sm font-semibold">{header}</h3>
+	<p class="mt-1 text-sm">{text}</p>
+	<div>
+		<Button
+			variant="outline"
+			class="cursor-pointer"
+			onclick={() => {
+				click();
+			}}
+		>
+			{buttonText}
+		</Button>
+	</div>
+</div>

packages/frontend/src/lib/components/custom/Footer.svelte

@@ -4,7 +4,8 @@
 	>
 		<div class="flex flex-col items-center gap-2">
 			<p class="  text-balance text-center text-xs font-medium leading-loose">
-				© {new Date().getFullYear()} Open Archiver. All rights reserved.
+				© {new Date().getFullYear()}
+				<a href="https://openarchiver.com/" target="_blank">Open Archiver</a>. All rights reserved.
 			</p>
 		</div>
 	</div>

packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte

@@ -2,9 +2,11 @@
 	import type { IngestionSource, CreateIngestionSourceDto } from '@open-archiver/types';
 	import { Button } from '$lib/components/ui/button';
 	import * as Dialog from '$lib/components/ui/dialog';
+	import { Checkbox } from '$lib/components/ui/checkbox';
 	import { Input } from '$lib/components/ui/input';
 	import { Label } from '$lib/components/ui/label';
 	import * as Select from '$lib/components/ui/select';
+	import * as Alert from '$lib/components/ui/alert/index.js';
 	import { Textarea } from '$lib/components/ui/textarea/index.js';
 
 	let {
@@ -12,40 +14,53 @@
 		onSubmit
 	}: {
 		source?: IngestionSource | null;
-		onSubmit: (data: CreateIngestionSourceDto) => void;
+		onSubmit: (data: CreateIngestionSourceDto) => Promise<void>;
 	} = $props();
 
 	const providerOptions = [
+		{ value: 'generic_imap', label: 'Generic IMAP' },
 		{ value: 'google_workspace', label: 'Google Workspace' },
-		{ value: 'microsoft_365', label: 'Microsoft 365' },
-		{ value: 'generic_imap', label: 'Generic IMAP' }
+		{ value: 'microsoft_365', label: 'Microsoft 365' }
 	];
 
 	let formData: CreateIngestionSourceDto = $state({
 		name: source?.name ?? '',
-		provider: source?.provider ?? 'google_workspace',
+		provider: source?.provider ?? 'generic_imap',
 		providerConfig: source?.credentials ?? {
-			type: source?.provider ?? 'google_workspace'
+			type: source?.provider ?? 'generic_imap',
+			secure: true
 		}
 	});
 
+	$effect(() => {
+		formData.providerConfig.type = formData.provider;
+		console.log(formData);
+	});
+
 	const triggerContent = $derived(
 		providerOptions.find((p) => p.value === formData.provider)?.label ?? 'Select a provider'
 	);
 
-	const handleSubmit = (event: Event) => {
+	let isSubmitting = $state(false);
+
+	const handleSubmit = async (event: Event) => {
 		event.preventDefault();
-		onSubmit(formData);
+		isSubmitting = true;
+		try {
+			await onSubmit(formData);
+		} finally {
+			isSubmitting = false;
+		}
 	};
 </script>
 
 <form onsubmit={handleSubmit} class="grid gap-4 py-4">
 	<div class="grid grid-cols-4 items-center gap-4">
-		<Label for="name" class="text-right">Name</Label>
+		<Label for="name" class="text-left">Name</Label>
 		<Input id="name" bind:value={formData.name} class="col-span-3" />
 	</div>
 	<div class="grid grid-cols-4 items-center gap-4">
-		<Label for="provider" class="text-right">Provider</Label>
+		<Label for="provider" class="text-left">Provider</Label>
 		<Select.Root name="provider" bind:value={formData.provider} type="single">
 			<Select.Trigger class="col-span-3">
 				{triggerContent}
@@ -60,7 +75,7 @@
 
 	{#if formData.provider === 'google_workspace'}
 		<div class="grid grid-cols-4 items-center gap-4">
-			<Label for="serviceAccountKeyJson" class="text-right">Service Account Key (JSON)</Label>
+			<Label for="serviceAccountKeyJson" class="text-left">Service Account Key (JSON)</Label>
 			<Textarea
 				placeholder="Paste your service account key JSON content"
 				id="serviceAccountKeyJson"
@@ -69,7 +84,7 @@
 			/>
 		</div>
 		<div class="grid grid-cols-4 items-center gap-4">
-			<Label for="impersonatedAdminEmail" class="text-right">Impersonated Admin Email</Label>
+			<Label for="impersonatedAdminEmail" class="text-left">Impersonated Admin Email</Label>
 			<Input
 				id="impersonatedAdminEmail"
 				bind:value={formData.providerConfig.impersonatedAdminEmail}
@@ -78,32 +93,38 @@
 		</div>
 	{:else if formData.provider === 'microsoft_365'}
 		<div class="grid grid-cols-4 items-center gap-4">
-			<Label for="clientId" class="text-right">Client ID</Label>
+			<Label for="clientId" class="text-left">Application (Client) ID</Label>
 			<Input id="clientId" bind:value={formData.providerConfig.clientId} class="col-span-3" />
 		</div>
 		<div class="grid grid-cols-4 items-center gap-4">
-			<Label for="clientSecret" class="text-right">Client Secret</Label>
+			<Label for="clientSecret" class="text-left">Client Secret Value</Label>
 			<Input
 				id="clientSecret"
+				type="password"
+				placeholder="Enter the secret Value, not the Secret ID"
 				bind:value={formData.providerConfig.clientSecret}
 				class="col-span-3"
 			/>
 		</div>
+		<div class="grid grid-cols-4 items-center gap-4">
+			<Label for="tenantId" class="text-left">Directory (Tenant) ID</Label>
+			<Input id="tenantId" bind:value={formData.providerConfig.tenantId} class="col-span-3" />
+		</div>
 	{:else if formData.provider === 'generic_imap'}
 		<div class="grid grid-cols-4 items-center gap-4">
-			<Label for="host" class="text-right">Host</Label>
+			<Label for="host" class="text-left">Host</Label>
 			<Input id="host" bind:value={formData.providerConfig.host} class="col-span-3" />
 		</div>
 		<div class="grid grid-cols-4 items-center gap-4">
-			<Label for="port" class="text-right">Port</Label>
+			<Label for="port" class="text-left">Port</Label>
 			<Input id="port" type="number" bind:value={formData.providerConfig.port} class="col-span-3" />
 		</div>
 		<div class="grid grid-cols-4 items-center gap-4">
-			<Label for="username" class="text-right">Username</Label>
+			<Label for="username" class="text-left">Username</Label>
 			<Input id="username" bind:value={formData.providerConfig.username} class="col-span-3" />
 		</div>
 		<div class="grid grid-cols-4 items-center gap-4">
-			<Label for="password" class="text-right">Password</Label>
+			<Label for="password" class="text-left">Password</Label>
 			<Input
 				id="password"
 				type="password"
@@ -111,8 +132,30 @@
 				class="col-span-3"
 			/>
 		</div>
+		<div class="grid grid-cols-4 items-center gap-4">
+			<Label for="secure" class="text-left">Use TLS</Label>
+			<Checkbox id="secure" bind:checked={formData.providerConfig.secure} />
+		</div>
+	{/if}
+	{#if formData.provider === 'google_workspace' || formData.provider === 'microsoft_365'}
+		<Alert.Root>
+			<Alert.Title>Heads up!</Alert.Title>
+			<Alert.Description>
+				<div class="my-1">
+					Please note that this is an organization-wide operation. This kind of ingestions will
+					import and index <b>all</b> email inboxes in your organization. If you want to import only
+					specific email inboxes, use the IMAP connector.
+				</div>
+			</Alert.Description>
+		</Alert.Root>
 	{/if}
 	<Dialog.Footer>
-		<Button type="submit">Save changes</Button>
+		<Button type="submit" disabled={isSubmitting}>
+			{#if isSubmitting}
+				Submitting...
+			{:else}
+				Submit
+			{/if}
+		</Button>
 	</Dialog.Footer>
 </form>

packages/frontend/src/lib/components/custom/alert/Alerts.svelte

@@ -0,0 +1,123 @@
+<script lang="ts">
+	import type { AlertType } from './alert-state.svelte';
+	import { initialAlertState, setAlert } from './alert-state.svelte';
+	let { type, title, message, duration, show = false }: AlertType = $props();
+	import { fade, fly } from 'svelte/transition';
+	import { bounceIn } from 'svelte/easing';
+	import Icon from '@iconify/svelte';
+	let timeout: NodeJS.Timeout;
+	let styleConfig: {
+		icon: string;
+		color: string;
+		messageColor: string;
+		bgColor: string;
+	} = $state({
+		icon: 'heroicons-outline:check-circle',
+		color: 'text-green-800',
+		messageColor: 'text-green-700',
+		bgColor: 'text-green-50'
+	});
+	$effect(() => {
+		show;
+		if (show) {
+			timeout = scheduleHide();
+		}
+	});
+	$effect(() => {
+		type;
+		if (type === 'success') {
+			styleConfig = {
+				icon: 'heroicons-outline:check-circle',
+				color: 'text-green-600',
+				messageColor: 'text-green-500',
+				bgColor: 'bg-green-50'
+			};
+		} else if (type === 'error') {
+			styleConfig = {
+				icon: 'heroicons-outline:exclamation-circle',
+				color: 'text-yellow-600',
+				messageColor: 'text-yellow-600',
+				bgColor: 'bg-yellow-50'
+			};
+		} else if (type === 'warning') {
+			styleConfig = {
+				icon: 'heroicons-outline:exclamation',
+				color: 'text-yellow-600',
+				messageColor: 'text-yellow-600',
+				bgColor: 'bg-yellow-50'
+			};
+		}
+	});
+	function scheduleHide(): NodeJS.Timeout {
+		return setTimeout(() => {
+			setAlert(initialAlertState);
+		}, duration);
+	}
+</script>
+
+<!-- Global notification live region, render this permanently at the end of the document -->
+{#if show}
+	<div
+		aria-live="assertive"
+		class="pointer-events-none fixed inset-0 flex px-4 py-6 items-start sm:p-6 z-999999"
+		in:fly={{ easing: bounceIn, x: 1000, duration: 500 }}
+		out:fade={{ duration: 100 }}
+		role="alert"
+		onmouseenter={() => {
+			clearTimeout(timeout);
+		}}
+		onmouseleave={() => {
+			timeout = scheduleHide();
+		}}
+	>
+		<div class="flex w-full flex-col items-center space-y-4 sm:items-end">
+			<!--
+         Notification panel, dynamically insert this into the live region when it needs to be displayed
+   
+         Entering: "transform ease-out duration-300 transition"
+           From: "translate-y-2 opacity-0 sm:translate-y-0 sm:translate-x-2"
+           To: "translate-y-0 opacity-100 sm:translate-x-0"
+         Leaving: "transition ease-in duration-100"
+           From: "opacity-100"
+           To: "opacity-0"
+       -->
+			<div
+				class=" pointer-events-auto w-full max-w-sm overflow-hidden rounded-lg shadow-lg ring-1 ring-black/5 {styleConfig.bgColor} "
+			>
+				<div class="p-4">
+					<div class="flex items-start">
+						<div class="shrink-0">
+							<Icon icon={styleConfig.icon} class="size-6 {styleConfig.color}"></Icon>
+						</div>
+						<div class="ml-3 w-0 flex-1 pt-0.5">
+							<p class="text-sm font-medium {styleConfig.color}">{title}</p>
+							<p class="mt-1 text-sm {styleConfig.messageColor}">{message}</p>
+						</div>
+						<div class="ml-4 flex shrink-0">
+							<button
+								type="button"
+								class="inline-flex rounded-md {styleConfig.color} cursor-pointer"
+								onclick={() => {
+									setAlert(initialAlertState);
+								}}
+							>
+								<span class="sr-only">Close</span>
+								<svg
+									class="size-5"
+									viewBox="0 0 20 20"
+									fill="currentColor"
+									aria-hidden="true"
+									data-slot="icon"
+								>
+									<path
+										d="M6.28 5.22a.75.75 0 0 0-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 1 0 1.06 1.06L10 11.06l3.72 3.72a.75.75 0 1 0 1.06-1.06L11.06 10l3.72-3.72a.75.75 0 0 0-1.06-1.06L10 8.94 6.28 5.22Z"
+									/>
+								</svg>
+							</button>
+						</div>
+					</div>
+				</div>
+			</div>
+		</div>
+	</div>
+{/if}

packages/frontend/src/lib/components/custom/alert/alert-state.svelte.ts

@@ -0,0 +1,25 @@
+export type AlertType = {
+    type: 'success' | 'warning' | 'error';
+    title: string;
+    message: string;
+    duration: number;
+    show: boolean;
+};
+
+export const initialAlertState: AlertType = {
+    type: 'success',
+    title: '',
+    message: '',
+    duration: 0,
+    show: false
+};
+
+let alertState = $state(initialAlertState);
+
+export function setAlert(alert: AlertType) {
+    alertState = alert;
+}
+
+export function getAlert() {
+    return alertState;
+}

packages/frontend/src/lib/components/custom/charts/IngestionHistoryChart.svelte

@@ -0,0 +1,54 @@
+<script lang="ts">
+	import * as Chart from '$lib/components/ui/chart/index.js';
+	import { AreaChart } from 'layerchart';
+	import { curveCatmullRom } from 'd3-shape';
+	import type { ChartConfig } from '$lib/components/ui/chart';
+
+	export let data: { date: Date; count: number }[];
+
+	const chartConfig = {
+		count: {
+			label: 'Emails Ingested',
+			color: 'var(--chart-1)'
+		}
+	} satisfies ChartConfig;
+</script>
+
+<Chart.Container config={chartConfig} class="min-h-[300px] w-full">
+	<AreaChart
+		{data}
+		x="date"
+		y="count"
+		yDomain={[0, Math.max(...data.map((d) => d.count)) * 1.1]}
+		axis
+		legend={false}
+		series={[
+			{
+				key: 'count',
+				...chartConfig.count
+			}
+		]}
+		cRange={[
+			'var(--color-chart-1)',
+			'var(--color-chart-2)',
+			'var(--color-chart-3)',
+			'var(--color-chart-4)',
+			'var(--color-chart-5)'
+		]}
+		labels={{}}
+		props={{
+			xAxis: {
+				format: (d) =>
+					new Date(d).toLocaleDateString('en-US', {
+						month: 'short',
+						day: 'numeric'
+					})
+			},
+			area: { curve: curveCatmullRom }
+		}}
+	>
+		{#snippet tooltip()}
+			<Chart.Tooltip />
+		{/snippet}
+	</AreaChart>
+</Chart.Container>

packages/frontend/src/lib/components/custom/charts/StorageBySourceChart.svelte

@@ -0,0 +1,35 @@
+<script lang="ts">
+	import * as Chart from '$lib/components/ui/chart/index.js';
+	import { PieChart } from 'layerchart';
+	import type { IngestionSourceStats } from '@open-archiver/types';
+	import type { ChartConfig } from '$lib/components/ui/chart';
+
+	export let data: IngestionSourceStats[];
+
+	const chartConfig = {
+		storageUsed: {
+			label: 'Storage Used'
+		}
+	} satisfies ChartConfig;
+</script>
+
+<Chart.Container config={chartConfig} class="h-full min-h-[300px] w-full">
+	<PieChart
+		{data}
+		key="name"
+		value="storageUsed"
+		label="name"
+		legend={{}}
+		cRange={[
+			'var(--color-chart-1)',
+			'var(--color-chart-2)',
+			'var(--color-chart-3)',
+			'var(--color-chart-4)',
+			'var(--color-chart-5)'
+		]}
+	>
+		{#snippet tooltip()}
+			<Chart.Tooltip></Chart.Tooltip>
+		{/snippet}
+	</PieChart>
+</Chart.Container>

packages/frontend/src/lib/components/custom/charts/TopSendersChart.svelte

@@ -0,0 +1,44 @@
+<script lang="ts">
+	import * as Chart from '$lib/components/ui/chart/index.js';
+	import { BarChart } from 'layerchart';
+	import type { TopSender } from '@open-archiver/types';
+	import type { ChartConfig } from '$lib/components/ui/chart';
+
+	export let data: TopSender[];
+
+	const chartConfig = {
+		count: {
+			label: 'Emails'
+		}
+	} satisfies ChartConfig;
+</script>
+
+<Chart.Container config={chartConfig} class="min-h-[300px] w-full">
+	<BarChart
+		{data}
+		x="count"
+		y="sender"
+		orientation="horizontal"
+		xDomain={[0, Math.max(...data.map((d) => d.count)) * 1.1]}
+		axis={'x'}
+		legend={false}
+		series={[
+			{
+				key: 'count',
+				...chartConfig.count
+			}
+		]}
+		cRange={[
+			'var(--color-chart-1)',
+			'var(--color-chart-2)',
+			'var(--color-chart-3)',
+			'var(--color-chart-4)',
+			'var(--color-chart-5)'
+		]}
+		labels={{}}
+	>
+		{#snippet tooltip()}
+			<Chart.Tooltip />
+		{/snippet}
+	</BarChart>
+</Chart.Container>

packages/frontend/src/lib/components/ui/alert/alert-description.svelte

@@ -0,0 +1,23 @@
+<script lang="ts">
+	import type { HTMLAttributes } from "svelte/elements";
+	import { cn, type WithElementRef } from "$lib/utils.js";
+
+	let {
+		ref = $bindable(null),
+		class: className,
+		children,
+		...restProps
+	}: WithElementRef<HTMLAttributes<HTMLDivElement>> = $props();
+</script>
+
+<div
+	bind:this={ref}
+	data-slot="alert-description"
+	class={cn(
+		"text-muted-foreground col-start-2 grid justify-items-start gap-1 text-sm [&_p]:leading-relaxed",
+		className
+	)}
+	{...restProps}
+>
+	{@render children?.()}
+</div>