Project wide format

This commit is contained in:
Wayne
2025-08-15 14:18:23 +03:00
parent 9873228d01
commit b2ca3ef0e1
163 changed files with 17955 additions and 18307 deletions

View File

@@ -41,7 +41,6 @@ Password: openarchiver_demo
## ✨ Key Features
- **Universal Ingestion**: Connect to any email provider to perform initial bulk imports and maintain continuous, real-time synchronization. Ingestion sources include:
- IMAP connection
- Google Workspace
- Microsoft 365

View File

@@ -7,24 +7,24 @@ export default defineConfig({
{
defer: '',
src: 'https://analytics.zenceipt.com/script.js',
'data-website-id': '2c8b452e-eab5-4f82-8ead-902d8f8b976f'
}
]
'data-website-id': '2c8b452e-eab5-4f82-8ead-902d8f8b976f',
},
],
],
title: 'Open Archiver',
description: 'Official documentation for the Open Archiver project.',
themeConfig: {
search: {
provider: 'local'
provider: 'local',
},
logo: {
src: '/logo-sq.svg'
src: '/logo-sq.svg',
},
nav: [
{ text: 'Home', link: '/' },
{ text: 'Github', link: 'https://github.com/LogicLabs-OU/OpenArchiver' },
{ text: "Website", link: 'https://openarchiver.com/' },
{ text: "Discord", link: 'https://discord.gg/MTtD7BhuTQ' }
{ text: 'Website', link: 'https://openarchiver.com/' },
{ text: 'Discord', link: 'https://discord.gg/MTtD7BhuTQ' },
],
sidebar: [
{
@@ -37,14 +37,23 @@ export default defineConfig({
link: '/user-guides/email-providers/',
collapsed: true,
items: [
{ text: 'Generic IMAP Server', link: '/user-guides/email-providers/imap' },
{ text: 'Google Workspace', link: '/user-guides/email-providers/google-workspace' },
{ text: 'Microsoft 365', link: '/user-guides/email-providers/microsoft-365' },
{
text: 'Generic IMAP Server',
link: '/user-guides/email-providers/imap',
},
{
text: 'Google Workspace',
link: '/user-guides/email-providers/google-workspace',
},
{
text: 'Microsoft 365',
link: '/user-guides/email-providers/microsoft-365',
},
{ text: 'EML Import', link: '/user-guides/email-providers/eml' },
{ text: 'PST Import', link: '/user-guides/email-providers/pst' }
]
}
]
{ text: 'PST Import', link: '/user-guides/email-providers/pst' },
],
},
],
},
{
text: 'API Reference',
@@ -56,16 +65,16 @@ export default defineConfig({
{ text: 'Dashboard', link: '/api/dashboard' },
{ text: 'Ingestion', link: '/api/ingestion' },
{ text: 'Search', link: '/api/search' },
{ text: 'Storage', link: '/api/storage' }
]
{ text: 'Storage', link: '/api/storage' },
],
},
{
text: 'Services',
items: [
{ text: 'Overview', link: '/services/' },
{ text: 'Storage Service', link: '/services/storage-service' }
]
}
]
}
{ text: 'Storage Service', link: '/services/storage-service' },
],
},
],
},
});

View File

@@ -38,9 +38,7 @@ Retrieves a paginated list of archived emails for a specific ingestion source.
"from": "sender@example.com",
"sentAt": "2023-10-27T10:00:00.000Z",
"hasAttachments": true,
"recipients": [
{ "name": "Recipient 1", "email": "recipient1@example.com" }
]
"recipients": [{ "name": "Recipient 1", "email": "recipient1@example.com" }]
}
],
"total": 100,
@@ -74,9 +72,7 @@ Retrieves a single archived email by its ID, including its raw content and attac
"from": "sender@example.com",
"sentAt": "2023-10-27T10:00:00.000Z",
"hasAttachments": true,
"recipients": [
{ "name": "Recipient 1", "email": "recipient1@example.com" }
],
"recipients": [{ "name": "Recipient 1", "email": "recipient1@example.com" }],
"raw": "...",
"attachments": [
{

View File

@@ -79,13 +79,7 @@ interface UpdateIngestionSourceDto {
name?: string;
provider?: 'google' | 'microsoft' | 'generic_imap';
providerConfig?: IngestionCredentials;
status?:
| 'pending_auth'
| 'auth_success'
| 'importing'
| 'active'
| 'paused'
| 'error';
status?: 'pending_auth' | 'auth_success' | 'importing' | 'active' | 'paused' | 'error';
}
```

View File

@@ -1,2 +1 @@
# services

View File

@@ -69,11 +69,7 @@ class IngestionService {
this.storageService = new StorageService();
}
public async archiveEmail(
rawEmail: Buffer,
userId: string,
messageId: string
): Promise<void> {
public async archiveEmail(rawEmail: Buffer, userId: string, messageId: string): Promise<void> {
// Define a structured, unique path for the email.
const archivePath = `${userId}/messages/${messageId}.eml`;

View File

@@ -24,19 +24,16 @@ The setup process involves three main parts:
In this part, you will create a service account and enable the APIs it needs to function.
1. **Create a Google Cloud Project:**
- Go to the [Google Cloud Console](https://console.cloud.google.com/).
- If you don't already have one, create a new project for the archiving service (e.g., "Email Archiver").
2. **Enable Required APIs:**
- In your selected project, navigate to the **"APIs & Services" > "Library"** section.
- Search for and enable the following two APIs:
- **Gmail API**
- **Admin SDK API**
3. **Create a Service Account:**
- Navigate to **"IAM & Admin" > "Service Accounts"**.
- Click **"Create Service Account"**.
- Give the service account a name (e.g., `email-archiver-service`) and a description.
@@ -77,13 +74,11 @@ To resolve this, you must have **Organization Administrator** permissions.
Now, you will authorize the service account you created to access data from your Google Workspace.
1. **Get the Service Account's Client ID:**
- Go back to the list of service accounts in the Google Cloud Console.
- Click on the service account you created.
- Under the **"Details"** tab, find and copy the **Unique ID** (this is the Client ID).
2. **Authorize the Client in Google Workspace:**
- Go to your **Google Workspace Admin Console** at [admin.google.com](https://admin.google.com).
- Navigate to **Security > Access and data control > API controls**.
- Under the "Domain-wide Delegation" section, click **"Manage Domain-wide Delegation"**.
@@ -112,7 +107,6 @@ Finally, you will provide the generated credentials to the application.
Click the **"Create New"** button.
3. **Fill in the Configuration Details:**
- **Name:** Give the source a name (e.g., "Google Workspace Archive").
- **Provider:** Select **"Google Workspace"** from the dropdown.
- **Service Account Key (JSON):** Open the JSON file you downloaded in Part 1. Copy the entire content of the file and paste it into this text area.

View File

@@ -12,7 +12,6 @@ This guide will walk you through connecting a standard IMAP email account as an
3. **Fill in the Configuration Details:**
You will see a form with several fields. Here is how to fill them out for an IMAP connection:
- **Name:** Give your ingestion source a descriptive name that you will easily recognize, such as "Work Email (IMAP)" or "Personal Gmail".
- **Provider:** From the dropdown menu, select **"Generic IMAP"**. This will reveal the specific fields required for an IMAP connection.

View File

@@ -75,7 +75,6 @@ You now have the three pieces of information required to configure the connectio
Click the **"Create New"** button.
3. **Fill in the Configuration Details:**
- **Name:** Give the source a name (e.g., "Microsoft 365 Archive").
- **Provider:** Select **"Microsoft 365"** from the dropdown.
- **Application (Client) ID:** Go to the **Overview** page of your app registration in the Entra admin center and copy this value.

View File

@@ -174,7 +174,6 @@ To do this, you will need to make a small modification to your `docker-compose.y
2. **Remove all `networks` sections** from the file. This includes the network configuration for each service and the top-level network definition.
Specifically, you need to remove:
- The `networks: - open-archiver-net` lines from the `open-archiver`, `postgres`, `valkey`, and `meilisearch` services.
- The entire `networks:` block at the end of the file.

View File

@@ -37,9 +37,7 @@ export class ArchivedEmailController {
public deleteArchivedEmail = async (req: Request, res: Response): Promise<Response> => {
if (config.app.isDemo) {
return res
.status(403)
.json({ message: 'This operation is not allowed in demo mode.' });
return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
}
try {
const { id } = req.params;

View File

@@ -6,7 +6,6 @@ import * as schema from '../../database/schema';
import { sql } from 'drizzle-orm';
import 'dotenv/config';
export class AuthController {
#authService: AuthService;
#userService: UserService;
@@ -29,14 +28,19 @@ export class AuthController {
}
try {
const userCountResult = await db.select({ count: sql<number>`count(*)` }).from(schema.users);
const userCountResult = await db
.select({ count: sql<number>`count(*)` })
.from(schema.users);
const userCount = Number(userCountResult[0].count);
if (userCount > 0) {
return res.status(403).json({ message: 'Setup has already been completed.' });
}
const newUser = await this.#userService.createAdminUser({ email, password, first_name, last_name }, true);
const newUser = await this.#userService.createAdminUser(
{ email, password, first_name, last_name },
true
);
const result = await this.#authService.login(email, password);
return res.status(201).json(result);
} catch (error) {
@@ -68,20 +72,22 @@ export class AuthController {
public status = async (req: Request, res: Response): Promise<Response> => {
try {
const userCountResult = await db.select({ count: sql<number>`count(*)` }).from(schema.users);
const userCountResult = await db
.select({ count: sql<number>`count(*)` })
.from(schema.users);
const userCount = Number(userCountResult[0].count);
const needsSetup = userCount === 0;
// in case user uses older version with admin user variables, we will create the admin user using those variables.
if (needsSetup && process.env.ADMIN_EMAIL && process.env.ADMIN_PASSWORD) {
await this.#userService.createAdminUser({
await this.#userService.createAdminUser(
{
email: process.env.ADMIN_EMAIL,
password: process.env.ADMIN_PASSWORD,
first_name: "Admin",
last_name: "User"
}, true);
first_name: 'Admin',
last_name: 'User',
},
true
);
return res.status(200).json({ needsSetup: false });
}
return res.status(200).json({ needsSetup });

View File

@@ -4,7 +4,7 @@ import {
CreateIngestionSourceDto,
UpdateIngestionSourceDto,
IngestionSource,
SafeIngestionSource
SafeIngestionSource,
} from '@open-archiver/types';
import { logger } from '../../config/logger';
import { config } from '../../config';
@@ -33,7 +33,13 @@ export class IngestionController {
} catch (error: any) {
logger.error({ err: error }, 'Create ingestion source error');
// Return a 400 Bad Request for connection errors
return res.status(400).json({ message: error.message || 'Failed to create ingestion source due to a connection error.' });
return res
.status(400)
.json({
message:
error.message ||
'Failed to create ingestion source due to a connection error.',
});
}
};

View File

@@ -22,7 +22,7 @@ export class SearchController {
query: keywords as string,
page: page ? parseInt(page as string) : 1,
limit: limit ? parseInt(limit as string) : 10,
matchingStrategy: matchingStrategy as MatchingStrategies
matchingStrategy: matchingStrategy as MatchingStrategies,
});
res.status(200).json(results);

View File

@@ -4,7 +4,7 @@ import * as path from 'path';
import { storage as storageConfig } from '../../config/storage';
export class StorageController {
constructor(private storageService: StorageService) { }
constructor(private storageService: StorageService) {}
public downloadFile = async (req: Request, res: Response): Promise<void> => {
const unsafePath = req.query.path as string;

View File

@@ -4,7 +4,6 @@ import { randomUUID } from 'crypto';
import busboy from 'busboy';
import { config } from '../../config/index';
export const uploadFile = async (req: Request, res: Response) => {
const storage = new StorageService();
const bb = busboy({ headers: req.headers });

View File

@@ -33,7 +33,9 @@ export const requireAuth = (authService: AuthService) => {
next();
} catch (error) {
console.error('Authentication error:', error);
return res.status(500).json({ message: 'An internal server error occurred during authentication' });
return res
.status(500)
.json({ message: 'An internal server error occurred during authentication' });
}
};
};

View File

@@ -3,7 +3,4 @@ import { ingestionQueue } from '../../jobs/queues';
const router: Router = Router();
export default router;

View File

@@ -5,5 +5,5 @@ export const app = {
port: process.env.PORT_BACKEND ? parseInt(process.env.PORT_BACKEND, 10) : 4000,
encryptionKey: process.env.ENCRYPTION_KEY,
isDemo: process.env.IS_DEMO === 'true',
syncFrequency: process.env.SYNC_FREQUENCY || '* * * * *' //default to 1 minute
syncFrequency: process.env.SYNC_FREQUENCY || '* * * * *', //default to 1 minute
};

View File

@@ -5,7 +5,7 @@ export const logger = pino({
transport: {
target: 'pino-pretty',
options: {
colorize: true
}
}
colorize: true,
},
},
});

View File

@@ -12,7 +12,7 @@ const connectionOptions: any = {
if (process.env.REDIS_TLS_ENABLED === 'true') {
connectionOptions.tls = {
rejectUnauthorized: false
rejectUnauthorized: false,
};
}

View File

@@ -12,7 +12,7 @@ if (storageType === 'local') {
storageConfig = {
type: 'local',
rootPath: process.env.STORAGE_LOCAL_ROOT_PATH,
openArchiverFolderName: openArchiverFolderName
openArchiverFolderName: openArchiverFolderName,
};
} else if (storageType === 's3') {
if (
@@ -31,7 +31,7 @@ if (storageType === 'local') {
secretAccessKey: process.env.STORAGE_S3_SECRET_ACCESS_KEY,
region: process.env.STORAGE_S3_REGION,
forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true',
openArchiverFolderName: openArchiverFolderName
openArchiverFolderName: openArchiverFolderName,
};
} else {
throw new Error(`Invalid STORAGE_TYPE: ${storageType}`);

View File

@@ -110,12 +110,8 @@
"name": "archived_emails_custodian_id_custodians_id_fk",
"tableFrom": "archived_emails",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -175,9 +171,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -207,12 +201,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -220,12 +210,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -233,10 +219,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -365,9 +348,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -442,12 +423,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -519,12 +496,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -532,12 +505,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -625,9 +594,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -686,9 +653,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -779,30 +744,17 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap"
]
"values": ["google_workspace", "microsoft_365", "generic_imap"]
},
"public.ingestion_status": {
"name": "ingestion_status",
"schema": "public",
"values": [
"active",
"paused",
"error",
"pending_auth",
"syncing"
]
"values": ["active", "paused", "error", "pending_auth", "syncing"]
}
},
"schemas": {},

View File

@@ -110,12 +110,8 @@
"name": "archived_emails_custodian_id_custodians_id_fk",
"tableFrom": "archived_emails",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -175,9 +171,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -207,12 +201,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -220,12 +210,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -233,10 +219,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -365,9 +348,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -442,12 +423,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -519,12 +496,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -532,12 +505,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -625,9 +594,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -686,9 +653,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -779,31 +744,17 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap"
]
"values": ["google_workspace", "microsoft_365", "generic_imap"]
},
"public.ingestion_status": {
"name": "ingestion_status",
"schema": "public",
"values": [
"active",
"paused",
"error",
"pending_auth",
"syncing",
"auth_success"
]
"values": ["active", "paused", "error", "pending_auth", "syncing", "auth_success"]
}
},
"schemas": {},

View File

@@ -110,12 +110,8 @@
"name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
"tableFrom": "archived_emails",
"tableTo": "ingestion_sources",
"columnsFrom": [
"ingestion_source_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["ingestion_source_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -175,9 +171,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -207,12 +201,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -220,12 +210,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -233,10 +219,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -365,9 +348,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -442,12 +423,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -519,12 +496,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -532,12 +505,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -625,9 +594,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -686,9 +653,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -779,31 +744,17 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap"
]
"values": ["google_workspace", "microsoft_365", "generic_imap"]
},
"public.ingestion_status": {
"name": "ingestion_status",
"schema": "public",
"values": [
"active",
"paused",
"error",
"pending_auth",
"syncing",
"auth_success"
]
"values": ["active", "paused", "error", "pending_auth", "syncing", "auth_success"]
}
},
"schemas": {},

View File

@@ -110,12 +110,8 @@
"name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
"tableFrom": "archived_emails",
"tableTo": "ingestion_sources",
"columnsFrom": [
"ingestion_source_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["ingestion_source_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -175,9 +171,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -207,12 +201,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -220,12 +210,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -233,10 +219,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -365,9 +348,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -442,12 +423,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -519,12 +496,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -532,12 +505,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -625,9 +594,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -686,9 +653,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -779,31 +744,17 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap"
]
"values": ["google_workspace", "microsoft_365", "generic_imap"]
},
"public.ingestion_status": {
"name": "ingestion_status",
"schema": "public",
"values": [
"active",
"paused",
"error",
"pending_auth",
"syncing",
"auth_success"
]
"values": ["active", "paused", "error", "pending_auth", "syncing", "auth_success"]
}
},
"schemas": {},

View File

@@ -110,12 +110,8 @@
"name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
"tableFrom": "archived_emails",
"tableTo": "ingestion_sources",
"columnsFrom": [
"ingestion_source_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["ingestion_source_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -175,9 +171,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -207,12 +201,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -220,12 +210,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -233,10 +219,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -365,9 +348,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -442,12 +423,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -519,12 +496,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -532,12 +505,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -625,9 +594,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -686,9 +653,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -779,19 +744,12 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap"
]
"values": ["google_workspace", "microsoft_365", "generic_imap"]
},
"public.ingestion_status": {
"name": "ingestion_status",

View File

@@ -110,12 +110,8 @@
"name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
"tableFrom": "archived_emails",
"tableTo": "ingestion_sources",
"columnsFrom": [
"ingestion_source_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["ingestion_source_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -175,9 +171,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -207,12 +201,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -220,12 +210,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -233,10 +219,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -365,9 +348,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -442,12 +423,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -519,12 +496,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -532,12 +505,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -625,9 +594,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -686,9 +653,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -779,19 +744,12 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap"
]
"values": ["google_workspace", "microsoft_365", "generic_imap"]
},
"public.ingestion_status": {
"name": "ingestion_status",

View File

@@ -110,12 +110,8 @@
"name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
"tableFrom": "archived_emails",
"tableTo": "ingestion_sources",
"columnsFrom": [
"ingestion_source_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["ingestion_source_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -175,9 +171,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -207,12 +201,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -220,12 +210,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -233,10 +219,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -365,9 +348,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -442,12 +423,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -519,12 +496,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -532,12 +505,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -625,9 +594,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -686,9 +653,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -785,19 +750,12 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap"
]
"values": ["google_workspace", "microsoft_365", "generic_imap"]
},
"public.ingestion_status": {
"name": "ingestion_status",

View File

@@ -110,12 +110,8 @@
"name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
"tableFrom": "archived_emails",
"tableTo": "ingestion_sources",
"columnsFrom": [
"ingestion_source_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["ingestion_source_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -175,9 +171,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -207,12 +201,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -220,12 +210,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -233,10 +219,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -365,9 +348,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -442,12 +423,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -519,12 +496,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -532,12 +505,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -625,9 +594,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -686,9 +653,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -785,19 +750,12 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap"
]
"values": ["google_workspace", "microsoft_365", "generic_imap"]
},
"public.ingestion_status": {
"name": "ingestion_status",

View File

@@ -138,12 +138,8 @@
"name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
"tableFrom": "archived_emails",
"tableTo": "ingestion_sources",
"columnsFrom": [
"ingestion_source_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["ingestion_source_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -203,9 +199,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -235,12 +229,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -248,12 +238,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -261,10 +247,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -393,9 +376,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -470,12 +451,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -547,12 +524,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -560,12 +533,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -653,9 +622,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -714,9 +681,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -813,19 +778,12 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap"
]
"values": ["google_workspace", "microsoft_365", "generic_imap"]
},
"public.ingestion_status": {
"name": "ingestion_status",

View File

@@ -138,12 +138,8 @@
"name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
"tableFrom": "archived_emails",
"tableTo": "ingestion_sources",
"columnsFrom": [
"ingestion_source_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["ingestion_source_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -203,9 +199,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -235,12 +229,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -248,12 +238,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -261,10 +247,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -393,9 +376,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -470,12 +451,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -547,12 +524,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -560,12 +533,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -653,9 +622,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -714,9 +681,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -854,9 +819,7 @@
"roles_name_unique": {
"name": "roles_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -892,12 +855,8 @@
"name": "sessions_user_id_users_id_fk",
"tableFrom": "sessions",
"tableTo": "users",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["user_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -931,12 +890,8 @@
"name": "user_roles_user_id_users_id_fk",
"tableFrom": "user_roles",
"tableTo": "users",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["user_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -944,12 +899,8 @@
"name": "user_roles_role_id_roles_id_fk",
"tableFrom": "user_roles",
"tableTo": "roles",
"columnsFrom": [
"role_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["role_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -957,10 +908,7 @@
"compositePrimaryKeys": {
"user_roles_user_id_role_id_pk": {
"name": "user_roles_user_id_role_id_pk",
"columns": [
"user_id",
"role_id"
]
"columns": ["user_id", "role_id"]
}
},
"uniqueConstraints": {},
@@ -1032,9 +980,7 @@
"users_email_unique": {
"name": "users_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -1046,19 +992,12 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap"
]
"values": ["google_workspace", "microsoft_365", "generic_imap"]
},
"public.ingestion_status": {
"name": "ingestion_status",

View File

@@ -138,12 +138,8 @@
"name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
"tableFrom": "archived_emails",
"tableTo": "ingestion_sources",
"columnsFrom": [
"ingestion_source_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["ingestion_source_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -203,9 +199,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -235,12 +229,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -248,12 +238,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -261,10 +247,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -393,9 +376,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -470,12 +451,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -547,12 +524,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -560,12 +533,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -653,9 +622,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -714,9 +681,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -854,9 +819,7 @@
"roles_name_unique": {
"name": "roles_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -892,12 +855,8 @@
"name": "sessions_user_id_users_id_fk",
"tableFrom": "sessions",
"tableTo": "users",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["user_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -931,12 +890,8 @@
"name": "user_roles_user_id_users_id_fk",
"tableFrom": "user_roles",
"tableTo": "users",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["user_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -944,12 +899,8 @@
"name": "user_roles_role_id_roles_id_fk",
"tableFrom": "user_roles",
"tableTo": "roles",
"columnsFrom": [
"role_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["role_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -957,10 +908,7 @@
"compositePrimaryKeys": {
"user_roles_user_id_role_id_pk": {
"name": "user_roles_user_id_role_id_pk",
"columns": [
"user_id",
"role_id"
]
"columns": ["user_id", "role_id"]
}
},
"uniqueConstraints": {},
@@ -1038,9 +986,7 @@
"users_email_unique": {
"name": "users_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -1052,19 +998,12 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap"
]
"values": ["google_workspace", "microsoft_365", "generic_imap"]
},
"public.ingestion_status": {
"name": "ingestion_status",

View File

@@ -138,12 +138,8 @@
"name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
"tableFrom": "archived_emails",
"tableTo": "ingestion_sources",
"columnsFrom": [
"ingestion_source_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["ingestion_source_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -203,9 +199,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -235,12 +229,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -248,12 +238,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -261,10 +247,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -393,9 +376,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -470,12 +451,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -547,12 +524,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -560,12 +533,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -653,9 +622,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -714,9 +681,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -854,9 +819,7 @@
"roles_name_unique": {
"name": "roles_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -892,12 +855,8 @@
"name": "sessions_user_id_users_id_fk",
"tableFrom": "sessions",
"tableTo": "users",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["user_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -931,12 +890,8 @@
"name": "user_roles_user_id_users_id_fk",
"tableFrom": "user_roles",
"tableTo": "users",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["user_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -944,12 +899,8 @@
"name": "user_roles_role_id_roles_id_fk",
"tableFrom": "user_roles",
"tableTo": "roles",
"columnsFrom": [
"role_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["role_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -957,10 +908,7 @@
"compositePrimaryKeys": {
"user_roles_user_id_role_id_pk": {
"name": "user_roles_user_id_role_id_pk",
"columns": [
"user_id",
"role_id"
]
"columns": ["user_id", "role_id"]
}
},
"uniqueConstraints": {},
@@ -1038,9 +986,7 @@
"users_email_unique": {
"name": "users_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -1052,20 +998,12 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap",
"pst_import"
]
"values": ["google_workspace", "microsoft_365", "generic_imap", "pst_import"]
},
"public.ingestion_status": {
"name": "ingestion_status",

View File

@@ -150,12 +150,8 @@
"name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
"tableFrom": "archived_emails",
"tableTo": "ingestion_sources",
"columnsFrom": [
"ingestion_source_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["ingestion_source_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -215,9 +211,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -247,12 +241,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -260,12 +250,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -273,10 +259,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -405,9 +388,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -482,12 +463,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -559,12 +536,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -572,12 +545,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -665,9 +634,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -726,9 +693,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -866,9 +831,7 @@
"roles_name_unique": {
"name": "roles_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -904,12 +867,8 @@
"name": "sessions_user_id_users_id_fk",
"tableFrom": "sessions",
"tableTo": "users",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["user_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -943,12 +902,8 @@
"name": "user_roles_user_id_users_id_fk",
"tableFrom": "user_roles",
"tableTo": "users",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["user_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -956,12 +911,8 @@
"name": "user_roles_role_id_roles_id_fk",
"tableFrom": "user_roles",
"tableTo": "roles",
"columnsFrom": [
"role_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["role_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -969,10 +920,7 @@
"compositePrimaryKeys": {
"user_roles_user_id_role_id_pk": {
"name": "user_roles_user_id_role_id_pk",
"columns": [
"user_id",
"role_id"
]
"columns": ["user_id", "role_id"]
}
},
"uniqueConstraints": {},
@@ -1050,9 +998,7 @@
"users_email_unique": {
"name": "users_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -1064,20 +1010,12 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",
"schema": "public",
"values": [
"google_workspace",
"microsoft_365",
"generic_imap",
"pst_import"
]
"values": ["google_workspace", "microsoft_365", "generic_imap", "pst_import"]
},
"public.ingestion_status": {
"name": "ingestion_status",

View File

@@ -150,12 +150,8 @@
"name": "archived_emails_ingestion_source_id_ingestion_sources_id_fk",
"tableFrom": "archived_emails",
"tableTo": "ingestion_sources",
"columnsFrom": [
"ingestion_source_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["ingestion_source_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -215,9 +211,7 @@
"attachments_content_hash_sha256_unique": {
"name": "attachments_content_hash_sha256_unique",
"nullsNotDistinct": false,
"columns": [
"content_hash_sha256"
]
"columns": ["content_hash_sha256"]
}
},
"policies": {},
@@ -247,12 +241,8 @@
"name": "email_attachments_email_id_archived_emails_id_fk",
"tableFrom": "email_attachments",
"tableTo": "archived_emails",
"columnsFrom": [
"email_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["email_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -260,12 +250,8 @@
"name": "email_attachments_attachment_id_attachments_id_fk",
"tableFrom": "email_attachments",
"tableTo": "attachments",
"columnsFrom": [
"attachment_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["attachment_id"],
"columnsTo": ["id"],
"onDelete": "restrict",
"onUpdate": "no action"
}
@@ -273,10 +259,7 @@
"compositePrimaryKeys": {
"email_attachments_email_id_attachment_id_pk": {
"name": "email_attachments_email_id_attachment_id_pk",
"columns": [
"email_id",
"attachment_id"
]
"columns": ["email_id", "attachment_id"]
}
},
"uniqueConstraints": {},
@@ -405,9 +388,7 @@
"ediscovery_cases_name_unique": {
"name": "ediscovery_cases_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -482,12 +463,8 @@
"name": "export_jobs_case_id_ediscovery_cases_id_fk",
"tableFrom": "export_jobs",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "set null",
"onUpdate": "no action"
}
@@ -559,12 +536,8 @@
"name": "legal_holds_case_id_ediscovery_cases_id_fk",
"tableFrom": "legal_holds",
"tableTo": "ediscovery_cases",
"columnsFrom": [
"case_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["case_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -572,12 +545,8 @@
"name": "legal_holds_custodian_id_custodians_id_fk",
"tableFrom": "legal_holds",
"tableTo": "custodians",
"columnsFrom": [
"custodian_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["custodian_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -665,9 +634,7 @@
"retention_policies_name_unique": {
"name": "retention_policies_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -726,9 +693,7 @@
"custodians_email_unique": {
"name": "custodians_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -866,9 +831,7 @@
"roles_name_unique": {
"name": "roles_name_unique",
"nullsNotDistinct": false,
"columns": [
"name"
]
"columns": ["name"]
}
},
"policies": {},
@@ -904,12 +867,8 @@
"name": "sessions_user_id_users_id_fk",
"tableFrom": "sessions",
"tableTo": "users",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["user_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -943,12 +902,8 @@
"name": "user_roles_user_id_users_id_fk",
"tableFrom": "user_roles",
"tableTo": "users",
"columnsFrom": [
"user_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["user_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
},
@@ -956,12 +911,8 @@
"name": "user_roles_role_id_roles_id_fk",
"tableFrom": "user_roles",
"tableTo": "roles",
"columnsFrom": [
"role_id"
],
"columnsTo": [
"id"
],
"columnsFrom": ["role_id"],
"columnsTo": ["id"],
"onDelete": "cascade",
"onUpdate": "no action"
}
@@ -969,10 +920,7 @@
"compositePrimaryKeys": {
"user_roles_user_id_role_id_pk": {
"name": "user_roles_user_id_role_id_pk",
"columns": [
"user_id",
"role_id"
]
"columns": ["user_id", "role_id"]
}
},
"uniqueConstraints": {},
@@ -1050,9 +998,7 @@
"users_email_unique": {
"name": "users_email_unique",
"nullsNotDistinct": false,
"columns": [
"email"
]
"columns": ["email"]
}
},
"policies": {},
@@ -1064,10 +1010,7 @@
"public.retention_action": {
"name": "retention_action",
"schema": "public",
"values": [
"delete_permanently",
"notify_admin"
]
"values": ["delete_permanently", "notify_admin"]
},
"public.ingestion_provider": {
"name": "ingestion_provider",

View File

@@ -33,6 +33,6 @@ export const archivedEmails = pgTable(
export const archivedEmailsRelations = relations(archivedEmails, ({ one }) => ({
ingestionSource: one(ingestionSources, {
fields: [archivedEmails.ingestionSourceId],
references: [ingestionSources.id]
})
references: [ingestionSources.id],
}),
}));

View File

@@ -11,12 +11,20 @@ export const attachments = pgTable('attachments', {
storagePath: text('storage_path').notNull(),
});
export const emailAttachments = pgTable('email_attachments', {
emailId: uuid('email_id').notNull().references(() => archivedEmails.id, { onDelete: 'cascade' }),
attachmentId: uuid('attachment_id').notNull().references(() => attachments.id, { onDelete: 'restrict' }),
}, (t) => ({
export const emailAttachments = pgTable(
'email_attachments',
{
emailId: uuid('email_id')
.notNull()
.references(() => archivedEmails.id, { onDelete: 'cascade' }),
attachmentId: uuid('attachment_id')
.notNull()
.references(() => attachments.id, { onDelete: 'restrict' }),
},
(t) => ({
pk: primaryKey({ columns: [t.emailId, t.attachmentId] }),
}));
})
);
export const attachmentsRelations = relations(attachments, ({ many }) => ({
emailAttachments: many(emailAttachments),

View File

@@ -1,10 +1,22 @@
import { relations } from 'drizzle-orm';
import { boolean, integer, jsonb, pgEnum, pgTable, text, timestamp, uuid } from 'drizzle-orm/pg-core';
import {
boolean,
integer,
jsonb,
pgEnum,
pgTable,
text,
timestamp,
uuid,
} from 'drizzle-orm/pg-core';
import { custodians } from './custodians';
// --- Enums ---
export const retentionActionEnum = pgEnum('retention_action', ['delete_permanently', 'notify_admin']);
export const retentionActionEnum = pgEnum('retention_action', [
'delete_permanently',
'notify_admin',
]);
// --- Tables ---
@@ -33,7 +45,9 @@ export const ediscoveryCases = pgTable('ediscovery_cases', {
export const legalHolds = pgTable('legal_holds', {
id: uuid('id').primaryKey().defaultRandom(),
caseId: uuid('case_id').notNull().references(() => ediscoveryCases.id, { onDelete: 'cascade' }),
caseId: uuid('case_id')
.notNull()
.references(() => ediscoveryCases.id, { onDelete: 'cascade' }),
custodianId: uuid('custodian_id').references(() => custodians.id, { onDelete: 'cascade' }),
holdCriteria: jsonb('hold_criteria'),
reason: text('reason'),

View File

@@ -7,5 +7,5 @@ export const custodians = pgTable('custodians', {
displayName: text('display_name'),
sourceType: ingestionProviderEnum('source_type').notNull(),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow()
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
});

View File

@@ -5,7 +5,7 @@ export const ingestionProviderEnum = pgEnum('ingestion_provider', [
'microsoft_365',
'generic_imap',
'pst_import',
'eml_import'
'eml_import',
]);
export const ingestionStatusEnum = pgEnum('ingestion_status', [
@@ -16,7 +16,7 @@ export const ingestionStatusEnum = pgEnum('ingestion_status', [
'syncing',
'importing',
'auth_success',
'imported'
'imported',
]);
export const ingestionSources = pgTable('ingestion_sources', {
@@ -30,5 +30,5 @@ export const ingestionSources = pgTable('ingestion_sources', {
lastSyncStatusMessage: text('last_sync_status_message'),
syncState: jsonb('sync_state'),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow()
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
});

View File

@@ -1,12 +1,5 @@
import { relations, sql } from 'drizzle-orm';
import {
pgTable,
text,
timestamp,
uuid,
primaryKey,
jsonb
} from 'drizzle-orm/pg-core';
import { pgTable, text, timestamp, uuid, primaryKey, jsonb } from 'drizzle-orm/pg-core';
import type { PolicyStatement } from '@open-archiver/types';
/**
@@ -21,7 +14,7 @@ export const users = pgTable('users', {
provider: text('provider').default('local'),
providerId: text('provider_id'),
createdAt: timestamp('created_at').defaultNow().notNull(),
updatedAt: timestamp('updated_at').defaultNow().notNull()
updatedAt: timestamp('updated_at').defaultNow().notNull(),
});
/**
@@ -35,8 +28,8 @@ export const sessions = pgTable('sessions', {
.references(() => users.id, { onDelete: 'cascade' }),
expiresAt: timestamp('expires_at', {
withTimezone: true,
mode: 'date'
}).notNull()
mode: 'date',
}).notNull(),
});
/**
@@ -46,9 +39,12 @@ export const sessions = pgTable('sessions', {
export const roles = pgTable('roles', {
id: uuid('id').primaryKey().defaultRandom(),
name: text('name').notNull().unique(),
policies: jsonb('policies').$type<PolicyStatement[]>().notNull().default(sql`'[]'::jsonb`),
policies: jsonb('policies')
.$type<PolicyStatement[]>()
.notNull()
.default(sql`'[]'::jsonb`),
createdAt: timestamp('created_at').defaultNow().notNull(),
updatedAt: timestamp('updated_at').defaultNow().notNull()
updatedAt: timestamp('updated_at').defaultNow().notNull(),
});
/**
@@ -63,27 +59,27 @@ export const userRoles = pgTable(
.references(() => users.id, { onDelete: 'cascade' }),
roleId: uuid('role_id')
.notNull()
.references(() => roles.id, { onDelete: 'cascade' })
.references(() => roles.id, { onDelete: 'cascade' }),
},
(t) => [primaryKey({ columns: [t.userId, t.roleId] })]
);
// Define relationships for Drizzle ORM
export const usersRelations = relations(users, ({ many }) => ({
userRoles: many(userRoles)
userRoles: many(userRoles),
}));
export const rolesRelations = relations(roles, ({ many }) => ({
userRoles: many(userRoles)
userRoles: many(userRoles),
}));
export const userRolesRelations = relations(userRoles, ({ one }) => ({
role: one(roles, {
fields: [userRoles.roleId],
references: [roles.id]
references: [roles.id],
}),
user: one(users, {
fields: [userRoles.userId],
references: [users.id]
})
references: [users.id],
}),
}));

View File

@@ -6,7 +6,7 @@ export const encodeDatabaseUrl = (databaseUrl: string): string => {
}
return url.toString();
} catch (error) {
console.error("Invalid DATABASE_URL, please check your .env file.", error);
throw new Error("Invalid DATABASE_URL");
console.error('Invalid DATABASE_URL, please check your .env file.', error);
throw new Error('Invalid DATABASE_URL');
}
};

View File

@@ -15,9 +15,7 @@ function extractTextFromPdf(buffer: Buffer): Promise<string> {
};
pdfParser.on('pdfParser_dataError', () => finish(''));
pdfParser.on('pdfParser_dataReady', () =>
finish(pdfParser.getRawTextContent())
);
pdfParser.on('pdfParser_dataReady', () => finish(pdfParser.getRawTextContent()));
try {
pdfParser.parseBuffer(buffer);
@@ -31,27 +29,20 @@ function extractTextFromPdf(buffer: Buffer): Promise<string> {
});
}
export async function extractText(
buffer: Buffer,
mimeType: string
): Promise<string> {
export async function extractText(buffer: Buffer, mimeType: string): Promise<string> {
try {
if (mimeType === 'application/pdf') {
return await extractTextFromPdf(buffer);
}
if (
mimeType ===
'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
mimeType === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
) {
const { value } = await mammoth.extractRawText({ buffer });
return value;
}
if (
mimeType ===
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
) {
if (mimeType === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') {
const workbook = xlsx.read(buffer, { type: 'buffer' });
let fullText = '';
for (const sheetName of workbook.SheetNames) {
@@ -70,10 +61,7 @@ export async function extractText(
return buffer.toString('utf-8');
}
} catch (error) {
console.error(
`Error extracting text from attachment with MIME type ${mimeType}:`,
error
);
console.error(`Error extracting text from attachment with MIME type ${mimeType}:`, error);
return ''; // Return empty string on failure
}

View File

@@ -33,7 +33,6 @@ const ARCHIVE_RESOURCES = {
CUSTODIAN: 'archive/custodian/*',
} as const;
// ===================================================================================
// SERVICE: ingestion
// ===================================================================================
@@ -51,7 +50,6 @@ const INGESTION_RESOURCES = {
SOURCE: 'ingestion-source/{sourceId}',
} as const;
// ===================================================================================
// SERVICE: system
// ===================================================================================
@@ -72,7 +70,6 @@ const SYSTEM_RESOURCES = {
USER: 'system/user/{userId}',
} as const;
// ===================================================================================
// SERVICE: dashboard
// ===================================================================================
@@ -85,7 +82,6 @@ const DASHBOARD_RESOURCES = {
ALL: 'dashboard/*',
} as const;
// ===================================================================================
// EXPORTED DEFINITIONS
// ===================================================================================

View File

@@ -18,7 +18,7 @@ export class PolicyValidator {
* @returns {{valid: boolean; reason?: string}} - An object containing a boolean `valid` property
* and an optional `reason` string if validation fails.
*/
public static isValid(statement: PolicyStatement): { valid: boolean; reason: string; } {
public static isValid(statement: PolicyStatement): { valid: boolean; reason: string } {
if (!statement || !statement.Action || !statement.Resource || !statement.Effect) {
return { valid: false, reason: 'Policy statement is missing required fields.' };
}
@@ -54,7 +54,7 @@ export class PolicyValidator {
* @param {string} action - The action string to validate.
* @returns {{valid: boolean; reason?: string}} - An object indicating validity and a reason for failure.
*/
private static isActionValid(action: string): { valid: boolean; reason: string; } {
private static isActionValid(action: string): { valid: boolean; reason: string } {
if (action === '*') {
return { valid: true, reason: 'valid' };
}
@@ -63,7 +63,10 @@ export class PolicyValidator {
if (service in ValidResourcePatterns) {
return { valid: true, reason: 'valid' };
}
return { valid: false, reason: `Invalid service '${service}' in action wildcard '${action}'.` };
return {
valid: false,
reason: `Invalid service '${service}' in action wildcard '${action}'.`,
};
}
if (ValidActions.has(action)) {
return { valid: true, reason: 'valid' };
@@ -83,7 +86,7 @@ export class PolicyValidator {
* @param {string} resource - The resource string to validate.
* @returns {{valid: boolean; reason?: string}} - An object indicating validity and a reason for failure.
*/
private static isResourceValid(resource: string): { valid: boolean; reason: string; } {
private static isResourceValid(resource: string): { valid: boolean; reason: string } {
const service = resource.split('/')[0];
if (service === '*') {
return { valid: true, reason: 'valid' };
@@ -93,7 +96,10 @@ export class PolicyValidator {
if (pattern.test(resource)) {
return { valid: true, reason: 'valid' };
}
return { valid: false, reason: `Resource '${resource}' does not match the expected format for the '${service}' service.` };
return {
valid: false,
reason: `Resource '${resource}' does not match the expected format for the '${service}' service.`,
};
}
return { valid: false, reason: `Invalid service '${service}' in resource '${resource}'.` };
}

View File

@@ -22,21 +22,16 @@ import { IamService } from './services/IamService';
import { StorageService } from './services/StorageService';
import { SearchService } from './services/SearchService';
// Load environment variables
dotenv.config();
// --- Environment Variable Validation ---
const {
PORT_BACKEND,
JWT_SECRET,
JWT_EXPIRES_IN
} = process.env;
const { PORT_BACKEND, JWT_SECRET, JWT_EXPIRES_IN } = process.env;
if (!PORT_BACKEND || !JWT_SECRET || !JWT_EXPIRES_IN) {
throw new Error('Missing required environment variables for the backend: PORT_BACKEND, JWT_SECRET, JWT_EXPIRES_IN.');
throw new Error(
'Missing required environment variables for the backend: PORT_BACKEND, JWT_SECRET, JWT_EXPIRES_IN.'
);
}
// --- Dependency Injection Setup ---
@@ -85,11 +80,11 @@ app.use('/v1/test', testRouter);
app.get('/v1/protected', requireAuth(authService), (req, res) => {
res.json({
message: 'You have accessed a protected route!',
user: req.user // The user payload is attached by the requireAuth middleware
user: req.user, // The user payload is attached by the requireAuth middleware
});
});
app.get("/", (req, res) => {
app.get('/', (req, res) => {
res.send('Backend is running!');
});

View File

@@ -11,7 +11,10 @@ export default async (job: Job<IContinuousSyncJob>) => {
const source = await IngestionService.findById(ingestionSourceId);
if (!source || !['error', 'active'].includes(source.status)) {
logger.warn({ ingestionSourceId, status: source?.status }, 'Skipping continuous sync for non-active or non-error source.');
logger.warn(
{ ingestionSourceId, status: source?.status },
'Skipping continuous sync for non-active or non-error source.'
);
return;
}
@@ -31,17 +34,17 @@ export default async (job: Job<IContinuousSyncJob>) => {
queueName: 'ingestion',
data: {
ingestionSourceId: source.id,
userEmail: user.primaryEmail
userEmail: user.primaryEmail,
},
opts: {
removeOnComplete: {
age: 60 * 10 // 10 minutes
age: 60 * 10, // 10 minutes
},
removeOnFail: {
age: 60 * 30 // 30 minutes
age: 60 * 30, // 30 minutes
},
timeout: 1000 * 60 * 30, // 30 minutes
},
timeout: 1000 * 60 * 30 // 30 minutes
}
});
}
}
@@ -53,26 +56,29 @@ export default async (job: Job<IContinuousSyncJob>) => {
queueName: 'ingestion',
data: {
ingestionSourceId,
isInitialImport: false
isInitialImport: false,
},
children: jobs,
opts: {
removeOnComplete: true,
removeOnFail: true
}
removeOnFail: true,
},
});
}
// The status will be set back to 'active' by the 'sync-cycle-finished' job
// once all the mailboxes have been processed.
logger.info({ ingestionSourceId }, 'Continuous sync job finished dispatching mailbox jobs.');
logger.info(
{ ingestionSourceId },
'Continuous sync job finished dispatching mailbox jobs.'
);
} catch (error) {
logger.error({ err: error, ingestionSourceId }, 'Continuous sync job failed.');
await IngestionService.update(ingestionSourceId, {
status: 'error',
lastSyncFinishedAt: new Date(),
lastSyncStatusMessage: error instanceof Error ? error.message : 'An unknown error occurred during sync.',
lastSyncStatusMessage:
error instanceof Error ? error.message : 'An unknown error occurred during sync.',
});
throw error;
}

View File

@@ -9,7 +9,7 @@ const storageService = new StorageService();
const databaseService = new DatabaseService();
const indexingService = new IndexingService(databaseService, searchService, storageService);
export default async function (job: Job<{ emailId: string; }>) {
export default async function (job: Job<{ emailId: string }>) {
const { emailId } = job.data;
console.log(`Indexing email with ID: ${emailId}`);
await indexingService.indexEmailById(emailId);

View File

@@ -5,7 +5,6 @@ import { EmailProviderFactory } from '../../services/EmailProviderFactory';
import { flowProducer } from '../queues';
import { logger } from '../../config/logger';
export default async (job: Job<IInitialImportJob>) => {
const { ingestionSourceId } = job.data;
logger.info({ ingestionSourceId }, 'Starting initial import master job');
@@ -18,7 +17,7 @@ export default async (job: Job<IInitialImportJob>) => {
await IngestionService.update(ingestionSourceId, {
status: 'importing',
lastSyncStatusMessage: 'Starting initial import...'
lastSyncStatusMessage: 'Starting initial import...',
});
const connector = EmailProviderFactory.createConnector(source);
@@ -37,43 +36,48 @@ export default async (job: Job<IInitialImportJob>) => {
},
opts: {
removeOnComplete: {
age: 60 * 10 // 10 minutes
age: 60 * 10, // 10 minutes
},
removeOnFail: {
age: 60 * 30 // 30 minutes
age: 60 * 30, // 30 minutes
},
attempts: 1,
// failParentOnFailure: true
}
},
});
userCount++;
}
}
if (jobs.length > 0) {
logger.info({ ingestionSourceId, userCount }, 'Adding sync-cycle-finished job to the queue');
logger.info(
{ ingestionSourceId, userCount },
'Adding sync-cycle-finished job to the queue'
);
await flowProducer.add({
name: 'sync-cycle-finished',
queueName: 'ingestion',
data: {
ingestionSourceId,
userCount,
isInitialImport: true
isInitialImport: true,
},
children: jobs,
opts: {
removeOnComplete: true,
removeOnFail: true
}
removeOnFail: true,
},
});
} else {
const fileBasedIngestions = IngestionService.returnFileBasedIngestions();
const finalStatus = fileBasedIngestions.includes(source.provider) ? 'imported' : 'active';
const finalStatus = fileBasedIngestions.includes(source.provider)
? 'imported'
: 'active';
// If there are no users, we can consider the import finished and set to active
await IngestionService.update(ingestionSourceId, {
status: finalStatus,
lastSyncFinishedAt: new Date(),
lastSyncStatusMessage: 'Initial import complete. No users found.'
lastSyncStatusMessage: 'Initial import complete. No users found.',
});
}
@@ -82,7 +86,7 @@ export default async (job: Job<IInitialImportJob>) => {
logger.error({ err: error, ingestionSourceId }, 'Error in initial import master job');
await IngestionService.update(ingestionSourceId, {
status: 'error',
lastSyncStatusMessage: `Initial import failed: ${error instanceof Error ? error.message : 'Unknown error'}`
lastSyncStatusMessage: `Initial import failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
});
throw error;
}

View File

@@ -46,7 +46,7 @@ export const processMailboxProcessor = async (job: Job<IProcessMailboxJob, SyncS
const errorMessage = error instanceof Error ? error.message : 'An unknown error occurred';
const processMailboxError: ProcessMailboxError = {
error: true,
message: `Failed to process mailbox for ${userEmail}: ${errorMessage}`
message: `Failed to process mailbox for ${userEmail}: ${errorMessage}`,
};
return processMailboxError;
}

View File

@@ -5,19 +5,12 @@ import { or, eq } from 'drizzle-orm';
import { ingestionQueue } from '../queues';
export default async (job: Job) => {
console.log(
'Scheduler running: Looking for active or error ingestion sources to sync.'
);
console.log('Scheduler running: Looking for active or error ingestion sources to sync.');
// find all sources that have the status of active or error for continuous syncing.
const sourcesToSync = await db
.select({ id: ingestionSources.id })
.from(ingestionSources)
.where(
or(
eq(ingestionSources.status, 'active'),
eq(ingestionSources.status, 'error')
)
);
.where(or(eq(ingestionSources.status, 'active'), eq(ingestionSources.status, 'error')));
for (const source of sourcesToSync) {
// The status field on the ingestion source is used to prevent duplicate syncs.

View File

@@ -1,7 +1,12 @@
import { Job } from 'bullmq';
import { IngestionService } from '../../services/IngestionService';
import { logger } from '../../config/logger';
import { SyncState, ProcessMailboxError, IngestionStatus, IngestionProvider } from '@open-archiver/types';
import {
SyncState,
ProcessMailboxError,
IngestionStatus,
IngestionProvider,
} from '@open-archiver/types';
import { db } from '../../database';
import { ingestionSources } from '../../database/schema';
import { eq } from 'drizzle-orm';
@@ -29,17 +34,24 @@ interface ISyncCycleFinishedJob {
*/
export default async (job: Job<ISyncCycleFinishedJob, any, string>) => {
const { ingestionSourceId, userCount, isInitialImport } = job.data;
logger.info({ ingestionSourceId, userCount, isInitialImport }, 'Sync cycle finished job started');
logger.info(
{ ingestionSourceId, userCount, isInitialImport },
'Sync cycle finished job started'
);
try {
const childrenValues = await job.getChildrenValues<SyncState | ProcessMailboxError>();
const allChildJobs = Object.values(childrenValues);
// if data has error property, it is a failed job
const failedJobs = allChildJobs.filter(v => v && (v as any).error) as ProcessMailboxError[];
const failedJobs = allChildJobs.filter(
(v) => v && (v as any).error
) as ProcessMailboxError[];
// if data doesn't have error property, it is a successful job with SyncState
const successfulJobs = allChildJobs.filter(v => !v || !(v as any).error) as SyncState[];
const successfulJobs = allChildJobs.filter((v) => !v || !(v as any).error) as SyncState[];
const finalSyncState = deepmerge(...successfulJobs.filter(s => s && Object.keys(s).length > 0));
const finalSyncState = deepmerge(
...successfulJobs.filter((s) => s && Object.keys(s).length > 0)
);
const source = await IngestionService.findById(ingestionSourceId);
let status: IngestionStatus = 'active';
@@ -51,18 +63,20 @@ export default async (job: Job<ISyncCycleFinishedJob, any, string>) => {
let message: string;
// Check for a specific rate-limit message from the successful jobs
const rateLimitMessage = successfulJobs.find(j => j.statusMessage)?.statusMessage;
const rateLimitMessage = successfulJobs.find((j) => j.statusMessage)?.statusMessage;
if (failedJobs.length > 0) {
status = 'error';
const errorMessages = failedJobs.map(j => j.message).join('\n');
const errorMessages = failedJobs.map((j) => j.message).join('\n');
message = `Sync cycle completed with ${failedJobs.length} error(s):\n${errorMessages}`;
logger.error({ ingestionSourceId, errors: errorMessages }, 'Sync cycle finished with errors.');
logger.error(
{ ingestionSourceId, errors: errorMessages },
'Sync cycle finished with errors.'
);
} else if (rateLimitMessage) {
message = rateLimitMessage;
logger.warn({ ingestionSourceId, message }, 'Sync cycle paused due to rate limiting.');
}
else {
} else {
message = 'Continuous sync cycle finished successfully.';
if (isInitialImport) {
message = `Initial import finished for ${userCount} mailboxes.`;
@@ -76,15 +90,18 @@ export default async (job: Job<ISyncCycleFinishedJob, any, string>) => {
status,
lastSyncFinishedAt: new Date(),
lastSyncStatusMessage: message,
syncState: finalSyncState
syncState: finalSyncState,
})
.where(eq(ingestionSources.id, ingestionSourceId));
} catch (error) {
logger.error({ err: error, ingestionSourceId }, 'An unexpected error occurred while finalizing the sync cycle.');
logger.error(
{ err: error, ingestionSourceId },
'An unexpected error occurred while finalizing the sync cycle.'
);
await IngestionService.update(ingestionSourceId, {
status: 'error',
lastSyncFinishedAt: new Date(),
lastSyncStatusMessage: 'An unexpected error occurred while finalizing the sync cycle.'
lastSyncStatusMessage: 'An unexpected error occurred while finalizing the sync cycle.',
});
}
};

View File

@@ -8,22 +8,22 @@ const defaultJobOptions = {
attempts: 5,
backoff: {
type: 'exponential',
delay: 1000
delay: 1000,
},
removeOnComplete: {
count: 1000
count: 1000,
},
removeOnFail: {
count: 5000
}
count: 5000,
},
};
export const ingestionQueue = new Queue('ingestion', {
connection,
defaultJobOptions
defaultJobOptions,
});
export const indexingQueue = new Queue('indexing', {
connection,
defaultJobOptions
defaultJobOptions,
});

View File

@@ -9,7 +9,7 @@ const scheduleContinuousSync = async () => {
{},
{
repeat: {
pattern: config.app.syncFrequency
pattern: config.app.syncFrequency,
},
}
);

View File

@@ -12,13 +12,11 @@ import { SearchService } from './SearchService';
import type { Readable } from 'stream';
interface DbRecipients {
to: { name: string; address: string; }[];
cc: { name: string; address: string; }[];
bcc: { name: string; address: string; }[];
to: { name: string; address: string }[];
cc: { name: string; address: string }[];
bcc: { name: string; address: string }[];
}
async function streamToBuffer(stream: Readable): Promise<Buffer> {
return new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
@@ -36,7 +34,7 @@ export class ArchivedEmailService {
return allRecipients.map((r) => ({
name: r.name,
email: r.address
email: r.address,
}));
}
@@ -49,7 +47,7 @@ export class ArchivedEmailService {
const [total] = await db
.select({
count: count(archivedEmails.id)
count: count(archivedEmails.id),
})
.from(archivedEmails)
.where(eq(archivedEmails.ingestionSourceId, ingestionSourceId));
@@ -67,11 +65,11 @@ export class ArchivedEmailService {
...item,
recipients: this.mapRecipients(item.recipients),
tags: (item.tags as string[] | null) || null,
path: item.path || null
path: item.path || null,
})),
total: total.count,
page,
limit
limit,
};
}
@@ -113,7 +111,7 @@ export class ArchivedEmailService {
raw,
thread: threadEmails,
tags: (email.tags as string[] | null) || null,
path: email.path || null
path: email.path || null,
};
if (email.hasAttachments) {
@@ -123,7 +121,7 @@ export class ArchivedEmailService {
filename: attachments.filename,
mimeType: attachments.mimeType,
sizeBytes: attachments.sizeBytes,
storagePath: attachments.storagePath
storagePath: attachments.storagePath,
})
.from(emailAttachments)
.innerJoin(attachments, eq(emailAttachments.attachmentId, attachments.id))
@@ -139,7 +137,7 @@ export class ArchivedEmailService {
return {
...mappedEmail,
attachments: emailAttachmentsResult
attachments: emailAttachmentsResult,
};
}

View File

@@ -49,11 +49,11 @@ export class AuthService {
const userRoles = await db.query.userRoles.findMany({
where: eq(schema.userRoles.userId, user.id),
with: {
role: true
}
role: true,
},
});
const roles = userRoles.map(ur => ur.role.name);
const roles = userRoles.map((ur) => ur.role.name);
const { password: _, ...userWithoutPassword } = user;

View File

@@ -34,7 +34,10 @@ export class CryptoService {
const data = Buffer.from(encrypted, 'hex');
const salt = data.subarray(0, SALT_LENGTH);
const iv = data.subarray(SALT_LENGTH, SALT_LENGTH + IV_LENGTH);
const tag = data.subarray(SALT_LENGTH + IV_LENGTH, SALT_LENGTH + IV_LENGTH + TAG_LENGTH);
const tag = data.subarray(
SALT_LENGTH + IV_LENGTH,
SALT_LENGTH + IV_LENGTH + TAG_LENGTH
);
const encryptedValue = data.subarray(SALT_LENGTH + IV_LENGTH + TAG_LENGTH);
const key = getKey(salt);

View File

@@ -36,7 +36,7 @@ class DashboardService {
return {
totalEmailsArchived: totalEmailsArchived[0].count,
totalStorageUsed: totalStorageUsed[0].sum || 0,
failedIngestionsLast7Days: failedIngestionsLast7Days[0].count
failedIngestionsLast7Days: failedIngestionsLast7Days[0].count,
};
}
@@ -47,7 +47,7 @@ class DashboardService {
const history = await this.#db
.select({
date: sql<string>`date_trunc('day', ${archivedEmails.archivedAt})`,
count: count()
count: count(),
})
.from(archivedEmails)
.where(gte(archivedEmails.archivedAt, thirtyDaysAgo))
@@ -64,7 +64,7 @@ class DashboardService {
name: ingestionSources.name,
provider: ingestionSources.provider,
status: ingestionSources.status,
storageUsed: sql<number>`sum(${archivedEmails.sizeBytes})`.mapWith(Number)
storageUsed: sql<number>`sum(${archivedEmails.sizeBytes})`.mapWith(Number),
})
.from(ingestionSources)
.leftJoin(archivedEmails, eq(ingestionSources.id, archivedEmails.ingestionSourceId))
@@ -81,7 +81,7 @@ class DashboardService {
public async getIndexedInsights(): Promise<IndexedInsights> {
const topSenders = await this.#searchService.getTopSenders(10);
return {
topSenders
topSenders,
};
}
}

View File

@@ -3,4 +3,3 @@ import { db } from '../database';
export class DatabaseService {
public db = db;
}

View File

@@ -7,7 +7,7 @@ import type {
EMLImportCredentials,
EmailObject,
SyncState,
MailboxUser
MailboxUser,
} from '@open-archiver/types';
import { GoogleWorkspaceConnector } from './ingestion-connectors/GoogleWorkspaceConnector';
import { MicrosoftConnector } from './ingestion-connectors/MicrosoftConnector';
@@ -18,7 +18,10 @@ import { EMLConnector } from './ingestion-connectors/EMLConnector';
// Define a common interface for all connectors
export interface IEmailConnector {
testConnection(): Promise<boolean>;
fetchEmails(userEmail: string, syncState?: SyncState | null): AsyncGenerator<EmailObject | null>;
fetchEmails(
userEmail: string,
syncState?: SyncState | null
): AsyncGenerator<EmailObject | null>;
getUpdatedSyncState(userEmail?: string): SyncState;
listAllUsers(): AsyncGenerator<MailboxUser>;
returnImapUserEmail?(): string;

View File

@@ -9,14 +9,14 @@ import { streamToBuffer } from '../helpers/streamToBuffer';
import { simpleParser } from 'mailparser';
interface DbRecipients {
to: { name: string; address: string; }[];
cc: { name: string; address: string; }[];
bcc: { name: string; address: string; }[];
to: { name: string; address: string }[];
cc: { name: string; address: string }[];
bcc: { name: string; address: string }[];
}
type AttachmentsType = {
filename: string,
buffer: Buffer,
filename: string;
buffer: Buffer;
mimeType: string;
}[];
@@ -31,7 +31,7 @@ export class IndexingService {
constructor(
dbService: DatabaseService,
searchService: SearchService,
storageService: StorageService,
storageService: StorageService
) {
this.dbService = dbService;
this.searchService = searchService;
@@ -73,18 +73,27 @@ export class IndexingService {
/**
* Indexes an email object directly, creates a search document, and indexes it.
*/
public async indexByEmail(email: EmailObject, ingestionSourceId: string, archivedEmailId: string): Promise<void> {
public async indexByEmail(
email: EmailObject,
ingestionSourceId: string,
archivedEmailId: string
): Promise<void> {
const attachments: AttachmentsType = [];
if (email.attachments && email.attachments.length > 0) {
for (const attachment of email.attachments) {
attachments.push({
buffer: attachment.content,
filename: attachment.filename,
mimeType: attachment.contentType
mimeType: attachment.contentType,
});
}
}
const document = await this.createEmailDocumentFromRaw(email, attachments, ingestionSourceId, archivedEmailId);
const document = await this.createEmailDocumentFromRaw(
email,
attachments,
ingestionSourceId,
archivedEmailId
);
await this.searchService.addDocuments('emails', [document], 'id');
}
@@ -100,10 +109,7 @@ export class IndexingService {
const extractedAttachments = [];
for (const attachment of attachments) {
try {
const textContent = await extractText(
attachment.buffer,
attachment.mimeType || ''
);
const textContent = await extractText(attachment.buffer, attachment.mimeType || '');
extractedAttachments.push({
filename: attachment.filename,
content: textContent,
@@ -126,11 +132,10 @@ export class IndexingService {
body: email.body || email.html || '',
attachments: extractedAttachments,
timestamp: new Date(email.receivedAt).getTime(),
ingestionSourceId: ingestionSourceId
ingestionSourceId: ingestionSourceId,
};
}
/**
* Creates a search document from a database email record and its attachments.
*/
@@ -161,7 +166,7 @@ export class IndexingService {
body: emailBodyText,
attachments: attachmentContents,
timestamp: new Date(email.sentAt).getTime(),
ingestionSourceId: email.ingestionSourceId
ingestionSourceId: email.ingestionSourceId,
};
}
@@ -170,18 +175,13 @@ export class IndexingService {
*/
private async extractAttachmentContents(
attachments: Attachment[]
): Promise<{ filename: string; content: string; }[]> {
): Promise<{ filename: string; content: string }[]> {
const extractedAttachments = [];
for (const attachment of attachments) {
try {
const fileStream = await this.storageService.get(
attachment.storagePath
);
const fileStream = await this.storageService.get(attachment.storagePath);
const fileBuffer = await streamToBuffer(fileStream);
const textContent = await extractText(
fileBuffer,
attachment.mimeType || ''
);
const textContent = await extractText(fileBuffer, attachment.mimeType || '');
extractedAttachments.push({
filename: attachment.filename,
content: textContent,
@@ -196,5 +196,4 @@ export class IndexingService {
}
return extractedAttachments;
}
}

View File

@@ -5,7 +5,7 @@ import type {
UpdateIngestionSourceDto,
IngestionSource,
IngestionCredentials,
IngestionProvider
IngestionProvider,
} from '@open-archiver/types';
import { and, desc, eq } from 'drizzle-orm';
import { CryptoService } from './CryptoService';
@@ -14,7 +14,11 @@ import { ingestionQueue } from '../jobs/queues';
import type { JobType } from 'bullmq';
import { StorageService } from './StorageService';
import type { IInitialImportJob, EmailObject } from '@open-archiver/types';
import { archivedEmails, attachments as attachmentsSchema, emailAttachments } from '../database/schema';
import {
archivedEmails,
attachments as attachmentsSchema,
emailAttachments,
} from '../database/schema';
import { createHash } from 'crypto';
import { logger } from '../config/logger';
import { IndexingService } from './IndexingService';
@@ -22,15 +26,19 @@ import { SearchService } from './SearchService';
import { DatabaseService } from './DatabaseService';
import { config } from '../config/index';
export class IngestionService {
private static decryptSource(source: typeof ingestionSources.$inferSelect): IngestionSource | null {
private static decryptSource(
source: typeof ingestionSources.$inferSelect
): IngestionSource | null {
const decryptedCredentials = CryptoService.decryptObject<IngestionCredentials>(
source.credentials as string
);
if (!decryptedCredentials) {
logger.error({ sourceId: source.id }, 'Failed to decrypt ingestion source credentials.');
logger.error(
{ sourceId: source.id },
'Failed to decrypt ingestion source credentials.'
);
return null;
}
@@ -48,7 +56,7 @@ export class IngestionService {
const valuesToInsert = {
...rest,
status: 'pending_auth' as const,
credentials: encryptedCredentials
credentials: encryptedCredentials,
};
const [newSource] = await db.insert(ingestionSources).values(valuesToInsert).returning();
@@ -56,7 +64,9 @@ export class IngestionService {
const decryptedSource = this.decryptSource(newSource);
if (!decryptedSource) {
await this.delete(newSource.id);
throw new Error('Failed to process newly created ingestion source due to a decryption error.');
throw new Error(
'Failed to process newly created ingestion source due to a decryption error.'
);
}
const connector = EmailProviderFactory.createConnector(decryptedSource);
@@ -72,15 +82,21 @@ export class IngestionService {
}
public static async findAll(): Promise<IngestionSource[]> {
const sources = await db.select().from(ingestionSources).orderBy(desc(ingestionSources.createdAt));
return sources.flatMap(source => {
const sources = await db
.select()
.from(ingestionSources)
.orderBy(desc(ingestionSources.createdAt));
return sources.flatMap((source) => {
const decrypted = this.decryptSource(source);
return decrypted ? [decrypted] : [];
});
}
public static async findById(id: string): Promise<IngestionSource> {
const [source] = await db.select().from(ingestionSources).where(eq(ingestionSources.id, id));
const [source] = await db
.select()
.from(ingestionSources)
.where(eq(ingestionSources.id, id));
if (!source) {
throw new Error('Ingestion source not found');
}
@@ -119,14 +135,13 @@ export class IngestionService {
const decryptedSource = this.decryptSource(updatedSource);
if (!decryptedSource) {
throw new Error('Failed to process updated ingestion source due to a decryption error.');
throw new Error(
'Failed to process updated ingestion source due to a decryption error.'
);
}
// If the status has changed to auth_success, trigger the initial import
if (
originalSource.status !== 'auth_success' &&
decryptedSource.status === 'auth_success'
) {
if (originalSource.status !== 'auth_success' && decryptedSource.status === 'auth_success') {
await this.triggerInitialImport(decryptedSource.id);
}
@@ -145,7 +160,8 @@ export class IngestionService {
await storage.delete(emailPath);
if (
(source.credentials.type === 'pst_import' || source.credentials.type === 'eml_import') &&
(source.credentials.type === 'pst_import' ||
source.credentials.type === 'eml_import') &&
source.credentials.uploadedFilePath &&
(await storage.exists(source.credentials.uploadedFilePath))
) {
@@ -170,7 +186,10 @@ export class IngestionService {
// Even if decryption fails, we should confirm deletion.
// We might return a simpler object or just a success message.
// For now, we'll indicate the issue but still confirm deletion happened.
logger.warn({ sourceId: deletedSource.id }, 'Could not decrypt credentials of deleted source, but deletion was successful.');
logger.warn(
{ sourceId: deletedSource.id },
'Could not decrypt credentials of deleted source, but deletion was successful.'
);
return { ...deletedSource, credentials: null } as unknown as IngestionSource;
}
return decryptedSource;
@@ -180,7 +199,6 @@ export class IngestionService {
const source = await this.findById(id);
await ingestionQueue.add('initial-import', { ingestionSourceId: source.id });
}
public static async triggerForceSync(id: string): Promise<void> {
@@ -197,7 +215,10 @@ export class IngestionService {
if (job.data.ingestionSourceId === id) {
try {
await job.remove();
logger.info({ jobId: job.id, ingestionSourceId: id }, 'Removed stale job during force sync.');
logger.info(
{ jobId: job.id, ingestionSourceId: id },
'Removed stale job during force sync.'
);
} catch (error) {
logger.error({ err: error, jobId: job.id }, 'Failed to remove stale job.');
}
@@ -205,8 +226,10 @@ export class IngestionService {
}
// Reset status to 'active'
await this.update(id, { status: 'active', lastSyncStatusMessage: 'Force sync triggered by user.' });
await this.update(id, {
status: 'active',
lastSyncStatusMessage: 'Force sync triggered by user.',
});
await ingestionQueue.add('continuous-sync', { ingestionSourceId: source.id });
}
@@ -221,7 +244,7 @@ export class IngestionService {
logger.info(`Starting bulk import for source: ${source.name} (${source.id})`);
await IngestionService.update(ingestionSourceId, {
status: 'importing',
lastSyncStartedAt: new Date()
lastSyncStartedAt: new Date(),
});
const connector = EmailProviderFactory.createConnector(source);
@@ -242,7 +265,10 @@ export class IngestionService {
// For single-mailbox providers, dispatch a single job
await ingestionQueue.add('process-mailbox', {
ingestionSourceId: source.id,
userEmail: source.credentials.type === 'generic_imap' ? source.credentials.username : 'Default'
userEmail:
source.credentials.type === 'generic_imap'
? source.credentials.username
: 'Default',
});
}
} catch (error) {
@@ -250,7 +276,8 @@ export class IngestionService {
await IngestionService.update(ingestionSourceId, {
status: 'error',
lastSyncFinishedAt: new Date(),
lastSyncStatusMessage: error instanceof Error ? error.message : 'An unknown error occurred.'
lastSyncStatusMessage:
error instanceof Error ? error.message : 'An unknown error occurred.',
});
throw error; // Re-throw to allow BullMQ to handle the job failure
}
@@ -273,18 +300,23 @@ export class IngestionService {
messageId = messageIdHeader;
}
if (!messageId) {
messageId = `generated-${createHash('sha256').update(email.eml ?? Buffer.from(email.body, 'utf-8')).digest('hex')}-${source.id}-${email.id}`;
messageId = `generated-${createHash('sha256')
.update(email.eml ?? Buffer.from(email.body, 'utf-8'))
.digest('hex')}-${source.id}-${email.id}`;
}
// Check if an email with the same message ID has already been imported for the current ingestion source. This is to prevent duplicate imports when an email is present in multiple mailboxes (e.g., "Inbox" and "All Mail").
const existingEmail = await db.query.archivedEmails.findFirst({
where: and(
eq(archivedEmails.messageIdHeader, messageId),
eq(archivedEmails.ingestionSourceId, source.id)
)
),
});
if (existingEmail) {
logger.info({ messageId, ingestionSourceId: source.id }, 'Skipping duplicate email');
logger.info(
{ messageId, ingestionSourceId: source.id },
'Skipping duplicate email'
);
return;
}
@@ -308,21 +340,23 @@ export class IngestionService {
recipients: {
to: email.to,
cc: email.cc,
bcc: email.bcc
bcc: email.bcc,
},
storagePath: emailPath,
storageHashSha256: emailHash,
sizeBytes: emlBuffer.length,
hasAttachments: email.attachments.length > 0,
path: email.path,
tags: email.tags
tags: email.tags,
})
.returning();
if (email.attachments.length > 0) {
for (const attachment of email.attachments) {
const attachmentBuffer = attachment.content;
const attachmentHash = createHash('sha256').update(attachmentBuffer).digest('hex');
const attachmentHash = createHash('sha256')
.update(attachmentBuffer)
.digest('hex');
const attachmentPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/attachments/${attachment.filename}`;
await storage.put(attachmentPath, attachmentBuffer);
@@ -333,11 +367,11 @@ export class IngestionService {
mimeType: attachment.contentType,
sizeBytes: attachment.size,
contentHashSha256: attachmentHash,
storagePath: attachmentPath
storagePath: attachmentPath,
})
.onConflictDoUpdate({
target: attachmentsSchema.contentHashSha256,
set: { filename: attachment.filename }
set: { filename: attachment.filename },
})
.returning();
@@ -345,7 +379,7 @@ export class IngestionService {
.insert(emailAttachments)
.values({
emailId: archivedEmail.id,
attachmentId: newAttachment.id
attachmentId: newAttachment.id,
})
.onConflictDoNothing();
}
@@ -359,14 +393,18 @@ export class IngestionService {
const searchService = new SearchService();
const storageService = new StorageService();
const databaseService = new DatabaseService();
const indexingService = new IndexingService(databaseService, searchService, storageService);
const indexingService = new IndexingService(
databaseService,
searchService,
storageService
);
await indexingService.indexByEmail(email, source.id, archivedEmail.id);
} catch (error) {
logger.error({
message: `Failed to process email ${email.id} for source ${source.id}`,
error,
emailId: email.id,
ingestionSourceId: source.id
ingestionSourceId: source.id,
});
}
}

View File

@@ -28,7 +28,11 @@ export class SearchService {
return index.addDocuments(documents);
}
public async search<T extends Record<string, any>>(indexName: string, query: string, options?: any) {
public async search<T extends Record<string, any>>(
indexName: string,
query: string,
options?: any
) {
const index = await this.getIndex<T>(indexName);
return index.search(query, options);
}
@@ -53,7 +57,7 @@ export class SearchService {
attributesToHighlight: ['*'],
showMatchesPosition: true,
sort: ['timestamp:desc'],
matchingStrategy
matchingStrategy,
};
if (filters) {
@@ -73,8 +77,10 @@ export class SearchService {
total: searchResults.estimatedTotalHits ?? searchResults.hits.length,
page,
limit,
totalPages: Math.ceil((searchResults.estimatedTotalHits ?? searchResults.hits.length) / limit),
processingTimeMs: searchResults.processingTimeMs
totalPages: Math.ceil(
(searchResults.estimatedTotalHits ?? searchResults.hits.length) / limit
),
processingTimeMs: searchResults.processingTimeMs,
};
}
@@ -82,7 +88,7 @@ export class SearchService {
const index = await this.getIndex<EmailDocument>('emails');
const searchResults = await index.search('', {
facets: ['from'],
limit: 0
limit: 0,
});
if (!searchResults.facetDistribution?.from) {
@@ -112,7 +118,7 @@ export class SearchService {
'attachments.content',
],
filterableAttributes: ['from', 'to', 'cc', 'bcc', 'timestamp', 'ingestionSourceId'],
sortableAttributes: ['timestamp']
sortableAttributes: ['timestamp'],
});
}
}

View File

@@ -11,9 +11,9 @@ export class UserService {
* @param email The email address of the user to find.
* @returns The user object if found, otherwise null.
*/
public async findByEmail(email: string): Promise<(typeof schema.users.$inferSelect) | null> {
public async findByEmail(email: string): Promise<typeof schema.users.$inferSelect | null> {
const user = await db.query.users.findFirst({
where: eq(schema.users.email, email)
where: eq(schema.users.email, email),
});
return user || null;
}
@@ -23,9 +23,9 @@ export class UserService {
* @param id The ID of the user to find.
* @returns The user object if found, otherwise null.
*/
public async findById(id: string): Promise<(typeof schema.users.$inferSelect) | null> {
public async findById(id: string): Promise<typeof schema.users.$inferSelect | null> {
const user = await db.query.users.findFirst({
where: eq(schema.users.id, id)
where: eq(schema.users.id, id),
});
return user || null;
}
@@ -39,48 +39,62 @@ export class UserService {
* @param isSetup Is this an initial setup?
* @returns The newly created user object.
*/
public async createAdminUser(userDetails: Pick<User, 'email' | 'first_name' | 'last_name'> & { password?: string; }, isSetup: boolean): Promise<(typeof schema.users.$inferSelect)> {
public async createAdminUser(
userDetails: Pick<User, 'email' | 'first_name' | 'last_name'> & { password?: string },
isSetup: boolean
): Promise<typeof schema.users.$inferSelect> {
if (!isSetup) {
throw Error('This operation is only allowed upon initial setup.');
}
const { email, first_name, last_name, password } = userDetails;
const userCountResult = await db.select({ count: sql<number>`count(*)` }).from(schema.users);
const userCountResult = await db
.select({ count: sql<number>`count(*)` })
.from(schema.users);
const isFirstUser = Number(userCountResult[0].count) === 0;
if (!isFirstUser) {
throw Error('This operation is only allowed upon initial setup.');
}
const hashedPassword = password ? await hash(password, 10) : undefined;
const newUser = await db.insert(schema.users).values({
const newUser = await db
.insert(schema.users)
.values({
email,
first_name,
last_name,
password: hashedPassword,
}).returning();
})
.returning();
// find super admin role
let superAdminRole = await db.query.roles.findFirst({
where: eq(schema.roles.name, 'Super Admin')
where: eq(schema.roles.name, 'Super Admin'),
});
if (!superAdminRole) {
const suerAdminPolicies: PolicyStatement[] = [{
const suerAdminPolicies: PolicyStatement[] = [
{
Effect: 'Allow',
Action: ['*'],
Resource: ['*']
}];
superAdminRole = (await db.insert(schema.roles).values({
Resource: ['*'],
},
];
superAdminRole = (
await db
.insert(schema.roles)
.values({
name: 'Super Admin',
policies: suerAdminPolicies
}).returning())[0];
policies: suerAdminPolicies,
})
.returning()
)[0];
}
await db.insert(schema.userRoles).values({
userId: newUser[0].id,
roleId: superAdminRole.id
roleId: superAdminRole.id,
});
return newUser[0];
}
}

View File

@@ -1,4 +1,10 @@
import type { EMLImportCredentials, EmailObject, EmailAddress, SyncState, MailboxUser } from '@open-archiver/types';
import type {
EMLImportCredentials,
EmailObject,
EmailAddress,
SyncState,
MailboxUser,
} from '@open-archiver/types';
import type { IEmailConnector } from '../EmailProviderFactory';
import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
import { logger } from '../../config/logger';
@@ -29,14 +35,14 @@ export class EMLConnector implements IEmailConnector {
public async testConnection(): Promise<boolean> {
try {
if (!this.credentials.uploadedFilePath) {
throw Error("EML file path not provided.");
throw Error('EML file path not provided.');
}
if (!this.credentials.uploadedFilePath.includes('.zip')) {
throw Error("Provided file is not in the ZIP format.");
throw Error('Provided file is not in the ZIP format.');
}
const fileExist = await this.storage.exists(this.credentials.uploadedFilePath);
if (!fileExist) {
throw Error("EML file upload not finished yet, please wait.");
throw Error('EML file upload not finished yet, please wait.');
}
return true;
@@ -47,7 +53,8 @@ export class EMLConnector implements IEmailConnector {
}
public async *listAllUsers(): AsyncGenerator<MailboxUser> {
const displayName = this.credentials.uploadedFileName || `eml-import-${new Date().getTime()}`;
const displayName =
this.credentials.uploadedFileName || `eml-import-${new Date().getTime()}`;
logger.info(`Found potential mailbox: ${displayName}`);
const constructedPrimaryEmail = `${displayName.replace(/ /g, '.').toLowerCase()}@eml.local`;
yield {
@@ -57,7 +64,10 @@ export class EMLConnector implements IEmailConnector {
};
}
public async *fetchEmails(userEmail: string, syncState?: SyncState | null): AsyncGenerator<EmailObject | null> {
public async *fetchEmails(
userEmail: string,
syncState?: SyncState | null
): AsyncGenerator<EmailObject | null> {
const fileStream = await this.storage.get(this.credentials.uploadedFilePath);
const tempDir = await fs.mkdtemp(join('/tmp', 'eml-import-'));
const unzippedPath = join(tempDir, 'unzipped');
@@ -93,7 +103,10 @@ export class EMLConnector implements IEmailConnector {
// logger.info({ file, messageId: emailObject.id }, 'Parsed email message.');
yield emailObject;
} catch (error) {
logger.error({ error, file }, 'Failed to process a single EML file. Skipping.');
logger.error(
{ error, file },
'Failed to process a single EML file. Skipping.'
);
}
}
}
@@ -120,7 +133,9 @@ export class EMLConnector implements IEmailConnector {
}
const entryPath = join(dest, fileName);
if (/\/$/.test(fileName)) {
fs.mkdir(entryPath, { recursive: true }).then(() => zipfile.readEntry()).catch(reject);
fs.mkdir(entryPath, { recursive: true })
.then(() => zipfile.readEntry())
.catch(reject);
} else {
zipfile.openReadStream(entry, (err, readStream) => {
if (err) reject(err);
@@ -158,13 +173,20 @@ export class EMLConnector implements IEmailConnector {
filename: attachment.filename || 'untitled',
contentType: attachment.contentType,
size: attachment.size,
content: attachment.content as Buffer
content: attachment.content as Buffer,
}));
const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
const mapAddresses = (
addresses: AddressObject | AddressObject[] | undefined
): EmailAddress[] => {
if (!addresses) return [];
const addressArray = Array.isArray(addresses) ? addresses : [addresses];
return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address?.replaceAll(`'`, '') || '' })));
return addressArray.flatMap((a) =>
a.value.map((v) => ({
name: v.name,
address: v.address?.replaceAll(`'`, '') || '',
}))
);
};
const threadId = getThreadId(parsedEmail.headers);
@@ -179,7 +201,6 @@ export class EMLConnector implements IEmailConnector {
from.push({ name: 'No Sender', address: 'No Sender' });
}
return {
id: messageId,
threadId: threadId,
@@ -194,7 +215,7 @@ export class EMLConnector implements IEmailConnector {
attachments,
receivedAt: parsedEmail.date || new Date(),
eml: emlBuffer,
path
path,
};
}

View File

@@ -5,7 +5,7 @@ import type {
EmailObject,
EmailAddress,
SyncState,
MailboxUser
MailboxUser,
} from '@open-archiver/types';
import type { IEmailConnector } from '../EmailProviderFactory';
import { logger } from '../../config/logger';
@@ -18,7 +18,7 @@ import { getThreadId } from './helpers/utils';
*/
export class GoogleWorkspaceConnector implements IEmailConnector {
private credentials: GoogleWorkspaceCredentials;
private serviceAccountCreds: { client_email: string; private_key: string; };
private serviceAccountCreds: { client_email: string; private_key: string };
private newHistoryId: string | undefined;
constructor(credentials: GoogleWorkspaceCredentials) {
@@ -31,7 +31,7 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
}
this.serviceAccountCreds = {
client_email: parsedKey.client_email,
private_key: parsedKey.private_key
private_key: parsedKey.private_key,
};
} catch (error) {
logger.error({ err: error }, 'Failed to parse Google Service Account JSON');
@@ -50,12 +50,11 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
email: this.serviceAccountCreds.client_email,
key: this.serviceAccountCreds.private_key,
scopes,
subject
subject,
});
return jwtClient;
}
/**
* Tests the connection and authentication by attempting to list the first user
* from the directory, impersonating the admin user.
@@ -63,19 +62,19 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
public async testConnection(): Promise<boolean> {
try {
const authClient = this.getAuthClient(this.credentials.impersonatedAdminEmail, [
'https://www.googleapis.com/auth/admin.directory.user.readonly'
'https://www.googleapis.com/auth/admin.directory.user.readonly',
]);
const admin = google.admin({
version: 'directory_v1',
auth: authClient
auth: authClient,
});
// Perform a simple, low-impact read operation to verify credentials.
await admin.users.list({
customer: 'my_customer',
maxResults: 1,
orderBy: 'email'
orderBy: 'email',
});
logger.info('Google Workspace connection test successful.');
@@ -93,18 +92,19 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
*/
public async *listAllUsers(): AsyncGenerator<MailboxUser> {
const authClient = this.getAuthClient(this.credentials.impersonatedAdminEmail, [
'https://www.googleapis.com/auth/admin.directory.user.readonly'
'https://www.googleapis.com/auth/admin.directory.user.readonly',
]);
const admin = google.admin({ version: 'directory_v1', auth: authClient });
let pageToken: string | undefined = undefined;
do {
const res: Common.GaxiosResponseWithHTTP2<admin_directory_v1.Schema$Users> = await admin.users.list({
const res: Common.GaxiosResponseWithHTTP2<admin_directory_v1.Schema$Users> =
await admin.users.list({
customer: 'my_customer',
maxResults: 500, // Max allowed per page
pageToken: pageToken,
orderBy: 'email'
orderBy: 'email',
});
const users = res.data.users;
@@ -114,7 +114,7 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
yield {
id: user.id,
primaryEmail: user.primaryEmail,
displayName: user.name.fullName
displayName: user.name.fullName,
};
}
}
@@ -135,7 +135,7 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
syncState?: SyncState | null
): AsyncGenerator<EmailObject> {
const authClient = this.getAuthClient(userEmail, [
'https://www.googleapis.com/auth/gmail.readonly'
'https://www.googleapis.com/auth/gmail.readonly',
]);
const gmail = google.gmail({ version: 'v1', auth: authClient });
let pageToken: string | undefined = undefined;
@@ -151,11 +151,12 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
this.newHistoryId = startHistoryId;
do {
const historyResponse: Common.GaxiosResponseWithHTTP2<gmail_v1.Schema$ListHistoryResponse> = await gmail.users.history.list({
const historyResponse: Common.GaxiosResponseWithHTTP2<gmail_v1.Schema$ListHistoryResponse> =
await gmail.users.history.list({
userId: userEmail,
startHistoryId: this.newHistoryId,
pageToken: pageToken,
historyTypes: ['messageAdded']
historyTypes: ['messageAdded'],
});
const histories = historyResponse.data.history;
@@ -173,29 +174,44 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
userId: userEmail,
id: messageId,
format: 'METADATA',
fields: 'labelIds'
fields: 'labelIds',
});
const labels = await this.getLabelDetails(gmail, userEmail, metadataResponse.data.labelIds || []);
const labels = await this.getLabelDetails(
gmail,
userEmail,
metadataResponse.data.labelIds || []
);
const msgResponse = await gmail.users.messages.get({
userId: userEmail,
id: messageId,
format: 'RAW'
format: 'RAW',
});
if (msgResponse.data.raw) {
const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
const parsedEmail: ParsedMail = await simpleParser(rawEmail);
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
const attachments = parsedEmail.attachments.map(
(attachment: Attachment) => ({
filename: attachment.filename || 'untitled',
contentType: attachment.contentType,
size: attachment.size,
content: attachment.content as Buffer
}));
const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
content: attachment.content as Buffer,
})
);
const mapAddresses = (
addresses: AddressObject | AddressObject[] | undefined
): EmailAddress[] => {
if (!addresses) return [];
const addressArray = Array.isArray(addresses) ? addresses : [addresses];
return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
const addressArray = Array.isArray(addresses)
? addresses
: [addresses];
return addressArray.flatMap((a) =>
a.value.map((v) => ({
name: v.name,
address: v.address || '',
}))
);
};
const threadId = getThreadId(parsedEmail.headers);
console.log('threadId', threadId);
@@ -215,12 +231,15 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
attachments,
receivedAt: parsedEmail.date || new Date(),
path: labels.path,
tags: labels.tags
tags: labels.tags,
};
}
} catch (error: any) {
if (error.code === 404) {
logger.warn({ messageId: messageAdded.message.id, userEmail }, 'Message not found, skipping.');
logger.warn(
{ messageId: messageAdded.message.id, userEmail },
'Message not found, skipping.'
);
} else {
throw error;
}
@@ -234,16 +253,19 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
if (historyResponse.data.historyId) {
this.newHistoryId = historyResponse.data.historyId;
}
} while (pageToken);
}
private async *fetchAllMessagesForUser(gmail: gmail_v1.Gmail, userEmail: string): AsyncGenerator<EmailObject> {
private async *fetchAllMessagesForUser(
gmail: gmail_v1.Gmail,
userEmail: string
): AsyncGenerator<EmailObject> {
let pageToken: string | undefined = undefined;
do {
const listResponse: Common.GaxiosResponseWithHTTP2<gmail_v1.Schema$ListMessagesResponse> = await gmail.users.messages.list({
const listResponse: Common.GaxiosResponseWithHTTP2<gmail_v1.Schema$ListMessagesResponse> =
await gmail.users.messages.list({
userId: userEmail,
pageToken: pageToken
pageToken: pageToken,
});
const messages = listResponse.data.messages;
@@ -259,29 +281,41 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
userId: userEmail,
id: messageId,
format: 'METADATA',
fields: 'labelIds'
fields: 'labelIds',
});
const labels = await this.getLabelDetails(gmail, userEmail, metadataResponse.data.labelIds || []);
const labels = await this.getLabelDetails(
gmail,
userEmail,
metadataResponse.data.labelIds || []
);
const msgResponse = await gmail.users.messages.get({
userId: userEmail,
id: messageId,
format: 'RAW'
format: 'RAW',
});
if (msgResponse.data.raw) {
const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
const parsedEmail: ParsedMail = await simpleParser(rawEmail);
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
const attachments = parsedEmail.attachments.map(
(attachment: Attachment) => ({
filename: attachment.filename || 'untitled',
contentType: attachment.contentType,
size: attachment.size,
content: attachment.content as Buffer
}));
const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
content: attachment.content as Buffer,
})
);
const mapAddresses = (
addresses: AddressObject | AddressObject[] | undefined
): EmailAddress[] => {
if (!addresses) return [];
const addressArray = Array.isArray(addresses) ? addresses : [addresses];
return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
const addressArray = Array.isArray(addresses)
? addresses
: [addresses];
return addressArray.flatMap((a) =>
a.value.map((v) => ({ name: v.name, address: v.address || '' }))
);
};
const threadId = getThreadId(parsedEmail.headers);
console.log('threadId', threadId);
@@ -301,12 +335,15 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
attachments,
receivedAt: parsedEmail.date || new Date(),
path: labels.path,
tags: labels.tags
tags: labels.tags,
};
}
} catch (error: any) {
if (error.code === 404) {
logger.warn({ messageId: message.id, userEmail }, 'Message not found during initial import, skipping.');
logger.warn(
{ messageId: message.id, userEmail },
'Message not found during initial import, skipping.'
);
} else {
throw error;
}
@@ -330,15 +367,19 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
return {
google: {
[userEmail]: {
historyId: this.newHistoryId
}
}
historyId: this.newHistoryId,
},
},
};
}
private labelCache: Map<string, gmail_v1.Schema$Label> = new Map();
private async getLabelDetails(gmail: gmail_v1.Gmail, userEmail: string, labelIds: string[]): Promise<{ path: string, tags: string[]; }> {
private async getLabelDetails(
gmail: gmail_v1.Gmail,
userEmail: string,
labelIds: string[]
): Promise<{ path: string; tags: string[] }> {
const tags: string[] = [];
let path = '';

View File

@@ -1,4 +1,10 @@
import type { GenericImapCredentials, EmailObject, EmailAddress, SyncState, MailboxUser } from '@open-archiver/types';
import type {
GenericImapCredentials,
EmailObject,
EmailAddress,
SyncState,
MailboxUser,
} from '@open-archiver/types';
import type { IEmailConnector } from '../EmailProviderFactory';
import { ImapFlow } from 'imapflow';
import { simpleParser, ParsedMail, Attachment, AddressObject, Headers } from 'mailparser';
@@ -7,7 +13,7 @@ import { getThreadId } from './helpers/utils';
export class ImapConnector implements IEmailConnector {
private client: ImapFlow;
private newMaxUids: { [mailboxPath: string]: number; } = {};
private newMaxUids: { [mailboxPath: string]: number } = {};
private isConnected = false;
private statusMessage: string | undefined;
@@ -94,9 +100,8 @@ export class ImapConnector implements IEmailConnector {
yield {
id: String(index),
primaryEmail: email,
displayName: email
displayName: email,
};
}
} finally {
await this.disconnect();
@@ -129,19 +134,22 @@ export class ImapConnector implements IEmailConnector {
const delay = Math.pow(2, attempt) * 1000;
const jitter = Math.random() * 1000;
logger.info(`Retrying in ${Math.round((delay + jitter) / 1000)}s`);
await new Promise(resolve => setTimeout(resolve, delay + jitter));
await new Promise((resolve) => setTimeout(resolve, delay + jitter));
}
}
// This line should be unreachable
throw new Error('IMAP operation failed after all retries.');
}
public async *fetchEmails(userEmail: string, syncState?: SyncState | null): AsyncGenerator<EmailObject | null> {
public async *fetchEmails(
userEmail: string,
syncState?: SyncState | null
): AsyncGenerator<EmailObject | null> {
// list all mailboxes first
const mailboxes = await this.withRetry(async () => await this.client.list());
await this.disconnect();
const processableMailboxes = mailboxes.filter(mailbox => {
const processableMailboxes = mailboxes.filter((mailbox) => {
// filter out trash and all mail emails
if (mailbox.specialUse) {
const specialUse = mailbox.specialUse.toLowerCase();
@@ -150,7 +158,12 @@ export class ImapConnector implements IEmailConnector {
}
}
// Fallback to checking flags
if (mailbox.flags.has('\\Noselect') || mailbox.flags.has('\\Trash') || mailbox.flags.has('\\Junk') || mailbox.flags.has('\\All')) {
if (
mailbox.flags.has('\\Noselect') ||
mailbox.flags.has('\\Trash') ||
mailbox.flags.has('\\Junk') ||
mailbox.flags.has('\\All')
) {
return false;
}
@@ -162,12 +175,16 @@ export class ImapConnector implements IEmailConnector {
logger.info({ mailboxPath }, 'Processing mailbox');
try {
const mailbox = await this.withRetry(async () => await this.client.mailboxOpen(mailboxPath));
const mailbox = await this.withRetry(
async () => await this.client.mailboxOpen(mailboxPath)
);
const lastUid = syncState?.imap?.[mailboxPath]?.maxUid;
let currentMaxUid = lastUid || 0;
if (mailbox.exists > 0) {
const lastMessage = await this.client.fetchOne(String(mailbox.exists), { uid: true });
const lastMessage = await this.client.fetchOne(String(mailbox.exists), {
uid: true,
});
if (lastMessage && lastMessage.uid > currentMaxUid) {
currentMaxUid = lastMessage.uid;
}
@@ -184,7 +201,12 @@ export class ImapConnector implements IEmailConnector {
const endUid = Math.min(startUid + BATCH_SIZE - 1, maxUidToFetch);
const searchCriteria = { uid: `${startUid}:${endUid}` };
for await (const msg of this.client.fetch(searchCriteria, { envelope: true, source: true, bodyStructure: true, uid: true })) {
for await (const msg of this.client.fetch(searchCriteria, {
envelope: true,
source: true,
bodyStructure: true,
uid: true,
})) {
if (lastUid && msg.uid <= lastUid) {
continue;
}
@@ -199,7 +221,10 @@ export class ImapConnector implements IEmailConnector {
try {
yield await this.parseMessage(msg, mailboxPath);
} catch (err: any) {
logger.error({ err, mailboxPath, uid: msg.uid }, 'Failed to parse message');
logger.error(
{ err, mailboxPath, uid: msg.uid },
'Failed to parse message'
);
throw err;
}
}
@@ -213,10 +238,10 @@ export class ImapConnector implements IEmailConnector {
logger.error({ err, mailboxPath }, 'Failed to process mailbox');
// Check if the error indicates a persistent failure after retries
if (err.message.includes('IMAP operation failed after all retries')) {
this.statusMessage = 'Sync paused due to reaching the mail server rate limit. The process will automatically resume later.';
this.statusMessage =
'Sync paused due to reaching the mail server rate limit. The process will automatically resume later.';
}
}
finally {
} finally {
await this.disconnect();
}
}
@@ -228,13 +253,17 @@ export class ImapConnector implements IEmailConnector {
filename: attachment.filename || 'untitled',
contentType: attachment.contentType,
size: attachment.size,
content: attachment.content as Buffer
content: attachment.content as Buffer,
}));
const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
const mapAddresses = (
addresses: AddressObject | AddressObject[] | undefined
): EmailAddress[] => {
if (!addresses) return [];
const addressArray = Array.isArray(addresses) ? addresses : [addresses];
return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
return addressArray.flatMap((a) =>
a.value.map((v) => ({ name: v.name, address: v.address || '' }))
);
};
const threadId = getThreadId(parsedEmail.headers);
@@ -253,17 +282,17 @@ export class ImapConnector implements IEmailConnector {
attachments,
receivedAt: parsedEmail.date || new Date(),
eml: msg.source,
path: mailboxPath
path: mailboxPath,
};
}
public getUpdatedSyncState(): SyncState {
const imapSyncState: { [mailboxPath: string]: { maxUid: number; }; } = {};
const imapSyncState: { [mailboxPath: string]: { maxUid: number } } = {};
for (const [path, uid] of Object.entries(this.newMaxUids)) {
imapSyncState[path] = { maxUid: uid };
}
const syncState: SyncState = {
imap: imapSyncState
imap: imapSyncState,
};
if (this.statusMessage) {

View File

@@ -4,7 +4,7 @@ import type {
EmailObject,
EmailAddress,
SyncState,
MailboxUser
MailboxUser,
} from '@open-archiver/types';
import type { IEmailConnector } from '../EmailProviderFactory';
import { logger } from '../../config/logger';
@@ -22,7 +22,7 @@ export class MicrosoftConnector implements IEmailConnector {
private credentials: Microsoft365Credentials;
private graphClient: Client;
// Store delta tokens for each folder during a sync operation.
private newDeltaTokens: { [folderId: string]: string; };
private newDeltaTokens: { [folderId: string]: string };
constructor(credentials: Microsoft365Credentials) {
this.credentials = credentials;
@@ -55,8 +55,8 @@ export class MicrosoftConnector implements IEmailConnector {
},
piiLoggingEnabled: false,
logLevel: LogLevel.Warning,
}
}
},
},
};
const msalClient = new ConfidentialClientApplication(msalConfig);
@@ -110,7 +110,7 @@ export class MicrosoftConnector implements IEmailConnector {
yield {
id: user.id,
primaryEmail: user.userPrincipalName,
displayName: user.displayName
displayName: user.displayName,
};
}
}
@@ -144,8 +144,16 @@ export class MicrosoftConnector implements IEmailConnector {
const folders = this.listAllFolders(userEmail);
for await (const folder of folders) {
if (folder.id && folder.path) {
logger.info({ userEmail, folderId: folder.id, folderName: folder.displayName }, 'Syncing folder');
yield* this.syncFolder(userEmail, folder.id, folder.path, this.newDeltaTokens[folder.id]);
logger.info(
{ userEmail, folderId: folder.id, folderName: folder.displayName },
'Syncing folder'
);
yield* this.syncFolder(
userEmail,
folder.id,
folder.path,
this.newDeltaTokens[folder.id]
);
}
}
} catch (error) {
@@ -159,7 +167,11 @@ export class MicrosoftConnector implements IEmailConnector {
* @param userEmail The user principal name or ID.
* @returns An async generator that yields each mail folder.
*/
private async *listAllFolders(userEmail: string, parentFolderId?: string, currentPath = ''): AsyncGenerator<MailFolder & { path: string; }> {
private async *listAllFolders(
userEmail: string,
parentFolderId?: string,
currentPath = ''
): AsyncGenerator<MailFolder & { path: string }> {
const requestUrl = parentFolderId
? `/users/${userEmail}/mailFolders/${parentFolderId}/childFolders`
: `/users/${userEmail}/mailFolders`;
@@ -169,7 +181,9 @@ export class MicrosoftConnector implements IEmailConnector {
while (response) {
for (const folder of response.value as MailFolder[]) {
const newPath = currentPath ? `${currentPath}/${folder.displayName || ''}` : folder.displayName || '';
const newPath = currentPath
? `${currentPath}/${folder.displayName || ''}`
: folder.displayName || '';
yield { ...folder, path: newPath || '' };
if (folder.childFolderCount && folder.childFolderCount > 0) {
@@ -214,15 +228,21 @@ export class MicrosoftConnector implements IEmailConnector {
while (requestUrl) {
try {
const response = await this.graphClient.api(requestUrl)
const response = await this.graphClient
.api(requestUrl)
.select('id,conversationId,@removed')
.get();
for (const message of response.value) {
if (message.id && !(message)['@removed']) {
if (message.id && !message['@removed']) {
const rawEmail = await this.getRawEmail(userEmail, message.id);
if (rawEmail) {
const emailObject = await this.parseEmail(rawEmail, message.id, userEmail, path);
const emailObject = await this.parseEmail(
rawEmail,
message.id,
userEmail,
path
);
emailObject.threadId = message.conversationId; // Add conversationId as threadId
yield emailObject;
}
@@ -244,30 +264,44 @@ export class MicrosoftConnector implements IEmailConnector {
private async getRawEmail(userEmail: string, messageId: string): Promise<Buffer | null> {
try {
const response = await this.graphClient.api(`/users/${userEmail}/messages/${messageId}/$value`).getStream();
const response = await this.graphClient
.api(`/users/${userEmail}/messages/${messageId}/$value`)
.getStream();
const chunks: any[] = [];
for await (const chunk of response) {
chunks.push(chunk);
}
return Buffer.concat(chunks);
} catch (error) {
logger.error({ err: error, userEmail, messageId }, 'Failed to fetch raw email content.');
logger.error(
{ err: error, userEmail, messageId },
'Failed to fetch raw email content.'
);
return null;
}
}
private async parseEmail(rawEmail: Buffer, messageId: string, userEmail: string, path: string): Promise<EmailObject> {
private async parseEmail(
rawEmail: Buffer,
messageId: string,
userEmail: string,
path: string
): Promise<EmailObject> {
const parsedEmail: ParsedMail = await simpleParser(rawEmail);
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
filename: attachment.filename || 'untitled',
contentType: attachment.contentType,
size: attachment.size,
content: attachment.content as Buffer
content: attachment.content as Buffer,
}));
const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
const mapAddresses = (
addresses: AddressObject | AddressObject[] | undefined
): EmailAddress[] => {
if (!addresses) return [];
const addressArray = Array.isArray(addresses) ? addresses : [addresses];
return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
return addressArray.flatMap((a) =>
a.value.map((v) => ({ name: v.name, address: v.address || '' }))
);
};
return {
@@ -284,7 +318,7 @@ export class MicrosoftConnector implements IEmailConnector {
headers: parsedEmail.headers,
attachments,
receivedAt: parsedEmail.date || new Date(),
path
path,
};
}
@@ -295,9 +329,9 @@ export class MicrosoftConnector implements IEmailConnector {
return {
microsoft: {
[userEmail]: {
deltaTokens: this.newDeltaTokens
}
}
deltaTokens: this.newDeltaTokens,
},
},
};
}
}

View File

@@ -1,4 +1,10 @@
import type { PSTImportCredentials, EmailObject, EmailAddress, SyncState, MailboxUser } from '@open-archiver/types';
import type {
PSTImportCredentials,
EmailObject,
EmailAddress,
SyncState,
MailboxUser,
} from '@open-archiver/types';
import type { IEmailConnector } from '../EmailProviderFactory';
import { PSTFile, PSTFolder, PSTMessage } from 'pst-extractor';
import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
@@ -20,42 +26,57 @@ const streamToBuffer = (stream: Readable): Promise<Buffer> => {
// We have to hardcode names for deleted and trash folders here as current lib doesn't support looking into PST properties.
const DELETED_FOLDERS = new Set([
// English
'deleted items', 'trash',
'deleted items',
'trash',
// Spanish
'elementos eliminados', 'papelera',
'elementos eliminados',
'papelera',
// French
'éléments supprimés', 'corbeille',
'éléments supprimés',
'corbeille',
// German
'gelöschte elemente', 'papierkorb',
'gelöschte elemente',
'papierkorb',
// Italian
'posta eliminata', 'cestino',
'posta eliminata',
'cestino',
// Portuguese
'itens excluídos', 'lixo',
'itens excluídos',
'lixo',
// Dutch
'verwijderde items', 'prullenbak',
'verwijderde items',
'prullenbak',
// Russian
'удаленные', 'корзина',
'удаленные',
'корзина',
// Polish
'usunięte elementy', 'kosz',
'usunięte elementy',
'kosz',
// Japanese
'削除済みアイテム',
// Czech
'odstraněná pošta', 'koš',
'odstraněná pošta',
'koš',
// Estonian
'kustutatud kirjad', 'prügikast',
'kustutatud kirjad',
'prügikast',
// Swedish
'borttagna objekt', 'skräp',
'borttagna objekt',
'skräp',
// Danish
'slettet post', 'papirkurv',
'slettet post',
'papirkurv',
// Norwegian
'slettede elementer',
// Finnish
'poistetut', 'roskakori'
'poistetut',
'roskakori',
]);
const JUNK_FOLDERS = new Set([
// English
'junk email', 'spam',
'junk email',
'spam',
// Spanish
'correo no deseado',
// French
@@ -69,11 +90,13 @@ const JUNK_FOLDERS = new Set([
// Dutch
'ongewenste e-mail',
// Russian
'нежелательная почта', 'спам',
'нежелательная почта',
'спам',
// Polish
'wiadomości-śmieci',
// Japanese
'迷惑メール', 'スパム',
'迷惑メール',
'スパム',
// Czech
'nevyžádaná pošta',
// Estonian
@@ -85,7 +108,7 @@ const JUNK_FOLDERS = new Set([
// Norwegian
'søppelpost',
// Finnish
'roskaposti'
'roskaposti',
]);
export class PSTConnector implements IEmailConnector {
@@ -109,14 +132,14 @@ export class PSTConnector implements IEmailConnector {
public async testConnection(): Promise<boolean> {
try {
if (!this.credentials.uploadedFilePath) {
throw Error("PST file path not provided.");
throw Error('PST file path not provided.');
}
if (!this.credentials.uploadedFilePath.includes('.pst')) {
throw Error("Provided file is not in the PST format.");
throw Error('Provided file is not in the PST format.');
}
const fileExist = await this.storage.exists(this.credentials.uploadedFilePath);
if (!fileExist) {
throw Error("PST file upload not finished yet, please wait.");
throw Error('PST file upload not finished yet, please wait.');
}
return true;
@@ -136,7 +159,8 @@ export class PSTConnector implements IEmailConnector {
try {
pstFile = await this.loadPstFile();
const root = pstFile.getRootFolder();
const displayName: string = root.displayName || pstFile.pstFilename || String(new Date().getTime());
const displayName: string =
root.displayName || pstFile.pstFilename || String(new Date().getTime());
logger.info(`Found potential mailbox: ${displayName}`);
const constructedPrimaryEmail = `${displayName.replace(/ /g, '.').toLowerCase()}@pst.local`;
yield {
@@ -154,7 +178,10 @@ export class PSTConnector implements IEmailConnector {
}
}
public async *fetchEmails(userEmail: string, syncState?: SyncState | null): AsyncGenerator<EmailObject | null> {
public async *fetchEmails(
userEmail: string,
syncState?: SyncState | null
): AsyncGenerator<EmailObject | null> {
let pstFile: PSTFile | null = null;
try {
pstFile = await this.loadPstFile();
@@ -164,14 +191,16 @@ export class PSTConnector implements IEmailConnector {
logger.error({ error }, 'Failed to fetch email.');
pstFile?.close();
throw error;
}
finally {
} finally {
pstFile?.close();
}
}
private async *processFolder(folder: PSTFolder, currentPath: string, userEmail: string): AsyncGenerator<EmailObject | null> {
private async *processFolder(
folder: PSTFolder,
currentPath: string,
userEmail: string
): AsyncGenerator<EmailObject | null> {
const folderName = folder.displayName.toLowerCase();
if (DELETED_FOLDERS.has(folderName) || JUNK_FOLDERS.has(folderName)) {
logger.info(`Skipping folder: ${folder.displayName}`);
@@ -200,7 +229,11 @@ export class PSTConnector implements IEmailConnector {
}
}
private async parseMessage(msg: PSTMessage, path: string, userEmail: string): Promise<EmailObject> {
private async parseMessage(
msg: PSTMessage,
path: string,
userEmail: string
): Promise<EmailObject> {
const emlContent = await this.constructEml(msg);
const emlBuffer = Buffer.from(emlContent, 'utf-8');
const parsedEmail: ParsedMail = await simpleParser(emlBuffer);
@@ -209,13 +242,20 @@ export class PSTConnector implements IEmailConnector {
filename: attachment.filename || 'untitled',
contentType: attachment.contentType,
size: attachment.size,
content: attachment.content as Buffer
content: attachment.content as Buffer,
}));
const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
const mapAddresses = (
addresses: AddressObject | AddressObject[] | undefined
): EmailAddress[] => {
if (!addresses) return [];
const addressArray = Array.isArray(addresses) ? addresses : [addresses];
return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address?.replaceAll(`'`, '') || '' })));
return addressArray.flatMap((a) =>
a.value.map((v) => ({
name: v.name,
address: v.address?.replaceAll(`'`, '') || '',
}))
);
};
const from = mapAddresses(parsedEmail.from);
@@ -228,7 +268,13 @@ export class PSTConnector implements IEmailConnector {
// generate a unique ID for this message
if (!messageId) {
messageId = `generated-${createHash('sha256').update(emlBuffer ?? Buffer.from(parsedEmail.text || parsedEmail.html || '', 'utf-8')).digest('hex')}-${createHash('sha256').update(emlBuffer ?? Buffer.from(msg.subject || '', 'utf-8')).digest('hex')}-${msg.clientSubmitTime?.getTime()}`;
messageId = `generated-${createHash('sha256')
.update(
emlBuffer ?? Buffer.from(parsedEmail.text || parsedEmail.html || '', 'utf-8')
)
.digest('hex')}-${createHash('sha256')
.update(emlBuffer ?? Buffer.from(msg.subject || '', 'utf-8'))
.digest('hex')}-${msg.clientSubmitTime?.getTime()}`;
}
return {
id: messageId,
@@ -244,7 +290,7 @@ export class PSTConnector implements IEmailConnector {
attachments,
receivedAt: parsedEmail.date || new Date(),
eml: emlBuffer,
path
path,
};
}

View File

@@ -1,4 +1,3 @@
import type { Headers } from 'mailparser';
function getHeaderValue(header: any): string | undefined {
@@ -15,7 +14,6 @@ function getHeaderValue(header: any): string | undefined {
}
export function getThreadId(headers: Headers): string | undefined {
const referencesHeader = headers.get('references');
if (referencesHeader) {

View File

@@ -33,7 +33,6 @@ const worker = new Worker('ingestion', processor, {
},
});
console.log('Ingestion worker started');
process.on('SIGINT', () => worker.close());

View File

@@ -15,4 +15,4 @@ declare global {
}
}
export { };
export {};

View File

@@ -3,8 +3,6 @@ import { jwtVerify } from 'jose';
import type { User } from '@open-archiver/types';
import 'dotenv/config';
const JWT_SECRET_ENCODED = new TextEncoder().encode(process.env.JWT_SECRET);
export const handle: Handle = async ({ event, resolve }) => {

View File

@@ -9,10 +9,7 @@ const BASE_URL = '/api/v1'; // Using a relative URL for proxying
* @param options The standard Fetch API options.
* @returns A Promise that resolves to the Fetch Response.
*/
export const api = async (
url: string,
options: RequestInit = {}
): Promise<Response> => {
export const api = async (url: string, options: RequestInit = {}): Promise<Response> => {
const { accessToken } = get(authStore);
const defaultHeaders: HeadersInit = {};
@@ -28,8 +25,8 @@ export const api = async (
...options,
headers: {
...defaultHeaders,
...options.headers
}
...options.headers,
},
};
return fetch(`${BASE_URL}${url}`, mergedOptions);

View File

@@ -4,8 +4,9 @@
let {
raw,
rawHtml
}: { raw?: Buffer | { type: 'Buffer'; data: number[] } | undefined; rawHtml?: string } = $props();
rawHtml,
}: { raw?: Buffer | { type: 'Buffer'; data: number[] } | undefined; rawHtml?: string } =
$props();
let parsedEmail: Email | null = $state(null);
let isLoading = $state(true);

View File

@@ -5,7 +5,7 @@
let {
thread,
currentEmailId
currentEmailId,
}: {
thread: ArchivedEmail['thread'];
currentEmailId: string;
@@ -45,7 +45,7 @@
onclick={(e) => {
e.preventDefault();
goto(`/dashboard/archived-emails/${item.id}`, {
invalidateAll: true
invalidateAll: true,
});
}}>{item.subject || 'No Subject'}</a
>
@@ -53,7 +53,9 @@
{item.subject || 'No Subject'}
{/if}
</h4>
<div class="flex flex-col space-y-2 text-sm font-normal leading-none text-gray-400">
<div
class="flex flex-col space-y-2 text-sm font-normal leading-none text-gray-400"
>
<span>From: {item.senderEmail}</span>
<time class="">{new Date(item.sentAt).toLocaleString()}</time>
</div>

View File

@@ -5,7 +5,7 @@
header,
text,
buttonText,
click
click,
}: {
header: string;
text: string;

View File

@@ -5,7 +5,8 @@
<div class="flex flex-col items-center gap-2">
<p class=" text-balance text-center text-xs font-medium leading-loose">
© {new Date().getFullYear()}
<a href="https://openarchiver.com/" target="_blank">Open Archiver</a>. All rights reserved.
<a href="https://openarchiver.com/" target="_blank">Open Archiver</a>. All rights
reserved.
</p>
</div>
</div>

View File

@@ -13,7 +13,7 @@
import { Loader2 } from 'lucide-svelte';
let {
source = null,
onSubmit
onSubmit,
}: {
source?: IngestionSource | null;
onSubmit: (data: CreateIngestionSourceDto) => Promise<void>;
@@ -24,7 +24,7 @@
{ value: 'google_workspace', label: 'Google Workspace' },
{ value: 'microsoft_365', label: 'Microsoft 365' },
{ value: 'pst_import', label: 'PST Import' },
{ value: 'eml_import', label: 'EML Import' }
{ value: 'eml_import', label: 'EML Import' },
];
let formData: CreateIngestionSourceDto = $state({
@@ -32,8 +32,8 @@
provider: source?.provider ?? 'generic_imap',
providerConfig: source?.credentials ?? {
type: source?.provider ?? 'generic_imap',
secure: true
}
secure: true,
},
});
$effect(() => {
@@ -74,7 +74,7 @@
try {
const response = await api('/upload', {
method: 'POST',
body: uploadFormData
body: uploadFormData,
});
const result = await response.json();
if (!response.ok) {
@@ -92,7 +92,7 @@
title: 'Upload Failed, please try again',
message: JSON.stringify(error),
duration: 5000,
show: true
show: true,
});
}
};
@@ -161,7 +161,12 @@
</div>
<div class="grid grid-cols-4 items-center gap-4">
<Label for="port" class="text-left">Port</Label>
<Input id="port" type="number" bind:value={formData.providerConfig.port} class="col-span-3" />
<Input
id="port"
type="number"
bind:value={formData.providerConfig.port}
class="col-span-3"
/>
</div>
<div class="grid grid-cols-4 items-center gap-4">
<Label for="username" class="text-left">Username</Label>
@@ -184,7 +189,13 @@
<div class="grid grid-cols-4 items-center gap-4">
<Label for="pst-file" class="text-left">PST File</Label>
<div class="col-span-3 flex flex-row items-center space-x-2">
<Input id="pst-file" type="file" class="" accept=".pst" onchange={handleFileChange} />
<Input
id="pst-file"
type="file"
class=""
accept=".pst"
onchange={handleFileChange}
/>
{#if fileUploading}
<span class=" text-primary animate-spin"><Loader2 /></span>
{/if}
@@ -194,7 +205,13 @@
<div class="grid grid-cols-4 items-center gap-4">
<Label for="eml-file" class="text-left">EML File</Label>
<div class="col-span-3 flex flex-row items-center space-x-2">
<Input id="eml-file" type="file" class="" accept=".zip" onchange={handleFileChange} />
<Input
id="eml-file"
type="file"
class=""
accept=".zip"
onchange={handleFileChange}
/>
{#if fileUploading}
<span class=" text-primary animate-spin"><Loader2 /></span>
{/if}
@@ -206,9 +223,9 @@
<Alert.Title>Heads up!</Alert.Title>
<Alert.Description>
<div class="my-1">
Please note that this is an organization-wide operation. This kind of ingestions will
import and index <b>all</b> email inboxes in your organization. If you want to import only
specific email inboxes, use the IMAP connector.
Please note that this is an organization-wide operation. This kind of ingestions
will import and index <b>all</b> email inboxes in your organization. If you want
to import only specific email inboxes, use the IMAP connector.
</div>
</Alert.Description>
</Alert.Root>

View File

@@ -15,7 +15,7 @@
icon: 'heroicons-outline:check-circle',
color: 'text-green-800',
messageColor: 'text-green-700',
bgColor: 'text-green-50'
bgColor: 'text-green-50',
});
$effect(() => {
show;
@@ -30,21 +30,21 @@
icon: 'heroicons-outline:check-circle',
color: 'text-green-600',
messageColor: 'text-green-500',
bgColor: 'bg-green-50'
bgColor: 'bg-green-50',
};
} else if (type === 'error') {
styleConfig = {
icon: 'heroicons-outline:exclamation-circle',
color: 'text-yellow-600',
messageColor: 'text-yellow-600',
bgColor: 'bg-yellow-50'
bgColor: 'bg-yellow-50',
};
} else if (type === 'warning') {
styleConfig = {
icon: 'heroicons-outline:exclamation',
color: 'text-yellow-600',
messageColor: 'text-yellow-600',
bgColor: 'bg-yellow-50'
bgColor: 'bg-yellow-50',
};
}
});
@@ -59,7 +59,7 @@
{#if show}
<div
aria-live="assertive"
class="pointer-events-none fixed inset-0 flex px-4 py-6 items-start sm:p-6 z-999999"
class="z-999999 pointer-events-none fixed inset-0 flex items-start px-4 py-6 sm:p-6"
in:fly={{ easing: bounceIn, x: 1000, duration: 500 }}
out:fade={{ duration: 100 }}
role="alert"

View File

@@ -11,7 +11,7 @@ export const initialAlertState: AlertType = {
title: '',
message: '',
duration: 0,
show: false
show: false,
};
let alertState = $state(initialAlertState);

View File

@@ -9,8 +9,8 @@
const chartConfig = {
count: {
label: 'Emails Ingested',
color: 'var(--chart-1)'
}
color: 'var(--chart-1)',
},
} satisfies ChartConfig;
</script>
@@ -25,15 +25,15 @@
series={[
{
key: 'count',
...chartConfig.count
}
...chartConfig.count,
},
]}
cRange={[
'var(--color-chart-1)',
'var(--color-chart-2)',
'var(--color-chart-3)',
'var(--color-chart-4)',
'var(--color-chart-5)'
'var(--color-chart-5)',
]}
labels={{}}
props={{
@@ -41,10 +41,10 @@
format: (d) =>
new Date(d).toLocaleDateString('en-US', {
month: 'short',
day: 'numeric'
})
day: 'numeric',
}),
},
area: { curve: curveCatmullRom }
area: { curve: curveCatmullRom },
}}
>
{#snippet tooltip()}

View File

@@ -8,8 +8,8 @@
const chartConfig = {
storageUsed: {
label: 'Storage Used'
}
label: 'Storage Used',
},
} satisfies ChartConfig;
</script>
@@ -25,7 +25,7 @@
'var(--color-chart-2)',
'var(--color-chart-3)',
'var(--color-chart-4)',
'var(--color-chart-5)'
'var(--color-chart-5)',
]}
>
{#snippet tooltip()}

View File

@@ -8,8 +8,8 @@
const chartConfig = {
count: {
label: 'Emails'
}
label: 'Emails',
},
} satisfies ChartConfig;
</script>
@@ -25,15 +25,15 @@
series={[
{
key: 'count',
...chartConfig.count
}
...chartConfig.count,
},
]}
cRange={[
'var(--color-chart-1)',
'var(--color-chart-2)',
'var(--color-chart-3)',
'var(--color-chart-4)',
'var(--color-chart-5)'
'var(--color-chart-5)',
]}
labels={{}}
>

View File

@@ -2,7 +2,6 @@ import type { RequestEvent } from '@sveltejs/kit';
const BASE_URL = '/api/v1'; // Using a relative URL for proxying
/**
* A custom fetch wrapper for the server-side to automatically handle authentication headers.
* @param url The URL to fetch, relative to the API base.
@@ -18,7 +17,7 @@ export const api = async (
const accessToken = event.cookies.get('accessToken');
const defaultHeaders: HeadersInit = {
'Content-Type': 'application/json'
'Content-Type': 'application/json',
};
if (accessToken) {
@@ -29,8 +28,8 @@ export const api = async (
...options,
headers: {
...defaultHeaders,
...options.headers
}
...options.headers,
},
};
return event.fetch(`${BASE_URL}${url}`, mergedOptions);

View File

@@ -30,16 +30,13 @@ const createAuthStore = () => {
}
set(initialValue);
},
syncWithServer: (
user: Omit<User, 'passwordHash'> | null,
accessToken: string | null
) => {
syncWithServer: (user: Omit<User, 'passwordHash'> | null, accessToken: string | null) => {
if (user && accessToken) {
set({ accessToken, user });
} else {
set(initialValue);
}
}
},
};
};

View File

@@ -1,5 +1,5 @@
import { clsx, type ClassValue } from "clsx";
import { twMerge } from "tailwind-merge";
import { clsx, type ClassValue } from 'clsx';
import { twMerge } from 'tailwind-merge';
export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs));
@@ -18,8 +18,8 @@ export function formatBytes(bytes: number, decimals = 2) {
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export type WithoutChild<T> = T extends { child?: any; } ? Omit<T, "child"> : T;
export type WithoutChild<T> = T extends { child?: any } ? Omit<T, 'child'> : T;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export type WithoutChildren<T> = T extends { children?: any; } ? Omit<T, "children"> : T;
export type WithoutChildren<T> = T extends { children?: any } ? Omit<T, 'children'> : T;
export type WithoutChildrenOrChild<T> = WithoutChildren<WithoutChild<T>>;
export type WithElementRef<T, U extends HTMLElement = HTMLElement> = T & { ref?: U | null; };
export type WithElementRef<T, U extends HTMLElement = HTMLElement> = T & { ref?: U | null };

View File

@@ -18,13 +18,11 @@ export const load: LayoutServerLoad = async (event) => {
}
} catch (error) {
throw error;
}
return {
user: locals.user,
accessToken: locals.accessToken,
isDemo: process.env.IS_DEMO === 'true'
isDemo: process.env.IS_DEMO === 'true',
};
};

View File

@@ -13,7 +13,7 @@ const handleRequest: RequestHandler = async ({ request, params }) => {
method: request.method,
headers: request.headers,
body: request.body,
duplex: 'half' // Required for streaming request bodies
duplex: 'half', // Required for streaming request bodies
} as RequestInit);
// Forward the request to the backend

View File

@@ -6,6 +6,6 @@ export const load: LayoutServerLoad = async ({ locals }) => {
throw redirect(302, '/signin');
}
return {
user: locals.user
user: locals.user,
};
};

View File

@@ -9,7 +9,7 @@
{ href: '/dashboard', label: 'Dashboard' },
{ href: '/dashboard/ingestions', label: 'Ingestions' },
{ href: '/dashboard/archived-emails', label: 'Archived emails' },
{ href: '/dashboard/search', label: 'Search' }
{ href: '/dashboard/search', label: 'Search' },
];
let { children } = $props();
function handleLogout() {

View File

@@ -5,7 +5,7 @@ import type {
IngestionHistory,
IngestionSourceStats,
RecentSync,
IndexedInsights
IndexedInsights,
} from '@open-archiver/types';
export const load: PageServerLoad = async (event) => {
@@ -70,7 +70,7 @@ export const load: PageServerLoad = async (event) => {
fetchIngestionHistory(),
fetchIngestionSources(),
fetchRecentSyncs(),
fetchIndexedInsights()
fetchIndexedInsights(),
]);
return {
@@ -78,6 +78,6 @@ export const load: PageServerLoad = async (event) => {
ingestionHistory,
ingestionSources,
recentSyncs,
indexedInsights
indexedInsights,
};
};

View File

@@ -13,7 +13,7 @@
const transformedHistory = $derived(
data.ingestionHistory?.history.map((item) => ({
...item,
date: new Date(item.date)
date: new Date(item.date),
})) ?? []
);
</script>
@@ -44,16 +44,24 @@
{#if data.stats}
<div class="grid gap-4 md:grid-cols-2 lg:grid-cols-3">
<Card.Root>
<Card.Header class="flex flex-row items-center justify-between space-y-0 pb-2">
<Card.Title class="text-sm font-medium">Total Emails Archived</Card.Title>
<Card.Header
class="flex flex-row items-center justify-between space-y-0 pb-2"
>
<Card.Title class="text-sm font-medium"
>Total Emails Archived</Card.Title
>
<Archive class="text-muted-foreground h-4 w-4" />
</Card.Header>
<Card.Content>
<div class="text-primary text-2xl font-bold">{data.stats.totalEmailsArchived}</div>
<div class="text-primary text-2xl font-bold">
{data.stats.totalEmailsArchived}
</div>
</Card.Content>
</Card.Root>
<Card.Root>
<Card.Header class="flex flex-row items-center justify-between space-y-0 pb-2">
<Card.Header
class="flex flex-row items-center justify-between space-y-0 pb-2"
>
<Card.Title class="text-sm font-medium">Total Storage Used</Card.Title>
<HardDrive class="text-muted-foreground h-4 w-4" />
</Card.Header>
@@ -64,8 +72,12 @@
</Card.Content>
</Card.Root>
<Card.Root class="">
<Card.Header class="flex flex-row items-center justify-between space-y-0 pb-2">
<Card.Title class="text-sm font-medium">Failed Ingestions (Last 7 Days)</Card.Title>
<Card.Header
class="flex flex-row items-center justify-between space-y-0 pb-2"
>
<Card.Title class="text-sm font-medium"
>Failed Ingestions (Last 7 Days)</Card.Title
>
<CircleAlert class=" text-muted-foreground h-4 w-4" />
</Card.Header>
<Card.Content>

View File

@@ -19,7 +19,7 @@ export const load: PageServerLoad = async (event) => {
items: [],
total: 0,
page: 1,
limit: 10
limit: 10,
};
const selectedIngestionSourceId = ingestionSourceId || ingestionSources[0]?.id;
@@ -38,7 +38,7 @@ export const load: PageServerLoad = async (event) => {
return {
ingestionSources,
archivedEmails,
selectedIngestionSourceId
selectedIngestionSourceId,
};
} catch (error) {
console.error('Failed to load archived emails page:', error);
@@ -48,9 +48,9 @@ export const load: PageServerLoad = async (event) => {
items: [],
total: 0,
page: 1,
limit: 10
limit: 10,
},
error: 'Failed to load data'
error: 'Failed to load data',
};
}
};

View File

@@ -60,7 +60,10 @@
};
let paginationItems = $derived(
getPaginationItems(archivedEmails.page, Math.ceil(archivedEmails.total / archivedEmails.limit))
getPaginationItems(
archivedEmails.page,
Math.ceil(archivedEmails.total / archivedEmails.limit)
)
);
</script>
@@ -125,7 +128,9 @@
<Table.Cell>{email.userEmail}</Table.Cell>
<Table.Cell>
{#if email.path}
<span class=" bg-muted truncate rounded p-1.5 text-xs">{email.path} </span>
<span class=" bg-muted truncate rounded p-1.5 text-xs"
>{email.path}
</span>
{/if}
</Table.Cell>
<Table.Cell class="text-right">
@@ -137,7 +142,9 @@
{/each}
{:else}
<Table.Row>
<Table.Cell colspan={5} class="text-center">No archived emails found.</Table.Cell>
<Table.Cell colspan={5} class="text-center"
>No archived emails found.</Table.Cell
>
</Table.Row>
{/if}
</Table.Body>
@@ -160,7 +167,9 @@
<a
href={`/dashboard/archived-emails?ingestionSourceId=${selectedIngestionSourceId}&page=${item}&limit=${archivedEmails.limit}`}
>
<Button variant={item === archivedEmails.page ? 'default' : 'outline'}>{item}</Button>
<Button variant={item === archivedEmails.page ? 'default' : 'outline'}
>{item}</Button
>
</a>
{:else}
<span class="px-4 py-2">...</span>
@@ -177,8 +186,8 @@
>
<Button
variant="outline"
disabled={archivedEmails.page === Math.ceil(archivedEmails.total / archivedEmails.limit)}
>Next</Button
disabled={archivedEmails.page ===
Math.ceil(archivedEmails.total / archivedEmails.limit)}>Next</Button
>
</a>
</div>

View File

@@ -11,13 +11,13 @@ export const load: PageServerLoad = async (event) => {
}
const email: ArchivedEmail = await response.json();
return {
email
email,
};
} catch (error) {
console.error('Failed to load archived email:', error);
return {
email: null,
error: 'Failed to load email'
error: 'Failed to load email',
};
}
};

View File

@@ -45,7 +45,7 @@
try {
isDeleting = true;
const response = await api(`/archived-emails/${email.id}`, {
method: 'DELETE'
method: 'DELETE',
});
if (!response.ok) {
const errorData = await response.json().catch(() => null);
@@ -81,7 +81,9 @@
<div class="space-y-1">
<h3 class="font-semibold">Recipients</h3>
<Card.Description>
<p>To: {email.recipients.map((r) => r.email || r.name).join(', ')}</p>
<p>
To: {email.recipients.map((r) => r.email || r.name).join(', ')}
</p>
</Card.Description>
</div>
<div class=" space-y-1">
@@ -99,7 +101,9 @@
<div class="flex flex-wrap items-center gap-2">
<span> Tags: </span>
{#each email.tags as tag}
<span class=" bg-muted truncate rounded p-1.5 text-xs">{tag}</span>
<span class=" bg-muted truncate rounded p-1.5 text-xs"
>{tag}</span
>
{/each}
</div>
{/if}
@@ -120,12 +124,20 @@
<h3 class="font-semibold">Attachments</h3>
<ul class="mt-2 space-y-2">
{#each email.attachments as attachment}
<li class="flex items-center justify-between rounded-md border p-2">
<span>{attachment.filename} ({attachment.sizeBytes} bytes)</span>
<li
class="flex items-center justify-between rounded-md border p-2"
>
<span
>{attachment.filename} ({attachment.sizeBytes} bytes)</span
>
<Button
variant="outline"
size="sm"
onclick={() => download(attachment.storagePath, attachment.filename)}
onclick={() =>
download(
attachment.storagePath,
attachment.filename
)}
>
Download
</Button>
@@ -144,13 +156,12 @@
<Card.Title>Actions</Card.Title>
</Card.Header>
<Card.Content class="space-y-2">
<Button onclick={() => download(email.storagePath, `${email.subject || 'email'}.eml`)}
<Button
onclick={() =>
download(email.storagePath, `${email.subject || 'email'}.eml`)}
>Download Email (.eml)</Button
>
<Button
variant="destructive"
onclick={() => (isDeleteDialogOpen = true)}
>
<Button variant="destructive" onclick={() => (isDeleteDialogOpen = true)}>
Delete Email
</Button>
</Card.Content>

View File

@@ -10,13 +10,13 @@ export const load: PageServerLoad = async (event) => {
}
const ingestionSources: IngestionSource[] = await response.json();
return {
ingestionSources
ingestionSources,
};
} catch (error) {
console.error('Failed to load ingestion sources:', error);
return {
ingestionSources: [],
error: 'Failed to load ingestion sources'
error: 'Failed to load ingestion sources',
};
}
};

View File

@@ -36,7 +36,7 @@
title: 'Demo mode',
message: 'Editing is not allowed in demo mode.',
duration: 5000,
show: true
show: true,
});
return;
}
@@ -61,7 +61,7 @@
title: 'Failed to delete ingestion',
message: errorBody.message || JSON.stringify(errorBody),
duration: 5000,
show: true
show: true,
});
return;
}
@@ -82,7 +82,7 @@
title: 'Failed to trigger force sync ingestion',
message: errorBody.message || JSON.stringify(errorBody),
duration: 5000,
show: true
show: true,
});
return;
}
@@ -107,7 +107,7 @@
} else {
await api(`/ingestion-sources/${source.id}`, {
method: 'PUT',
body: JSON.stringify({ status: 'active' })
body: JSON.stringify({ status: 'active' }),
});
}
@@ -123,7 +123,7 @@
title: 'Failed to trigger force sync ingestion',
message: e instanceof Error ? e.message : JSON.stringify(e),
duration: 5000,
show: true
show: true,
});
}
};
@@ -140,7 +140,7 @@
title: `Failed to delete ingestion ${id}`,
message: errorBody.message || JSON.stringify(errorBody),
duration: 5000,
show: true
show: true,
});
}
}
@@ -163,7 +163,7 @@
title: `Failed to trigger force sync for ingestion ${id}`,
message: errorBody.message || JSON.stringify(errorBody),
duration: 5000,
show: true
show: true,
});
}
}
@@ -181,7 +181,7 @@
title: 'Failed to trigger force sync',
message: e instanceof Error ? e.message : JSON.stringify(e),
duration: 5000,
show: true
show: true,
});
}
};
@@ -192,7 +192,7 @@
// Update
const response = await api(`/ingestion-sources/${selectedSource.id}`, {
method: 'PUT',
body: JSON.stringify(formData)
body: JSON.stringify(formData),
});
if (!response.ok) {
const errorData = await response.json();
@@ -206,7 +206,7 @@
// Create
const response = await api('/ingestion-sources', {
method: 'POST',
body: JSON.stringify(formData)
body: JSON.stringify(formData),
});
if (!response.ok) {
const errorData = await response.json();
@@ -226,7 +226,7 @@
title: 'Authentication Failed',
message,
duration: 5000,
show: true
show: true,
});
}
};
@@ -276,7 +276,10 @@
<RefreshCw class="mr-2 h-4 w-4" />
Force Sync
</DropdownMenu.Item>
<DropdownMenu.Item class="text-red-600" onclick={() => (isBulkDeleteDialogOpen = true)}>
<DropdownMenu.Item
class="text-red-600"
onclick={() => (isBulkDeleteDialogOpen = true)}
>
<Trash class="mr-2 h-4 w-4" />
Delete
</DropdownMenu.Item>
@@ -300,7 +303,8 @@
selectedIds = [];
}
}}
checked={ingestionSources.length > 0 && selectedIds.length === ingestionSources.length
checked={ingestionSources.length > 0 &&
selectedIds.length === ingestionSources.length
? true
: ((selectedIds.length > 0 ? 'indeterminate' : false) as any)}
/>
@@ -322,7 +326,9 @@
checked={selectedIds.includes(source.id)}
onCheckedChange={() => {
if (selectedIds.includes(source.id)) {
selectedIds = selectedIds.filter((id) => id !== source.id);
selectedIds = selectedIds.filter(
(id) => id !== source.id
);
} else {
selectedIds = [...selectedIds, source.id];
}
@@ -330,15 +336,23 @@
/>
</Table.Cell>
<Table.Cell>
<a class="link" href="/dashboard/archived-emails?ingestionSourceId={source.id}"
<a
class="link"
href="/dashboard/archived-emails?ingestionSourceId={source.id}"
>{source.name}</a
>
</Table.Cell>
<Table.Cell class="capitalize">{source.provider.split('_').join(' ')}</Table.Cell>
<Table.Cell class="capitalize"
>{source.provider.split('_').join(' ')}</Table.Cell
>
<Table.Cell class="min-w-24">
<HoverCard.Root>
<HoverCard.Trigger>
<Badge class="{getStatusClasses(source.status)} cursor-pointer capitalize">
<Badge
class="{getStatusClasses(
source.status
)} cursor-pointer capitalize"
>
{source.status.split('_').join(' ')}
</Badge>
</HoverCard.Trigger>
@@ -358,10 +372,13 @@
class="cursor-pointer"
checked={source.status !== 'paused'}
onCheckedChange={() => handleToggle(source)}
disabled={source.status === 'importing' || source.status === 'syncing'}
disabled={source.status === 'importing' ||
source.status === 'syncing'}
/>
</Table.Cell>
<Table.Cell>{new Date(source.createdAt).toLocaleDateString()}</Table.Cell>
<Table.Cell
>{new Date(source.createdAt).toLocaleDateString()}</Table.Cell
>
<Table.Cell class="text-right">
<DropdownMenu.Root>
<DropdownMenu.Trigger>
@@ -379,7 +396,9 @@
>Force sync</DropdownMenu.Item
>
<DropdownMenu.Separator />
<DropdownMenu.Item class="text-red-600" onclick={() => openDeleteDialog(source)}
<DropdownMenu.Item
class="text-red-600"
onclick={() => openDeleteDialog(source)}
>Delete</DropdownMenu.Item
>
</DropdownMenu.Content>
@@ -409,7 +428,8 @@
>Read <a
class="text-primary underline underline-offset-2"
target="_blank"
href="https://docs.openarchiver.com/user-guides/email-providers/">docs here</a
href="https://docs.openarchiver.com/user-guides/email-providers/"
>docs here</a
>.</span
>
</Dialog.Description>
@@ -423,12 +443,17 @@
<Dialog.Header>
<Dialog.Title>Are you sure you want to delete this ingestion?</Dialog.Title>
<Dialog.Description>
This will delete all archived emails, attachments, indexing, and files associated with this
ingestion. If you only want to stop syncing new emails, you can pause the ingestion instead.
This will delete all archived emails, attachments, indexing, and files associated
with this ingestion. If you only want to stop syncing new emails, you can pause the
ingestion instead.
</Dialog.Description>
</Dialog.Header>
<Dialog.Footer class="sm:justify-start">
<Button type="button" variant="destructive" onclick={confirmDelete} disabled={isDeleting}
<Button
type="button"
variant="destructive"
onclick={confirmDelete}
disabled={isDeleting}
>{#if isDeleting}Deleting...{:else}Confirm{/if}</Button
>
<Dialog.Close>
@@ -445,13 +470,17 @@
>Are you sure you want to delete {selectedIds.length} selected ingestions?</Dialog.Title
>
<Dialog.Description>
This will delete all archived emails, attachments, indexing, and files associated with these
ingestions. If you only want to stop syncing new emails, you can pause the ingestions
instead.
This will delete all archived emails, attachments, indexing, and files associated
with these ingestions. If you only want to stop syncing new emails, you can pause
the ingestions instead.
</Dialog.Description>
</Dialog.Header>
<Dialog.Footer class="sm:justify-start">
<Button type="button" variant="destructive" onclick={handleBulkDelete} disabled={isDeleting}
<Button
type="button"
variant="destructive"
onclick={handleBulkDelete}
disabled={isDeleting}
>{#if isDeleting}Deleting...{:else}Confirm{/if}</Button
>
<Dialog.Close>

Some files were not shown because too many files have changed in this diff Show More