mirror of
https://github.com/LogicLabs-OU/OpenArchiver.git
synced 2026-04-06 00:31:57 +02:00
feat: add preserve-original-file mode for email ingestion for GoBD compliance
- Add `preserveOriginalFile` option to ingestion sources and connectors - Stream original EML/MBOX/PST emails to temp files instead of holding full buffers in memory, reducing memory allocation during ingestion - Skip attachment binary extraction and EML re-serialization when preserve mode is enabled; use raw file on disk as source of truth - Update `EmailObject` to use `tempFilePath` instead of in-memory `eml` buffer across all connectors (EML, MBOX, PST) - Add new database migration (0032) for `preserve_original_file` column - Add frontend UI toggle with tooltip (tippy.js) for the new option - Replace console.warn calls with structured pino logger in connectors
This commit is contained in:
wayneshn
@@ -0,0 +1 @@
|
||||
ALTER TABLE "ingestion_sources" ADD COLUMN "preserve_original_file" boolean DEFAULT false NOT NULL;
|
||||
1840
packages/backend/src/database/migrations/meta/0031_snapshot.json
Normal file
1840
packages/backend/src/database/migrations/meta/0031_snapshot.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,223 +1,230 @@
|
||||
{
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"entries": [
|
||||
{
|
||||
"idx": 0,
|
||||
"version": "7",
|
||||
"when": 1752225352591,
|
||||
"tag": "0000_amusing_namora",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 1,
|
||||
"version": "7",
|
||||
"when": 1752326803882,
|
||||
"tag": "0001_odd_night_thrasher",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 2,
|
||||
"version": "7",
|
||||
"when": 1752332648392,
|
||||
"tag": "0002_lethal_quentin_quire",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 3,
|
||||
"version": "7",
|
||||
"when": 1752332967084,
|
||||
"tag": "0003_petite_wrecker",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 4,
|
||||
"version": "7",
|
||||
"when": 1752606108876,
|
||||
"tag": "0004_sleepy_paper_doll",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 5,
|
||||
"version": "7",
|
||||
"when": 1752606327253,
|
||||
"tag": "0005_chunky_sue_storm",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 6,
|
||||
"version": "7",
|
||||
"when": 1753112018514,
|
||||
"tag": "0006_majestic_caretaker",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 7,
|
||||
"version": "7",
|
||||
"when": 1753190159356,
|
||||
"tag": "0007_handy_archangel",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 8,
|
||||
"version": "7",
|
||||
"when": 1753370737317,
|
||||
"tag": "0008_eminent_the_spike",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 9,
|
||||
"version": "7",
|
||||
"when": 1754337938241,
|
||||
"tag": "0009_late_lenny_balinger",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 10,
|
||||
"version": "7",
|
||||
"when": 1754420780849,
|
||||
"tag": "0010_perpetual_lightspeed",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 11,
|
||||
"version": "7",
|
||||
"when": 1754422064158,
|
||||
"tag": "0011_tan_blackheart",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 12,
|
||||
"version": "7",
|
||||
"when": 1754476962901,
|
||||
"tag": "0012_warm_the_stranger",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 13,
|
||||
"version": "7",
|
||||
"when": 1754659373517,
|
||||
"tag": "0013_classy_talkback",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 14,
|
||||
"version": "7",
|
||||
"when": 1754831765718,
|
||||
"tag": "0014_foamy_vapor",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 15,
|
||||
"version": "7",
|
||||
"when": 1755443936046,
|
||||
"tag": "0015_wakeful_norman_osborn",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 16,
|
||||
"version": "7",
|
||||
"when": 1755780572342,
|
||||
"tag": "0016_lonely_mariko_yashida",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 17,
|
||||
"version": "7",
|
||||
"when": 1755961566627,
|
||||
"tag": "0017_tranquil_shooting_star",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 18,
|
||||
"version": "7",
|
||||
"when": 1756911118035,
|
||||
"tag": "0018_flawless_owl",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 19,
|
||||
"version": "7",
|
||||
"when": 1756937533843,
|
||||
"tag": "0019_confused_scream",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 20,
|
||||
"version": "7",
|
||||
"when": 1757860242528,
|
||||
"tag": "0020_panoramic_wolverine",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 21,
|
||||
"version": "7",
|
||||
"when": 1759412986134,
|
||||
"tag": "0021_nosy_veda",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 22,
|
||||
"version": "7",
|
||||
"when": 1759701622932,
|
||||
"tag": "0022_complete_triton",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 23,
|
||||
"version": "7",
|
||||
"when": 1760354094610,
|
||||
"tag": "0023_swift_swordsman",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 24,
|
||||
"version": "7",
|
||||
"when": 1772842674479,
|
||||
"tag": "0024_careful_black_panther",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 25,
|
||||
"version": "7",
|
||||
"when": 1773013461190,
|
||||
"tag": "0025_peaceful_grim_reaper",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 26,
|
||||
"version": "7",
|
||||
"when": 1773326266420,
|
||||
"tag": "0026_pink_fantastic_four",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 27,
|
||||
"version": "7",
|
||||
"when": 1773768709477,
|
||||
"tag": "0027_black_morph",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 28,
|
||||
"version": "7",
|
||||
"when": 1773770326402,
|
||||
"tag": "0028_youthful_kitty_pryde",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 29,
|
||||
"version": "7",
|
||||
"when": 1773927678269,
|
||||
"tag": "0029_lethal_brood",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 30,
|
||||
"version": "7",
|
||||
"when": 1774440788278,
|
||||
"tag": "0030_strong_ultron",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"entries": [
|
||||
{
|
||||
"idx": 0,
|
||||
"version": "7",
|
||||
"when": 1752225352591,
|
||||
"tag": "0000_amusing_namora",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 1,
|
||||
"version": "7",
|
||||
"when": 1752326803882,
|
||||
"tag": "0001_odd_night_thrasher",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 2,
|
||||
"version": "7",
|
||||
"when": 1752332648392,
|
||||
"tag": "0002_lethal_quentin_quire",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 3,
|
||||
"version": "7",
|
||||
"when": 1752332967084,
|
||||
"tag": "0003_petite_wrecker",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 4,
|
||||
"version": "7",
|
||||
"when": 1752606108876,
|
||||
"tag": "0004_sleepy_paper_doll",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 5,
|
||||
"version": "7",
|
||||
"when": 1752606327253,
|
||||
"tag": "0005_chunky_sue_storm",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 6,
|
||||
"version": "7",
|
||||
"when": 1753112018514,
|
||||
"tag": "0006_majestic_caretaker",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 7,
|
||||
"version": "7",
|
||||
"when": 1753190159356,
|
||||
"tag": "0007_handy_archangel",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 8,
|
||||
"version": "7",
|
||||
"when": 1753370737317,
|
||||
"tag": "0008_eminent_the_spike",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 9,
|
||||
"version": "7",
|
||||
"when": 1754337938241,
|
||||
"tag": "0009_late_lenny_balinger",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 10,
|
||||
"version": "7",
|
||||
"when": 1754420780849,
|
||||
"tag": "0010_perpetual_lightspeed",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 11,
|
||||
"version": "7",
|
||||
"when": 1754422064158,
|
||||
"tag": "0011_tan_blackheart",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 12,
|
||||
"version": "7",
|
||||
"when": 1754476962901,
|
||||
"tag": "0012_warm_the_stranger",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 13,
|
||||
"version": "7",
|
||||
"when": 1754659373517,
|
||||
"tag": "0013_classy_talkback",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 14,
|
||||
"version": "7",
|
||||
"when": 1754831765718,
|
||||
"tag": "0014_foamy_vapor",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 15,
|
||||
"version": "7",
|
||||
"when": 1755443936046,
|
||||
"tag": "0015_wakeful_norman_osborn",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 16,
|
||||
"version": "7",
|
||||
"when": 1755780572342,
|
||||
"tag": "0016_lonely_mariko_yashida",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 17,
|
||||
"version": "7",
|
||||
"when": 1755961566627,
|
||||
"tag": "0017_tranquil_shooting_star",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 18,
|
||||
"version": "7",
|
||||
"when": 1756911118035,
|
||||
"tag": "0018_flawless_owl",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 19,
|
||||
"version": "7",
|
||||
"when": 1756937533843,
|
||||
"tag": "0019_confused_scream",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 20,
|
||||
"version": "7",
|
||||
"when": 1757860242528,
|
||||
"tag": "0020_panoramic_wolverine",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 21,
|
||||
"version": "7",
|
||||
"when": 1759412986134,
|
||||
"tag": "0021_nosy_veda",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 22,
|
||||
"version": "7",
|
||||
"when": 1759701622932,
|
||||
"tag": "0022_complete_triton",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 23,
|
||||
"version": "7",
|
||||
"when": 1760354094610,
|
||||
"tag": "0023_swift_swordsman",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 24,
|
||||
"version": "7",
|
||||
"when": 1772842674479,
|
||||
"tag": "0024_careful_black_panther",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 25,
|
||||
"version": "7",
|
||||
"when": 1773013461190,
|
||||
"tag": "0025_peaceful_grim_reaper",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 26,
|
||||
"version": "7",
|
||||
"when": 1773326266420,
|
||||
"tag": "0026_pink_fantastic_four",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 27,
|
||||
"version": "7",
|
||||
"when": 1773768709477,
|
||||
"tag": "0027_black_morph",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 28,
|
||||
"version": "7",
|
||||
"when": 1773770326402,
|
||||
"tag": "0028_youthful_kitty_pryde",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 29,
|
||||
"version": "7",
|
||||
"when": 1773927678269,
|
||||
"tag": "0029_lethal_brood",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 30,
|
||||
"version": "7",
|
||||
"when": 1774440788278,
|
||||
"tag": "0030_strong_ultron",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 31,
|
||||
"version": "7",
|
||||
"when": 1774623960683,
|
||||
"tag": "0031_bouncy_boomerang",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { jsonb, pgEnum, pgTable, text, timestamp, uuid } from 'drizzle-orm/pg-core';
|
||||
import { boolean, jsonb, pgEnum, pgTable, text, timestamp, uuid } from 'drizzle-orm/pg-core';
|
||||
import { users } from './users';
|
||||
import { relations } from 'drizzle-orm';
|
||||
|
||||
@@ -34,6 +34,7 @@ export const ingestionSources = pgTable('ingestion_sources', {
|
||||
lastSyncFinishedAt: timestamp('last_sync_finished_at', { withTimezone: true }),
|
||||
lastSyncStatusMessage: text('last_sync_status_message'),
|
||||
syncState: jsonb('sync_state'),
|
||||
preserveOriginalFile: boolean('preserve_original_file').notNull().default(false),
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
});
|
||||
|
||||
@@ -21,6 +21,7 @@ import { AuditService } from './AuditService';
|
||||
import { User } from '@open-archiver/types';
|
||||
import { checkDeletionEnabled } from '../helpers/deletionGuard';
|
||||
import { RetentionHook } from '../hooks/RetentionHook';
|
||||
import { logger } from '../config/logger';
|
||||
|
||||
interface DbRecipients {
|
||||
to: { name: string; address: string }[];
|
||||
@@ -263,7 +264,13 @@ export class ArchivedEmailService {
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to delete email attachments', error);
|
||||
logger.error(
|
||||
{
|
||||
emailId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
'Failed to delete email attachments'
|
||||
);
|
||||
throw new Error('Failed to delete email attachments');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,18 @@ import { PSTConnector } from './ingestion-connectors/PSTConnector';
|
||||
import { EMLConnector } from './ingestion-connectors/EMLConnector';
|
||||
import { MboxConnector } from './ingestion-connectors/MboxConnector';
|
||||
|
||||
/**
|
||||
* Options passed to connectors to control ingestion behaviour.
|
||||
* Currently used to skip extracting full attachment binary content
|
||||
* in preserve-original-file (GoBD) mode, where attachments are never
|
||||
* stored separately and the raw EML is kept as-is.
|
||||
*/
|
||||
export interface ConnectorOptions {
|
||||
/** When true, connectors omit attachment binary content from the
|
||||
* yielded EmailObject to avoid unnecessary memory allocation. */
|
||||
preserveOriginalFile: boolean;
|
||||
}
|
||||
|
||||
// Define a common interface for all connectors
|
||||
export interface IEmailConnector {
|
||||
testConnection(): Promise<boolean>;
|
||||
@@ -34,20 +46,26 @@ export class EmailProviderFactory {
|
||||
static createConnector(source: IngestionSource): IEmailConnector {
|
||||
// Credentials are now decrypted by the IngestionService before being passed around
|
||||
const credentials = source.credentials;
|
||||
const options: ConnectorOptions = {
|
||||
preserveOriginalFile: source.preserveOriginalFile ?? false,
|
||||
};
|
||||
|
||||
switch (source.provider) {
|
||||
case 'google_workspace':
|
||||
return new GoogleWorkspaceConnector(credentials as GoogleWorkspaceCredentials);
|
||||
return new GoogleWorkspaceConnector(
|
||||
credentials as GoogleWorkspaceCredentials,
|
||||
options
|
||||
);
|
||||
case 'microsoft_365':
|
||||
return new MicrosoftConnector(credentials as Microsoft365Credentials);
|
||||
return new MicrosoftConnector(credentials as Microsoft365Credentials, options);
|
||||
case 'generic_imap':
|
||||
return new ImapConnector(credentials as GenericImapCredentials);
|
||||
return new ImapConnector(credentials as GenericImapCredentials, options);
|
||||
case 'pst_import':
|
||||
return new PSTConnector(credentials as PSTImportCredentials);
|
||||
return new PSTConnector(credentials as PSTImportCredentials, options);
|
||||
case 'eml_import':
|
||||
return new EMLConnector(credentials as EMLImportCredentials);
|
||||
return new EMLConnector(credentials as EMLImportCredentials, options);
|
||||
case 'mbox_import':
|
||||
return new MboxConnector(credentials as MboxImportCredentials);
|
||||
return new MboxConnector(credentials as MboxImportCredentials, options);
|
||||
default:
|
||||
throw new Error(`Unsupported provider: ${source.provider}`);
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import { DatabaseService } from './DatabaseService';
|
||||
import { archivedEmails, attachments, emailAttachments } from '../database/schema';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { streamToBuffer } from '../helpers/streamToBuffer';
|
||||
import { simpleParser } from 'mailparser';
|
||||
import { simpleParser, type Attachment as ParsedAttachment } from 'mailparser';
|
||||
import { logger } from '../config/logger';
|
||||
|
||||
interface DbRecipients {
|
||||
@@ -351,9 +351,13 @@ export class IndexingService {
|
||||
content: textContent,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`Failed to extract text from attachment: ${attachment.filename}`,
|
||||
error
|
||||
logger.error(
|
||||
{
|
||||
filename: attachment.filename,
|
||||
mimeType: attachment.mimeType,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
'Failed to extract text from attachment'
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -378,8 +382,6 @@ export class IndexingService {
|
||||
attachments: Attachment[],
|
||||
userEmail: string //the owner of the email inbox
|
||||
): Promise<EmailDocument> {
|
||||
const attachmentContents = await this.extractAttachmentContents(attachments);
|
||||
|
||||
const emailBodyStream = await this.storageService.get(email.storagePath);
|
||||
const emailBodyBuffer = await streamToBuffer(emailBodyStream);
|
||||
const parsedEmail = await simpleParser(emailBodyBuffer);
|
||||
@@ -389,6 +391,20 @@ export class IndexingService {
|
||||
(await extractText(emailBodyBuffer, 'text/plain')) ||
|
||||
'';
|
||||
|
||||
// If there are linked attachment records, extract text from storage (default mode).
|
||||
// Otherwise, if the email has attachments but no records (preserve original file mode),
|
||||
// extract attachment text directly from the parsed EML body.
|
||||
let attachmentContents: { filename: string; content: string }[];
|
||||
if (attachments.length > 0) {
|
||||
attachmentContents = await this.extractAttachmentContents(attachments);
|
||||
} else if (email.hasAttachments && parsedEmail.attachments.length > 0) {
|
||||
attachmentContents = await this.extractInlineAttachmentContents(
|
||||
parsedEmail.attachments
|
||||
);
|
||||
} else {
|
||||
attachmentContents = [];
|
||||
}
|
||||
|
||||
const recipients = email.recipients as DbRecipients;
|
||||
// console.log('email.userEmail', email.userEmail);
|
||||
return {
|
||||
@@ -406,6 +422,40 @@ export class IndexingService {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts text content from attachments embedded in the parsed EML.
|
||||
* Used in preserve-original-file (GoBD) mode where no separate attachment
|
||||
* records exist — the full MIME body is stored unmodified, so we parse
|
||||
* attachments directly from the in-memory parsed email.
|
||||
*/
|
||||
private async extractInlineAttachmentContents(
|
||||
parsedAttachments: ParsedAttachment[]
|
||||
): Promise<{ filename: string; content: string }[]> {
|
||||
const extracted: { filename: string; content: string }[] = [];
|
||||
for (const attachment of parsedAttachments) {
|
||||
try {
|
||||
const textContent = await extractText(
|
||||
attachment.content,
|
||||
attachment.contentType || ''
|
||||
);
|
||||
extracted.push({
|
||||
filename: attachment.filename || 'untitled',
|
||||
content: textContent,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
{
|
||||
filename: attachment.filename,
|
||||
mimeType: attachment.contentType,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
'Failed to extract text from inline attachment in preserve-original mode'
|
||||
);
|
||||
}
|
||||
}
|
||||
return extracted;
|
||||
}
|
||||
|
||||
private async extractAttachmentContents(
|
||||
attachments: Attachment[]
|
||||
): Promise<{ filename: string; content: string }[]> {
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
emailAttachments,
|
||||
} from '../database/schema';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import { readFile, unlink } from 'fs/promises';
|
||||
import { logger } from '../config/logger';
|
||||
import { SearchService } from './SearchService';
|
||||
import { config } from '../config/index';
|
||||
@@ -420,6 +421,9 @@ export class IngestionService {
|
||||
userEmail: string
|
||||
): Promise<PendingEmail | null> {
|
||||
try {
|
||||
// Read the raw bytes from the temp file written by the connector
|
||||
const rawEmlBuffer = await readFile(email.tempFilePath);
|
||||
|
||||
// Generate a unique message ID for the email. If the email already has a message-id header, use that.
|
||||
// Otherwise, generate a new one based on the email's hash, source ID, and email ID.
|
||||
const messageIdHeader = email.headers.get('message-id');
|
||||
@@ -431,7 +435,7 @@ export class IngestionService {
|
||||
}
|
||||
if (!messageId) {
|
||||
messageId = `generated-${createHash('sha256')
|
||||
.update(email.eml ?? Buffer.from(email.body, 'utf-8'))
|
||||
.update(rawEmlBuffer)
|
||||
.digest('hex')}-${source.id}-${email.id}`;
|
||||
}
|
||||
// Check if an email with the same message ID has already been imported for the current ingestion source. This is to prevent duplicate imports when an email is present in multiple mailboxes (e.g., "Inbox" and "All Mail").
|
||||
@@ -450,13 +454,70 @@ export class IngestionService {
|
||||
return null;
|
||||
}
|
||||
|
||||
const rawEmlBuffer = email.eml ?? Buffer.from(email.body, 'utf-8');
|
||||
// Strip non-inline attachments from the .eml to avoid double-storing
|
||||
const sanitizedPath = email.path ? email.path : '';
|
||||
const emailPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${sanitizedPath}${email.id}.eml`;
|
||||
|
||||
// GoBD / Preserve Original File mode: store the unmodified raw EML as-is.
|
||||
// No attachment stripping, no attachment table records — the full MIME body
|
||||
// including attachments is preserved in the single .eml file.
|
||||
if (source.preserveOriginalFile) {
|
||||
const emailHash = createHash('sha256').update(rawEmlBuffer).digest('hex');
|
||||
|
||||
// Message-level deduplication by file hash
|
||||
const hashDuplicate = await db.query.archivedEmails.findFirst({
|
||||
where: and(
|
||||
eq(archivedEmails.storageHashSha256, emailHash),
|
||||
eq(archivedEmails.ingestionSourceId, source.id)
|
||||
),
|
||||
columns: { id: true },
|
||||
});
|
||||
|
||||
if (hashDuplicate) {
|
||||
logger.info(
|
||||
{ emailHash, ingestionSourceId: source.id },
|
||||
'Skipping duplicate email (hash-level dedup, preserve original mode)'
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Store the unmodified raw buffer — no modifications
|
||||
await storage.put(emailPath, rawEmlBuffer);
|
||||
|
||||
const [archivedEmail] = await db
|
||||
.insert(archivedEmails)
|
||||
.values({
|
||||
ingestionSourceId: source.id,
|
||||
userEmail,
|
||||
threadId: email.threadId,
|
||||
messageIdHeader: messageId,
|
||||
providerMessageId: email.id,
|
||||
sentAt: email.receivedAt,
|
||||
subject: email.subject,
|
||||
senderName: email.from[0]?.name,
|
||||
senderEmail: email.from[0]?.address,
|
||||
recipients: {
|
||||
to: email.to,
|
||||
cc: email.cc,
|
||||
bcc: email.bcc,
|
||||
},
|
||||
storagePath: emailPath,
|
||||
storageHashSha256: emailHash,
|
||||
sizeBytes: rawEmlBuffer.length,
|
||||
hasAttachments: email.attachments.length > 0,
|
||||
path: email.path,
|
||||
tags: email.tags,
|
||||
})
|
||||
.returning();
|
||||
|
||||
return {
|
||||
archivedEmailId: archivedEmail.id,
|
||||
};
|
||||
}
|
||||
|
||||
// Default mode: strip non-inline attachments from the .eml to avoid double-storing
|
||||
// attachment data (attachments are stored separately).
|
||||
const emlBuffer = await stripAttachmentsFromEml(rawEmlBuffer);
|
||||
const emailHash = createHash('sha256').update(emlBuffer).digest('hex');
|
||||
const sanitizedPath = email.path ? email.path : '';
|
||||
const emailPath = `${config.storage.openArchiverFolderName}/${source.name.replaceAll(' ', '-')}-${source.id}/emails/${sanitizedPath}${email.id}.eml`;
|
||||
await storage.put(emailPath, emlBuffer);
|
||||
|
||||
const [archivedEmail] = await db
|
||||
@@ -564,6 +625,14 @@ export class IngestionService {
|
||||
ingestionSourceId: source.id,
|
||||
});
|
||||
return null;
|
||||
} finally {
|
||||
// Always clean up the temp file, regardless of success or failure
|
||||
await unlink(email.tempFilePath).catch((err) =>
|
||||
logger.warn(
|
||||
{ err, tempFilePath: email.tempFilePath },
|
||||
'Failed to delete temp email file'
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,10 +5,11 @@ import type {
|
||||
SyncState,
|
||||
MailboxUser,
|
||||
} from '@open-archiver/types';
|
||||
import type { IEmailConnector } from '../EmailProviderFactory';
|
||||
import type { IEmailConnector, ConnectorOptions } from '../EmailProviderFactory';
|
||||
import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
|
||||
import { logger } from '../../config/logger';
|
||||
import { getThreadId } from './helpers/utils';
|
||||
import { writeEmailToTempFile } from './helpers/tempFile';
|
||||
import { StorageService } from '../StorageService';
|
||||
import { Readable } from 'stream';
|
||||
import { createHash } from 'crypto';
|
||||
@@ -27,8 +28,13 @@ const streamToBuffer = (stream: Readable): Promise<Buffer> => {
|
||||
|
||||
export class EMLConnector implements IEmailConnector {
|
||||
private storage: StorageService;
|
||||
private options: ConnectorOptions;
|
||||
|
||||
constructor(private credentials: EMLImportCredentials) {
|
||||
constructor(
|
||||
private credentials: EMLImportCredentials,
|
||||
options?: ConnectorOptions
|
||||
) {
|
||||
this.options = options ?? { preserveOriginalFile: false };
|
||||
this.storage = new StorageService();
|
||||
}
|
||||
|
||||
@@ -266,13 +272,18 @@ export class EMLConnector implements IEmailConnector {
|
||||
emlBuffer = await streamToBuffer(input);
|
||||
}
|
||||
|
||||
const tempFilePath = await writeEmailToTempFile(emlBuffer);
|
||||
const parsedEmail: ParsedMail = await simpleParser(emlBuffer);
|
||||
|
||||
// In preserve-original mode, skip extracting full attachment binary content
|
||||
// to avoid unnecessary memory allocation — the raw EML on disk is the source of truth.
|
||||
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
|
||||
filename: attachment.filename || 'untitled',
|
||||
contentType: attachment.contentType,
|
||||
size: attachment.size,
|
||||
content: attachment.content as Buffer,
|
||||
content: this.options.preserveOriginalFile
|
||||
? Buffer.alloc(0)
|
||||
: (attachment.content as Buffer),
|
||||
}));
|
||||
|
||||
const mapAddresses = (
|
||||
@@ -313,7 +324,7 @@ export class EMLConnector implements IEmailConnector {
|
||||
headers: parsedEmail.headers,
|
||||
attachments,
|
||||
receivedAt: parsedEmail.date || new Date(),
|
||||
eml: emlBuffer,
|
||||
tempFilePath,
|
||||
path,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -7,10 +7,11 @@ import type {
|
||||
SyncState,
|
||||
MailboxUser,
|
||||
} from '@open-archiver/types';
|
||||
import type { IEmailConnector } from '../EmailProviderFactory';
|
||||
import type { IEmailConnector, ConnectorOptions } from '../EmailProviderFactory';
|
||||
import { logger } from '../../config/logger';
|
||||
import { simpleParser, ParsedMail, Attachment, AddressObject, Headers } from 'mailparser';
|
||||
import { getThreadId } from './helpers/utils';
|
||||
import { writeEmailToTempFile } from './helpers/tempFile';
|
||||
|
||||
/**
|
||||
* A connector for Google Workspace that uses a service account with domain-wide delegation
|
||||
@@ -20,9 +21,11 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
|
||||
private credentials: GoogleWorkspaceCredentials;
|
||||
private serviceAccountCreds: { client_email: string; private_key: string };
|
||||
private newHistoryId: string | undefined;
|
||||
private options: ConnectorOptions;
|
||||
|
||||
constructor(credentials: GoogleWorkspaceCredentials) {
|
||||
constructor(credentials: GoogleWorkspaceCredentials, options?: ConnectorOptions) {
|
||||
this.credentials = credentials;
|
||||
this.options = options ?? { preserveOriginalFile: false };
|
||||
try {
|
||||
// Pre-parse the JSON key to catch errors early.
|
||||
const parsedKey = JSON.parse(this.credentials.serviceAccountKeyJson);
|
||||
@@ -201,48 +204,13 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
|
||||
|
||||
if (msgResponse.data.raw) {
|
||||
const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
|
||||
const parsedEmail: ParsedMail = await simpleParser(rawEmail);
|
||||
const attachments = parsedEmail.attachments.map(
|
||||
(attachment: Attachment) => ({
|
||||
filename: attachment.filename || 'untitled',
|
||||
contentType: attachment.contentType,
|
||||
size: attachment.size,
|
||||
content: attachment.content as Buffer,
|
||||
})
|
||||
yield this.parseRawEmail(
|
||||
rawEmail,
|
||||
msgResponse.data.id!,
|
||||
userEmail,
|
||||
labels.path,
|
||||
labels.tags
|
||||
);
|
||||
const mapAddresses = (
|
||||
addresses: AddressObject | AddressObject[] | undefined
|
||||
): EmailAddress[] => {
|
||||
if (!addresses) return [];
|
||||
const addressArray = Array.isArray(addresses)
|
||||
? addresses
|
||||
: [addresses];
|
||||
return addressArray.flatMap((a) =>
|
||||
a.value.map((v) => ({
|
||||
name: v.name,
|
||||
address: v.address || '',
|
||||
}))
|
||||
);
|
||||
};
|
||||
const threadId = getThreadId(parsedEmail.headers);
|
||||
yield {
|
||||
id: msgResponse.data.id!,
|
||||
threadId,
|
||||
userEmail: userEmail,
|
||||
eml: rawEmail,
|
||||
from: mapAddresses(parsedEmail.from),
|
||||
to: mapAddresses(parsedEmail.to),
|
||||
cc: mapAddresses(parsedEmail.cc),
|
||||
bcc: mapAddresses(parsedEmail.bcc),
|
||||
subject: parsedEmail.subject || '',
|
||||
body: parsedEmail.text || '',
|
||||
html: parsedEmail.html || '',
|
||||
headers: parsedEmail.headers,
|
||||
attachments,
|
||||
receivedAt: parsedEmail.date || new Date(),
|
||||
path: labels.path,
|
||||
tags: labels.tags,
|
||||
};
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code === 404) {
|
||||
@@ -326,45 +294,13 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
|
||||
|
||||
if (msgResponse.data.raw) {
|
||||
const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
|
||||
const parsedEmail: ParsedMail = await simpleParser(rawEmail);
|
||||
const attachments = parsedEmail.attachments.map(
|
||||
(attachment: Attachment) => ({
|
||||
filename: attachment.filename || 'untitled',
|
||||
contentType: attachment.contentType,
|
||||
size: attachment.size,
|
||||
content: attachment.content as Buffer,
|
||||
})
|
||||
yield this.parseRawEmail(
|
||||
rawEmail,
|
||||
msgResponse.data.id!,
|
||||
userEmail,
|
||||
labels.path,
|
||||
labels.tags
|
||||
);
|
||||
const mapAddresses = (
|
||||
addresses: AddressObject | AddressObject[] | undefined
|
||||
): EmailAddress[] => {
|
||||
if (!addresses) return [];
|
||||
const addressArray = Array.isArray(addresses)
|
||||
? addresses
|
||||
: [addresses];
|
||||
return addressArray.flatMap((a) =>
|
||||
a.value.map((v) => ({ name: v.name, address: v.address || '' }))
|
||||
);
|
||||
};
|
||||
const threadId = getThreadId(parsedEmail.headers);
|
||||
yield {
|
||||
id: msgResponse.data.id!,
|
||||
threadId,
|
||||
userEmail: userEmail,
|
||||
eml: rawEmail,
|
||||
from: mapAddresses(parsedEmail.from),
|
||||
to: mapAddresses(parsedEmail.to),
|
||||
cc: mapAddresses(parsedEmail.cc),
|
||||
bcc: mapAddresses(parsedEmail.bcc),
|
||||
subject: parsedEmail.subject || '',
|
||||
body: parsedEmail.text || '',
|
||||
html: parsedEmail.html || '',
|
||||
headers: parsedEmail.headers,
|
||||
attachments,
|
||||
receivedAt: parsedEmail.date || new Date(),
|
||||
path: labels.path,
|
||||
tags: labels.tags,
|
||||
};
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code === 404) {
|
||||
@@ -382,6 +318,63 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
|
||||
} while (pageToken);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a raw email buffer into an EmailObject, extracting metadata via simpleParser.
|
||||
* In preserve-original mode, attachment binary content is omitted to save memory.
|
||||
*/
|
||||
private async parseRawEmail(
|
||||
rawEmail: Buffer,
|
||||
messageId: string,
|
||||
userEmail: string,
|
||||
path: string,
|
||||
tags: string[]
|
||||
): Promise<EmailObject> {
|
||||
const tempFilePath = await writeEmailToTempFile(rawEmail);
|
||||
const parsedEmail: ParsedMail = await simpleParser(rawEmail);
|
||||
|
||||
// In preserve-original mode, skip extracting full attachment binary content
|
||||
// to avoid unnecessary memory allocation — the raw EML on disk is the source of truth.
|
||||
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
|
||||
filename: attachment.filename || 'untitled',
|
||||
contentType: attachment.contentType,
|
||||
size: attachment.size,
|
||||
content: this.options.preserveOriginalFile
|
||||
? Buffer.alloc(0)
|
||||
: (attachment.content as Buffer),
|
||||
}));
|
||||
|
||||
const mapAddresses = (
|
||||
addresses: AddressObject | AddressObject[] | undefined
|
||||
): EmailAddress[] => {
|
||||
if (!addresses) return [];
|
||||
const addressArray = Array.isArray(addresses) ? addresses : [addresses];
|
||||
return addressArray.flatMap((a) =>
|
||||
a.value.map((v) => ({ name: v.name, address: v.address || '' }))
|
||||
);
|
||||
};
|
||||
|
||||
const threadId = getThreadId(parsedEmail.headers);
|
||||
|
||||
return {
|
||||
id: messageId,
|
||||
threadId,
|
||||
userEmail,
|
||||
tempFilePath,
|
||||
from: mapAddresses(parsedEmail.from),
|
||||
to: mapAddresses(parsedEmail.to),
|
||||
cc: mapAddresses(parsedEmail.cc),
|
||||
bcc: mapAddresses(parsedEmail.bcc),
|
||||
subject: parsedEmail.subject || '',
|
||||
body: parsedEmail.text || '',
|
||||
html: parsedEmail.html || '',
|
||||
headers: parsedEmail.headers,
|
||||
attachments,
|
||||
receivedAt: parsedEmail.date || new Date(),
|
||||
path,
|
||||
tags,
|
||||
};
|
||||
}
|
||||
|
||||
public getUpdatedSyncState(userEmail: string): SyncState {
|
||||
if (!this.newHistoryId) {
|
||||
return {};
|
||||
|
||||
@@ -5,19 +5,25 @@ import type {
|
||||
SyncState,
|
||||
MailboxUser,
|
||||
} from '@open-archiver/types';
|
||||
import type { IEmailConnector } from '../EmailProviderFactory';
|
||||
import type { IEmailConnector, ConnectorOptions } from '../EmailProviderFactory';
|
||||
import { ImapFlow } from 'imapflow';
|
||||
import { simpleParser, ParsedMail, Attachment, AddressObject, Headers } from 'mailparser';
|
||||
import { config } from '../../config';
|
||||
import { logger } from '../../config/logger';
|
||||
import { getThreadId } from './helpers/utils';
|
||||
import { writeEmailToTempFile } from './helpers/tempFile';
|
||||
|
||||
export class ImapConnector implements IEmailConnector {
|
||||
private client: ImapFlow;
|
||||
private newMaxUids: { [mailboxPath: string]: number } = {};
|
||||
private statusMessage: string | undefined;
|
||||
private options: ConnectorOptions;
|
||||
|
||||
constructor(private credentials: GenericImapCredentials) {
|
||||
constructor(
|
||||
private credentials: GenericImapCredentials,
|
||||
options?: ConnectorOptions
|
||||
) {
|
||||
this.options = options ?? { preserveOriginalFile: false };
|
||||
this.client = this.createClient();
|
||||
}
|
||||
|
||||
@@ -298,12 +304,21 @@ export class ImapConnector implements IEmailConnector {
|
||||
}
|
||||
|
||||
private async parseMessage(msg: any, mailboxPath: string): Promise<EmailObject> {
|
||||
// Write raw bytes to temp file to keep large buffers off the JS heap
|
||||
const tempFilePath = await writeEmailToTempFile(msg.source);
|
||||
|
||||
// Parse only for metadata extraction (read-only)
|
||||
const parsedEmail: ParsedMail = await simpleParser(msg.source);
|
||||
|
||||
// In preserve-original mode, skip extracting full attachment binary content
|
||||
// to avoid unnecessary memory allocation — the raw EML on disk is the source of truth.
|
||||
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
|
||||
filename: attachment.filename || 'untitled',
|
||||
contentType: attachment.contentType,
|
||||
size: attachment.size,
|
||||
content: attachment.content as Buffer,
|
||||
content: this.options.preserveOriginalFile
|
||||
? Buffer.alloc(0)
|
||||
: (attachment.content as Buffer),
|
||||
}));
|
||||
|
||||
const mapAddresses = (
|
||||
@@ -331,7 +346,7 @@ export class ImapConnector implements IEmailConnector {
|
||||
headers: parsedEmail.headers,
|
||||
attachments,
|
||||
receivedAt: parsedEmail.date || new Date(),
|
||||
eml: msg.source,
|
||||
tempFilePath,
|
||||
path: mailboxPath,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -5,10 +5,11 @@ import type {
|
||||
SyncState,
|
||||
MailboxUser,
|
||||
} from '@open-archiver/types';
|
||||
import type { IEmailConnector } from '../EmailProviderFactory';
|
||||
import type { IEmailConnector, ConnectorOptions } from '../EmailProviderFactory';
|
||||
import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
|
||||
import { logger } from '../../config/logger';
|
||||
import { getThreadId } from './helpers/utils';
|
||||
import { writeEmailToTempFile } from './helpers/tempFile';
|
||||
import { StorageService } from '../StorageService';
|
||||
import { Readable, Transform } from 'stream';
|
||||
import { createHash } from 'crypto';
|
||||
@@ -54,8 +55,13 @@ class MboxSplitter extends Transform {
|
||||
|
||||
export class MboxConnector implements IEmailConnector {
|
||||
private storage: StorageService;
|
||||
private options: ConnectorOptions;
|
||||
|
||||
constructor(private credentials: MboxImportCredentials) {
|
||||
constructor(
|
||||
private credentials: MboxImportCredentials,
|
||||
options?: ConnectorOptions
|
||||
) {
|
||||
this.options = options ?? { preserveOriginalFile: false };
|
||||
this.storage = new StorageService();
|
||||
}
|
||||
|
||||
@@ -164,14 +170,42 @@ export class MboxConnector implements IEmailConnector {
|
||||
}
|
||||
}
|
||||
|
||||
private async parseMessage(emlBuffer: Buffer, path: string): Promise<EmailObject> {
|
||||
/**
|
||||
* Strips the mbox "From " envelope line from the raw buffer.
|
||||
* The mbox format prepends each message with a "From sender@... timestamp\n"
|
||||
* line that is NOT part of the RFC 5322 message. Storing this line in the
|
||||
* .eml would produce an invalid file and corrupt the SHA-256 hash for GoBD
|
||||
* compliance purposes.
|
||||
*/
|
||||
private stripMboxEnvelope(buffer: Buffer): Buffer {
|
||||
// The "From " line ends at the first \n — everything after is the real RFC 5322 message.
|
||||
const fromPrefix = Buffer.from('From ');
|
||||
if (buffer.subarray(0, fromPrefix.length).equals(fromPrefix)) {
|
||||
const newlineIndex = buffer.indexOf(0x0a); // \n
|
||||
if (newlineIndex !== -1) {
|
||||
return buffer.subarray(newlineIndex + 1);
|
||||
}
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
private async parseMessage(rawMboxBuffer: Buffer, path: string): Promise<EmailObject> {
|
||||
// Strip the mbox "From " envelope line before writing to temp file.
|
||||
// This line is an mbox transport artifact, not part of the RFC 5322 message.
|
||||
const emlBuffer = this.stripMboxEnvelope(rawMboxBuffer);
|
||||
|
||||
const tempFilePath = await writeEmailToTempFile(emlBuffer);
|
||||
const parsedEmail: ParsedMail = await simpleParser(emlBuffer);
|
||||
|
||||
// In preserve-original mode, skip extracting full attachment binary content
|
||||
// to avoid unnecessary memory allocation — the raw EML on disk is the source of truth.
|
||||
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
|
||||
filename: attachment.filename || 'untitled',
|
||||
contentType: attachment.contentType,
|
||||
size: attachment.size,
|
||||
content: attachment.content as Buffer,
|
||||
content: this.options.preserveOriginalFile
|
||||
? Buffer.alloc(0)
|
||||
: (attachment.content as Buffer),
|
||||
}));
|
||||
|
||||
const mapAddresses = (
|
||||
@@ -226,7 +260,7 @@ export class MboxConnector implements IEmailConnector {
|
||||
headers: parsedEmail.headers,
|
||||
attachments,
|
||||
receivedAt: parsedEmail.date || new Date(),
|
||||
eml: emlBuffer,
|
||||
tempFilePath,
|
||||
path: finalPath,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -6,9 +6,10 @@ import type {
|
||||
SyncState,
|
||||
MailboxUser,
|
||||
} from '@open-archiver/types';
|
||||
import type { IEmailConnector } from '../EmailProviderFactory';
|
||||
import type { IEmailConnector, ConnectorOptions } from '../EmailProviderFactory';
|
||||
import { logger } from '../../config/logger';
|
||||
import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
|
||||
import { writeEmailToTempFile } from './helpers/tempFile';
|
||||
import { ConfidentialClientApplication, Configuration, LogLevel } from '@azure/msal-node';
|
||||
import { Client } from '@microsoft/microsoft-graph-client';
|
||||
import type { User, MailFolder } from 'microsoft-graph';
|
||||
@@ -23,9 +24,11 @@ export class MicrosoftConnector implements IEmailConnector {
|
||||
private graphClient: Client;
|
||||
// Store delta tokens for each folder during a sync operation.
|
||||
private newDeltaTokens: { [folderId: string]: string };
|
||||
private options: ConnectorOptions;
|
||||
|
||||
constructor(credentials: Microsoft365Credentials) {
|
||||
constructor(credentials: Microsoft365Credentials, options?: ConnectorOptions) {
|
||||
this.credentials = credentials;
|
||||
this.options = options ?? { preserveOriginalFile: false };
|
||||
this.newDeltaTokens = {}; // Initialize as an empty object
|
||||
|
||||
const msalConfig: Configuration = {
|
||||
@@ -299,12 +302,18 @@ export class MicrosoftConnector implements IEmailConnector {
|
||||
userEmail: string,
|
||||
path: string
|
||||
): Promise<EmailObject> {
|
||||
const tempFilePath = await writeEmailToTempFile(rawEmail);
|
||||
const parsedEmail: ParsedMail = await simpleParser(rawEmail);
|
||||
|
||||
// In preserve-original mode, skip extracting full attachment binary content
|
||||
// to avoid unnecessary memory allocation — the raw EML on disk is the source of truth.
|
||||
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
|
||||
filename: attachment.filename || 'untitled',
|
||||
contentType: attachment.contentType,
|
||||
size: attachment.size,
|
||||
content: attachment.content as Buffer,
|
||||
content: this.options.preserveOriginalFile
|
||||
? Buffer.alloc(0)
|
||||
: (attachment.content as Buffer),
|
||||
}));
|
||||
const mapAddresses = (
|
||||
addresses: AddressObject | AddressObject[] | undefined
|
||||
@@ -319,7 +328,7 @@ export class MicrosoftConnector implements IEmailConnector {
|
||||
return {
|
||||
id: messageId,
|
||||
userEmail: userEmail,
|
||||
eml: rawEmail,
|
||||
tempFilePath,
|
||||
from: mapAddresses(parsedEmail.from),
|
||||
to: mapAddresses(parsedEmail.to),
|
||||
cc: mapAddresses(parsedEmail.cc),
|
||||
|
||||
@@ -5,13 +5,13 @@ import type {
|
||||
SyncState,
|
||||
MailboxUser,
|
||||
} from '@open-archiver/types';
|
||||
import type { IEmailConnector } from '../EmailProviderFactory';
|
||||
import type { IEmailConnector, ConnectorOptions } from '../EmailProviderFactory';
|
||||
import { PSTFile, PSTFolder, PSTMessage } from 'pst-extractor';
|
||||
import { simpleParser, ParsedMail, Attachment, AddressObject } from 'mailparser';
|
||||
import { logger } from '../../config/logger';
|
||||
import { getThreadId } from './helpers/utils';
|
||||
import { writeEmailToTempFile } from './helpers/tempFile';
|
||||
import { StorageService } from '../StorageService';
|
||||
import { Readable } from 'stream';
|
||||
import { createHash } from 'crypto';
|
||||
import { join } from 'path';
|
||||
import { createWriteStream, createReadStream, promises as fs } from 'fs';
|
||||
@@ -106,8 +106,13 @@ const JUNK_FOLDERS = new Set([
|
||||
|
||||
export class PSTConnector implements IEmailConnector {
|
||||
private storage: StorageService;
|
||||
private options: ConnectorOptions;
|
||||
|
||||
constructor(private credentials: PSTImportCredentials) {
|
||||
constructor(
|
||||
private credentials: PSTImportCredentials,
|
||||
options?: ConnectorOptions
|
||||
) {
|
||||
this.options = options ?? { preserveOriginalFile: false };
|
||||
this.storage = new StorageService();
|
||||
}
|
||||
|
||||
@@ -263,7 +268,10 @@ export class PSTConnector implements IEmailConnector {
|
||||
try {
|
||||
email = folder.getNextChild();
|
||||
} catch (error) {
|
||||
console.warn("Folder doesn't have child");
|
||||
logger.warn(
|
||||
{ folder: folder.displayName, error },
|
||||
"Folder doesn't have child or failed to read next child."
|
||||
);
|
||||
email = null;
|
||||
}
|
||||
}
|
||||
@@ -283,13 +291,18 @@ export class PSTConnector implements IEmailConnector {
|
||||
): Promise<EmailObject> {
|
||||
const emlContent = await this.constructEml(msg);
|
||||
const emlBuffer = Buffer.from(emlContent, 'utf-8');
|
||||
const tempFilePath = await writeEmailToTempFile(emlBuffer);
|
||||
const parsedEmail: ParsedMail = await simpleParser(emlBuffer);
|
||||
|
||||
// In preserve-original mode, skip extracting full attachment binary content
|
||||
// to avoid unnecessary memory allocation — the raw EML on disk is the source of truth.
|
||||
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
|
||||
filename: attachment.filename || 'untitled',
|
||||
contentType: attachment.contentType,
|
||||
size: attachment.size,
|
||||
content: attachment.content as Buffer,
|
||||
content: this.options.preserveOriginalFile
|
||||
? Buffer.alloc(0)
|
||||
: (attachment.content as Buffer),
|
||||
}));
|
||||
|
||||
const mapAddresses = (
|
||||
@@ -336,7 +349,7 @@ export class PSTConnector implements IEmailConnector {
|
||||
headers: parsedEmail.headers,
|
||||
attachments,
|
||||
receivedAt: parsedEmail.date || new Date(),
|
||||
eml: emlBuffer,
|
||||
tempFilePath,
|
||||
path,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { writeFile } from 'fs/promises';
|
||||
import { randomUUID } from 'crypto';
|
||||
|
||||
/**
|
||||
* Writes a raw email buffer to a temporary file on disk and returns the path.
|
||||
* This keeps large buffers off the JS heap between connector yield and processEmail().
|
||||
* The caller (IngestionService.processEmail) is responsible for deleting the file.
|
||||
*/
|
||||
export async function writeEmailToTempFile(buffer: Buffer): Promise<string> {
|
||||
const tempFilePath = join(tmpdir(), `oa-email-${randomUUID()}.eml`);
|
||||
await writeFile(tempFilePath, buffer);
|
||||
return tempFilePath;
|
||||
}
|
||||
@@ -28,7 +28,8 @@
|
||||
"svelte-persisted-store": "^0.12.0",
|
||||
"sveltekit-i18n": "^2.4.2",
|
||||
"tailwind-merge": "^3.3.1",
|
||||
"tailwind-variants": "^1.0.0"
|
||||
"tailwind-variants": "^1.0.0",
|
||||
"tippy.js": "^6.3.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@internationalized/date": "^3.8.2",
|
||||
|
||||
@@ -11,7 +11,9 @@
|
||||
import { Textarea } from '$lib/components/ui/textarea/index.js';
|
||||
import { setAlert } from '$lib/components/custom/alert/alert-state.svelte';
|
||||
import { api } from '$lib/api.client';
|
||||
import { Loader2 } from 'lucide-svelte';
|
||||
import { Loader2, Info } from 'lucide-svelte';
|
||||
import tippy from 'tippy.js';
|
||||
import 'tippy.js/dist/tippy.css';
|
||||
import { t } from '$lib/translations';
|
||||
let {
|
||||
source = null,
|
||||
@@ -56,6 +58,7 @@
|
||||
secure: true,
|
||||
allowInsecureCert: false,
|
||||
},
|
||||
preserveOriginalFile: source?.preserveOriginalFile ?? false,
|
||||
});
|
||||
|
||||
$effect(() => {
|
||||
@@ -439,6 +442,29 @@
|
||||
</Alert.Description>
|
||||
</Alert.Root>
|
||||
{/if}
|
||||
|
||||
<div class="grid grid-cols-4 items-center gap-4">
|
||||
<div class="flex items-center gap-1 text-left">
|
||||
<Label for="preserveOriginalFile"
|
||||
>{$t('app.components.ingestion_source_form.preserve_original_file')}</Label
|
||||
>
|
||||
<span
|
||||
use:tippy={{
|
||||
allowHTML: true,
|
||||
content: $t(
|
||||
'app.components.ingestion_source_form.preserve_original_file_tooltip'
|
||||
),
|
||||
interactive: true,
|
||||
delay: 500,
|
||||
}}
|
||||
class="text-muted-foreground cursor-help"
|
||||
>
|
||||
<Info class="h-4 w-4" />
|
||||
</span>
|
||||
</div>
|
||||
<Checkbox id="preserveOriginalFile" bind:checked={formData.preserveOriginalFile} />
|
||||
</div>
|
||||
|
||||
<Dialog.Footer>
|
||||
<Button type="submit" disabled={isSubmitting || fileUploading}>
|
||||
{#if isSubmitting}
|
||||
|
||||
@@ -52,7 +52,11 @@
|
||||
"retention_matching_policies": "Matching Policies",
|
||||
"retention_delete_permanently": "Permanent Deletion",
|
||||
"retention_scheduled_deletion": "Scheduled Deletion",
|
||||
"retention_policy_overridden_by_label": "This policy is overridden by retention label "
|
||||
"retention_policy_overridden_by_label": "This policy is overridden by retention label ",
|
||||
"embedded_attachments": "Embedded Attachments",
|
||||
"embedded": "Embedded",
|
||||
"embedded_attachment_title": "Embedded Attachment",
|
||||
"embedded_attachment_description": "This attachment is embedded in the original email file and cannot be downloaded separately. To get this attachment, download the complete email (.eml) file."
|
||||
},
|
||||
"ingestions": {
|
||||
"title": "Ingestion Sources",
|
||||
@@ -233,7 +237,9 @@
|
||||
"heads_up": "Heads up!",
|
||||
"org_wide_warning": "Please note that this is an organization-wide operation. This kind of ingestions will import and index <b>all</b> email inboxes in your organization. If you want to import only specific email inboxes, use the IMAP connector.",
|
||||
"upload_failed": "Upload Failed, please try again",
|
||||
"upload_network_error": "The server could not process the upload. The file may exceed the configured upload size limit (BODY_SIZE_LIMIT). For very large files, use the Local Path option instead."
|
||||
"upload_network_error": "The server could not process the upload. The file may exceed the configured upload size limit (BODY_SIZE_LIMIT). For very large files, use the Local Path option instead.",
|
||||
"preserve_original_file": "Preserve Original File",
|
||||
"preserve_original_file_tooltip": "When checked: Stores the exact, unmodified email file as received from the server. No attachments are stripped. Required for GoBD (Germany) and SEC 17a-4 compliance.<br><br>When unchecked: Strips non-inline attachments and stores them separately with deduplication, saving storage space."
|
||||
},
|
||||
"role_form": {
|
||||
"policies_json": "Policies (JSON)",
|
||||
|
||||
@@ -29,6 +29,8 @@
|
||||
import { page } from '$app/state';
|
||||
import { enhance } from '$app/forms';
|
||||
import type { LegalHold, EmailLegalHoldInfo } from '@open-archiver/types';
|
||||
import PostalMime, { type Attachment as PostalAttachment } from 'postal-mime';
|
||||
import { Paperclip } from 'lucide-svelte';
|
||||
|
||||
let { data, form }: { data: PageData; form: ActionData } = $props();
|
||||
let email = $derived(data.email);
|
||||
@@ -77,6 +79,51 @@
|
||||
// --- Integrity report PDF download state (enterprise only) ---
|
||||
let isDownloadingReport = $state(false);
|
||||
|
||||
// --- Embedded attachment state (parsed from raw EML) ---
|
||||
/** Non-inline attachments parsed from the raw EML via postal-mime */
|
||||
let embeddedAttachments = $state<PostalAttachment[]>([]);
|
||||
let isEmbeddedAttachmentDialogOpen = $state(false);
|
||||
let selectedEmbeddedFilename = $state('');
|
||||
|
||||
/** Parse raw EML to extract non-inline attachments for display */
|
||||
$effect(() => {
|
||||
async function parseEmlAttachments() {
|
||||
const raw = email?.raw;
|
||||
if (!raw) return;
|
||||
|
||||
try {
|
||||
let buffer: Uint8Array;
|
||||
if (raw && typeof raw === 'object' && 'type' in raw && raw.type === 'Buffer') {
|
||||
buffer = new Uint8Array(
|
||||
(raw as unknown as { type: 'Buffer'; data: number[] }).data
|
||||
);
|
||||
} else {
|
||||
buffer = new Uint8Array(raw as unknown as ArrayLike<number>);
|
||||
}
|
||||
|
||||
const parsed = await new PostalMime().parse(buffer);
|
||||
// Filter to non-inline attachments (those with a filename and no contentId,
|
||||
// or with disposition=attachment)
|
||||
embeddedAttachments = parsed.attachments.filter(
|
||||
(att) => att.filename && (att.disposition === 'attachment' || !att.contentId)
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('Failed to parse EML for embedded attachments:', error);
|
||||
}
|
||||
}
|
||||
parseEmlAttachments();
|
||||
});
|
||||
|
||||
/**
|
||||
* Opens the confirmation dialog when a user tries to download an
|
||||
* embedded attachment. Since embedded attachments are not stored
|
||||
* separately, the user must download the entire EML file.
|
||||
*/
|
||||
function handleEmbeddedAttachmentDownload(filename: string) {
|
||||
selectedEmbeddedFilename = filename;
|
||||
isEmbeddedAttachmentDialogOpen = true;
|
||||
}
|
||||
|
||||
// React to form results for label and hold actions
|
||||
$effect(() => {
|
||||
if (form) {
|
||||
@@ -313,6 +360,53 @@
|
||||
</ul>
|
||||
</div>
|
||||
{/if}
|
||||
{#if embeddedAttachments.length > 0}
|
||||
<div>
|
||||
<h3 class="font-semibold">
|
||||
{$t('app.archive.embedded_attachments')}
|
||||
</h3>
|
||||
<ul class="mt-2 space-y-2">
|
||||
{#each embeddedAttachments as attachment}
|
||||
<li
|
||||
class="flex items-center justify-between rounded-md border p-2"
|
||||
>
|
||||
<div class="flex min-w-0 items-center gap-2">
|
||||
<Paperclip
|
||||
class="text-muted-foreground h-4 w-4 flex-shrink-0"
|
||||
/>
|
||||
<span class="truncate">
|
||||
{attachment.filename}
|
||||
{#if typeof attachment.content === 'string'}
|
||||
({formatBytes(attachment.content.length)})
|
||||
{:else if attachment.content}
|
||||
({formatBytes(
|
||||
attachment.content.byteLength
|
||||
)})
|
||||
{/if}
|
||||
</span>
|
||||
<Badge
|
||||
variant="secondary"
|
||||
class="flex-shrink-0 text-xs"
|
||||
>
|
||||
{$t('app.archive.embedded')}
|
||||
</Badge>
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
class="ml-2 flex-shrink-0 text-xs"
|
||||
onclick={() =>
|
||||
handleEmbeddedAttachmentDownload(
|
||||
attachment.filename || 'attachment'
|
||||
)}
|
||||
>
|
||||
{$t('app.archive.download')}
|
||||
</Button>
|
||||
</li>
|
||||
{/each}
|
||||
</ul>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
</Card.Content>
|
||||
</Card.Root>
|
||||
@@ -920,6 +1014,38 @@
|
||||
</Dialog.Footer>
|
||||
</Dialog.Content>
|
||||
</Dialog.Root>
|
||||
|
||||
<!-- Embedded attachment download confirmation modal -->
|
||||
<Dialog.Root bind:open={isEmbeddedAttachmentDialogOpen}>
|
||||
<Dialog.Content class="sm:max-w-lg">
|
||||
<Dialog.Header>
|
||||
<Dialog.Title>
|
||||
{$t('app.archive.embedded_attachment_title')}
|
||||
</Dialog.Title>
|
||||
<Dialog.Description>
|
||||
<span class="font-medium">{selectedEmbeddedFilename}</span>
|
||||
<br /><br />
|
||||
{$t('app.archive.embedded_attachment_description')}
|
||||
</Dialog.Description>
|
||||
</Dialog.Header>
|
||||
<Dialog.Footer class="sm:justify-start">
|
||||
<Button
|
||||
type="button"
|
||||
onclick={() => {
|
||||
download(email.storagePath, `${email.subject || 'email'}.eml`);
|
||||
isEmbeddedAttachmentDialogOpen = false;
|
||||
}}
|
||||
>
|
||||
{$t('app.archive.download_eml')}
|
||||
</Button>
|
||||
<Dialog.Close>
|
||||
<Button type="button" variant="secondary">
|
||||
{$t('app.archive.cancel')}
|
||||
</Button>
|
||||
</Dialog.Close>
|
||||
</Dialog.Footer>
|
||||
</Dialog.Content>
|
||||
</Dialog.Root>
|
||||
{:else}
|
||||
<p>{$t('app.archive.not_found')}</p>
|
||||
{/if}
|
||||
|
||||
@@ -45,8 +45,10 @@ export interface EmailObject {
|
||||
attachments: EmailAttachment[];
|
||||
/** The date and time when the email was received. */
|
||||
receivedAt: Date;
|
||||
/** An optional buffer containing the full raw EML content of the email, which is useful for archival and compliance purposes. */
|
||||
eml?: Buffer;
|
||||
/** Path to a temporary file on disk containing the raw EML bytes.
|
||||
* Connectors write the raw email to tmpdir() and pass only the path,
|
||||
* keeping large buffers off the JS heap between yield and processEmail(). */
|
||||
tempFilePath: string;
|
||||
/** The email address of the user whose mailbox this email belongs to. */
|
||||
userEmail?: string;
|
||||
/** The folder path of the email in the source mailbox. */
|
||||
|
||||
@@ -120,6 +120,9 @@ export interface IngestionSource {
|
||||
lastSyncFinishedAt?: Date | null;
|
||||
lastSyncStatusMessage?: string | null;
|
||||
syncState?: SyncState | null;
|
||||
/** When true, the raw EML file is stored without any modification (no attachment
|
||||
* stripping). Required for GoBD / SEC 17a-4 compliance. Defaults to false. */
|
||||
preserveOriginalFile: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -133,6 +136,8 @@ export interface CreateIngestionSourceDto {
|
||||
name: string;
|
||||
provider: IngestionProvider;
|
||||
providerConfig: Record<string, any>;
|
||||
/** Store the unmodified raw EML for GoBD compliance. Defaults to false. */
|
||||
preserveOriginalFile?: boolean;
|
||||
}
|
||||
|
||||
export interface UpdateIngestionSourceDto {
|
||||
|
||||
15
pnpm-lock.yaml
generated
15
pnpm-lock.yaml
generated
@@ -334,6 +334,9 @@ importers:
|
||||
tailwind-variants:
|
||||
specifier: ^1.0.0
|
||||
version: 1.0.0(tailwindcss@4.1.11)
|
||||
tippy.js:
|
||||
specifier: ^6.3.7
|
||||
version: 6.3.7
|
||||
devDependencies:
|
||||
'@internationalized/date':
|
||||
specifier: ^3.8.2
|
||||
@@ -1313,6 +1316,9 @@ packages:
|
||||
'@polka/url@1.0.0-next.29':
|
||||
resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==}
|
||||
|
||||
'@popperjs/core@2.11.8':
|
||||
resolution: {integrity: sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==}
|
||||
|
||||
'@rollup/plugin-commonjs@28.0.6':
|
||||
resolution: {integrity: sha512-XSQB1K7FUU5QP+3lOQmVCE3I0FcbbNvmNT4VJSj93iUjayaARrTQeoRdiYQoftAJBLrR9t2agwAd3ekaTgHNlw==}
|
||||
engines: {node: '>=16.0.0 || 14 >= 14.17'}
|
||||
@@ -4580,6 +4586,9 @@ packages:
|
||||
resolution: {integrity: sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ==}
|
||||
engines: {node: '>=12.0.0'}
|
||||
|
||||
tippy.js@6.3.7:
|
||||
resolution: {integrity: sha512-E1d3oP2emgJ9dRQZdf3Kkn0qJgI6ZLpyS5z6ZkY1DF3kaQaBsGZsndEpHwx+eC+tYM41HaSNvNtLx8tU57FzTQ==}
|
||||
|
||||
tlds@1.259.0:
|
||||
resolution: {integrity: sha512-AldGGlDP0PNgwppe2quAvuBl18UcjuNtOnDuUkqhd6ipPqrYYBt3aTxK1QTsBVknk97lS2JcafWMghjGWFtunw==}
|
||||
hasBin: true
|
||||
@@ -5996,6 +6005,8 @@ snapshots:
|
||||
|
||||
'@polka/url@1.0.0-next.29': {}
|
||||
|
||||
'@popperjs/core@2.11.8': {}
|
||||
|
||||
'@rollup/plugin-commonjs@28.0.6(rollup@4.44.2)':
|
||||
dependencies:
|
||||
'@rollup/pluginutils': 5.2.0(rollup@4.44.2)
|
||||
@@ -9535,6 +9546,10 @@ snapshots:
|
||||
fdir: 6.4.6(picomatch@4.0.2)
|
||||
picomatch: 4.0.2
|
||||
|
||||
tippy.js@6.3.7:
|
||||
dependencies:
|
||||
'@popperjs/core': 2.11.8
|
||||
|
||||
tlds@1.259.0: {}
|
||||
|
||||
to-regex-range@5.0.1:
|
||||
|
||||
Reference in New Issue
Block a user