diff --git a/.env.example b/.env.example index 5055822..ed0d4a0 100644 --- a/.env.example +++ b/.env.example @@ -33,6 +33,8 @@ REDIS_TLS_ENABLED=false # --- Storage Settings --- # Choose your storage backend. Valid options are 'local' or 's3'. STORAGE_TYPE=local +# The maximum request body size to accept in bytes including while streaming. The body size can also be specified with a unit suffix for kilobytes (K), megabytes (M), or gigabytes (G). For example, 512K or 1M. Defaults to 512kb. Or the value of Infinity if you don't want any upload limit. +FRONTEND_BODY_SIZE_LIMIT=100M # --- Local Storage Settings --- # The path inside the container where files will be stored. diff --git a/packages/backend/src/services/ingestion-connectors/EMLConnector.ts b/packages/backend/src/services/ingestion-connectors/EMLConnector.ts index afd7d5e..51bb255 100644 --- a/packages/backend/src/services/ingestion-connectors/EMLConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/EMLConnector.ts @@ -174,11 +174,16 @@ export class EMLConnector implements IEmailConnector { messageId = `generated-${createHash('sha256').update(emlBuffer).digest('hex')}`; } + const from = mapAddresses(parsedEmail.from); + if (from.length === 0) { + from.push({ name: 'No Sender', address: 'No Sender' }); + } + return { id: messageId, threadId: threadId, - from: mapAddresses(parsedEmail.from), + from, to: mapAddresses(parsedEmail.to), cc: mapAddresses(parsedEmail.cc), bcc: mapAddresses(parsedEmail.bcc), diff --git a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts index 459dde9..2989624 100644 --- a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts @@ -159,7 +159,7 @@ export class PSTConnector implements IEmailConnector { try { pstFile = await this.loadPstFile(); const root = pstFile.getRootFolder(); - yield* this.processFolder(root, ''); + yield* this.processFolder(root, '', userEmail); } catch (error) { logger.error({ error }, 'Failed to fetch email.'); pstFile?.close(); @@ -171,7 +171,7 @@ export class PSTConnector implements IEmailConnector { } } - private async *processFolder(folder: PSTFolder, currentPath: string): AsyncGenerator { + private async *processFolder(folder: PSTFolder, currentPath: string, userEmail: string): AsyncGenerator { const folderName = folder.displayName.toLowerCase(); if (DELETED_FOLDERS.has(folderName) || JUNK_FOLDERS.has(folderName)) { logger.info(`Skipping folder: ${folder.displayName}`); @@ -183,7 +183,7 @@ export class PSTConnector implements IEmailConnector { if (folder.contentCount > 0) { let email: PSTMessage | null = folder.getNextChild(); while (email != null) { - yield await this.parseMessage(email, newPath); + yield await this.parseMessage(email, newPath, userEmail); try { email = folder.getNextChild(); } catch (error) { @@ -195,12 +195,12 @@ export class PSTConnector implements IEmailConnector { if (folder.hasSubfolders) { for (const subFolder of folder.getSubFolders()) { - yield* this.processFolder(subFolder, newPath); + yield* this.processFolder(subFolder, newPath, userEmail); } } } - private async parseMessage(msg: PSTMessage, path: string): Promise { + private async parseMessage(msg: PSTMessage, path: string, userEmail: string): Promise { const emlContent = await this.constructEml(msg); const emlBuffer = Buffer.from(emlContent, 'utf-8'); const parsedEmail: ParsedMail = await simpleParser(emlBuffer); @@ -218,6 +218,11 @@ export class PSTConnector implements IEmailConnector { return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address?.replaceAll(`'`, '') || '' }))); }; + const from = mapAddresses(parsedEmail.from); + if (from.length === 0) { + from.push({ name: 'No Sender', address: 'No Sender' }); + } + const threadId = getThreadId(parsedEmail.headers); let messageId = msg.internetMessageId; // generate a unique ID for this message @@ -228,7 +233,7 @@ export class PSTConnector implements IEmailConnector { return { id: messageId, threadId: threadId, - from: mapAddresses(parsedEmail.from), + from, to: mapAddresses(parsedEmail.to), cc: mapAddresses(parsedEmail.cc), bcc: mapAddresses(parsedEmail.bcc), diff --git a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte index 1a91530..79b8704 100644 --- a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte +++ b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte @@ -76,22 +76,21 @@ method: 'POST', body: uploadFormData }); - + const result = await response.json(); if (!response.ok) { - throw new Error('File upload failed'); + throw new Error(`File upload failed + ${result}`); } - const result = await response.json(); formData.providerConfig.uploadedFilePath = result.filePath; formData.providerConfig.uploadedFileName = file.name; - console.log(formData.providerConfig.uploadedFilePath); + fileUploading = false; } catch (error) { fileUploading = false; setAlert({ type: 'error', - title: 'Upload Failed', - message: 'PST file upload failed. Please try again.', + title: 'Upload Failed, please try again', + message: JSON.stringify(error), duration: 5000, show: true }); diff --git a/packages/frontend/svelte.config.js b/packages/frontend/svelte.config.js index 03c17f2..4b4f698 100644 --- a/packages/frontend/svelte.config.js +++ b/packages/frontend/svelte.config.js @@ -1,12 +1,16 @@ import adapter from '@sveltejs/adapter-node'; import { vitePreprocess } from '@sveltejs/vite-plugin-svelte'; - +import 'dotenv/config'; /** @type {import('@sveltejs/kit').Config} */ const config = { // Consult https://svelte.dev/docs/kit/integrations // for more information about preprocessors preprocess: vitePreprocess(), - kit: { adapter: adapter() } + kit: { + adapter: adapter({ + bodySizeLimit: process.env.FRONTEND_BODY_SIZE_LIMIT || '100M' + }) + } }; export default config;