From b03791d9a6a9a5af93d9cd9d8c9e22f871d70402 Mon Sep 17 00:00:00 2001 From: Wayne <5291640+ringoinca@users.noreply.github.com> Date: Wed, 13 Aug 2025 19:20:19 +0300 Subject: [PATCH 1/2] adding FRONTEND_BODY_SIZE_LIMIT to allow bigger file upload for the frontend. This is to fix the pst file upload error. --- .env.example | 2 ++ .../src/lib/components/custom/IngestionSourceForm.svelte | 7 +++---- packages/frontend/svelte.config.js | 8 ++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 5055822..ed0d4a0 100644 --- a/.env.example +++ b/.env.example @@ -33,6 +33,8 @@ REDIS_TLS_ENABLED=false # --- Storage Settings --- # Choose your storage backend. Valid options are 'local' or 's3'. STORAGE_TYPE=local +# The maximum request body size to accept in bytes including while streaming. The body size can also be specified with a unit suffix for kilobytes (K), megabytes (M), or gigabytes (G). For example, 512K or 1M. Defaults to 512kb. Or the value of Infinity if you don't want any upload limit. +FRONTEND_BODY_SIZE_LIMIT=100M # --- Local Storage Settings --- # The path inside the container where files will be stored. diff --git a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte index 1a91530..b085bb6 100644 --- a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte +++ b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte @@ -76,12 +76,11 @@ method: 'POST', body: uploadFormData }); - + const result = await response.json(); if (!response.ok) { - throw new Error('File upload failed'); + throw new Error(`File upload failed + ${result}`); } - const result = await response.json(); formData.providerConfig.uploadedFilePath = result.filePath; formData.providerConfig.uploadedFileName = file.name; console.log(formData.providerConfig.uploadedFilePath); @@ -91,7 +90,7 @@ setAlert({ type: 'error', title: 'Upload Failed', - message: 'PST file upload failed. Please try again.', + message: 'PST file upload failed. Please try again.' + error.message, duration: 5000, show: true }); diff --git a/packages/frontend/svelte.config.js b/packages/frontend/svelte.config.js index 03c17f2..4b4f698 100644 --- a/packages/frontend/svelte.config.js +++ b/packages/frontend/svelte.config.js @@ -1,12 +1,16 @@ import adapter from '@sveltejs/adapter-node'; import { vitePreprocess } from '@sveltejs/vite-plugin-svelte'; - +import 'dotenv/config'; /** @type {import('@sveltejs/kit').Config} */ const config = { // Consult https://svelte.dev/docs/kit/integrations // for more information about preprocessors preprocess: vitePreprocess(), - kit: { adapter: adapter() } + kit: { + adapter: adapter({ + bodySizeLimit: process.env.FRONTEND_BODY_SIZE_LIMIT || '100M' + }) + } }; export default config; From d2b4337be9ac9a14eb6c5349708d4c7f59716d1b Mon Sep 17 00:00:00 2001 From: Wayne <5291640+ringoinca@users.noreply.github.com> Date: Wed, 13 Aug 2025 20:46:02 +0300 Subject: [PATCH 2/2] Fix error when pst file emails don't include senders --- .../ingestion-connectors/EMLConnector.ts | 7 ++++++- .../ingestion-connectors/PSTConnector.ts | 17 +++++++++++------ .../custom/IngestionSourceForm.svelte | 6 +++--- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/backend/src/services/ingestion-connectors/EMLConnector.ts b/packages/backend/src/services/ingestion-connectors/EMLConnector.ts index afd7d5e..51bb255 100644 --- a/packages/backend/src/services/ingestion-connectors/EMLConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/EMLConnector.ts @@ -174,11 +174,16 @@ export class EMLConnector implements IEmailConnector { messageId = `generated-${createHash('sha256').update(emlBuffer).digest('hex')}`; } + const from = mapAddresses(parsedEmail.from); + if (from.length === 0) { + from.push({ name: 'No Sender', address: 'No Sender' }); + } + return { id: messageId, threadId: threadId, - from: mapAddresses(parsedEmail.from), + from, to: mapAddresses(parsedEmail.to), cc: mapAddresses(parsedEmail.cc), bcc: mapAddresses(parsedEmail.bcc), diff --git a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts index 459dde9..2989624 100644 --- a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts @@ -159,7 +159,7 @@ export class PSTConnector implements IEmailConnector { try { pstFile = await this.loadPstFile(); const root = pstFile.getRootFolder(); - yield* this.processFolder(root, ''); + yield* this.processFolder(root, '', userEmail); } catch (error) { logger.error({ error }, 'Failed to fetch email.'); pstFile?.close(); @@ -171,7 +171,7 @@ export class PSTConnector implements IEmailConnector { } } - private async *processFolder(folder: PSTFolder, currentPath: string): AsyncGenerator { + private async *processFolder(folder: PSTFolder, currentPath: string, userEmail: string): AsyncGenerator { const folderName = folder.displayName.toLowerCase(); if (DELETED_FOLDERS.has(folderName) || JUNK_FOLDERS.has(folderName)) { logger.info(`Skipping folder: ${folder.displayName}`); @@ -183,7 +183,7 @@ export class PSTConnector implements IEmailConnector { if (folder.contentCount > 0) { let email: PSTMessage | null = folder.getNextChild(); while (email != null) { - yield await this.parseMessage(email, newPath); + yield await this.parseMessage(email, newPath, userEmail); try { email = folder.getNextChild(); } catch (error) { @@ -195,12 +195,12 @@ export class PSTConnector implements IEmailConnector { if (folder.hasSubfolders) { for (const subFolder of folder.getSubFolders()) { - yield* this.processFolder(subFolder, newPath); + yield* this.processFolder(subFolder, newPath, userEmail); } } } - private async parseMessage(msg: PSTMessage, path: string): Promise { + private async parseMessage(msg: PSTMessage, path: string, userEmail: string): Promise { const emlContent = await this.constructEml(msg); const emlBuffer = Buffer.from(emlContent, 'utf-8'); const parsedEmail: ParsedMail = await simpleParser(emlBuffer); @@ -218,6 +218,11 @@ export class PSTConnector implements IEmailConnector { return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address?.replaceAll(`'`, '') || '' }))); }; + const from = mapAddresses(parsedEmail.from); + if (from.length === 0) { + from.push({ name: 'No Sender', address: 'No Sender' }); + } + const threadId = getThreadId(parsedEmail.headers); let messageId = msg.internetMessageId; // generate a unique ID for this message @@ -228,7 +233,7 @@ export class PSTConnector implements IEmailConnector { return { id: messageId, threadId: threadId, - from: mapAddresses(parsedEmail.from), + from, to: mapAddresses(parsedEmail.to), cc: mapAddresses(parsedEmail.cc), bcc: mapAddresses(parsedEmail.bcc), diff --git a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte index b085bb6..79b8704 100644 --- a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte +++ b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte @@ -83,14 +83,14 @@ formData.providerConfig.uploadedFilePath = result.filePath; formData.providerConfig.uploadedFileName = file.name; - console.log(formData.providerConfig.uploadedFilePath); + fileUploading = false; } catch (error) { fileUploading = false; setAlert({ type: 'error', - title: 'Upload Failed', - message: 'PST file upload failed. Please try again.' + error.message, + title: 'Upload Failed, please try again', + message: JSON.stringify(error), duration: 5000, show: true });