Merge pull request #33 from LogicLabs-OU/dev

Increase file upload limit and improve ingestion robustness

.env.example

@@ -33,6 +33,8 @@ REDIS_TLS_ENABLED=false
 # --- Storage Settings ---
 # Choose your storage backend. Valid options are 'local' or 's3'.
 STORAGE_TYPE=local
+# The maximum request body size to accept in bytes including while streaming. The body size can also be specified with a unit suffix for kilobytes (K), megabytes (M), or gigabytes (G). For example, 512K or 1M. Defaults to 512kb. Or the value of  Infinity if you don't want any upload limit.
+FRONTEND_BODY_SIZE_LIMIT=100M
 
 # --- Local Storage Settings ---
 # The path inside the container where files will be stored.

packages/backend/src/services/ingestion-connectors/EMLConnector.ts

@@ -174,11 +174,16 @@ export class EMLConnector implements IEmailConnector {
             messageId = `generated-${createHash('sha256').update(emlBuffer).digest('hex')}`;
         }
 
+        const from = mapAddresses(parsedEmail.from);
+        if (from.length === 0) {
+            from.push({ name: 'No Sender', address: 'No Sender' });
+        }
+
 
         return {
             id: messageId,
             threadId: threadId,
-            from: mapAddresses(parsedEmail.from),
+            from,
             to: mapAddresses(parsedEmail.to),
             cc: mapAddresses(parsedEmail.cc),
             bcc: mapAddresses(parsedEmail.bcc),

packages/backend/src/services/ingestion-connectors/PSTConnector.ts

@@ -159,7 +159,7 @@ export class PSTConnector implements IEmailConnector {
         try {
             pstFile = await this.loadPstFile();
             const root = pstFile.getRootFolder();
-            yield* this.processFolder(root, '');
+            yield* this.processFolder(root, '', userEmail);
         } catch (error) {
             logger.error({ error }, 'Failed to fetch email.');
             pstFile?.close();
@@ -171,7 +171,7 @@ export class PSTConnector implements IEmailConnector {
         }
     }
 
-    private async *processFolder(folder: PSTFolder, currentPath: string): AsyncGenerator<EmailObject | null> {
+    private async *processFolder(folder: PSTFolder, currentPath: string, userEmail: string): AsyncGenerator<EmailObject | null> {
         const folderName = folder.displayName.toLowerCase();
         if (DELETED_FOLDERS.has(folderName) || JUNK_FOLDERS.has(folderName)) {
             logger.info(`Skipping folder: ${folder.displayName}`);
@@ -183,7 +183,7 @@ export class PSTConnector implements IEmailConnector {
         if (folder.contentCount > 0) {
             let email: PSTMessage | null = folder.getNextChild();
             while (email != null) {
-                yield await this.parseMessage(email, newPath);
+                yield await this.parseMessage(email, newPath, userEmail);
                 try {
                     email = folder.getNextChild();
                 } catch (error) {
@@ -195,12 +195,12 @@ export class PSTConnector implements IEmailConnector {
 
         if (folder.hasSubfolders) {
             for (const subFolder of folder.getSubFolders()) {
-                yield* this.processFolder(subFolder, newPath);
+                yield* this.processFolder(subFolder, newPath, userEmail);
             }
         }
     }
 
-    private async parseMessage(msg: PSTMessage, path: string): Promise<EmailObject> {
+    private async parseMessage(msg: PSTMessage, path: string, userEmail: string): Promise<EmailObject> {
         const emlContent = await this.constructEml(msg);
         const emlBuffer = Buffer.from(emlContent, 'utf-8');
         const parsedEmail: ParsedMail = await simpleParser(emlBuffer);
@@ -218,6 +218,11 @@ export class PSTConnector implements IEmailConnector {
             return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address?.replaceAll(`'`, '') || '' })));
         };
 
+        const from = mapAddresses(parsedEmail.from);
+        if (from.length === 0) {
+            from.push({ name: 'No Sender', address: 'No Sender' });
+        }
+
         const threadId = getThreadId(parsedEmail.headers);
         let messageId = msg.internetMessageId;
         // generate a unique ID for this message
@@ -228,7 +233,7 @@ export class PSTConnector implements IEmailConnector {
         return {
             id: messageId,
             threadId: threadId,
-            from: mapAddresses(parsedEmail.from),
+            from,
             to: mapAddresses(parsedEmail.to),
             cc: mapAddresses(parsedEmail.cc),
             bcc: mapAddresses(parsedEmail.bcc),

packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte

@@ -76,22 +76,21 @@
 				method: 'POST',
 				body: uploadFormData
 			});
-
+			const result = await response.json();
 			if (!response.ok) {
-				throw new Error('File upload failed');
+				throw new Error(`File upload failed + ${result}`);
 			}
 
-			const result = await response.json();
 			formData.providerConfig.uploadedFilePath = result.filePath;
 			formData.providerConfig.uploadedFileName = file.name;
-			console.log(formData.providerConfig.uploadedFilePath);
+
 			fileUploading = false;
 		} catch (error) {
 			fileUploading = false;
 			setAlert({
 				type: 'error',
-				title: 'Upload Failed',
-				message: 'PST file upload failed. Please try again.',
+				title: 'Upload Failed, please try again',
+				message: JSON.stringify(error),
 				duration: 5000,
 				show: true
 			});

packages/frontend/svelte.config.js

@@ -1,12 +1,16 @@
 import adapter from '@sveltejs/adapter-node';
 import { vitePreprocess } from '@sveltejs/vite-plugin-svelte';
-
+import 'dotenv/config';
 /** @type {import('@sveltejs/kit').Config} */
 const config = {
 	// Consult https://svelte.dev/docs/kit/integrations
 	// for more information about preprocessors
 	preprocess: vitePreprocess(),
-	kit: { adapter: adapter() }
+	kit: {
+		adapter: adapter({
+			bodySizeLimit: process.env.FRONTEND_BODY_SIZE_LIMIT || '100M'
+		})
+	}
 };
 
 export default config;