v0.3.0 release (#76)

* Remove extra ports in Docker Compose file

* Allow self-assigned cert

* Adding allow insecure cert option

* fix(IMAP): Share connections between each fetch email action

* Update docs: troubleshooting CORS error

---------

Co-authored-by: Wayne <5291640+ringoinca@users.noreply.github.com>

docker-compose.yml

@@ -6,7 +6,6 @@ services:
         container_name: open-archiver
         restart: unless-stopped
         ports:
-            - '4000:4000' # Backend
             - '3000:3000' # Frontend
         env_file:
             - .env
@@ -29,8 +28,6 @@ services:
             POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password}
         volumes:
             - pgdata:/var/lib/postgresql/data
-        ports:
-            - '5432:5432'
         networks:
             - open-archiver-net
 
@@ -39,8 +36,6 @@ services:
         container_name: valkey
         restart: unless-stopped
         command: valkey-server --requirepass ${REDIS_PASSWORD}
-        ports:
-            - '6379:6379'
         volumes:
             - valkeydata:/data
         networks:
@@ -52,8 +47,6 @@ services:
         restart: unless-stopped
         environment:
             MEILI_MASTER_KEY: ${MEILI_MASTER_KEY:-aSampleMasterKey}
-        ports:
-            - '7700:7700'
         volumes:
             - meilidata:/meili_data
         networks:

docs/user-guides/installation.md

@@ -297,3 +297,31 @@ After you've saved the changes, run the following command in your terminal to ap
     ```
 
 After this, any new data will be saved directly into the `./data/open-archiver` folder in your project directory.
+
+## Troubleshooting
+
+### 403 Cross-Site POST Forbidden Error
+
+If you are running the application behind a reverse proxy or have mapped the application to a different port (e.g., `3005:3000`), you may encounter a `403 Cross-site POST from submissions are forbidden` error when uploading files.
+
+To resolve this, you must set the `ORIGIN` environment variable to the URL of your application. This ensures that the backend can verify the origin of requests and prevent cross-site request forgery (CSRF) attacks.
+
+Add the following line to your `.env` file, replacing `<your_host>` and `<your_port>` with your specific values:
+
+```bash
+ORIGIN=http://<your_host>:<your_port>
+```
+
+For example, if your application is accessible at `http://localhost:3005`, you would set the variable as follows:
+
+```bash
+ORIGIN=http://localhost:3005
+```
+
+After adding the `ORIGIN` variable, restart your Docker containers for the changes to take effect:
+
+```bash
+docker-compose up -d --force-recreate
+```
+
+This will ensure that your file uploads are correctly authorized.

packages/backend/src/services/IndexingService.ts

@@ -99,7 +99,7 @@ export class IndexingService {
 			archivedEmailId,
 			email.userEmail || ''
 		);
-		console.log(document);
+		// console.log(document);
 		await this.searchService.addDocuments('emails', [document], 'id');
 	}
 
@@ -129,7 +129,7 @@ export class IndexingService {
 				//  skip attachment or fail the job
 			}
 		}
-		console.log('email.userEmail', userEmail);
+		// console.log('email.userEmail', userEmail);
 		return {
 			id: archivedEmailId,
 			userEmail: userEmail,
@@ -165,7 +165,7 @@ export class IndexingService {
 			'';
 
 		const recipients = email.recipients as DbRecipients;
-		console.log('email.userEmail', email.userEmail);
+		// console.log('email.userEmail', email.userEmail);
 		return {
 			id: email.id,
 			userEmail: userEmail,

packages/backend/src/services/ingestion-connectors/ImapConnector.ts

@@ -26,6 +26,10 @@ export class ImapConnector implements IEmailConnector {
 			host: this.credentials.host,
 			port: this.credentials.port,
 			secure: this.credentials.secure,
+			tls: {
+				rejectUnauthorized: this.credentials.allowInsecureCert,
+				requestCert: true,
+			},
 			auth: {
 				user: this.credentials.username,
 				pass: this.credentials.password,
@@ -145,107 +149,108 @@ export class ImapConnector implements IEmailConnector {
 		userEmail: string,
 		syncState?: SyncState | null
 	): AsyncGenerator<EmailObject | null> {
-		// list all mailboxes first
-		const mailboxes = await this.withRetry(async () => await this.client.list());
-		await this.disconnect();
+		try {
+			// list all mailboxes first
+			const mailboxes = await this.withRetry(async () => await this.client.list());
 
-		const processableMailboxes = mailboxes.filter((mailbox) => {
-			// filter out trash and all mail emails
-			if (mailbox.specialUse) {
-				const specialUse = mailbox.specialUse.toLowerCase();
-				if (specialUse === '\\junk' || specialUse === '\\trash' || specialUse === '\\all') {
+			const processableMailboxes = mailboxes.filter((mailbox) => {
+				// filter out trash and all mail emails
+				if (mailbox.specialUse) {
+					const specialUse = mailbox.specialUse.toLowerCase();
+					if (specialUse === '\\junk' || specialUse === '\\trash' || specialUse === '\\all') {
+						return false;
+					}
+				}
+				// Fallback to checking flags
+				if (
+					mailbox.flags.has('\\Noselect') ||
+					mailbox.flags.has('\\Trash') ||
+					mailbox.flags.has('\\Junk') ||
+					mailbox.flags.has('\\All')
+				) {
 					return false;
 				}
-			}
-			// Fallback to checking flags
-			if (
-				mailbox.flags.has('\\Noselect') ||
-				mailbox.flags.has('\\Trash') ||
-				mailbox.flags.has('\\Junk') ||
-				mailbox.flags.has('\\All')
-			) {
-				return false;
-			}
 
-			return true;
-		});
+				return true;
+			});
 
-		for (const mailboxInfo of processableMailboxes) {
-			const mailboxPath = mailboxInfo.path;
-			logger.info({ mailboxPath }, 'Processing mailbox');
+			for (const mailboxInfo of processableMailboxes) {
+				const mailboxPath = mailboxInfo.path;
+				logger.info({ mailboxPath }, 'Processing mailbox');
 
-			try {
-				const mailbox = await this.withRetry(
-					async () => await this.client.mailboxOpen(mailboxPath)
-				);
-				const lastUid = syncState?.imap?.[mailboxPath]?.maxUid;
-				let currentMaxUid = lastUid || 0;
+				try {
+					const mailbox = await this.withRetry(
+						async () => await this.client.mailboxOpen(mailboxPath)
+					);
+					const lastUid = syncState?.imap?.[mailboxPath]?.maxUid;
+					let currentMaxUid = lastUid || 0;
 
-				if (mailbox.exists > 0) {
-					const lastMessage = await this.client.fetchOne(String(mailbox.exists), {
-						uid: true,
-					});
-					if (lastMessage && lastMessage.uid > currentMaxUid) {
-						currentMaxUid = lastMessage.uid;
-					}
-				}
-
-				// Initialize with last synced UID, not the maximum UID in mailbox
-				this.newMaxUids[mailboxPath] = lastUid || 0;
-
-				// Only fetch if the mailbox has messages, to avoid errors on empty mailboxes with some IMAP servers.
-				if (mailbox.exists > 0) {
-					const BATCH_SIZE = 250; // A configurable batch size
-					let startUid = (lastUid || 0) + 1;
-					const maxUidToFetch = currentMaxUid;
-
-					while (startUid <= maxUidToFetch) {
-						const endUid = Math.min(startUid + BATCH_SIZE - 1, maxUidToFetch);
-						const searchCriteria = { uid: `${startUid}:${endUid}` };
-
-						for await (const msg of this.client.fetch(searchCriteria, {
-							envelope: true,
-							source: true,
-							bodyStructure: true,
+					if (mailbox.exists > 0) {
+						const lastMessage = await this.client.fetchOne(String(mailbox.exists), {
 							uid: true,
-						})) {
-							if (lastUid && msg.uid <= lastUid) {
-								continue;
-							}
+						});
+						if (lastMessage && lastMessage.uid > currentMaxUid) {
+							currentMaxUid = lastMessage.uid;
+						}
+					}
 
-							if (msg.uid > this.newMaxUids[mailboxPath]) {
-								this.newMaxUids[mailboxPath] = msg.uid;
-							}
+					// Initialize with last synced UID, not the maximum UID in mailbox
+					this.newMaxUids[mailboxPath] = lastUid || 0;
 
-							logger.debug({ mailboxPath, uid: msg.uid }, 'Processing message');
+					// Only fetch if the mailbox has messages, to avoid errors on empty mailboxes with some IMAP servers.
+					if (mailbox.exists > 0) {
+						const BATCH_SIZE = 250; // A configurable batch size
+						let startUid = (lastUid || 0) + 1;
+						const maxUidToFetch = currentMaxUid;
 
-							if (msg.envelope && msg.source) {
-								try {
-									yield await this.parseMessage(msg, mailboxPath);
-								} catch (err: any) {
-									logger.error(
-										{ err, mailboxPath, uid: msg.uid },
-										'Failed to parse message'
-									);
-									throw err;
+						while (startUid <= maxUidToFetch) {
+							const endUid = Math.min(startUid + BATCH_SIZE - 1, maxUidToFetch);
+							const searchCriteria = { uid: `${startUid}:${endUid}` };
+
+							for await (const msg of this.client.fetch(searchCriteria, {
+								envelope: true,
+								source: true,
+								bodyStructure: true,
+								uid: true,
+							})) {
+								if (lastUid && msg.uid <= lastUid) {
+									continue;
+								}
+
+								if (msg.uid > this.newMaxUids[mailboxPath]) {
+									this.newMaxUids[mailboxPath] = msg.uid;
+								}
+
+								logger.debug({ mailboxPath, uid: msg.uid }, 'Processing message');
+
+								if (msg.envelope && msg.source) {
+									try {
+										yield await this.parseMessage(msg, mailboxPath);
+									} catch (err: any) {
+										logger.error(
+											{ err, mailboxPath, uid: msg.uid },
+											'Failed to parse message'
+										);
+										throw err;
+									}
 								}
 							}
-						}
 
-						// Move to the next batch
-						startUid = endUid + 1;
+							// Move to the next batch
+							startUid = endUid + 1;
+						}
+					}
+				} catch (err: any) {
+					logger.error({ err, mailboxPath }, 'Failed to process mailbox');
+					// Check if the error indicates a persistent failure after retries
+					if (err.message.includes('IMAP operation failed after all retries')) {
+						this.statusMessage =
+							'Sync paused due to reaching the mail server rate limit. The process will automatically resume later.';
 					}
 				}
-			} catch (err: any) {
-				logger.error({ err, mailboxPath }, 'Failed to process mailbox');
-				// Check if the error indicates a persistent failure after retries
-				if (err.message.includes('IMAP operation failed after all retries')) {
-					this.statusMessage =
-						'Sync paused due to reaching the mail server rate limit. The process will automatically resume later.';
-				}
-			} finally {
-				await this.disconnect();
 			}
+		} finally {
+			await this.disconnect();
 		}
 	}
 

packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte

@@ -49,6 +49,7 @@
 		providerConfig: source?.credentials ?? {
 			type: source?.provider ?? 'generic_imap',
 			secure: true,
+			allowInsecureCert: false,
 		},
 	});
 
@@ -222,6 +223,12 @@
 			>
 			<Checkbox id="secure" bind:checked={formData.providerConfig.secure} />
 		</div>
+		<div class="grid grid-cols-4 items-center gap-4">
+			<Label for="secure" class="text-left"
+				>{$t('app.components.ingestion_source_form.allow_insecure_cert')}</Label
+			>
+			<Checkbox id="secure" bind:checked={formData.providerConfig.allowInsecureCert} />
+		</div>
 	{:else if formData.provider === 'pst_import'}
 		<div class="grid grid-cols-4 items-center gap-4">
 			<Label for="pst-file" class="text-left"

packages/frontend/src/lib/translations/en.json

@@ -183,6 +183,7 @@
 				"port": "Port",
 				"username": "Username",
 				"use_tls": "Use TLS",
+				"allow_insecure_cert": "Allow insecure cert",
 				"pst_file": "PST File",
 				"eml_file": "EML File",
 				"heads_up": "Heads up!",

packages/types/src/ingestion.types.ts

@@ -44,6 +44,7 @@ export interface GenericImapCredentials extends BaseIngestionCredentials {
 	host: string;
 	port: number;
 	secure: boolean;
+	allowInsecureCert: boolean;
 	username: string;
 	password?: string;
 }