Copy locale files in backend build

This commit introduces internationalization (i18n) to the backend API using the `i18next` library.

Hardcoded error and response messages in the API controllers have been replaced with translation keys, which are processed by the new i18next middleware. This allows for API responses to be translated into different languages.

The following dependencies were added:
- `i18next`
- `i18next-fs-backend`
- `i18next-http-middleware`

docker-compose.yml

@@ -6,6 +6,7 @@ services:
         container_name: open-archiver
         restart: unless-stopped
         ports:
+            - '4000:4000' # Backend
             - '3000:3000' # Frontend
         env_file:
             - .env
@@ -28,6 +29,8 @@ services:
             POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password}
         volumes:
             - pgdata:/var/lib/postgresql/data
+        ports:
+            - '5432:5432'
         networks:
             - open-archiver-net
 
@@ -36,6 +39,8 @@ services:
         container_name: valkey
         restart: unless-stopped
         command: valkey-server --requirepass ${REDIS_PASSWORD}
+        ports:
+            - '6379:6379'
         volumes:
             - valkeydata:/data
         networks:
@@ -47,6 +52,8 @@ services:
         restart: unless-stopped
         environment:
             MEILI_MASTER_KEY: ${MEILI_MASTER_KEY:-aSampleMasterKey}
+        ports:
+            - '7700:7700'
         volumes:
             - meilidata:/meili_data
         networks:

docs/user-guides/installation.md

@@ -297,31 +297,3 @@ After you've saved the changes, run the following command in your terminal to ap
     ```
 
 After this, any new data will be saved directly into the `./data/open-archiver` folder in your project directory.
-
-## Troubleshooting
-
-### 403 Cross-Site POST Forbidden Error
-
-If you are running the application behind a reverse proxy or have mapped the application to a different port (e.g., `3005:3000`), you may encounter a `403 Cross-site POST from submissions are forbidden` error when uploading files.
-
-To resolve this, you must set the `ORIGIN` environment variable to the URL of your application. This ensures that the backend can verify the origin of requests and prevent cross-site request forgery (CSRF) attacks.
-
-Add the following line to your `.env` file, replacing `<your_host>` and `<your_port>` with your specific values:
-
-```bash
-ORIGIN=http://<your_host>:<your_port>
-```
-
-For example, if your application is accessible at `http://localhost:3005`, you would set the variable as follows:
-
-```bash
-ORIGIN=http://localhost:3005
-```
-
-After adding the `ORIGIN` variable, restart your Docker containers for the changes to take effect:
-
-```bash
-docker-compose up -d --force-recreate
-```
-
-This will ensure that your file uploads are correctly authorized.

packages/backend/src/services/IndexingService.ts

@@ -99,7 +99,7 @@ export class IndexingService {
 			archivedEmailId,
 			email.userEmail || ''
 		);
-		// console.log(document);
+		console.log(document);
 		await this.searchService.addDocuments('emails', [document], 'id');
 	}
 
@@ -129,7 +129,7 @@ export class IndexingService {
 				//  skip attachment or fail the job
 			}
 		}
-		// console.log('email.userEmail', userEmail);
+		console.log('email.userEmail', userEmail);
 		return {
 			id: archivedEmailId,
 			userEmail: userEmail,
@@ -165,7 +165,7 @@ export class IndexingService {
 			'';
 
 		const recipients = email.recipients as DbRecipients;
-		// console.log('email.userEmail', email.userEmail);
+		console.log('email.userEmail', email.userEmail);
 		return {
 			id: email.id,
 			userEmail: userEmail,

packages/backend/src/services/ingestion-connectors/ImapConnector.ts

@@ -26,10 +26,6 @@ export class ImapConnector implements IEmailConnector {
 			host: this.credentials.host,
 			port: this.credentials.port,
 			secure: this.credentials.secure,
-			tls: {
-				rejectUnauthorized: this.credentials.allowInsecureCert,
-				requestCert: true,
-			},
 			auth: {
 				user: this.credentials.username,
 				pass: this.credentials.password,
@@ -149,108 +145,107 @@ export class ImapConnector implements IEmailConnector {
 		userEmail: string,
 		syncState?: SyncState | null
 	): AsyncGenerator<EmailObject | null> {
-		try {
-			// list all mailboxes first
-			const mailboxes = await this.withRetry(async () => await this.client.list());
+		// list all mailboxes first
+		const mailboxes = await this.withRetry(async () => await this.client.list());
+		await this.disconnect();
 
-			const processableMailboxes = mailboxes.filter((mailbox) => {
-				// filter out trash and all mail emails
-				if (mailbox.specialUse) {
-					const specialUse = mailbox.specialUse.toLowerCase();
-					if (specialUse === '\\junk' || specialUse === '\\trash' || specialUse === '\\all') {
-						return false;
-					}
-				}
-				// Fallback to checking flags
-				if (
-					mailbox.flags.has('\\Noselect') ||
-					mailbox.flags.has('\\Trash') ||
-					mailbox.flags.has('\\Junk') ||
-					mailbox.flags.has('\\All')
-				) {
+		const processableMailboxes = mailboxes.filter((mailbox) => {
+			// filter out trash and all mail emails
+			if (mailbox.specialUse) {
+				const specialUse = mailbox.specialUse.toLowerCase();
+				if (specialUse === '\\junk' || specialUse === '\\trash' || specialUse === '\\all') {
 					return false;
 				}
+			}
+			// Fallback to checking flags
+			if (
+				mailbox.flags.has('\\Noselect') ||
+				mailbox.flags.has('\\Trash') ||
+				mailbox.flags.has('\\Junk') ||
+				mailbox.flags.has('\\All')
+			) {
+				return false;
+			}
 
-				return true;
-			});
+			return true;
+		});
 
-			for (const mailboxInfo of processableMailboxes) {
-				const mailboxPath = mailboxInfo.path;
-				logger.info({ mailboxPath }, 'Processing mailbox');
+		for (const mailboxInfo of processableMailboxes) {
+			const mailboxPath = mailboxInfo.path;
+			logger.info({ mailboxPath }, 'Processing mailbox');
 
-				try {
-					const mailbox = await this.withRetry(
-						async () => await this.client.mailboxOpen(mailboxPath)
-					);
-					const lastUid = syncState?.imap?.[mailboxPath]?.maxUid;
-					let currentMaxUid = lastUid || 0;
+			try {
+				const mailbox = await this.withRetry(
+					async () => await this.client.mailboxOpen(mailboxPath)
+				);
+				const lastUid = syncState?.imap?.[mailboxPath]?.maxUid;
+				let currentMaxUid = lastUid || 0;
 
-					if (mailbox.exists > 0) {
-						const lastMessage = await this.client.fetchOne(String(mailbox.exists), {
-							uid: true,
-						});
-						if (lastMessage && lastMessage.uid > currentMaxUid) {
-							currentMaxUid = lastMessage.uid;
-						}
-					}
-
-					// Initialize with last synced UID, not the maximum UID in mailbox
-					this.newMaxUids[mailboxPath] = lastUid || 0;
-
-					// Only fetch if the mailbox has messages, to avoid errors on empty mailboxes with some IMAP servers.
-					if (mailbox.exists > 0) {
-						const BATCH_SIZE = 250; // A configurable batch size
-						let startUid = (lastUid || 0) + 1;
-						const maxUidToFetch = currentMaxUid;
-
-						while (startUid <= maxUidToFetch) {
-							const endUid = Math.min(startUid + BATCH_SIZE - 1, maxUidToFetch);
-							const searchCriteria = { uid: `${startUid}:${endUid}` };
-
-							for await (const msg of this.client.fetch(searchCriteria, {
-								envelope: true,
-								source: true,
-								bodyStructure: true,
-								uid: true,
-							})) {
-								if (lastUid && msg.uid <= lastUid) {
-									continue;
-								}
-
-								if (msg.uid > this.newMaxUids[mailboxPath]) {
-									this.newMaxUids[mailboxPath] = msg.uid;
-								}
-
-								logger.debug({ mailboxPath, uid: msg.uid }, 'Processing message');
-
-								if (msg.envelope && msg.source) {
-									try {
-										yield await this.parseMessage(msg, mailboxPath);
-									} catch (err: any) {
-										logger.error(
-											{ err, mailboxPath, uid: msg.uid },
-											'Failed to parse message'
-										);
-										throw err;
-									}
-								}
-							}
-
-							// Move to the next batch
-							startUid = endUid + 1;
-						}
-					}
-				} catch (err: any) {
-					logger.error({ err, mailboxPath }, 'Failed to process mailbox');
-					// Check if the error indicates a persistent failure after retries
-					if (err.message.includes('IMAP operation failed after all retries')) {
-						this.statusMessage =
-							'Sync paused due to reaching the mail server rate limit. The process will automatically resume later.';
+				if (mailbox.exists > 0) {
+					const lastMessage = await this.client.fetchOne(String(mailbox.exists), {
+						uid: true,
+					});
+					if (lastMessage && lastMessage.uid > currentMaxUid) {
+						currentMaxUid = lastMessage.uid;
 					}
 				}
+
+				// Initialize with last synced UID, not the maximum UID in mailbox
+				this.newMaxUids[mailboxPath] = lastUid || 0;
+
+				// Only fetch if the mailbox has messages, to avoid errors on empty mailboxes with some IMAP servers.
+				if (mailbox.exists > 0) {
+					const BATCH_SIZE = 250; // A configurable batch size
+					let startUid = (lastUid || 0) + 1;
+					const maxUidToFetch = currentMaxUid;
+
+					while (startUid <= maxUidToFetch) {
+						const endUid = Math.min(startUid + BATCH_SIZE - 1, maxUidToFetch);
+						const searchCriteria = { uid: `${startUid}:${endUid}` };
+
+						for await (const msg of this.client.fetch(searchCriteria, {
+							envelope: true,
+							source: true,
+							bodyStructure: true,
+							uid: true,
+						})) {
+							if (lastUid && msg.uid <= lastUid) {
+								continue;
+							}
+
+							if (msg.uid > this.newMaxUids[mailboxPath]) {
+								this.newMaxUids[mailboxPath] = msg.uid;
+							}
+
+							logger.debug({ mailboxPath, uid: msg.uid }, 'Processing message');
+
+							if (msg.envelope && msg.source) {
+								try {
+									yield await this.parseMessage(msg, mailboxPath);
+								} catch (err: any) {
+									logger.error(
+										{ err, mailboxPath, uid: msg.uid },
+										'Failed to parse message'
+									);
+									throw err;
+								}
+							}
+						}
+
+						// Move to the next batch
+						startUid = endUid + 1;
+					}
+				}
+			} catch (err: any) {
+				logger.error({ err, mailboxPath }, 'Failed to process mailbox');
+				// Check if the error indicates a persistent failure after retries
+				if (err.message.includes('IMAP operation failed after all retries')) {
+					this.statusMessage =
+						'Sync paused due to reaching the mail server rate limit. The process will automatically resume later.';
+				}
+			} finally {
+				await this.disconnect();
 			}
-		} finally {
-			await this.disconnect();
 		}
 	}
 

packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte

@@ -49,7 +49,6 @@
 		providerConfig: source?.credentials ?? {
 			type: source?.provider ?? 'generic_imap',
 			secure: true,
-			allowInsecureCert: false,
 		},
 	});
 
@@ -223,12 +222,6 @@
 			>
 			<Checkbox id="secure" bind:checked={formData.providerConfig.secure} />
 		</div>
-		<div class="grid grid-cols-4 items-center gap-4">
-			<Label for="secure" class="text-left"
-				>{$t('app.components.ingestion_source_form.allow_insecure_cert')}</Label
-			>
-			<Checkbox id="secure" bind:checked={formData.providerConfig.allowInsecureCert} />
-		</div>
 	{:else if formData.provider === 'pst_import'}
 		<div class="grid grid-cols-4 items-center gap-4">
 			<Label for="pst-file" class="text-left"

packages/frontend/src/lib/translations/en.json

@@ -183,7 +183,6 @@
 				"port": "Port",
 				"username": "Username",
 				"use_tls": "Use TLS",
-				"allow_insecure_cert": "Allow insecure cert",
 				"pst_file": "PST File",
 				"eml_file": "EML File",
 				"heads_up": "Heads up!",

packages/types/src/ingestion.types.ts

@@ -44,7 +44,6 @@ export interface GenericImapCredentials extends BaseIngestionCredentials {
 	host: string;
 	port: number;
 	secure: boolean;
-	allowInsecureCert: boolean;
 	username: string;
 	password?: string;
 }