From 94021eab690d44f28c12cfd148e070e90c79c348 Mon Sep 17 00:00:00 2001 From: "Wei S." <5291640+wayneshn@users.noreply.github.com> Date: Mon, 1 Sep 2025 12:44:22 +0300 Subject: [PATCH] v0.3.0 release (#76) * Remove extra ports in Docker Compose file * Allow self-assigned cert * Adding allow insecure cert option * fix(IMAP): Share connections between each fetch email action * Update docs: troubleshooting CORS error --------- Co-authored-by: Wayne <5291640+ringoinca@users.noreply.github.com> --- docker-compose.yml | 7 - docs/user-guides/installation.md | 28 +++ .../backend/src/services/IndexingService.ts | 6 +- .../ingestion-connectors/ImapConnector.ts | 173 +++++++++--------- .../custom/IngestionSourceForm.svelte | 7 + .../frontend/src/lib/translations/en.json | 1 + packages/types/src/ingestion.types.ts | 1 + 7 files changed, 129 insertions(+), 94 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f38eea2..d00714c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,6 @@ services: container_name: open-archiver restart: unless-stopped ports: - - '4000:4000' # Backend - '3000:3000' # Frontend env_file: - .env @@ -29,8 +28,6 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password} volumes: - pgdata:/var/lib/postgresql/data - ports: - - '5432:5432' networks: - open-archiver-net @@ -39,8 +36,6 @@ services: container_name: valkey restart: unless-stopped command: valkey-server --requirepass ${REDIS_PASSWORD} - ports: - - '6379:6379' volumes: - valkeydata:/data networks: @@ -52,8 +47,6 @@ services: restart: unless-stopped environment: MEILI_MASTER_KEY: ${MEILI_MASTER_KEY:-aSampleMasterKey} - ports: - - '7700:7700' volumes: - meilidata:/meili_data networks: diff --git a/docs/user-guides/installation.md b/docs/user-guides/installation.md index 0a40cd3..5ab5bbf 100644 --- a/docs/user-guides/installation.md +++ b/docs/user-guides/installation.md @@ -297,3 +297,31 @@ After you've saved the changes, run the following command in your terminal to ap ``` After this, any new data will be saved directly into the `./data/open-archiver` folder in your project directory. + +## Troubleshooting + +### 403 Cross-Site POST Forbidden Error + +If you are running the application behind a reverse proxy or have mapped the application to a different port (e.g., `3005:3000`), you may encounter a `403 Cross-site POST from submissions are forbidden` error when uploading files. + +To resolve this, you must set the `ORIGIN` environment variable to the URL of your application. This ensures that the backend can verify the origin of requests and prevent cross-site request forgery (CSRF) attacks. + +Add the following line to your `.env` file, replacing `` and `` with your specific values: + +```bash +ORIGIN=http://: +``` + +For example, if your application is accessible at `http://localhost:3005`, you would set the variable as follows: + +```bash +ORIGIN=http://localhost:3005 +``` + +After adding the `ORIGIN` variable, restart your Docker containers for the changes to take effect: + +```bash +docker-compose up -d --force-recreate +``` + +This will ensure that your file uploads are correctly authorized. diff --git a/packages/backend/src/services/IndexingService.ts b/packages/backend/src/services/IndexingService.ts index dce50a1..d947f83 100644 --- a/packages/backend/src/services/IndexingService.ts +++ b/packages/backend/src/services/IndexingService.ts @@ -99,7 +99,7 @@ export class IndexingService { archivedEmailId, email.userEmail || '' ); - console.log(document); + // console.log(document); await this.searchService.addDocuments('emails', [document], 'id'); } @@ -129,7 +129,7 @@ export class IndexingService { // skip attachment or fail the job } } - console.log('email.userEmail', userEmail); + // console.log('email.userEmail', userEmail); return { id: archivedEmailId, userEmail: userEmail, @@ -165,7 +165,7 @@ export class IndexingService { ''; const recipients = email.recipients as DbRecipients; - console.log('email.userEmail', email.userEmail); + // console.log('email.userEmail', email.userEmail); return { id: email.id, userEmail: userEmail, diff --git a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts index a097b5c..37b55e1 100644 --- a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts @@ -26,6 +26,10 @@ export class ImapConnector implements IEmailConnector { host: this.credentials.host, port: this.credentials.port, secure: this.credentials.secure, + tls: { + rejectUnauthorized: this.credentials.allowInsecureCert, + requestCert: true, + }, auth: { user: this.credentials.username, pass: this.credentials.password, @@ -145,107 +149,108 @@ export class ImapConnector implements IEmailConnector { userEmail: string, syncState?: SyncState | null ): AsyncGenerator { - // list all mailboxes first - const mailboxes = await this.withRetry(async () => await this.client.list()); - await this.disconnect(); + try { + // list all mailboxes first + const mailboxes = await this.withRetry(async () => await this.client.list()); - const processableMailboxes = mailboxes.filter((mailbox) => { - // filter out trash and all mail emails - if (mailbox.specialUse) { - const specialUse = mailbox.specialUse.toLowerCase(); - if (specialUse === '\\junk' || specialUse === '\\trash' || specialUse === '\\all') { + const processableMailboxes = mailboxes.filter((mailbox) => { + // filter out trash and all mail emails + if (mailbox.specialUse) { + const specialUse = mailbox.specialUse.toLowerCase(); + if (specialUse === '\\junk' || specialUse === '\\trash' || specialUse === '\\all') { + return false; + } + } + // Fallback to checking flags + if ( + mailbox.flags.has('\\Noselect') || + mailbox.flags.has('\\Trash') || + mailbox.flags.has('\\Junk') || + mailbox.flags.has('\\All') + ) { return false; } - } - // Fallback to checking flags - if ( - mailbox.flags.has('\\Noselect') || - mailbox.flags.has('\\Trash') || - mailbox.flags.has('\\Junk') || - mailbox.flags.has('\\All') - ) { - return false; - } - return true; - }); + return true; + }); - for (const mailboxInfo of processableMailboxes) { - const mailboxPath = mailboxInfo.path; - logger.info({ mailboxPath }, 'Processing mailbox'); + for (const mailboxInfo of processableMailboxes) { + const mailboxPath = mailboxInfo.path; + logger.info({ mailboxPath }, 'Processing mailbox'); - try { - const mailbox = await this.withRetry( - async () => await this.client.mailboxOpen(mailboxPath) - ); - const lastUid = syncState?.imap?.[mailboxPath]?.maxUid; - let currentMaxUid = lastUid || 0; + try { + const mailbox = await this.withRetry( + async () => await this.client.mailboxOpen(mailboxPath) + ); + const lastUid = syncState?.imap?.[mailboxPath]?.maxUid; + let currentMaxUid = lastUid || 0; - if (mailbox.exists > 0) { - const lastMessage = await this.client.fetchOne(String(mailbox.exists), { - uid: true, - }); - if (lastMessage && lastMessage.uid > currentMaxUid) { - currentMaxUid = lastMessage.uid; - } - } - - // Initialize with last synced UID, not the maximum UID in mailbox - this.newMaxUids[mailboxPath] = lastUid || 0; - - // Only fetch if the mailbox has messages, to avoid errors on empty mailboxes with some IMAP servers. - if (mailbox.exists > 0) { - const BATCH_SIZE = 250; // A configurable batch size - let startUid = (lastUid || 0) + 1; - const maxUidToFetch = currentMaxUid; - - while (startUid <= maxUidToFetch) { - const endUid = Math.min(startUid + BATCH_SIZE - 1, maxUidToFetch); - const searchCriteria = { uid: `${startUid}:${endUid}` }; - - for await (const msg of this.client.fetch(searchCriteria, { - envelope: true, - source: true, - bodyStructure: true, + if (mailbox.exists > 0) { + const lastMessage = await this.client.fetchOne(String(mailbox.exists), { uid: true, - })) { - if (lastUid && msg.uid <= lastUid) { - continue; - } + }); + if (lastMessage && lastMessage.uid > currentMaxUid) { + currentMaxUid = lastMessage.uid; + } + } - if (msg.uid > this.newMaxUids[mailboxPath]) { - this.newMaxUids[mailboxPath] = msg.uid; - } + // Initialize with last synced UID, not the maximum UID in mailbox + this.newMaxUids[mailboxPath] = lastUid || 0; - logger.debug({ mailboxPath, uid: msg.uid }, 'Processing message'); + // Only fetch if the mailbox has messages, to avoid errors on empty mailboxes with some IMAP servers. + if (mailbox.exists > 0) { + const BATCH_SIZE = 250; // A configurable batch size + let startUid = (lastUid || 0) + 1; + const maxUidToFetch = currentMaxUid; - if (msg.envelope && msg.source) { - try { - yield await this.parseMessage(msg, mailboxPath); - } catch (err: any) { - logger.error( - { err, mailboxPath, uid: msg.uid }, - 'Failed to parse message' - ); - throw err; + while (startUid <= maxUidToFetch) { + const endUid = Math.min(startUid + BATCH_SIZE - 1, maxUidToFetch); + const searchCriteria = { uid: `${startUid}:${endUid}` }; + + for await (const msg of this.client.fetch(searchCriteria, { + envelope: true, + source: true, + bodyStructure: true, + uid: true, + })) { + if (lastUid && msg.uid <= lastUid) { + continue; + } + + if (msg.uid > this.newMaxUids[mailboxPath]) { + this.newMaxUids[mailboxPath] = msg.uid; + } + + logger.debug({ mailboxPath, uid: msg.uid }, 'Processing message'); + + if (msg.envelope && msg.source) { + try { + yield await this.parseMessage(msg, mailboxPath); + } catch (err: any) { + logger.error( + { err, mailboxPath, uid: msg.uid }, + 'Failed to parse message' + ); + throw err; + } } } - } - // Move to the next batch - startUid = endUid + 1; + // Move to the next batch + startUid = endUid + 1; + } + } + } catch (err: any) { + logger.error({ err, mailboxPath }, 'Failed to process mailbox'); + // Check if the error indicates a persistent failure after retries + if (err.message.includes('IMAP operation failed after all retries')) { + this.statusMessage = + 'Sync paused due to reaching the mail server rate limit. The process will automatically resume later.'; } } - } catch (err: any) { - logger.error({ err, mailboxPath }, 'Failed to process mailbox'); - // Check if the error indicates a persistent failure after retries - if (err.message.includes('IMAP operation failed after all retries')) { - this.statusMessage = - 'Sync paused due to reaching the mail server rate limit. The process will automatically resume later.'; - } - } finally { - await this.disconnect(); } + } finally { + await this.disconnect(); } } diff --git a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte index e0d8acd..65d5913 100644 --- a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte +++ b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte @@ -49,6 +49,7 @@ providerConfig: source?.credentials ?? { type: source?.provider ?? 'generic_imap', secure: true, + allowInsecureCert: false, }, }); @@ -222,6 +223,12 @@ > +
+ + +
{:else if formData.provider === 'pst_import'}