Remove SUPER_API_KEY support

Co-authored-by: Wayne <5291640+ringoinca@users.noreply.github.com>

.env.example

@@ -59,8 +59,6 @@ STORAGE_S3_FORCE_PATH_STYLE=false
 JWT_SECRET=a-very-secret-key-that-you-should-change
 JWT_EXPIRES_IN="7d"
 
-# Set the credentials for the initial admin user.
-SUPER_API_KEY=
 
 # Master Encryption Key for sensitive data (Such as Ingestion source credentials and passwords)
 # IMPORTANT: Generate a secure, random 32-byte hex string for this

docker-compose.yml

@@ -6,7 +6,6 @@ services:
         container_name: open-archiver
         restart: unless-stopped
         ports:
-            - '4000:4000' # Backend
             - '3000:3000' # Frontend
         env_file:
             - .env
@@ -29,8 +28,6 @@ services:
             POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password}
         volumes:
             - pgdata:/var/lib/postgresql/data
-        ports:
-            - '5432:5432'
         networks:
             - open-archiver-net
 
@@ -39,8 +36,6 @@ services:
         container_name: valkey
         restart: unless-stopped
         command: valkey-server --requirepass ${REDIS_PASSWORD}
-        ports:
-            - '6379:6379'
         volumes:
             - valkeydata:/data
         networks:
@@ -52,8 +47,6 @@ services:
         restart: unless-stopped
         environment:
             MEILI_MASTER_KEY: ${MEILI_MASTER_KEY:-aSampleMasterKey}
-        ports:
-            - '7700:7700'
         volumes:
             - meilidata:/meili_data
         networks:

docs/api/authentication.md

@@ -43,18 +43,3 @@ Authorization: Bearer your.jwt.token
 ```
 
 If the token is missing, expired, or invalid, the API will respond with a `401 Unauthorized` status code.
-
-## Using a Super API Key
-
-Alternatively, for server-to-server communication or scripts, you can use a super API key. This key provides unrestricted access to the API and should be kept secret.
-
-You can set the `SUPER_API_KEY` in your `.env` file.
-
-To authenticate using the super API key, include it in the `Authorization` header as a Bearer token.
-
-**Example:**
-
-```http
-GET /api/v1/dashboard/stats
-Authorization: Bearer your-super-secret-api-key
-```

docs/user-guides/installation.md

@@ -105,12 +105,12 @@ These variables are used by `docker-compose.yml` to configure the services.
 
 #### Security & Authentication
 
-| Variable         | Description                                                         | Default Value                              |
-| ---------------- | ------------------------------------------------------------------- | ------------------------------------------ |
-| `JWT_SECRET`     | A secret key for signing JWT tokens.                                | `a-very-secret-key-that-you-should-change` |
-| `JWT_EXPIRES_IN` | The expiration time for JWT tokens.                                 | `7d`                                       |
-| `SUPER_API_KEY`  | An API key with super admin privileges.                             |                                            |
-| `ENCRYPTION_KEY` | A 32-byte hex string for encrypting sensitive data in the database. |                                            |
+| Variable                         | Description                                                                                                                                    | Default Value                              |
+| -------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ |
+| `JWT_SECRET`                     | A secret key for signing JWT tokens.                                                                                                           | `a-very-secret-key-that-you-should-change` |
+| `JWT_EXPIRES_IN`                 | The expiration time for JWT tokens.                                                                                                            | `7d`                                       |
+| ~~`SUPER_API_KEY`~~ (Deprecated) | An API key with super admin privileges. (The SUPER_API_KEY is deprecated since v0.3.0 after we roll out the role-based access control system.) |                                            |
+| `ENCRYPTION_KEY`                 | A 32-byte hex string for encrypting sensitive data in the database.                                                                            |                                            |
 
 ## 3. Run the Application
 
@@ -297,3 +297,31 @@ After you've saved the changes, run the following command in your terminal to ap
     ```
 
 After this, any new data will be saved directly into the `./data/open-archiver` folder in your project directory.
+
+## Troubleshooting
+
+### 403 Cross-Site POST Forbidden Error
+
+If you are running the application behind a reverse proxy or have mapped the application to a different port (e.g., `3005:3000`), you may encounter a `403 Cross-site POST from submissions are forbidden` error when uploading files.
+
+To resolve this, you must set the `ORIGIN` environment variable to the URL of your application. This ensures that the backend can verify the origin of requests and prevent cross-site request forgery (CSRF) attacks.
+
+Add the following line to your `.env` file, replacing `<your_host>` and `<your_port>` with your specific values:
+
+```bash
+ORIGIN=http://<your_host>:<your_port>
+```
+
+For example, if your application is accessible at `http://localhost:3005`, you would set the variable as follows:
+
+```bash
+ORIGIN=http://localhost:3005
+```
+
+After adding the `ORIGIN` variable, restart your Docker containers for the changes to take effect:
+
+```bash
+docker-compose up -d --force-recreate
+```
+
+This will ensure that your file uploads are correctly authorized.

packages/backend/src/api/controllers/auth.controller.ts

@@ -121,7 +121,7 @@ export class AuthController {
 				);
 				return res.status(200).json({ needsSetup: false });
 			}
-			return res.status(200).json({ needsSetupUser });
+			return res.status(200).json({ needsSetup: needsSetupUser });
 		} catch (error) {
 			console.error('Status check error:', error);
 			return res.status(500).json({ message: req.t('errors.internalServerError') });

packages/backend/src/api/controllers/settings.controller.ts

@@ -1,5 +1,6 @@
 import type { Request, Response } from 'express';
 import { SettingsService } from '../../services/SettingsService';
+import { config } from '../../config';
 
 const settingsService = new SettingsService();
 
@@ -16,6 +17,9 @@ export const getSettings = async (req: Request, res: Response) => {
 export const updateSettings = async (req: Request, res: Response) => {
 	try {
 		// Basic validation can be performed here if necessary
+		if (config.app.isDemo) {
+			return res.status(403).json({ message: req.t('errors.demoMode') });
+		}
 		const updatedSettings = await settingsService.updateSettings(req.body);
 		res.status(200).json(updatedSettings);
 	} catch (error) {

packages/backend/src/api/middleware/requireAuth.ts

@@ -20,11 +20,6 @@ export const requireAuth = (authService: AuthService) => {
 		}
 		const token = authHeader.split(' ')[1];
 		try {
-			// use a SUPER_API_KEY for all authentications. add process.env.SUPER_API_KEY conditional check in case user didn't set a SUPER_API_KEY.
-			if (process.env.SUPER_API_KEY && token === process.env.SUPER_API_KEY) {
-				next();
-				return;
-			}
 			const payload = await authService.verifyToken(token);
 			if (!payload) {
 				return res.status(401).json({ message: 'Unauthorized: Invalid token' });

packages/backend/src/services/IndexingService.ts

@@ -99,7 +99,7 @@ export class IndexingService {
 			archivedEmailId,
 			email.userEmail || ''
 		);
-		console.log(document);
+		// console.log(document);
 		await this.searchService.addDocuments('emails', [document], 'id');
 	}
 
@@ -129,7 +129,7 @@ export class IndexingService {
 				//  skip attachment or fail the job
 			}
 		}
-		console.log('email.userEmail', userEmail);
+		// console.log('email.userEmail', userEmail);
 		return {
 			id: archivedEmailId,
 			userEmail: userEmail,
@@ -165,7 +165,7 @@ export class IndexingService {
 			'';
 
 		const recipients = email.recipients as DbRecipients;
-		console.log('email.userEmail', email.userEmail);
+		// console.log('email.userEmail', email.userEmail);
 		return {
 			id: email.id,
 			userEmail: userEmail,

packages/backend/src/services/ingestion-connectors/ImapConnector.ts

@@ -26,6 +26,10 @@ export class ImapConnector implements IEmailConnector {
 			host: this.credentials.host,
 			port: this.credentials.port,
 			secure: this.credentials.secure,
+			tls: {
+				rejectUnauthorized: this.credentials.allowInsecureCert,
+				requestCert: true,
+			},
 			auth: {
 				user: this.credentials.username,
 				pass: this.credentials.password,
@@ -145,107 +149,112 @@ export class ImapConnector implements IEmailConnector {
 		userEmail: string,
 		syncState?: SyncState | null
 	): AsyncGenerator<EmailObject | null> {
-		// list all mailboxes first
-		const mailboxes = await this.withRetry(async () => await this.client.list());
-		await this.disconnect();
+		try {
+			// list all mailboxes first
+			const mailboxes = await this.withRetry(async () => await this.client.list());
 
-		const processableMailboxes = mailboxes.filter((mailbox) => {
-			// filter out trash and all mail emails
-			if (mailbox.specialUse) {
-				const specialUse = mailbox.specialUse.toLowerCase();
-				if (specialUse === '\\junk' || specialUse === '\\trash' || specialUse === '\\all') {
+			const processableMailboxes = mailboxes.filter((mailbox) => {
+				// filter out trash and all mail emails
+				if (mailbox.specialUse) {
+					const specialUse = mailbox.specialUse.toLowerCase();
+					if (
+						specialUse === '\\junk' ||
+						specialUse === '\\trash' ||
+						specialUse === '\\all'
+					) {
+						return false;
+					}
+				}
+				// Fallback to checking flags
+				if (
+					mailbox.flags.has('\\Noselect') ||
+					mailbox.flags.has('\\Trash') ||
+					mailbox.flags.has('\\Junk') ||
+					mailbox.flags.has('\\All')
+				) {
 					return false;
 				}
-			}
-			// Fallback to checking flags
-			if (
-				mailbox.flags.has('\\Noselect') ||
-				mailbox.flags.has('\\Trash') ||
-				mailbox.flags.has('\\Junk') ||
-				mailbox.flags.has('\\All')
-			) {
-				return false;
-			}
 
-			return true;
-		});
+				return true;
+			});
 
-		for (const mailboxInfo of processableMailboxes) {
-			const mailboxPath = mailboxInfo.path;
-			logger.info({ mailboxPath }, 'Processing mailbox');
+			for (const mailboxInfo of processableMailboxes) {
+				const mailboxPath = mailboxInfo.path;
+				logger.info({ mailboxPath }, 'Processing mailbox');
 
-			try {
-				const mailbox = await this.withRetry(
-					async () => await this.client.mailboxOpen(mailboxPath)
-				);
-				const lastUid = syncState?.imap?.[mailboxPath]?.maxUid;
-				let currentMaxUid = lastUid || 0;
+				try {
+					const mailbox = await this.withRetry(
+						async () => await this.client.mailboxOpen(mailboxPath)
+					);
+					const lastUid = syncState?.imap?.[mailboxPath]?.maxUid;
+					let currentMaxUid = lastUid || 0;
 
-				if (mailbox.exists > 0) {
-					const lastMessage = await this.client.fetchOne(String(mailbox.exists), {
-						uid: true,
-					});
-					if (lastMessage && lastMessage.uid > currentMaxUid) {
-						currentMaxUid = lastMessage.uid;
-					}
-				}
-
-				// Initialize with last synced UID, not the maximum UID in mailbox
-				this.newMaxUids[mailboxPath] = lastUid || 0;
-
-				// Only fetch if the mailbox has messages, to avoid errors on empty mailboxes with some IMAP servers.
-				if (mailbox.exists > 0) {
-					const BATCH_SIZE = 250; // A configurable batch size
-					let startUid = (lastUid || 0) + 1;
-					const maxUidToFetch = currentMaxUid;
-
-					while (startUid <= maxUidToFetch) {
-						const endUid = Math.min(startUid + BATCH_SIZE - 1, maxUidToFetch);
-						const searchCriteria = { uid: `${startUid}:${endUid}` };
-
-						for await (const msg of this.client.fetch(searchCriteria, {
-							envelope: true,
-							source: true,
-							bodyStructure: true,
+					if (mailbox.exists > 0) {
+						const lastMessage = await this.client.fetchOne(String(mailbox.exists), {
 							uid: true,
-						})) {
-							if (lastUid && msg.uid <= lastUid) {
-								continue;
-							}
+						});
+						if (lastMessage && lastMessage.uid > currentMaxUid) {
+							currentMaxUid = lastMessage.uid;
+						}
+					}
 
-							if (msg.uid > this.newMaxUids[mailboxPath]) {
-								this.newMaxUids[mailboxPath] = msg.uid;
-							}
+					// Initialize with last synced UID, not the maximum UID in mailbox
+					this.newMaxUids[mailboxPath] = lastUid || 0;
 
-							logger.debug({ mailboxPath, uid: msg.uid }, 'Processing message');
+					// Only fetch if the mailbox has messages, to avoid errors on empty mailboxes with some IMAP servers.
+					if (mailbox.exists > 0) {
+						const BATCH_SIZE = 250; // A configurable batch size
+						let startUid = (lastUid || 0) + 1;
+						const maxUidToFetch = currentMaxUid;
 
-							if (msg.envelope && msg.source) {
-								try {
-									yield await this.parseMessage(msg, mailboxPath);
-								} catch (err: any) {
-									logger.error(
-										{ err, mailboxPath, uid: msg.uid },
-										'Failed to parse message'
-									);
-									throw err;
+						while (startUid <= maxUidToFetch) {
+							const endUid = Math.min(startUid + BATCH_SIZE - 1, maxUidToFetch);
+							const searchCriteria = { uid: `${startUid}:${endUid}` };
+
+							for await (const msg of this.client.fetch(searchCriteria, {
+								envelope: true,
+								source: true,
+								bodyStructure: true,
+								uid: true,
+							})) {
+								if (lastUid && msg.uid <= lastUid) {
+									continue;
+								}
+
+								if (msg.uid > this.newMaxUids[mailboxPath]) {
+									this.newMaxUids[mailboxPath] = msg.uid;
+								}
+
+								logger.debug({ mailboxPath, uid: msg.uid }, 'Processing message');
+
+								if (msg.envelope && msg.source) {
+									try {
+										yield await this.parseMessage(msg, mailboxPath);
+									} catch (err: any) {
+										logger.error(
+											{ err, mailboxPath, uid: msg.uid },
+											'Failed to parse message'
+										);
+										throw err;
+									}
 								}
 							}
-						}
 
-						// Move to the next batch
-						startUid = endUid + 1;
+							// Move to the next batch
+							startUid = endUid + 1;
+						}
+					}
+				} catch (err: any) {
+					logger.error({ err, mailboxPath }, 'Failed to process mailbox');
+					// Check if the error indicates a persistent failure after retries
+					if (err.message.includes('IMAP operation failed after all retries')) {
+						this.statusMessage =
+							'Sync paused due to reaching the mail server rate limit. The process will automatically resume later.';
 					}
 				}
-			} catch (err: any) {
-				logger.error({ err, mailboxPath }, 'Failed to process mailbox');
-				// Check if the error indicates a persistent failure after retries
-				if (err.message.includes('IMAP operation failed after all retries')) {
-					this.statusMessage =
-						'Sync paused due to reaching the mail server rate limit. The process will automatically resume later.';
-				}
-			} finally {
-				await this.disconnect();
 			}
+		} finally {
+			await this.disconnect();
 		}
 	}
 

packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte

@@ -49,6 +49,7 @@
 		providerConfig: source?.credentials ?? {
 			type: source?.provider ?? 'generic_imap',
 			secure: true,
+			allowInsecureCert: false,
 		},
 	});
 
@@ -222,6 +223,12 @@
 			>
 			<Checkbox id="secure" bind:checked={formData.providerConfig.secure} />
 		</div>
+		<div class="grid grid-cols-4 items-center gap-4">
+			<Label for="secure" class="text-left"
+				>{$t('app.components.ingestion_source_form.allow_insecure_cert')}</Label
+			>
+			<Checkbox id="secure" bind:checked={formData.providerConfig.allowInsecureCert} />
+		</div>
 	{:else if formData.provider === 'pst_import'}
 		<div class="grid grid-cols-4 items-center gap-4">
 			<Label for="pst-file" class="text-left"

packages/frontend/src/lib/translations/en.json

@@ -183,6 +183,7 @@
 				"port": "Port",
 				"username": "Username",
 				"use_tls": "Use TLS",
+				"allow_insecure_cert": "Allow insecure cert",
 				"pst_file": "PST File",
 				"eml_file": "EML File",
 				"heads_up": "Heads up!",

packages/types/src/ingestion.types.ts

@@ -44,6 +44,7 @@ export interface GenericImapCredentials extends BaseIngestionCredentials {
 	host: string;
 	port: number;
 	secure: boolean;
+	allowInsecureCert: boolean;
 	username: string;
 	password?: string;
 }