diff --git a/.env.example b/.env.example index 4f2b2c4..8c3997e 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,8 @@ REDIS_PORT=6379 REDIS_PASSWORD=defaultredispassword # If you run Valkey service from Docker Compose, set the REDIS_TLS_ENABLED variable to false. REDIS_TLS_ENABLED=false +# Redis username. Only required if not using the default user. +REDIS_USER=notdefaultuser # --- Storage Settings --- diff --git a/docker-compose.yml b/docker-compose.yml index b9468c0..e73b9bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,7 @@ services: restart: unless-stopped environment: MEILI_MASTER_KEY: ${MEILI_MASTER_KEY:-aSampleMasterKey} + MEILI_SCHEDULE_SNAPSHOT: ${MEILI_SCHEDULE_SNAPSHOT:-86400} volumes: - meilidata:/meili_data networks: diff --git a/docs/api/ingestion.md b/docs/api/ingestion.md index 4071bc4..a8eaad4 100644 --- a/docs/api/ingestion.md +++ b/docs/api/ingestion.md @@ -24,6 +24,40 @@ interface CreateIngestionSourceDto { } ``` +#### Example: Creating an Mbox Import Source with File Upload + +```json +{ + "name": "My Mbox Import", + "provider": "mbox_import", + "providerConfig": { + "type": "mbox_import", + "uploadedFileName": "emails.mbox", + "uploadedFilePath": "open-archiver/tmp/uuid-emails.mbox" + } +} +``` + +#### Example: Creating an Mbox Import Source with Local File Path + +```json +{ + "name": "My Mbox Import", + "provider": "mbox_import", + "providerConfig": { + "type": "mbox_import", + "localFilePath": "/path/to/emails.mbox" + } +} +``` + +**Note:** When using `localFilePath`, the file will not be deleted after import. When using `uploadedFilePath` (via the upload API), the file will be automatically deleted after import. The same applies to `pst_import` and `eml_import` providers. + +**Important regarding `localFilePath`:** When running OpenArchiver in a Docker container (which is the standard deployment), `localFilePath` refers to the path **inside the Docker container**, not on the host machine. +To use a local file: +1. **Recommended:** Place your file inside the directory defined by `STORAGE_LOCAL_ROOT_PATH` (e.g., inside a `temp` folder). Since this directory is already mounted as a volume, the file will be accessible at the same path inside the container. +2. **Alternative:** Mount a specific directory containing your files as a volume in `docker-compose.yml`. For example, add `- /path/to/my/files:/imports` to the `volumes` section and use `/imports/myfile.pst` as the `localFilePath`. + #### Responses - **201 Created:** The newly created ingestion source. diff --git a/docs/user-guides/email-providers/eml.md b/docs/user-guides/email-providers/eml.md index 157cb35..16fb7c3 100644 --- a/docs/user-guides/email-providers/eml.md +++ b/docs/user-guides/email-providers/eml.md @@ -30,7 +30,14 @@ archive.zip 2. Click the **Create New** button. 3. Select **EML Import** as the provider. 4. Enter a name for the ingestion source. -5. Click the **Choose File** button and select the zip archive containing your EML files. +5. **Choose Import Method:** + * **Upload File:** Click **Choose File** and select the zip archive containing your EML files. (Best for smaller archives) + * **Local Path:** Enter the path to the zip file **inside the container**. (Best for large archives) + + > **Note on Local Path:** When using Docker, the "Local Path" is relative to the container's filesystem. + > * **Recommended:** Place your zip file in a `temp` folder inside your configured storage directory (`STORAGE_LOCAL_ROOT_PATH`). This path is already mounted. For example, if your storage path is `/data`, put the file in `/data/temp/emails.zip` and enter `/data/temp/emails.zip` as the path. + > * **Alternative:** Mount a separate volume in `docker-compose.yml` (e.g., `- /host/path:/container/path`) and use the container path. + 6. Click the **Submit** button. OpenArchiver will then start importing the EML files from the zip archive. The ingestion process may take some time, depending on the size of the archive. diff --git a/docs/user-guides/email-providers/mbox.md b/docs/user-guides/email-providers/mbox.md index bced06c..587d89b 100644 --- a/docs/user-guides/email-providers/mbox.md +++ b/docs/user-guides/email-providers/mbox.md @@ -17,7 +17,13 @@ Once you have your `.mbox` file, you can upload it to OpenArchiver through the w 1. Navigate to the **Ingestion** page. 2. Click on the **New Ingestion** button. 3. Select **Mbox** as the source type. -4. Upload your `.mbox` file. +4. **Choose Import Method:** + * **Upload File:** Upload your `.mbox` file. + * **Local Path:** Enter the path to the mbox file **inside the container**. + + > **Note on Local Path:** When using Docker, the "Local Path" is relative to the container's filesystem. + > * **Recommended:** Place your mbox file in a `temp` folder inside your configured storage directory (`STORAGE_LOCAL_ROOT_PATH`). This path is already mounted. For example, if your storage path is `/data`, put the file in `/data/temp/emails.mbox` and enter `/data/temp/emails.mbox` as the path. + > * **Alternative:** Mount a separate volume in `docker-compose.yml` (e.g., `- /host/path:/container/path`) and use the container path. ## 3. Folder Structure diff --git a/docs/user-guides/email-providers/pst.md b/docs/user-guides/email-providers/pst.md index 4180daa..007e0ed 100644 --- a/docs/user-guides/email-providers/pst.md +++ b/docs/user-guides/email-providers/pst.md @@ -15,7 +15,14 @@ To ensure a successful import, you should prepare your PST file according to the 2. Click the **Create New** button. 3. Select **PST Import** as the provider. 4. Enter a name for the ingestion source. -5. Click the **Choose File** button and select the PST file. +5. **Choose Import Method:** + * **Upload File:** Click **Choose File** and select the PST file from your computer. (Best for smaller files) + * **Local Path:** Enter the path to the PST file **inside the container**. (Best for large files) + + > **Note on Local Path:** When using Docker, the "Local Path" is relative to the container's filesystem. + > * **Recommended:** Place your file in a `temp` folder inside your configured storage directory (`STORAGE_LOCAL_ROOT_PATH`). This path is already mounted. For example, if your storage path is `/data`, put the file in `/data/temp/archive.pst` and enter `/data/temp/archive.pst` as the path. + > * **Alternative:** Mount a separate volume in `docker-compose.yml` (e.g., `- /host/path:/container/path`) and use the container path. + 6. Click the **Submit** button. OpenArchiver will then start importing the emails from the PST file. The ingestion process may take some time, depending on the size of the file. diff --git a/docs/user-guides/installation.md b/docs/user-guides/installation.md index 1b66c1a..623c857 100644 --- a/docs/user-guides/installation.md +++ b/docs/user-guides/installation.md @@ -115,6 +115,7 @@ These variables are used by `docker-compose.yml` to configure the services. | `MEILI_INDEXING_BATCH` | The number of emails to batch together for indexing. | `500` | | `REDIS_HOST` | The host for the Valkey (Redis) service. | `valkey` | | `REDIS_PORT` | The port for the Valkey (Redis) service. | `6379` | +| `REDIS_USER` | Optional Redis username if ACLs are used. | | | `REDIS_PASSWORD` | The password for the Valkey (Redis) service. | `defaultredispassword` | | `REDIS_TLS_ENABLED` | Enable or disable TLS for Redis. | `false` | diff --git a/docs/user-guides/upgrade-and-migration/meilisearch-upgrade.md b/docs/user-guides/upgrade-and-migration/meilisearch-upgrade.md index eb4d0ee..e0213b3 100644 --- a/docs/user-guides/upgrade-and-migration/meilisearch-upgrade.md +++ b/docs/user-guides/upgrade-and-migration/meilisearch-upgrade.md @@ -4,9 +4,57 @@ Meilisearch, the search engine used by Open Archiver, requires a manual data mig If an Open Archiver upgrade includes a major Meilisearch version change, you will need to migrate your search index by following the process below. -## Migration Process Overview +## Experimental: Dumpless Upgrade -For self-hosted instances using Docker Compose (as recommended), the migration process involves creating a data dump from your current Meilisearch instance, upgrading the Docker image, and then importing that dump into the new version. +> **Warning:** This feature is currently **experimental**. We do not recommend using it for production environments until it is marked as stable. Please use the [standard migration process](#standard-migration-process-recommended) instead. Proceed with caution. + +Meilisearch recently introduced an experimental "dumpless" upgrade method. This allows you to migrate the database to a new Meilisearch version without manually creating and importing a dump. However, please note that **dumpless upgrades are not currently atomic**. If the process fails, your database may become corrupted, resulting in data loss. + +**Prerequisite: Create a Snapshot** + +Before attempting a dumpless upgrade, you **must** take a snapshot of your instance. This ensures you have a recovery point if the upgrade fails. Learn how to create snapshots in the [official Meilisearch documentation](https://www.meilisearch.com/docs/learn/data_backup/snapshots). + +### How to Enable + +To perform a dumpless upgrade, you need to configure your Meilisearch instance with the experimental flag. You can do this in one of two ways: + +**Option 1: Using an Environment Variable** + +Add the `MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE` environment variable to your `docker-compose.yml` file for the Meilisearch service. + +```yaml +services: + meilisearch: + image: getmeili/meilisearch:v1.x # The new version you want to upgrade to + environment: + - MEILI_MASTER_KEY=${MEILI_MASTER_KEY} + - MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE=true +``` + +**Option 2: Using a CLI Option** + +Alternatively, you can pass the `--experimental-dumpless-upgrade` flag in the command section of your `docker-compose.yml`. + +```yaml +services: + meilisearch: + image: getmeili/meilisearch:v1.x # The new version you want to upgrade to + command: meilisearch --experimental-dumpless-upgrade +``` + +After updating your configuration, restart your container: + +```bash +docker compose up -d +``` + +Meilisearch will attempt to migrate your database to the new version automatically. + +--- + +## Standard Migration Process (Recommended) + +For self-hosted instances using Docker Compose, the recommended migration process involves creating a data dump from your current Meilisearch instance, upgrading the Docker image, and then importing that dump into the new version. ### Step 1: Create a Dump diff --git a/open-archiver.yml b/open-archiver.yml index 9fdbb42..c4673d2 100644 --- a/open-archiver.yml +++ b/open-archiver.yml @@ -22,6 +22,7 @@ services: - MEILI_HOST=http://meilisearch:7700 - REDIS_HOST=valkey - REDIS_PORT=6379 + - REDIS_USER=default - REDIS_PASSWORD=${SERVICE_PASSWORD_VALKEY} - REDIS_TLS_ENABLED=false - STORAGE_TYPE=${STORAGE_TYPE:-local} @@ -73,5 +74,6 @@ services: image: getmeili/meilisearch:v1.15 environment: - MEILI_MASTER_KEY=${SERVICE_PASSWORD_MEILISEARCH} + - MEILI_SCHEDULE_SNAPSHOT=86400 volumes: - meilidata:/meili_data diff --git a/package.json b/package.json index 65b3bbb..df69c50 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "open-archiver", - "version": "0.4.1", + "version": "0.4.2", "private": true, "license": "SEE LICENSE IN LICENSE file", "scripts": { diff --git a/packages/backend/src/config/redis.ts b/packages/backend/src/config/redis.ts index ec21ca8..1f2e4bf 100644 --- a/packages/backend/src/config/redis.ts +++ b/packages/backend/src/config/redis.ts @@ -1,15 +1,20 @@ import 'dotenv/config'; +import { type ConnectionOptions } from 'bullmq'; /** * @see https://github.com/taskforcesh/bullmq/blob/master/docs/gitbook/guide/connections.md */ -const connectionOptions: any = { +const connectionOptions: ConnectionOptions = { host: process.env.REDIS_HOST || 'localhost', port: (process.env.REDIS_PORT && parseInt(process.env.REDIS_PORT, 10)) || 6379, password: process.env.REDIS_PASSWORD, enableReadyCheck: true, }; +if (process.env.REDIS_USER) { + connectionOptions.username = process.env.REDIS_USER; +} + if (process.env.REDIS_TLS_ENABLED === 'true') { connectionOptions.tls = { rejectUnauthorized: false, diff --git a/packages/backend/src/locales/bg/translation.json b/packages/backend/src/locales/bg/translation.json new file mode 100644 index 0000000..bf33ebe --- /dev/null +++ b/packages/backend/src/locales/bg/translation.json @@ -0,0 +1,69 @@ +{ + "auth": { + "setup": { + "allFieldsRequired": "Изискват се поща, парола и име", + "alreadyCompleted": "Настройката вече е завършена." + }, + "login": { + "emailAndPasswordRequired": "Изискват се поща и парола", + "invalidCredentials": "Невалидни идентификационни данни" + } + }, + "errors": { + "internalServerError": "Възникна вътрешна грешка в сървъра", + "demoMode": "Тази операция не е разрешена в демо режим", + "unauthorized": "Неоторизирано", + "unknown": "Възникна неизвестна грешка", + "noPermissionToAction": "Нямате разрешение да извършите текущото действие." + }, + "user": { + "notFound": "Потребителят не е открит", + "cannotDeleteOnlyUser": "Опитвате се да изтриете единствения потребител в базата данни, това не е позволено.", + "requiresSuperAdminRole": "За управление на потребители е необходима роля на супер администратор." + }, + "iam": { + "failedToGetRoles": "Неуспешно получаване на роли.", + "roleNotFound": "Ролята не е намерена.", + "failedToGetRole": "Неуспешно получаване на роля.", + "missingRoleFields": "Липсват задължителни полета: име и политика.", + "invalidPolicy": "Невалидно твърдение за политика:", + "failedToCreateRole": "Създаването на роля неуспешно.", + "failedToDeleteRole": "Изтриването на роля неуспешно.", + "missingUpdateFields": "Липсват полета за актуализиране: име или политики.", + "failedToUpdateRole": "Актуализирането на ролята неуспешно.", + "requiresSuperAdminRole": "За управление на роли е необходима роля на супер администратор." + }, + "settings": { + "failedToRetrieve": "Неуспешно извличане на настройките", + "failedToUpdate": "Неуспешно актуализиране на настройките", + "noPermissionToUpdate": "Нямате разрешение да актуализирате системните настройки." + }, + "dashboard": { + "permissionRequired": "Необходимо ви е разрешение за четене на таблото, за да видите данните от него." + }, + "ingestion": { + "failedToCreate": "Създаването на източник за приемане не бе успешно поради грешка при свързване.", + "notFound": "Източникът за приемане не е намерен", + "initialImportTriggered": "Първоначалният импорт е задействан успешно.", + "forceSyncTriggered": "Принудителното синхронизиране е задействано успешно." + }, + "archivedEmail": { + "notFound": "Архивираната поща не е намерена" + }, + "search": { + "keywordsRequired": "Ключовите думи са задължителни" + }, + "storage": { + "filePathRequired": "Пътят към файла е задължителен", + "invalidFilePath": "Невалиден път към файла", + "fileNotFound": "Файлът не е намерен", + "downloadError": "Грешка при изтегляне на файла" + }, + "apiKeys": { + "generateSuccess": "API ключът е генериран успешно.", + "deleteSuccess": "API ключът е успешно изтрит." + }, + "api": { + "requestBodyInvalid": "Невалидно съдържание на заявката." + } +} diff --git a/packages/backend/src/services/IngestionService.ts b/packages/backend/src/services/IngestionService.ts index 8f95bc2..00f83eb 100644 --- a/packages/backend/src/services/IngestionService.ts +++ b/packages/backend/src/services/IngestionService.ts @@ -219,7 +219,8 @@ export class IngestionService { if ( (source.credentials.type === 'pst_import' || - source.credentials.type === 'eml_import') && + source.credentials.type === 'eml_import' || + source.credentials.type === 'mbox_import') && source.credentials.uploadedFilePath && (await storage.exists(source.credentials.uploadedFilePath)) ) { diff --git a/packages/backend/src/services/ingestion-connectors/EMLConnector.ts b/packages/backend/src/services/ingestion-connectors/EMLConnector.ts index 4209ab3..0aff453 100644 --- a/packages/backend/src/services/ingestion-connectors/EMLConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/EMLConnector.ts @@ -32,17 +32,52 @@ export class EMLConnector implements IEmailConnector { this.storage = new StorageService(); } + private getFilePath(): string { + return this.credentials.localFilePath || this.credentials.uploadedFilePath || ''; + } + + private getDisplayName(): string { + if (this.credentials.uploadedFileName) { + return this.credentials.uploadedFileName; + } + if (this.credentials.localFilePath) { + const parts = this.credentials.localFilePath.split('/'); + return parts[parts.length - 1].replace('.zip', ''); + } + return `eml-import-${new Date().getTime()}`; + } + + private async getFileStream(): Promise { + if (this.credentials.localFilePath) { + return createReadStream(this.credentials.localFilePath); + } + return this.storage.get(this.getFilePath()); + } + public async testConnection(): Promise { try { - if (!this.credentials.uploadedFilePath) { + const filePath = this.getFilePath(); + if (!filePath) { throw Error('EML file path not provided.'); } - if (!this.credentials.uploadedFilePath.includes('.zip')) { + if (!filePath.includes('.zip')) { throw Error('Provided file is not in the ZIP format.'); } - const fileExist = await this.storage.exists(this.credentials.uploadedFilePath); + + let fileExist = false; + if (this.credentials.localFilePath) { + try { + await fs.access(this.credentials.localFilePath); + fileExist = true; + } catch { + fileExist = false; + } + } else { + fileExist = await this.storage.exists(filePath); + } + if (!fileExist) { - throw Error('EML file upload not finished yet, please wait.'); + throw Error('EML file not found or upload not finished yet, please wait.'); } return true; @@ -53,8 +88,7 @@ export class EMLConnector implements IEmailConnector { } public async *listAllUsers(): AsyncGenerator { - const displayName = - this.credentials.uploadedFileName || `eml-import-${new Date().getTime()}`; + const displayName = this.getDisplayName(); logger.info(`Found potential mailbox: ${displayName}`); const constructedPrimaryEmail = `${displayName.replace(/ /g, '.').toLowerCase()}@eml.local`; yield { @@ -68,10 +102,8 @@ export class EMLConnector implements IEmailConnector { userEmail: string, syncState?: SyncState | null ): AsyncGenerator { - const fileStream = await this.storage.get(this.credentials.uploadedFilePath); + const fileStream = await this.getFileStream(); const tempDir = await fs.mkdtemp(join('/tmp', `eml-import-${new Date().getTime()}`)); - const unzippedPath = join(tempDir, 'unzipped'); - await fs.mkdir(unzippedPath); const zipFilePath = join(tempDir, 'eml.zip'); try { @@ -82,99 +114,150 @@ export class EMLConnector implements IEmailConnector { dest.on('error', reject); }); - await this.extract(zipFilePath, unzippedPath); - - const files = await this.getAllFiles(unzippedPath); - - for (const file of files) { - if (file.endsWith('.eml')) { - try { - // logger.info({ file }, 'Processing EML file.'); - const stream = createReadStream(file); - const content = await streamToBuffer(stream); - // logger.info({ file, size: content.length }, 'Read file to buffer.'); - let relativePath = file.substring(unzippedPath.length + 1); - if (dirname(relativePath) === '.') { - relativePath = ''; - } else { - relativePath = dirname(relativePath); - } - const emailObject = await this.parseMessage(content, relativePath); - // logger.info({ file, messageId: emailObject.id }, 'Parsed email message.'); - yield emailObject; - } catch (error) { - logger.error( - { error, file }, - 'Failed to process a single EML file. Skipping.' - ); - } - } - } + yield* this.processZipEntries(zipFilePath); } catch (error) { logger.error({ error }, 'Failed to fetch email.'); throw error; } finally { await fs.rm(tempDir, { recursive: true, force: true }); - try { - await this.storage.delete(this.credentials.uploadedFilePath); - } catch (error) { - logger.error( - { error, file: this.credentials.uploadedFilePath }, - 'Failed to delete EML file after processing.' - ); + if (this.credentials.uploadedFilePath && !this.credentials.localFilePath) { + try { + await this.storage.delete(this.credentials.uploadedFilePath); + } catch (error) { + logger.error( + { error, file: this.credentials.uploadedFilePath }, + 'Failed to delete EML file after processing.' + ); + } } } } - private extract(zipFilePath: string, dest: string): Promise { - return new Promise((resolve, reject) => { + private async *processZipEntries(zipFilePath: string): AsyncGenerator { + // Open the ZIP file. + // Note: yauzl requires random access, so we must use the file on disk. + const zipfile = await new Promise((resolve, reject) => { yauzl.open(zipFilePath, { lazyEntries: true, decodeStrings: false }, (err, zipfile) => { - if (err) reject(err); - zipfile.on('error', reject); - zipfile.readEntry(); - zipfile.on('entry', (entry) => { - const fileName = entry.fileName.toString('utf8'); - // Ignore macOS-specific metadata files. - if (fileName.startsWith('__MACOSX/')) { - zipfile.readEntry(); - return; - } - const entryPath = join(dest, fileName); - if (/\/$/.test(fileName)) { - fs.mkdir(entryPath, { recursive: true }) - .then(() => zipfile.readEntry()) - .catch(reject); - } else { - zipfile.openReadStream(entry, (err, readStream) => { - if (err) reject(err); - const writeStream = createWriteStream(entryPath); - readStream.pipe(writeStream); - writeStream.on('finish', () => zipfile.readEntry()); - writeStream.on('error', reject); - }); - } - }); - zipfile.on('end', () => resolve()); + if (err || !zipfile) return reject(err); + resolve(zipfile); }); }); - } - private async getAllFiles(dirPath: string, arrayOfFiles: string[] = []): Promise { - const files = await fs.readdir(dirPath); + // Create an async iterator for zip entries + const entryIterator = this.zipEntryGenerator(zipfile); - for (const file of files) { - const fullPath = join(dirPath, file); - if ((await fs.stat(fullPath)).isDirectory()) { - await this.getAllFiles(fullPath, arrayOfFiles); - } else { - arrayOfFiles.push(fullPath); + for await (const { entry, openReadStream } of entryIterator) { + const fileName = entry.fileName.toString(); + if (fileName.startsWith('__MACOSX/') || /\/$/.test(fileName)) { + continue; + } + + if (fileName.endsWith('.eml')) { + try { + const readStream = await openReadStream(); + const relativePath = dirname(fileName) === '.' ? '' : dirname(fileName); + const emailObject = await this.parseMessage(readStream, relativePath); + yield emailObject; + } catch (error) { + logger.error( + { error, file: fileName }, + 'Failed to process a single EML file from zip. Skipping.' + ); + } } } - - return arrayOfFiles; } - private async parseMessage(emlBuffer: Buffer, path: string): Promise { + private async *zipEntryGenerator( + zipfile: yauzl.ZipFile + ): AsyncGenerator<{ entry: yauzl.Entry; openReadStream: () => Promise }> { + let resolveNext: ((value: any) => void) | null = null; + let rejectNext: ((reason?: any) => void) | null = null; + let finished = false; + const queue: yauzl.Entry[] = []; + + zipfile.readEntry(); + + zipfile.on('entry', (entry) => { + if (resolveNext) { + const resolve = resolveNext; + resolveNext = null; + rejectNext = null; + resolve(entry); + } else { + queue.push(entry); + } + }); + + zipfile.on('end', () => { + finished = true; + if (resolveNext) { + const resolve = resolveNext; + resolveNext = null; + rejectNext = null; + resolve(null); // Signal end + } + }); + + zipfile.on('error', (err) => { + finished = true; + if (rejectNext) { + const reject = rejectNext; + resolveNext = null; + rejectNext = null; + reject(err); + } + }); + + while (!finished || queue.length > 0) { + if (queue.length > 0) { + const entry = queue.shift()!; + yield { + entry, + openReadStream: () => + new Promise((resolve, reject) => { + zipfile.openReadStream(entry, (err, stream) => { + if (err || !stream) return reject(err); + resolve(stream); + }); + }), + }; + zipfile.readEntry(); // Read next entry only after yielding + } else { + const entry = await new Promise((resolve, reject) => { + resolveNext = resolve; + rejectNext = reject; + }); + if (entry) { + yield { + entry, + openReadStream: () => + new Promise((resolve, reject) => { + zipfile.openReadStream(entry, (err, stream) => { + if (err || !stream) return reject(err); + resolve(stream); + }); + }), + }; + zipfile.readEntry(); // Read next entry only after yielding + } else { + break; // End of zip + } + } + } + } + + private async parseMessage( + input: Buffer | Readable, + path: string + ): Promise { + let emlBuffer: Buffer; + if (Buffer.isBuffer(input)) { + emlBuffer = input; + } else { + emlBuffer = await streamToBuffer(input); + } + const parsedEmail: ParsedMail = await simpleParser(emlBuffer); const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({ diff --git a/packages/backend/src/services/ingestion-connectors/MboxConnector.ts b/packages/backend/src/services/ingestion-connectors/MboxConnector.ts index fa03c42..dab5eec 100644 --- a/packages/backend/src/services/ingestion-connectors/MboxConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/MboxConnector.ts @@ -12,6 +12,7 @@ import { getThreadId } from './helpers/utils'; import { StorageService } from '../StorageService'; import { Readable, Transform } from 'stream'; import { createHash } from 'crypto'; +import { promises as fs, createReadStream } from 'fs'; class MboxSplitter extends Transform { private buffer: Buffer = Buffer.alloc(0); @@ -60,15 +61,28 @@ export class MboxConnector implements IEmailConnector { public async testConnection(): Promise { try { - if (!this.credentials.uploadedFilePath) { + const filePath = this.getFilePath(); + if (!filePath) { throw Error('Mbox file path not provided.'); } - if (!this.credentials.uploadedFilePath.includes('.mbox')) { + if (!filePath.includes('.mbox')) { throw Error('Provided file is not in the MBOX format.'); } - const fileExist = await this.storage.exists(this.credentials.uploadedFilePath); + + let fileExist = false; + if (this.credentials.localFilePath) { + try { + await fs.access(this.credentials.localFilePath); + fileExist = true; + } catch { + fileExist = false; + } + } else { + fileExist = await this.storage.exists(filePath); + } + if (!fileExist) { - throw Error('Mbox file upload not finished yet, please wait.'); + throw Error('Mbox file not found or upload not finished yet, please wait.'); } return true; @@ -78,9 +92,19 @@ export class MboxConnector implements IEmailConnector { } } + private getFilePath(): string { + return this.credentials.localFilePath || this.credentials.uploadedFilePath || ''; + } + + private async getFileStream(): Promise { + if (this.credentials.localFilePath) { + return createReadStream(this.credentials.localFilePath); + } + return this.storage.getStream(this.getFilePath()); + } + public async *listAllUsers(): AsyncGenerator { - const displayName = - this.credentials.uploadedFileName || `mbox-import-${new Date().getTime()}`; + const displayName = this.getDisplayName(); logger.info(`Found potential mailbox: ${displayName}`); const constructedPrimaryEmail = `${displayName.replace(/ /g, '.').toLowerCase()}@mbox.local`; yield { @@ -90,11 +114,23 @@ export class MboxConnector implements IEmailConnector { }; } + private getDisplayName(): string { + if (this.credentials.uploadedFileName) { + return this.credentials.uploadedFileName; + } + if (this.credentials.localFilePath) { + const parts = this.credentials.localFilePath.split('/'); + return parts[parts.length - 1].replace('.mbox', ''); + } + return `mbox-import-${new Date().getTime()}`; + } + public async *fetchEmails( userEmail: string, syncState?: SyncState | null ): AsyncGenerator { - const fileStream = await this.storage.getStream(this.credentials.uploadedFilePath); + const filePath = this.getFilePath(); + const fileStream = await this.getFileStream(); const mboxSplitter = new MboxSplitter(); const emailStream = fileStream.pipe(mboxSplitter); @@ -104,22 +140,21 @@ export class MboxConnector implements IEmailConnector { yield emailObject; } catch (error) { logger.error( - { error, file: this.credentials.uploadedFilePath }, + { error, file: filePath }, 'Failed to process a single message from mbox file. Skipping.' ); } } - // After the stream is fully consumed, delete the file. - // The `for await...of` loop ensures streams are properly closed on completion, - // so we can safely delete the file here without causing a hang. - try { - await this.storage.delete(this.credentials.uploadedFilePath); - } catch (error) { - logger.error( - { error, file: this.credentials.uploadedFilePath }, - 'Failed to delete mbox file after processing.' - ); + if (this.credentials.uploadedFilePath && !this.credentials.localFilePath) { + try { + await this.storage.delete(filePath); + } catch (error) { + logger.error( + { error, file: filePath }, + 'Failed to delete mbox file after processing.' + ); + } } } diff --git a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts index d843423..d0f6171 100644 --- a/packages/backend/src/services/ingestion-connectors/PSTConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/PSTConnector.ts @@ -14,7 +14,7 @@ import { StorageService } from '../StorageService'; import { Readable } from 'stream'; import { createHash } from 'crypto'; import { join } from 'path'; -import { createWriteStream, promises as fs } from 'fs'; +import { createWriteStream, createReadStream, promises as fs } from 'fs'; // We have to hardcode names for deleted and trash folders here as current lib doesn't support looking into PST properties. const DELETED_FOLDERS = new Set([ @@ -111,8 +111,19 @@ export class PSTConnector implements IEmailConnector { this.storage = new StorageService(); } + private getFilePath(): string { + return this.credentials.localFilePath || this.credentials.uploadedFilePath || ''; + } + + private async getFileStream(): Promise { + if (this.credentials.localFilePath) { + return createReadStream(this.credentials.localFilePath); + } + return this.storage.getStream(this.getFilePath()); + } + private async loadPstFile(): Promise<{ pstFile: PSTFile; tempDir: string }> { - const fileStream = await this.storage.getStream(this.credentials.uploadedFilePath); + const fileStream = await this.getFileStream(); const tempDir = await fs.mkdtemp(join('/tmp', `pst-import-${new Date().getTime()}`)); const tempFilePath = join(tempDir, 'temp.pst'); @@ -129,15 +140,28 @@ export class PSTConnector implements IEmailConnector { public async testConnection(): Promise { try { - if (!this.credentials.uploadedFilePath) { + const filePath = this.getFilePath(); + if (!filePath) { throw Error('PST file path not provided.'); } - if (!this.credentials.uploadedFilePath.includes('.pst')) { + if (!filePath.includes('.pst')) { throw Error('Provided file is not in the PST format.'); } - const fileExist = await this.storage.exists(this.credentials.uploadedFilePath); + + let fileExist = false; + if (this.credentials.localFilePath) { + try { + await fs.access(this.credentials.localFilePath); + fileExist = true; + } catch { + fileExist = false; + } + } else { + fileExist = await this.storage.exists(filePath); + } + if (!fileExist) { - throw Error('PST file upload not finished yet, please wait.'); + throw Error('PST file not found or upload not finished yet, please wait.'); } return true; } catch (error) { @@ -200,13 +224,15 @@ export class PSTConnector implements IEmailConnector { if (tempDir) { await fs.rm(tempDir, { recursive: true, force: true }); } - try { - await this.storage.delete(this.credentials.uploadedFilePath); - } catch (error) { - logger.error( - { error, file: this.credentials.uploadedFilePath }, - 'Failed to delete PST file after processing.' - ); + if (this.credentials.uploadedFilePath && !this.credentials.localFilePath) { + try { + await this.storage.delete(this.credentials.uploadedFilePath); + } catch (error) { + logger.error( + { error, file: this.credentials.uploadedFilePath }, + 'Failed to delete PST file after processing.' + ); + } } } } diff --git a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte index a8242f8..2f423a7 100644 --- a/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte +++ b/packages/frontend/src/lib/components/custom/IngestionSourceForm.svelte @@ -7,6 +7,7 @@ import { Label } from '$lib/components/ui/label'; import * as Select from '$lib/components/ui/select'; import * as Alert from '$lib/components/ui/alert/index.js'; + import * as RadioGroup from '$lib/components/ui/radio-group/index.js'; import { Textarea } from '$lib/components/ui/textarea/index.js'; import { setAlert } from '$lib/components/custom/alert/alert-state.svelte'; import { api } from '$lib/api.client'; @@ -70,6 +71,27 @@ let fileUploading = $state(false); + let importMethod = $state<'upload' | 'local'>( + source?.credentials && 'localFilePath' in source.credentials && source.credentials.localFilePath + ? 'local' + : 'upload' + ); + + $effect(() => { + if (importMethod === 'upload') { + if ('localFilePath' in formData.providerConfig) { + delete formData.providerConfig.localFilePath; + } + } else { + if ('uploadedFilePath' in formData.providerConfig) { + delete formData.providerConfig.uploadedFilePath; + } + if ('uploadedFileName' in formData.providerConfig) { + delete formData.providerConfig.uploadedFileName; + } + } + }); + const handleSubmit = async (event: Event) => { event.preventDefault(); isSubmitting = true; @@ -236,59 +258,143 @@ /> {:else if formData.provider === 'pst_import'} -
- -
- - {#if fileUploading} - - {/if} -
+
+ + +
+ + +
+
+ + +
+
+ + {#if importMethod === 'upload'} +
+ +
+ + {#if fileUploading} + + {/if} +
+
+ {:else} +
+ + +
+ {/if} {:else if formData.provider === 'eml_import'} -
- -
- - {#if fileUploading} - - {/if} -
+
+ + +
+ + +
+
+ + +
+
+ + {#if importMethod === 'upload'} +
+ +
+ + {#if fileUploading} + + {/if} +
+
+ {:else} +
+ + +
+ {/if} {:else if formData.provider === 'mbox_import'} -
- -
- - {#if fileUploading} - - {/if} -
+
+ + +
+ + +
+
+ + +
+
+ + {#if importMethod === 'upload'} +
+ +
+ + {#if fileUploading} + + {/if} +
+
+ {:else} +
+ + +
+ {/if} {/if} {#if formData.provider === 'google_workspace' || formData.provider === 'microsoft_365'} diff --git a/packages/frontend/src/lib/translations/bg.json b/packages/frontend/src/lib/translations/bg.json new file mode 100644 index 0000000..9eab6e7 --- /dev/null +++ b/packages/frontend/src/lib/translations/bg.json @@ -0,0 +1,292 @@ +{ + "app": { + "auth": { + "login": "Вход", + "login_tip": "Въведете имейл адреса си по-долу, за да влезете в профила си.", + "email": "Имейл", + "password": "Парола" + }, + "common": { + "working": "Работата е в ход" + }, + "archive": { + "title": "Архив", + "no_subject": "Без тема", + "from": "От", + "sent": "Изпратени", + "recipients": "Получатели", + "to": "До", + "meta_data": "Метаданни", + "folder": "Папка", + "tags": "Етикети", + "size": "Размер", + "email_preview": "Преглед на имейла", + "attachments": "Прикачени файлове", + "download": "Изтегляне", + "actions": "Действия", + "download_eml": "Изтегляне на съобщения (.eml)", + "delete_email": "Изтриване на съобщения", + "email_thread": "Нишка от съобщения", + "delete_confirmation_title": "Сигурни ли сте, че искате да изтриете това съобщение?", + "delete_confirmation_description": "Това действие не може да бъде отменено и ще премахне имейла и прикачените към него файлове за постоянно.", + "deleting": "Изтриване", + "confirm": "Потвърдете", + "cancel": "Отказ", + "not_found": "Съобщението не е открито." + }, + "ingestions": { + "title": "Източници за приемане", + "ingestion_sources": "Източници за приемане", + "bulk_actions": "Групови действия", + "force_sync": "Принудително синхронизиране", + "delete": "Изтрийте", + "create_new": "Създайте нов", + "name": "Име", + "provider": "Доставчик", + "status": "Статус", + "active": "Активно", + "created_at": "Създадено в", + "actions": "Действия", + "last_sync_message": "Последно синхронизирано съобщение", + "empty": "Празно", + "open_menu": "Отворете менюто", + "edit": "Редактиране", + "create": "Създайте", + "ingestion_source": "Източници за приемане", + "edit_description": "Направете промени в източника си за приемане тук.", + "create_description": "Добавете нов източник за приемане, за да започнете да архивирате съобщения.", + "read": "Прочетете", + "docs_here": "тук се намира документацията", + "delete_confirmation_title": "Наистина ли искате да изтриете това приемане?", + "delete_confirmation_description": "Това ще изтрие всички архивирани съобщения, прикачени файлове, индексирания и файлове, свързани с това приемане. Ако искате да спрете само синхронизирането на нови съобщения, можете вместо това да поставите приемането на пауза.", + "deleting": "Изтриване", + "confirm": "Потвърдете", + "cancel": "Отказ", + "bulk_delete_confirmation_title": "Сигурни ли сте, че искате да изтриете {{count}} избраните приемания?", + "bulk_delete_confirmation_description": "Това ще изтрие всички архивирани съобщения, прикачени файлове, индексирания и файлове, свързани с тези приемания. Ако искате да спрете само синхронизирането на нови съобщения, можете вместо това да поставите приеманията на пауза." + }, + "search": { + "title": "Търсене", + "description": "Търсене на архивирани съобщения.", + "email_search": "Претърсване на имейл", + "placeholder": "Търсене по ключова дума, подател, получател...", + "search_button": "Търсене", + "search_options": "Опции за търсене", + "strategy_fuzzy": "Размито", + "strategy_verbatim": "Дословно", + "strategy_frequency": "Честота", + "select_strategy": "Изберете стратегия", + "error": "Грешка", + "found_results_in": "Намерени {{total}} резултати за {{seconds}}и", + "found_results": "Намерени {{total}} резултати", + "from": "От", + "to": "До", + "in_email_body": "В съдържанието на имейла", + "in_attachment": "В прикачения файл: {{filename}}", + "prev": "Предишен", + "next": "Следващ" + }, + "roles": { + "title": "Управление на ролите", + "role_management": "Управление на ролите", + "create_new": "Създаване на нова", + "name": "Име", + "created_at": "Създадено в", + "actions": "Действия", + "open_menu": "Отваряне на менюто", + "view_policy": "Преглед на политиката", + "edit": "Редактиране", + "delete": "Изтриване", + "no_roles_found": "Няма намерени роли.", + "role_policy": "Политика за ролите", + "viewing_policy_for_role": "Преглед на политиката за роля: {{name}}", + "create": "Създаване", + "role": "Роля", + "edit_description": "Направете промени в ролята тук.", + "create_description": "Добавете нова роля към системата.", + "delete_confirmation_title": "Сигурни ли сте, че искате да изтриете тази роля?", + "delete_confirmation_description": "Това действие не може да бъде отменено. Това ще изтрие ролята за постоянно.", + "deleting": "Изтриване", + "confirm": "Потвърдете", + "cancel": "Отказ" + }, + "system_settings": { + "title": "Системни настройки", + "system_settings": "Системни настройки", + "description": "Управление на глобалните настройки на приложението.", + "language": "Език", + "default_theme": "Тема по подразбиране", + "light": "Светла", + "dark": "Тъмна", + "system": "Система", + "support_email": "Имейл за поддръжка", + "saving": "Съхранява се", + "save_changes": "Съхранете промените" + }, + "users": { + "title": "Управление на потребителите", + "user_management": "Управление на потребителите", + "create_new": "Създаване на нов", + "name": "Име", + "email": "Имейл", + "role": "Роля", + "created_at": "Създадено в", + "actions": "Действия", + "open_menu": "Отваряне на меню", + "edit": "Редактиране", + "delete": "Изтриване", + "no_users_found": "Няма открити потребители.", + "create": "Създаване", + "user": "Потребител", + "edit_description": "Направете промени на потребителя тук.", + "create_description": "Добавете нов потребител към системата.", + "delete_confirmation_title": "Сигурни ли сте, че искате да изтриете този потребител?", + "delete_confirmation_description": "Това действие не може да бъде отменено. Това ще изтрие потребителя за постоянно и ще премахне данните му от нашите сървъри.", + "deleting": "Изтриване", + "confirm": "Потвърдете", + "cancel": "Отказ" + }, + "components": { + "charts": { + "emails_ingested": "Приети съобщения", + "storage_used": "Използвано пространство", + "emails": "Съобщения" + }, + "common": { + "submitting": "Изпращане...", + "submit": "Изпратете", + "save": "Съхранете" + }, + "email_preview": { + "loading": "Зареждане на визуализацията на имейла...", + "render_error": "Не можа да се изобрази визуализация на имейла.", + "not_available": "Необработеният .eml файл не е наличен за този имейл." + }, + "footer": { + "all_rights_reserved": "Всички права запазени.", + "new_version_available": "Налична е нова версия" + }, + "ingestion_source_form": { + "provider_generic_imap": "Общ IMAP", + "provider_google_workspace": "Google Workspace", + "provider_microsoft_365": "Microsoft 365", + "provider_pst_import": "PST Импортиране", + "provider_eml_import": "EML Импортиране", + "provider_mbox_import": "Mbox Импортиране", + "select_provider": "Изберете доставчик", + "service_account_key": "Ключ за сервизен акаунт (JSON)", + "service_account_key_placeholder": "Поставете JSON съдържанието на ключа на вашия сервизен акаунт", + "impersonated_admin_email": "Имейл адрес на администратор, използван като идентификатор", + "client_id": "Приложение (Клиент) ID", + "client_secret": "Клиентски таен ключ", + "client_secret_placeholder": "Въведете клиентския таен ключ като стойност, а не ID тайната", + "tenant_id": "Директория (Наемател) ID", + "host": "Хост", + "port": "Порт", + "username": "Потребителско им", + "use_tls": "Използвайте TLS", + "allow_insecure_cert": "Разрешаване на несигурен сертификат", + "pst_file": "PST Файл", + "eml_file": "EML Файл", + "mbox_file": "Mbox файл", + "heads_up": "Внимание!", + "org_wide_warning": "Моля, обърнете внимание, че това е операция за цялата организация. Този вид приемане ще импортира и индексира всички имейл входящи кутии във вашата организация. Ако искате да импортирате само конкретни имейл входящи кутии, използвайте IMAP инструмента за свързване.", + "upload_failed": "Качването не бе успешно, моля, опитайте отново" + }, + "role_form": { + "policies_json": "Политики (JSON)", + "invalid_json": "Невалиден JSON формат за политики." + }, + "theme_switcher": { + "toggle_theme": "Превключване на тема" + }, + "user_form": { + "select_role": "Изберете роля" + } + }, + "setup": { + "title": "Настройка", + "description": "Настройте първоначалния администраторски акаунт за Open Archiver.", + "welcome": "Добре дошли", + "create_admin_account": "Създайте първия администраторски акаунт, за да започнете.", + "first_name": "Име", + "last_name": "Фамилия", + "email": "Имейл", + "password": "Парола", + "creating_account": "Създаване на акаунт", + "create_account": "Създаване на акаунт" + }, + "layout": { + "dashboard": "Табло за управление", + "ingestions": "Приети", + "archived_emails": "Архивирани съобщения", + "search": "Търсене", + "settings": "Настройки", + "system": "Система", + "users": "Потребители", + "roles": "Роли", + "api_keys": "API ключове", + "logout": "Изход" + }, + "api_keys_page": { + "title": "API ключове", + "header": "API ключове", + "generate_new_key": "Генериране на нов ключ", + "name": "Име", + "key": "Ключ", + "expires_at": "Изтича на", + "created_at": "Създаден на", + "actions": "Действия", + "delete": "Изтриване", + "no_keys_found": "Няма намерени API ключове.", + "generate_modal_title": "Генериране на нов API ключ", + "generate_modal_description": "Моля, посочете име и срок на валидност за новия си API ключ.", + "expires_in": "Изтича след", + "select_expiration": "Изберете срок на валидност", + "30_days": "30 дни", + "60_days": "60 дни", + "6_months": "6 месеца", + "12_months": "12 месеца", + "24_months": "24 месеца", + "generate": "Генериране", + "new_api_key": "Нов API ключ", + "failed_to_delete": "Изтриването на API ключ е неуспешно", + "api_key_deleted": "API ключът е изтрит", + "generated_title": "API ключът е генериран", + "generated_message": "Вашият API ключ е генериран, моля, копирайте го и го запазете на сигурно място. Този ключ ще бъде показан само веднъж." + }, + "archived_emails_page": { + "title": "Архивирани съобщения", + "header": "Архивирани съобщения", + "select_ingestion_source": "Изберете източник за приемане", + "date": "Дата", + "subject": "Тема", + "sender": "Подател", + "inbox": "Входяща поща", + "path": "Път", + "actions": "Действия", + "view": "Преглед", + "no_emails_found": "Няма намерени архивирани съобщения.", + "prev": "Предишен", + "next": "Следващ" + }, + "dashboard_page": { + "title": "Табло за управление", + "meta_description": "Общ преглед на вашия имейл архив.", + "header": "Табло за управление", + "create_ingestion": "Създаване на приемане", + "no_ingestion_header": "Нямате настроен източник за приемане.", + "no_ingestion_text": "Добавете източник за приемане, за да започнете да архивирате входящите си кутии.", + "total_emails_archived": "Общо архивирани съобщения", + "total_storage_used": "Общо използвано място за съхранение", + "failed_ingestions": "Неуспешни приемания (последните 7 дни)", + "ingestion_history": "История на приеманията", + "no_ingestion_history": "Няма налична история на приеманията.", + "storage_by_source": "Съхранение по източник на приемане", + "no_ingestion_sources": "Няма налични източници за приемане.", + "indexed_insights": "Индексирани данни", + "top_10_senders": "Топ 10 податели", + "no_indexed_insights": "Няма налични индексирани данни." + } + } +} diff --git a/packages/frontend/src/lib/translations/en.json b/packages/frontend/src/lib/translations/en.json index 1452f45..c9e0e33 100644 --- a/packages/frontend/src/lib/translations/en.json +++ b/packages/frontend/src/lib/translations/en.json @@ -199,6 +199,10 @@ "provider_eml_import": "EML Import", "provider_mbox_import": "Mbox Import", "select_provider": "Select a provider", + "import_method": "Import Method", + "upload_file": "Upload File", + "local_path": "Local Path (Recommended for large files)", + "local_file_path": "Local File Path", "service_account_key": "Service Account Key (JSON)", "service_account_key_placeholder": "Paste your service account key JSON content", "impersonated_admin_email": "Impersonated Admin Email", diff --git a/packages/frontend/src/lib/translations/index.ts b/packages/frontend/src/lib/translations/index.ts index 3b7daad..46b1bcc 100644 --- a/packages/frontend/src/lib/translations/index.ts +++ b/packages/frontend/src/lib/translations/index.ts @@ -12,6 +12,7 @@ import nl from './nl.json'; import ja from './ja.json'; import et from './et.json'; import el from './el.json'; +import bg from './bg.json' // This is your config object. // It defines the languages and how to load them. const config: Config = { @@ -77,6 +78,12 @@ const config: Config = { key: 'app', loader: async () => el.app, }, + // Bulgarian 🇧🇬 + { + locale: 'bg', + key: 'app', + loader: async () => bg.app, + }, ], fallbackLocale: 'en', }; diff --git a/packages/frontend/src/routes/dashboard/settings/system/+page.svelte b/packages/frontend/src/routes/dashboard/settings/system/+page.svelte index c94bc53..7ccc904 100644 --- a/packages/frontend/src/routes/dashboard/settings/system/+page.svelte +++ b/packages/frontend/src/routes/dashboard/settings/system/+page.svelte @@ -24,6 +24,7 @@ { value: 'pt', label: '🇵🇹 Português' }, { value: 'nl', label: '🇳🇱 Nederlands' }, { value: 'el', label: '🇬🇷 Ελληνικά' }, + { value: 'bg', label: '🇧🇬 български' }, { value: 'ja', label: '🇯🇵 日本語' }, ]; diff --git a/packages/types/src/ingestion.types.ts b/packages/types/src/ingestion.types.ts index 83256f9..40ca2c6 100644 --- a/packages/types/src/ingestion.types.ts +++ b/packages/types/src/ingestion.types.ts @@ -72,20 +72,23 @@ export interface Microsoft365Credentials extends BaseIngestionCredentials { export interface PSTImportCredentials extends BaseIngestionCredentials { type: 'pst_import'; - uploadedFileName: string; - uploadedFilePath: string; + uploadedFileName?: string; + uploadedFilePath?: string; + localFilePath?: string; } export interface EMLImportCredentials extends BaseIngestionCredentials { type: 'eml_import'; - uploadedFileName: string; - uploadedFilePath: string; + uploadedFileName?: string; + uploadedFilePath?: string; + localFilePath?: string; } export interface MboxImportCredentials extends BaseIngestionCredentials { type: 'mbox_import'; - uploadedFileName: string; - uploadedFilePath: string; + uploadedFileName?: string; + uploadedFilePath?: string; + localFilePath?: string; } // Discriminated union for all possible credential types