From 4156abcdfa24cf11f7856c0ba00a0b0978ccc12a Mon Sep 17 00:00:00 2001 From: Wayne <5291640+ringoinca@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:24:46 +0300 Subject: [PATCH] Error handling, force sync, UI improvement --- README.md | 2 +- .../src/api/controllers/auth.controller.ts | 1 - .../api/controllers/ingestion.controller.ts | 8 +- .../processors/continuous-sync.processor.ts | 7 +- .../processors/initial-import.processor.ts | 9 +- .../processors/process-mailbox.processor.ts | 18 +- .../schedule-continuous-sync.processor.ts | 16 +- .../sync-cycle-finished.processor.ts | 56 +++++-- .../backend/src/services/CryptoService.ts | 39 +++-- .../backend/src/services/IngestionService.ts | 58 +++++-- packages/backend/src/services/UserService.ts | 4 +- .../GoogleWorkspaceConnector.ts | 154 ++++++++++-------- .../ingestion-connectors/ImapConnector.ts | 7 +- .../MicrosoftConnector.ts | 2 +- packages/frontend/package.json | 9 +- .../src/lib/components/custom/Footer.svelte | 3 +- .../lib/components/custom/alert/Alerts.svelte | 123 ++++++++++++++ .../custom/alert/alert-state.svelte.ts | 25 +++ .../ui/hover-card/hover-card-content.svelte | 29 ++++ .../ui/hover-card/hover-card-trigger.svelte | 7 + .../src/lib/components/ui/hover-card/index.ts | 14 ++ .../src/lib/components/ui/sonner/index.ts | 1 + .../lib/components/ui/sonner/sonner.svelte | 13 ++ packages/frontend/src/routes/+layout.svelte | 3 + .../routes/dashboard/ingestions/+page.svelte | 81 ++++++--- .../frontend/src/routes/signin/+page.svelte | 39 +++-- packages/types/src/ingestion.types.ts | 6 + pnpm-lock.yaml | 74 +++++++++ 28 files changed, 635 insertions(+), 173 deletions(-) create mode 100644 packages/frontend/src/lib/components/custom/alert/Alerts.svelte create mode 100644 packages/frontend/src/lib/components/custom/alert/alert-state.svelte.ts create mode 100644 packages/frontend/src/lib/components/ui/hover-card/hover-card-content.svelte create mode 100644 packages/frontend/src/lib/components/ui/hover-card/hover-card-trigger.svelte create mode 100644 packages/frontend/src/lib/components/ui/hover-card/index.ts create mode 100644 packages/frontend/src/lib/components/ui/sonner/index.ts create mode 100644 packages/frontend/src/lib/components/ui/sonner/sonner.svelte diff --git a/README.md b/README.md index 2705503..99eee84 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ _Full-text search across all your emails and attachments_ ## Community -Join our community to ask questions, share your projects, and connect with other developers. +We are committed to build an engaging community around Open Archiver, and we are inviting all of you to join our community on Discord to get real-time support and connect with the team. [![Discord](https://img.shields.io/badge/Join%20our%20Discord-7289DA?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/Qpv4BmHp) diff --git a/packages/backend/src/api/controllers/auth.controller.ts b/packages/backend/src/api/controllers/auth.controller.ts index 48a0e7b..557ccca 100644 --- a/packages/backend/src/api/controllers/auth.controller.ts +++ b/packages/backend/src/api/controllers/auth.controller.ts @@ -24,7 +24,6 @@ export class AuthController { return res.status(200).json(result); } catch (error) { - // In a real application, you'd want to log this error. console.error('Login error:', error); return res.status(500).json({ message: 'An internal server error occurred' }); } diff --git a/packages/backend/src/api/controllers/ingestion.controller.ts b/packages/backend/src/api/controllers/ingestion.controller.ts index 76039a8..aa719cc 100644 --- a/packages/backend/src/api/controllers/ingestion.controller.ts +++ b/packages/backend/src/api/controllers/ingestion.controller.ts @@ -1,6 +1,7 @@ import { Request, Response } from 'express'; import { IngestionService } from '../../services/IngestionService'; import { CreateIngestionSourceDto, UpdateIngestionSourceDto } from '@open-archiver/types'; +import { logger } from '../../config/logger'; export class IngestionController { public create = async (req: Request, res: Response): Promise => { @@ -8,9 +9,10 @@ export class IngestionController { const dto: CreateIngestionSourceDto = req.body; const newSource = await IngestionService.create(dto); return res.status(201).json(newSource); - } catch (error) { - console.error('Create ingestion source error:', error); - return res.status(500).json({ message: 'An internal server error occurred' }); + } catch (error: any) { + logger.error({ err: error }, 'Create ingestion source error'); + // Return a 400 Bad Request for connection errors + return res.status(400).json({ message: error.message || 'Failed to create ingestion source due to a connection error.' }); } }; diff --git a/packages/backend/src/jobs/processors/continuous-sync.processor.ts b/packages/backend/src/jobs/processors/continuous-sync.processor.ts index 78f0f39..7df62de 100644 --- a/packages/backend/src/jobs/processors/continuous-sync.processor.ts +++ b/packages/backend/src/jobs/processors/continuous-sync.processor.ts @@ -10,8 +10,8 @@ export default async (job: Job) => { logger.info({ ingestionSourceId }, 'Starting continuous sync job.'); const source = await IngestionService.findById(ingestionSourceId); - if (!source || source.status !== 'active') { - logger.warn({ ingestionSourceId, status: source?.status }, 'Skipping continuous sync for non-active source.'); + if (!source || !['error', 'active'].includes(source.status)) { + logger.warn({ ingestionSourceId, status: source?.status }, 'Skipping continuous sync for non-active or non-error source.'); return; } @@ -39,7 +39,8 @@ export default async (job: Job) => { }, removeOnFail: { age: 60 * 30 // 30 minutes - } + }, + timeout: 1000 * 60 * 30 // 30 minutes } }); } diff --git a/packages/backend/src/jobs/processors/initial-import.processor.ts b/packages/backend/src/jobs/processors/initial-import.processor.ts index f0b49bd..2e34d31 100644 --- a/packages/backend/src/jobs/processors/initial-import.processor.ts +++ b/packages/backend/src/jobs/processors/initial-import.processor.ts @@ -1,10 +1,11 @@ -import { Job } from 'bullmq'; +import { Job, FlowChildJob } from 'bullmq'; import { IngestionService } from '../../services/IngestionService'; import { IInitialImportJob } from '@open-archiver/types'; import { EmailProviderFactory } from '../../services/EmailProviderFactory'; import { flowProducer } from '../queues'; import { logger } from '../../config/logger'; + export default async (job: Job) => { const { ingestionSourceId } = job.data; logger.info({ ingestionSourceId }, 'Starting initial import master job'); @@ -23,7 +24,7 @@ export default async (job: Job) => { const connector = EmailProviderFactory.createConnector(source); // if (connector instanceof GoogleWorkspaceConnector || connector instanceof MicrosoftConnector) { - const jobs = []; + const jobs: FlowChildJob[] = []; let userCount = 0; for await (const user of connector.listAllUsers()) { if (user.primaryEmail) { @@ -40,7 +41,9 @@ export default async (job: Job) => { }, removeOnFail: { age: 60 * 30 // 30 minutes - } + }, + attempts: 1, + // failParentOnFailure: true } }); userCount++; diff --git a/packages/backend/src/jobs/processors/process-mailbox.processor.ts b/packages/backend/src/jobs/processors/process-mailbox.processor.ts index 67ee2a2..bc129d0 100644 --- a/packages/backend/src/jobs/processors/process-mailbox.processor.ts +++ b/packages/backend/src/jobs/processors/process-mailbox.processor.ts @@ -1,10 +1,18 @@ import { Job } from 'bullmq'; -import { IProcessMailboxJob, SyncState } from '@open-archiver/types'; +import { IProcessMailboxJob, SyncState, ProcessMailboxError } from '@open-archiver/types'; import { IngestionService } from '../../services/IngestionService'; import { logger } from '../../config/logger'; import { EmailProviderFactory } from '../../services/EmailProviderFactory'; import { StorageService } from '../../services/StorageService'; +/** + * This processor handles the ingestion of emails for a single user's mailbox. + * If an error occurs during processing (e.g., an API failure), + * it catches the exception and returns a structured error object instead of throwing. + * This prevents a single failed mailbox from halting the entire sync cycle for all users. + * The parent 'sync-cycle-finished' job is responsible for inspecting the results of all + * 'process-mailbox' jobs, aggregating successes, and reporting detailed failures. + */ export const processMailboxProcessor = async (job: Job) => { const { ingestionSourceId, userEmail } = job.data; @@ -28,7 +36,6 @@ export const processMailboxProcessor = async (job: Job { console.log( - 'Scheduler running: Looking for active ingestion sources to sync.' + 'Scheduler running: Looking for active or error ingestion sources to sync.' ); - const activeSources = await db + // find all sources that have the status of active or error for continuous syncing. + const sourcesToSync = await db .select({ id: ingestionSources.id }) .from(ingestionSources) - .where(eq(ingestionSources.status, 'active')); + .where( + or( + eq(ingestionSources.status, 'active'), + eq(ingestionSources.status, 'error') + ) + ); - for (const source of activeSources) { + for (const source of sourcesToSync) { // The status field on the ingestion source is used to prevent duplicate syncs. await ingestionQueue.add('continuous-sync', { ingestionSourceId: source.id }); } diff --git a/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts b/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts index cb07e99..a497663 100644 --- a/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts +++ b/packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts @@ -1,7 +1,7 @@ -import { Job, FlowJob } from 'bullmq'; +import { Job } from 'bullmq'; import { IngestionService } from '../../services/IngestionService'; import { logger } from '../../config/logger'; -import { SyncState } from '@open-archiver/types'; +import { SyncState, ProcessMailboxError } from '@open-archiver/types'; import { db } from '../../database'; import { ingestionSources } from '../../database/schema'; import { eq } from 'drizzle-orm'; @@ -13,41 +13,65 @@ interface ISyncCycleFinishedJob { isInitialImport: boolean; } +/** + * This processor runs after all 'process-mailbox' jobs for a sync cycle have completed. + * It is responsible for aggregating the results and finalizing the sync status. + * It inspects the return values of all child jobs to identify successes and failures. + * + * If any child jobs returned an error object, this processor will: + * 1. Mark the overall ingestion status as 'error'. + * 2. Aggregate the detailed error messages from all failed jobs. + * 3. Save the sync state from any jobs that *did* succeed, preserving partial progress. + * + * If all child jobs succeeded, it marks the ingestion as 'active' and saves the final + * aggregated sync state from all children. + * + */ export default async (job: Job) => { const { ingestionSourceId, userCount, isInitialImport } = job.data; logger.info({ ingestionSourceId, userCount, isInitialImport }, 'Sync cycle finished job started'); try { - const childrenJobs = await job.getChildrenValues(); - const allSyncStates = Object.values(childrenJobs); + const childrenValues = await job.getChildrenValues(); + const allChildJobs = Object.values(childrenValues); + // if data has error property, it is a failed job + const failedJobs = allChildJobs.filter(v => v && (v as any).error) as ProcessMailboxError[]; + // if data doesn't have error property, it is a successful job with SyncState + const successfulJobs = allChildJobs.filter(v => !v || !(v as any).error) as SyncState[]; - // Merge all sync states from children jobs into one - const finalSyncState = deepmerge(...allSyncStates.filter(s => s && Object.keys(s).length > 0)); + const finalSyncState = deepmerge(...successfulJobs.filter(s => s && Object.keys(s).length > 0)); - let message = 'Continuous sync cycle finished successfully.'; - if (isInitialImport) { - message = `Initial import finished for ${userCount} mailboxes.`; + let status: 'active' | 'error' = 'active'; + let message: string; + + if (failedJobs.length > 0) { + status = 'error'; + const errorMessages = failedJobs.map(j => j.message).join('\n'); + message = `Sync cycle completed with ${failedJobs.length} error(s):\n${errorMessages}`; + logger.error({ ingestionSourceId, errors: errorMessages }, 'Sync cycle finished with errors.'); + } else { + message = 'Continuous sync cycle finished successfully.'; + if (isInitialImport) { + message = `Initial import finished for ${userCount} mailboxes.`; + } + logger.info({ ingestionSourceId }, 'Successfully updated status and final sync state.'); } - // Update the database with the final aggregated sync state await db .update(ingestionSources) .set({ - status: 'active', + status, lastSyncFinishedAt: new Date(), lastSyncStatusMessage: message, syncState: finalSyncState }) .where(eq(ingestionSources.id, ingestionSourceId)); - - logger.info({ ingestionSourceId }, 'Successfully updated status and final sync state.'); } catch (error) { - logger.error({ err: error, ingestionSourceId }, 'Failed to process finished sync cycle.'); - // If this fails, we should probably set the status to 'error' to indicate a problem. + logger.error({ err: error, ingestionSourceId }, 'An unexpected error occurred while finalizing the sync cycle.'); await IngestionService.update(ingestionSourceId, { status: 'error', lastSyncFinishedAt: new Date(), - lastSyncStatusMessage: 'Failed to finalize sync cycle and update sync state.' + lastSyncStatusMessage: 'An unexpected error occurred while finalizing the sync cycle.' }); } }; diff --git a/packages/backend/src/services/CryptoService.ts b/packages/backend/src/services/CryptoService.ts index a8558b3..c52865d 100644 --- a/packages/backend/src/services/CryptoService.ts +++ b/packages/backend/src/services/CryptoService.ts @@ -29,20 +29,25 @@ export class CryptoService { return Buffer.concat([salt, iv, tag, encrypted]).toString('hex'); } - public static decrypt(encrypted: string): string { - const data = Buffer.from(encrypted, 'hex'); - const salt = data.subarray(0, SALT_LENGTH); - const iv = data.subarray(SALT_LENGTH, SALT_LENGTH + IV_LENGTH); - const tag = data.subarray(SALT_LENGTH + IV_LENGTH, SALT_LENGTH + IV_LENGTH + TAG_LENGTH); - const encryptedValue = data.subarray(SALT_LENGTH + IV_LENGTH + TAG_LENGTH); + public static decrypt(encrypted: string): string | null { + try { + const data = Buffer.from(encrypted, 'hex'); + const salt = data.subarray(0, SALT_LENGTH); + const iv = data.subarray(SALT_LENGTH, SALT_LENGTH + IV_LENGTH); + const tag = data.subarray(SALT_LENGTH + IV_LENGTH, SALT_LENGTH + IV_LENGTH + TAG_LENGTH); + const encryptedValue = data.subarray(SALT_LENGTH + IV_LENGTH + TAG_LENGTH); - const key = getKey(salt); - const decipher = createDecipheriv(ALGORITHM, key, iv); - decipher.setAuthTag(tag); + const key = getKey(salt); + const decipher = createDecipheriv(ALGORITHM, key, iv); + decipher.setAuthTag(tag); - const decrypted = Buffer.concat([decipher.update(encryptedValue), decipher.final()]); + const decrypted = Buffer.concat([decipher.update(encryptedValue), decipher.final()]); - return decrypted.toString('utf8'); + return decrypted.toString('utf8'); + } catch (error) { + console.error('Decryption failed:', error); + return null; + } } public static encryptObject(obj: T): string { @@ -50,8 +55,16 @@ export class CryptoService { return this.encrypt(jsonString); } - public static decryptObject(encrypted: string): T { + public static decryptObject(encrypted: string): T | null { const decryptedString = this.decrypt(encrypted); - return JSON.parse(decryptedString) as T; + if (!decryptedString) { + return null; + } + try { + return JSON.parse(decryptedString) as T; + } catch (error) { + console.error('Failed to parse decrypted JSON:', error); + return null; + } } } diff --git a/packages/backend/src/services/IngestionService.ts b/packages/backend/src/services/IngestionService.ts index 6bb3508..963e5a4 100644 --- a/packages/backend/src/services/IngestionService.ts +++ b/packages/backend/src/services/IngestionService.ts @@ -6,7 +6,7 @@ import type { IngestionSource, IngestionCredentials } from '@open-archiver/types'; -import { and, eq } from 'drizzle-orm'; +import { and, desc, eq } from 'drizzle-orm'; import { CryptoService } from './CryptoService'; import { EmailProviderFactory } from './EmailProviderFactory'; import { ingestionQueue } from '../jobs/queues'; @@ -22,10 +22,16 @@ import { DatabaseService } from './DatabaseService'; export class IngestionService { - private static decryptSource(source: typeof ingestionSources.$inferSelect): IngestionSource { + private static decryptSource(source: typeof ingestionSources.$inferSelect): IngestionSource | null { const decryptedCredentials = CryptoService.decryptObject( source.credentials as string ); + + if (!decryptedCredentials) { + logger.error({ sourceId: source.id }, 'Failed to decrypt ingestion source credentials.'); + return null; + } + return { ...source, credentials: decryptedCredentials } as IngestionSource; } @@ -43,21 +49,29 @@ export class IngestionService { const [newSource] = await db.insert(ingestionSources).values(valuesToInsert).returning(); const decryptedSource = this.decryptSource(newSource); - - // Test the connection - const connector = EmailProviderFactory.createConnector(decryptedSource); - const isConnected = await connector.testConnection(); - - if (isConnected) { - return await this.update(decryptedSource.id, { status: 'auth_success' }); + if (!decryptedSource) { + await this.delete(newSource.id); + throw new Error('Failed to process newly created ingestion source due to a decryption error.'); } + const connector = EmailProviderFactory.createConnector(decryptedSource); - return decryptedSource; + try { + await connector.testConnection(); + // If connection succeeds, update status to auth_success, which triggers the initial import. + return await this.update(decryptedSource.id, { status: 'auth_success' }); + } catch (error) { + // If connection fails, delete the newly created source and throw the error. + await this.delete(decryptedSource.id); + throw error; + } } public static async findAll(): Promise { - const sources = await db.select().from(ingestionSources); - return sources.map(this.decryptSource); + const sources = await db.select().from(ingestionSources).orderBy(desc(ingestionSources.createdAt)); + return sources.flatMap(source => { + const decrypted = this.decryptSource(source); + return decrypted ? [decrypted] : []; + }); } public static async findById(id: string): Promise { @@ -65,7 +79,11 @@ export class IngestionService { if (!source) { throw new Error('Ingestion source not found'); } - return this.decryptSource(source); + const decryptedSource = this.decryptSource(source); + if (!decryptedSource) { + throw new Error('Failed to decrypt ingestion source credentials.'); + } + return decryptedSource; } public static async update( @@ -95,6 +113,10 @@ export class IngestionService { const decryptedSource = this.decryptSource(updatedSource); + if (!decryptedSource) { + throw new Error('Failed to process updated ingestion source due to a decryption error.'); + } + // If the status has changed to auth_success, trigger the initial import if ( originalSource.status !== 'auth_success' && @@ -131,7 +153,15 @@ export class IngestionService { .where(eq(ingestionSources.id, id)) .returning(); - return this.decryptSource(deletedSource); + const decryptedSource = this.decryptSource(deletedSource); + if (!decryptedSource) { + // Even if decryption fails, we should confirm deletion. + // We might return a simpler object or just a success message. + // For now, we'll indicate the issue but still confirm deletion happened. + logger.warn({ sourceId: deletedSource.id }, 'Could not decrypt credentials of deleted source, but deletion was successful.'); + return { ...deletedSource, credentials: null } as unknown as IngestionSource; + } + return decryptedSource; } public static async triggerInitialImport(id: string): Promise { diff --git a/packages/backend/src/services/UserService.ts b/packages/backend/src/services/UserService.ts index cc30a31..a3b9e69 100644 --- a/packages/backend/src/services/UserService.ts +++ b/packages/backend/src/services/UserService.ts @@ -3,7 +3,7 @@ import type { User } from '@open-archiver/types'; import type { IUserService } from './AuthService'; // This is a mock implementation of the IUserService. -// In a real application, this service would interact with a database. +// Later on, this service would interact with a database. export class AdminUserService implements IUserService { #users: User[] = []; @@ -24,7 +24,7 @@ export class AdminUserService implements IUserService { } public async findByEmail(email: string): Promise { - // In a real implementation, this would be a database query. + // once user service is ready, this would be a database query. const user = this.#users.find(u => u.email === email); return user || null; } diff --git a/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts b/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts index 0013af6..66d3bd7 100644 --- a/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts @@ -80,7 +80,7 @@ export class GoogleWorkspaceConnector implements IEmailConnector { return true; } catch (error) { logger.error({ err: error }, 'Failed to verify Google Workspace connection'); - return false; + throw error; } } @@ -165,41 +165,49 @@ export class GoogleWorkspaceConnector implements IEmailConnector { if (historyRecord.messagesAdded) { for (const messageAdded of historyRecord.messagesAdded) { if (messageAdded.message?.id) { - const msgResponse = await gmail.users.messages.get({ - userId: 'me', - id: messageAdded.message.id, - format: 'RAW' - }); + try { + const msgResponse = await gmail.users.messages.get({ + userId: 'me', + id: messageAdded.message.id, + format: 'RAW' + }); - if (msgResponse.data.raw) { - const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url'); - const parsedEmail: ParsedMail = await simpleParser(rawEmail); - const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({ - filename: attachment.filename || 'untitled', - contentType: attachment.contentType, - size: attachment.size, - content: attachment.content as Buffer - })); - const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => { - if (!addresses) return []; - const addressArray = Array.isArray(addresses) ? addresses : [addresses]; - return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' }))); - }; - yield { - id: msgResponse.data.id!, - userEmail: userEmail, - eml: rawEmail, - from: mapAddresses(parsedEmail.from), - to: mapAddresses(parsedEmail.to), - cc: mapAddresses(parsedEmail.cc), - bcc: mapAddresses(parsedEmail.bcc), - subject: parsedEmail.subject || '', - body: parsedEmail.text || '', - html: parsedEmail.html || '', - headers: parsedEmail.headers, - attachments, - receivedAt: parsedEmail.date || new Date(), - }; + if (msgResponse.data.raw) { + const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url'); + const parsedEmail: ParsedMail = await simpleParser(rawEmail); + const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({ + filename: attachment.filename || 'untitled', + contentType: attachment.contentType, + size: attachment.size, + content: attachment.content as Buffer + })); + const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => { + if (!addresses) return []; + const addressArray = Array.isArray(addresses) ? addresses : [addresses]; + return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' }))); + }; + yield { + id: msgResponse.data.id!, + userEmail: userEmail, + eml: rawEmail, + from: mapAddresses(parsedEmail.from), + to: mapAddresses(parsedEmail.to), + cc: mapAddresses(parsedEmail.cc), + bcc: mapAddresses(parsedEmail.bcc), + subject: parsedEmail.subject || '', + body: parsedEmail.text || '', + html: parsedEmail.html || '', + headers: parsedEmail.headers, + attachments, + receivedAt: parsedEmail.date || new Date(), + }; + } + } catch (error: any) { + if (error.code === 404) { + logger.warn({ messageId: messageAdded.message.id, userEmail }, 'Message not found, skipping.'); + } else { + throw error; + } } } } @@ -229,41 +237,49 @@ export class GoogleWorkspaceConnector implements IEmailConnector { for (const message of messages) { if (message.id) { - const msgResponse = await gmail.users.messages.get({ - userId: 'me', - id: message.id, - format: 'RAW' - }); + try { + const msgResponse = await gmail.users.messages.get({ + userId: 'me', + id: message.id, + format: 'RAW' + }); - if (msgResponse.data.raw) { - const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url'); - const parsedEmail: ParsedMail = await simpleParser(rawEmail); - const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({ - filename: attachment.filename || 'untitled', - contentType: attachment.contentType, - size: attachment.size, - content: attachment.content as Buffer - })); - const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => { - if (!addresses) return []; - const addressArray = Array.isArray(addresses) ? addresses : [addresses]; - return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' }))); - }; - yield { - id: msgResponse.data.id!, - userEmail: userEmail, - eml: rawEmail, - from: mapAddresses(parsedEmail.from), - to: mapAddresses(parsedEmail.to), - cc: mapAddresses(parsedEmail.cc), - bcc: mapAddresses(parsedEmail.bcc), - subject: parsedEmail.subject || '', - body: parsedEmail.text || '', - html: parsedEmail.html || '', - headers: parsedEmail.headers, - attachments, - receivedAt: parsedEmail.date || new Date(), - }; + if (msgResponse.data.raw) { + const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url'); + const parsedEmail: ParsedMail = await simpleParser(rawEmail); + const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({ + filename: attachment.filename || 'untitled', + contentType: attachment.contentType, + size: attachment.size, + content: attachment.content as Buffer + })); + const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => { + if (!addresses) return []; + const addressArray = Array.isArray(addresses) ? addresses : [addresses]; + return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' }))); + }; + yield { + id: msgResponse.data.id!, + userEmail: userEmail, + eml: rawEmail, + from: mapAddresses(parsedEmail.from), + to: mapAddresses(parsedEmail.to), + cc: mapAddresses(parsedEmail.cc), + bcc: mapAddresses(parsedEmail.bcc), + subject: parsedEmail.subject || '', + body: parsedEmail.text || '', + html: parsedEmail.html || '', + headers: parsedEmail.headers, + attachments, + receivedAt: parsedEmail.date || new Date(), + }; + } + } catch (error: any) { + if (error.code === 404) { + logger.warn({ messageId: message.id, userEmail }, 'Message not found during initial import, skipping.'); + } else { + throw error; + } } } } diff --git a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts index 56601ec..5429e53 100644 --- a/packages/backend/src/services/ingestion-connectors/ImapConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/ImapConnector.ts @@ -38,9 +38,12 @@ export class ImapConnector implements IEmailConnector { try { await this.client.connect(); this.isConnected = true; - } catch (err) { + } catch (err: any) { this.isConnected = false; logger.error({ err }, 'IMAP connection failed'); + if (err.responseText) { + throw new Error(`IMAP Connection Error: ${err.responseText}`); + } throw err; } } @@ -62,7 +65,7 @@ export class ImapConnector implements IEmailConnector { return true; } catch (error) { logger.error({ error }, 'Failed to verify IMAP connection'); - return false; + throw error; } } diff --git a/packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts b/packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts index 602e15b..1d82f64 100644 --- a/packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts +++ b/packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts @@ -90,7 +90,7 @@ export class MicrosoftConnector implements IEmailConnector { return true; } catch (error) { logger.error({ err: error }, 'Failed to verify Microsoft 365 connection'); - return false; + throw error; } } diff --git a/packages/frontend/package.json b/packages/frontend/package.json index b37da40..7ac6cdd 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -15,15 +15,16 @@ "lint": "prettier --check ." }, "dependencies": { + "@iconify/svelte": "^5.0.1", "@open-archiver/types": "workspace:*", + "@sveltejs/kit": "^2.16.0", + "bits-ui": "^2.8.10", + "clsx": "^2.1.1", "d3-shape": "^3.2.0", "jose": "^6.0.1", "lucide-svelte": "^0.525.0", "postal-mime": "^2.4.4", "svelte-persisted-store": "^0.12.0", - "@sveltejs/kit": "^2.16.0", - "bits-ui": "^2.8.10", - "clsx": "^2.1.1", "tailwind-merge": "^3.3.1", "tailwind-variants": "^1.0.0" }, @@ -37,11 +38,13 @@ "@types/d3-shape": "^3.1.7", "dotenv": "^17.2.0", "layerchart": "2.0.0-next.27", + "mode-watcher": "^1.1.0", "prettier": "^3.4.2", "prettier-plugin-svelte": "^3.3.3", "prettier-plugin-tailwindcss": "^0.6.11", "svelte": "^5.0.0", "svelte-check": "^4.0.0", + "svelte-sonner": "^1.0.5", "tailwindcss": "^4.0.0", "tw-animate-css": "^1.3.5", "typescript": "^5.0.0", diff --git a/packages/frontend/src/lib/components/custom/Footer.svelte b/packages/frontend/src/lib/components/custom/Footer.svelte index 4e9f5f8..1226b7f 100644 --- a/packages/frontend/src/lib/components/custom/Footer.svelte +++ b/packages/frontend/src/lib/components/custom/Footer.svelte @@ -4,7 +4,8 @@ >

- © {new Date().getFullYear()} Open Archiver. All rights reserved. + © {new Date().getFullYear()} + Open Archiver. All rights reserved.

diff --git a/packages/frontend/src/lib/components/custom/alert/Alerts.svelte b/packages/frontend/src/lib/components/custom/alert/Alerts.svelte new file mode 100644 index 0000000..9acce51 --- /dev/null +++ b/packages/frontend/src/lib/components/custom/alert/Alerts.svelte @@ -0,0 +1,123 @@ + + + +{#if show} + +{/if} diff --git a/packages/frontend/src/lib/components/custom/alert/alert-state.svelte.ts b/packages/frontend/src/lib/components/custom/alert/alert-state.svelte.ts new file mode 100644 index 0000000..136f670 --- /dev/null +++ b/packages/frontend/src/lib/components/custom/alert/alert-state.svelte.ts @@ -0,0 +1,25 @@ +export type AlertType = { + type: 'success' | 'warning' | 'error'; + title: string; + message: string; + duration: number; + show: boolean; +}; + +export const initialAlertState: AlertType = { + type: 'success', + title: '', + message: '', + duration: 0, + show: false +}; + +let alertState = $state(initialAlertState); + +export function setAlert(alert: AlertType) { + alertState = alert; +} + +export function getAlert() { + return alertState; +} \ No newline at end of file diff --git a/packages/frontend/src/lib/components/ui/hover-card/hover-card-content.svelte b/packages/frontend/src/lib/components/ui/hover-card/hover-card-content.svelte new file mode 100644 index 0000000..e1d592a --- /dev/null +++ b/packages/frontend/src/lib/components/ui/hover-card/hover-card-content.svelte @@ -0,0 +1,29 @@ + + + + + diff --git a/packages/frontend/src/lib/components/ui/hover-card/hover-card-trigger.svelte b/packages/frontend/src/lib/components/ui/hover-card/hover-card-trigger.svelte new file mode 100644 index 0000000..322172b --- /dev/null +++ b/packages/frontend/src/lib/components/ui/hover-card/hover-card-trigger.svelte @@ -0,0 +1,7 @@ + + + diff --git a/packages/frontend/src/lib/components/ui/hover-card/index.ts b/packages/frontend/src/lib/components/ui/hover-card/index.ts new file mode 100644 index 0000000..85f3949 --- /dev/null +++ b/packages/frontend/src/lib/components/ui/hover-card/index.ts @@ -0,0 +1,14 @@ +import { LinkPreview as HoverCardPrimitive } from "bits-ui"; +import Content from "./hover-card-content.svelte"; +import Trigger from "./hover-card-trigger.svelte"; + +const Root = HoverCardPrimitive.Root; + +export { + Root, + Content, + Trigger, + Root as HoverCard, + Content as HoverCardContent, + Trigger as HoverCardTrigger, +}; diff --git a/packages/frontend/src/lib/components/ui/sonner/index.ts b/packages/frontend/src/lib/components/ui/sonner/index.ts new file mode 100644 index 0000000..1ad9f4a --- /dev/null +++ b/packages/frontend/src/lib/components/ui/sonner/index.ts @@ -0,0 +1 @@ +export { default as Toaster } from "./sonner.svelte"; diff --git a/packages/frontend/src/lib/components/ui/sonner/sonner.svelte b/packages/frontend/src/lib/components/ui/sonner/sonner.svelte new file mode 100644 index 0000000..1f50e1e --- /dev/null +++ b/packages/frontend/src/lib/components/ui/sonner/sonner.svelte @@ -0,0 +1,13 @@ + + + diff --git a/packages/frontend/src/routes/+layout.svelte b/packages/frontend/src/routes/+layout.svelte index b926d6c..b7e6b43 100644 --- a/packages/frontend/src/routes/+layout.svelte +++ b/packages/frontend/src/routes/+layout.svelte @@ -4,6 +4,8 @@ import { theme } from '$lib/stores/theme.store'; import { browser } from '$app/environment'; import Footer from '$lib/components/custom/Footer.svelte'; + import { getAlert } from '$lib/components/custom/alert/alert-state.svelte'; + import Alerts from '$lib/components/custom/alert/Alerts.svelte'; let { data, children } = $props(); @@ -21,6 +23,7 @@ }); +
{@render children()} diff --git a/packages/frontend/src/routes/dashboard/ingestions/+page.svelte b/packages/frontend/src/routes/dashboard/ingestions/+page.svelte index 4fef355..97019be 100644 --- a/packages/frontend/src/routes/dashboard/ingestions/+page.svelte +++ b/packages/frontend/src/routes/dashboard/ingestions/+page.svelte @@ -10,9 +10,10 @@ import { api } from '$lib/api.client'; import type { IngestionSource, CreateIngestionSourceDto } from '@open-archiver/types'; import Badge from '$lib/components/ui/badge/badge.svelte'; + import { setAlert } from '$lib/components/custom/alert/alert-state.svelte'; + import * as HoverCard from '$lib/components/ui/hover-card/index.js'; let { data }: { data: PageData } = $props(); - let ingestionSources = $state(data.ingestionSources); let isDialogOpen = $state(false); let isDeleteDialogOpen = $state(false); @@ -81,26 +82,48 @@ }; const handleFormSubmit = async (formData: CreateIngestionSourceDto) => { - if (selectedSource) { - // Update - const response = await api(`/ingestion-sources/${selectedSource.id}`, { - method: 'PUT', - body: JSON.stringify(formData) + try { + if (selectedSource) { + // Update + const response = await api(`/ingestion-sources/${selectedSource.id}`, { + method: 'PUT', + body: JSON.stringify(formData) + }); + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.message || 'Failed to update source.'); + } + const updatedSource = await response.json(); + ingestionSources = ingestionSources.map((s) => + s.id === updatedSource.id ? updatedSource : s + ); + } else { + // Create + const response = await api('/ingestion-sources', { + method: 'POST', + body: JSON.stringify(formData) + }); + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.message || 'Failed to create source.'); + } + const newSource = await response.json(); + ingestionSources = [...ingestionSources, newSource]; + } + isDialogOpen = false; + } catch (error) { + let message = 'An unknown error occurred.'; + if (error instanceof Error) { + message = error.message; + } + setAlert({ + type: 'error', + title: 'Authentication Failed', + message, + duration: 5000, + show: true }); - const updatedSource = await response.json(); - ingestionSources = ingestionSources.map((s) => - s.id === updatedSource.id ? updatedSource : s - ); - } else { - // Create - const response = await api('/ingestion-sources', { - method: 'POST', - body: JSON.stringify(formData) - }); - const newSource = await response.json(); - ingestionSources = [...ingestionSources, newSource]; } - isDialogOpen = false; }; function getStatusClasses(status: IngestionSource['status']): string { @@ -156,9 +179,21 @@ {source.provider.split('_').join(' ')} - - {source.status.split('_').join(' ')} - + + + + {source.status.split('_').join(' ')} + + + +
+

+ Last sync message: + {source.lastSyncStatusMessage || 'Empty'} +

+
+ + handleToggle(source)} - disabled={source.status !== 'active' && source.status !== 'paused'} + disabled={source.status === 'importing' || source.status === 'syncing'} /> {new Date(source.createdAt).toLocaleDateString()} diff --git a/packages/frontend/src/routes/signin/+page.svelte b/packages/frontend/src/routes/signin/+page.svelte index ea68691..9458efa 100644 --- a/packages/frontend/src/routes/signin/+page.svelte +++ b/packages/frontend/src/routes/signin/+page.svelte @@ -7,23 +7,28 @@ import { api } from '$lib/api.client'; import { authStore } from '$lib/stores/auth.store'; import type { LoginResponse } from '@open-archiver/types'; + import { setAlert } from '$lib/components/custom/alert/alert-state.svelte'; let email = ''; let password = ''; - let error: string | null = null; let isLoading = false; async function handleSubmit() { isLoading = true; - error = null; try { const response = await api('/auth/login', { method: 'POST', body: JSON.stringify({ email, password }) }); if (!response.ok) { - const errorData = await response.json(); - throw new Error(errorData.message || 'Failed to login'); + let errorMessage = 'Failed to login'; + try { + const errorData = await response.json(); + errorMessage = errorData.message || errorMessage; + } catch (e) { + errorMessage = response.statusText; + } + throw new Error(errorMessage); } const loginData: LoginResponse = await response.json(); @@ -31,7 +36,13 @@ // Redirect to a protected page after login goto('/dashboard'); } catch (e: any) { - error = e.message; + setAlert({ + type: 'error', + title: 'Login Failed', + message: e.message, + duration: 5000, + show: true + }); } finally { isLoading = false; } @@ -43,7 +54,19 @@ -
+
+ Login @@ -60,10 +83,6 @@
- {#if error} -

{error}

- {/if} - diff --git a/packages/types/src/ingestion.types.ts b/packages/types/src/ingestion.types.ts index 32b62f1..4b1dc7a 100644 --- a/packages/types/src/ingestion.types.ts +++ b/packages/types/src/ingestion.types.ts @@ -116,3 +116,9 @@ export type MailboxUser = { primaryEmail: string; displayName: string; }; + + +export type ProcessMailboxError = { + error: boolean; + message: string; +}; \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 27b1150..c2a522a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -151,6 +151,9 @@ importers: packages/frontend: dependencies: + '@iconify/svelte': + specifier: ^5.0.1 + version: 5.0.1(svelte@5.35.5) '@open-archiver/types': specifier: workspace:* version: link:../types @@ -212,6 +215,9 @@ importers: layerchart: specifier: 2.0.0-next.27 version: 2.0.0-next.27(svelte@5.35.5) + mode-watcher: + specifier: ^1.1.0 + version: 1.1.0(svelte@5.35.5) prettier: specifier: ^3.4.2 version: 3.6.2 @@ -227,6 +233,9 @@ importers: svelte-check: specifier: ^4.0.0 version: 4.2.2(picomatch@4.0.2)(svelte@5.35.5)(typescript@5.8.3) + svelte-sonner: + specifier: ^1.0.5 + version: 1.0.5(svelte@5.35.5) tailwindcss: specifier: ^4.0.0 version: 4.1.11 @@ -1014,6 +1023,11 @@ packages: '@iconify-json/simple-icons@1.2.44': resolution: {integrity: sha512-CdWgSPygwDlDbKtDWjvi3NtUefnkoepXv90n3dQxJerqzD9kI+nEJOiWUBM+eOyMYQKtxBpLWFBrgeotF0IZKw==} + '@iconify/svelte@5.0.1': + resolution: {integrity: sha512-nR1ApafyeRDcsx6ytd+0mzaY0ASYfG5YBW9dN8rUWxwi792/HX401qAZEkXSjT0iHm2Dgua+PwXDs/3ttJhkqQ==} + peerDependencies: + svelte: '>4.0.0' + '@iconify/types@2.0.0': resolution: {integrity: sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==} @@ -1365,6 +1379,7 @@ packages: '@smithy/middleware-endpoint@4.1.14': resolution: {integrity: sha512-+BGLpK5D93gCcSEceaaYhUD/+OCGXM1IDaq/jKUQ+ujB0PTWlWN85noodKw/IPFZhIKFCNEe19PGd/reUMeLSQ==} engines: {node: '>=18.0.0'} + deprecated: Please upgrade to @smithy/middleware-endpoint@4.1.15 or higher to fix a bug preventing the resolution of ENV and config file custom endpoints https://github.com/smithy-lang/smithy-typescript/issues/1645 '@smithy/middleware-retry@4.1.15': resolution: {integrity: sha512-iKYUJpiyTQ33U2KlOZeUb0GwtzWR3C0soYcKuCnTmJrvt6XwTPQZhMfsjJZNw7PpQ3TU4Ati1qLSrkSJxnnSMQ==} @@ -3357,6 +3372,11 @@ packages: engines: {node: '>=10'} hasBin: true + mode-watcher@1.1.0: + resolution: {integrity: sha512-mUT9RRGPDYenk59qJauN1rhsIMKBmWA3xMF+uRwE8MW/tjhaDSCCARqkSuDTq8vr4/2KcAxIGVjACxTjdk5C3g==} + peerDependencies: + svelte: ^5.27.0 + mri@1.2.0: resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} engines: {node: '>=4'} @@ -3839,6 +3859,16 @@ packages: resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} engines: {node: '>= 18'} + runed@0.23.4: + resolution: {integrity: sha512-9q8oUiBYeXIDLWNK5DfCWlkL0EW3oGbk845VdKlPeia28l751VpfesaB/+7pI6rnbx1I6rqoZ2fZxptOJLxILA==} + peerDependencies: + svelte: ^5.7.0 + + runed@0.25.0: + resolution: {integrity: sha512-7+ma4AG9FT2sWQEA0Egf6mb7PBT2vHyuHail1ie8ropfSjvZGtEAx8YTmUjv/APCsdRRxEVvArNjALk9zFSOrg==} + peerDependencies: + svelte: ^5.7.0 + runed@0.28.0: resolution: {integrity: sha512-k2xx7RuO9hWcdd9f+8JoBeqWtYrm5CALfgpkg2YDB80ds/QE4w0qqu34A7fqiAwiBBSBQOid7TLxwxVC27ymWQ==} peerDependencies: @@ -4098,6 +4128,17 @@ packages: peerDependencies: svelte: ^3.48.0 || ^4 || ^5 + svelte-sonner@1.0.5: + resolution: {integrity: sha512-9dpGPFqKb/QWudYqGnEz93vuY+NgCEvyNvxoCLMVGw6sDN/3oVeKV1xiEirW2E1N3vJEyj5imSBNOGltQHA7mg==} + peerDependencies: + svelte: ^5.0.0 + + svelte-toolbelt@0.7.1: + resolution: {integrity: sha512-HcBOcR17Vx9bjaOceUvxkY3nGmbBmCBBbuWLLEWO6jtmWH8f/QoWmbyUfQZrpDINH39en1b8mptfPQT9VKQ1xQ==} + engines: {node: '>=18', pnpm: '>=8.7.0'} + peerDependencies: + svelte: ^5.0.0 + svelte-toolbelt@0.9.3: resolution: {integrity: sha512-HCSWxCtVmv+c6g1ACb8LTwHVbDqLKJvHpo6J8TaqwUme2hj9ATJCpjCPNISR1OCq2Q4U1KT41if9ON0isINQZw==} engines: {node: '>=18', pnpm: '>=8.7.0'} @@ -5386,6 +5427,11 @@ snapshots: dependencies: '@iconify/types': 2.0.0 + '@iconify/svelte@5.0.1(svelte@5.35.5)': + dependencies: + '@iconify/types': 2.0.0 + svelte: 5.35.5 + '@iconify/types@2.0.0': {} '@internationalized/date@3.8.2': @@ -7985,6 +8031,12 @@ snapshots: mkdirp@3.0.1: {} + mode-watcher@1.1.0(svelte@5.35.5): + dependencies: + runed: 0.25.0(svelte@5.35.5) + svelte: 5.35.5 + svelte-toolbelt: 0.7.1(svelte@5.35.5) + mri@1.2.0: {} mrmime@2.0.1: {} @@ -8434,6 +8486,16 @@ snapshots: transitivePeerDependencies: - supports-color + runed@0.23.4(svelte@5.35.5): + dependencies: + esm-env: 1.2.2 + svelte: 5.35.5 + + runed@0.25.0(svelte@5.35.5): + dependencies: + esm-env: 1.2.2 + svelte: 5.35.5 + runed@0.28.0(svelte@5.35.5): dependencies: esm-env: 1.2.2 @@ -8737,6 +8799,18 @@ snapshots: dependencies: svelte: 5.35.5 + svelte-sonner@1.0.5(svelte@5.35.5): + dependencies: + runed: 0.28.0(svelte@5.35.5) + svelte: 5.35.5 + + svelte-toolbelt@0.7.1(svelte@5.35.5): + dependencies: + clsx: 2.1.1 + runed: 0.23.4(svelte@5.35.5) + style-to-object: 1.0.9 + svelte: 5.35.5 + svelte-toolbelt@0.9.3(svelte@5.35.5): dependencies: clsx: 2.1.1