github/OpenArchiver
1
1
Fork 0
mirror of https://github.com/LogicLabs-OU/OpenArchiver.git synced 2026-04-06 00:31:57 +02:00
Code Issues 161 Packages Projects Releases Wiki Activity

Error handling, force sync, UI improvement

Browse Source
This commit is contained in:
Wayne
2025-08-04 13:24:46 +03:00
parent d47f0c5b08
commit 4156abcdfa
28 changed files with 635 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -21,7 +21,7 @@ _Full-text search across all your emails and attachments_
## Community
Join our community to ask questions, share your projects, and connect with other developers.
We are committed to build an engaging community around Open Archiver, and we are inviting all of you to join our community on Discord to get real-time support and connect with the team.
[![Discord](https://img.shields.io/badge/Join%20our%20Discord-7289DA?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/Qpv4BmHp)

1
packages/backend/src/api/controllers/auth.controller.ts
View File

@@ -24,7 +24,6 @@ export class AuthController {
return res.status(200).json(result);
} catch (error) {
// In a real application, you'd want to log this error.
console.error('Login error:', error);
return res.status(500).json({ message: 'An internal server error occurred' });
}

8
packages/backend/src/api/controllers/ingestion.controller.ts
View File

@@ -1,6 +1,7 @@
import { Request, Response } from 'express';
import { IngestionService } from '../../services/IngestionService';
import { CreateIngestionSourceDto, UpdateIngestionSourceDto } from '@open-archiver/types';
import { logger } from '../../config/logger';
export class IngestionController {
public create = async (req: Request, res: Response): Promise<Response> => {
@@ -8,9 +9,10 @@ export class IngestionController {
const dto: CreateIngestionSourceDto = req.body;
const newSource = await IngestionService.create(dto);
return res.status(201).json(newSource);
} catch (error) {
console.error('Create ingestion source error:', error);
return res.status(500).json({ message: 'An internal server error occurred' });
} catch (error: any) {
logger.error({ err: error }, 'Create ingestion source error');
// Return a 400 Bad Request for connection errors
return res.status(400).json({ message: error.message || 'Failed to create ingestion source due to a connection error.' });
}
};

7
packages/backend/src/jobs/processors/continuous-sync.processor.ts
View File

@@ -10,8 +10,8 @@ export default async (job: Job<IContinuousSyncJob>) => {
logger.info({ ingestionSourceId }, 'Starting continuous sync job.');
const source = await IngestionService.findById(ingestionSourceId);
if (!source || source.status !== 'active') {
logger.warn({ ingestionSourceId, status: source?.status }, 'Skipping continuous sync for non-active source.');
if (!source || !['error', 'active'].includes(source.status)) {
logger.warn({ ingestionSourceId, status: source?.status }, 'Skipping continuous sync for non-active or non-error source.');
return;
}
@@ -39,7 +39,8 @@ export default async (job: Job<IContinuousSyncJob>) => {
},
removeOnFail: {
age: 60 * 30 // 30 minutes
}
},
timeout: 1000 * 60 * 30 // 30 minutes
}
});
}

9
packages/backend/src/jobs/processors/initial-import.processor.ts
View File

@@ -1,10 +1,11 @@
import { Job } from 'bullmq';
import { Job, FlowChildJob } from 'bullmq';
import { IngestionService } from '../../services/IngestionService';
import { IInitialImportJob } from '@open-archiver/types';
import { EmailProviderFactory } from '../../services/EmailProviderFactory';
import { flowProducer } from '../queues';
import { logger } from '../../config/logger';
export default async (job: Job<IInitialImportJob>) => {
const { ingestionSourceId } = job.data;
logger.info({ ingestionSourceId }, 'Starting initial import master job');
@@ -23,7 +24,7 @@ export default async (job: Job<IInitialImportJob>) => {
const connector = EmailProviderFactory.createConnector(source);
// if (connector instanceof GoogleWorkspaceConnector || connector instanceof MicrosoftConnector) {
const jobs = [];
const jobs: FlowChildJob[] = [];
let userCount = 0;
for await (const user of connector.listAllUsers()) {
if (user.primaryEmail) {
@@ -40,7 +41,9 @@ export default async (job: Job<IInitialImportJob>) => {
},
removeOnFail: {
age: 60 * 30 // 30 minutes
}
},
attempts: 1,
// failParentOnFailure: true
}
});
userCount++;

18
packages/backend/src/jobs/processors/process-mailbox.processor.ts
View File

@@ -1,10 +1,18 @@
import { Job } from 'bullmq';
import { IProcessMailboxJob, SyncState } from '@open-archiver/types';
import { IProcessMailboxJob, SyncState, ProcessMailboxError } from '@open-archiver/types';
import { IngestionService } from '../../services/IngestionService';
import { logger } from '../../config/logger';
import { EmailProviderFactory } from '../../services/EmailProviderFactory';
import { StorageService } from '../../services/StorageService';
/**
* This processor handles the ingestion of emails for a single user's mailbox.
* If an error occurs during processing (e.g., an API failure),
* it catches the exception and returns a structured error object instead of throwing.
* This prevents a single failed mailbox from halting the entire sync cycle for all users.
* The parent 'sync-cycle-finished' job is responsible for inspecting the results of all
* 'process-mailbox' jobs, aggregating successes, and reporting detailed failures.
*/
export const processMailboxProcessor = async (job: Job<IProcessMailboxJob, SyncState, string>) => {
const { ingestionSourceId, userEmail } = job.data;
@@ -28,7 +36,6 @@ export const processMailboxProcessor = async (job: Job<IProcessMailboxJob, SyncS
}
const newSyncState = connector.getUpdatedSyncState(userEmail);
console.log('newSyncState, ', newSyncState);
logger.info({ ingestionSourceId, userEmail }, `Finished processing mailbox for user`);
@@ -36,6 +43,11 @@ export const processMailboxProcessor = async (job: Job<IProcessMailboxJob, SyncS
return newSyncState;
} catch (error) {
logger.error({ err: error, ingestionSourceId, userEmail }, 'Error processing mailbox');
throw error;
const errorMessage = error instanceof Error ? error.message : 'An unknown error occurred';
const processMailboxError: ProcessMailboxError = {
error: true,
message: `Failed to process mailbox for ${userEmail}: ${errorMessage}`
};
return processMailboxError;
}
};

16
packages/backend/src/jobs/processors/schedule-continuous-sync.processor.ts
View File

@@ -1,19 +1,25 @@
import { Job } from 'bullmq';
import { db } from '../../database';
import { ingestionSources } from '../../database/schema';
import { eq } from 'drizzle-orm';
import { or, eq } from 'drizzle-orm';
import { ingestionQueue } from '../queues';
export default async (job: Job) => {
console.log(
'Scheduler running: Looking for active ingestion sources to sync.'
'Scheduler running: Looking for active or error ingestion sources to sync.'
);
const activeSources = await db
// find all sources that have the status of active or error for continuous syncing.
const sourcesToSync = await db
.select({ id: ingestionSources.id })
.from(ingestionSources)
.where(eq(ingestionSources.status, 'active'));
.where(
or(
eq(ingestionSources.status, 'active'),
eq(ingestionSources.status, 'error')
)
);
for (const source of activeSources) {
for (const source of sourcesToSync) {
// The status field on the ingestion source is used to prevent duplicate syncs.
await ingestionQueue.add('continuous-sync', { ingestionSourceId: source.id });
}

56
packages/backend/src/jobs/processors/sync-cycle-finished.processor.ts
View File

@@ -1,7 +1,7 @@
import { Job, FlowJob } from 'bullmq';
import { Job } from 'bullmq';
import { IngestionService } from '../../services/IngestionService';
import { logger } from '../../config/logger';
import { SyncState } from '@open-archiver/types';
import { SyncState, ProcessMailboxError } from '@open-archiver/types';
import { db } from '../../database';
import { ingestionSources } from '../../database/schema';
import { eq } from 'drizzle-orm';
@@ -13,41 +13,65 @@ interface ISyncCycleFinishedJob {
isInitialImport: boolean;
}
/**
* This processor runs after all 'process-mailbox' jobs for a sync cycle have completed.
* It is responsible for aggregating the results and finalizing the sync status.
* It inspects the return values of all child jobs to identify successes and failures.
*
* If any child jobs returned an error object, this processor will:
* 1. Mark the overall ingestion status as 'error'.
* 2. Aggregate the detailed error messages from all failed jobs.
* 3. Save the sync state from any jobs that *did* succeed, preserving partial progress.
*
* If all child jobs succeeded, it marks the ingestion as 'active' and saves the final
* aggregated sync state from all children.
*
*/
export default async (job: Job<ISyncCycleFinishedJob, any, string>) => {
const { ingestionSourceId, userCount, isInitialImport } = job.data;
logger.info({ ingestionSourceId, userCount, isInitialImport }, 'Sync cycle finished job started');
try {
const childrenJobs = await job.getChildrenValues<SyncState>();
const allSyncStates = Object.values(childrenJobs);
const childrenValues = await job.getChildrenValues<SyncState | ProcessMailboxError>();
const allChildJobs = Object.values(childrenValues);
// if data has error property, it is a failed job
const failedJobs = allChildJobs.filter(v => v && (v as any).error) as ProcessMailboxError[];
// if data doesn't have error property, it is a successful job with SyncState
const successfulJobs = allChildJobs.filter(v => !v || !(v as any).error) as SyncState[];
// Merge all sync states from children jobs into one
const finalSyncState = deepmerge(...allSyncStates.filter(s => s && Object.keys(s).length > 0));
const finalSyncState = deepmerge(...successfulJobs.filter(s => s && Object.keys(s).length > 0));
let message = 'Continuous sync cycle finished successfully.';
if (isInitialImport) {
message = `Initial import finished for ${userCount} mailboxes.`;
let status: 'active' | 'error' = 'active';
let message: string;
if (failedJobs.length > 0) {
status = 'error';
const errorMessages = failedJobs.map(j => j.message).join('\n');
message = `Sync cycle completed with ${failedJobs.length} error(s):\n${errorMessages}`;
logger.error({ ingestionSourceId, errors: errorMessages }, 'Sync cycle finished with errors.');
} else {
message = 'Continuous sync cycle finished successfully.';
if (isInitialImport) {
message = `Initial import finished for ${userCount} mailboxes.`;
}
logger.info({ ingestionSourceId }, 'Successfully updated status and final sync state.');
}
// Update the database with the final aggregated sync state
await db
.update(ingestionSources)
.set({
status: 'active',
status,
lastSyncFinishedAt: new Date(),
lastSyncStatusMessage: message,
syncState: finalSyncState
})
.where(eq(ingestionSources.id, ingestionSourceId));
logger.info({ ingestionSourceId }, 'Successfully updated status and final sync state.');
} catch (error) {
logger.error({ err: error, ingestionSourceId }, 'Failed to process finished sync cycle.');
// If this fails, we should probably set the status to 'error' to indicate a problem.
logger.error({ err: error, ingestionSourceId }, 'An unexpected error occurred while finalizing the sync cycle.');
await IngestionService.update(ingestionSourceId, {
status: 'error',
lastSyncFinishedAt: new Date(),
lastSyncStatusMessage: 'Failed to finalize sync cycle and update sync state.'
lastSyncStatusMessage: 'An unexpected error occurred while finalizing the sync cycle.'
});
}
};

39
packages/backend/src/services/CryptoService.ts
View File

@@ -29,20 +29,25 @@ export class CryptoService {
return Buffer.concat([salt, iv, tag, encrypted]).toString('hex');
}
public static decrypt(encrypted: string): string {
const data = Buffer.from(encrypted, 'hex');
const salt = data.subarray(0, SALT_LENGTH);
const iv = data.subarray(SALT_LENGTH, SALT_LENGTH + IV_LENGTH);
const tag = data.subarray(SALT_LENGTH + IV_LENGTH, SALT_LENGTH + IV_LENGTH + TAG_LENGTH);
const encryptedValue = data.subarray(SALT_LENGTH + IV_LENGTH + TAG_LENGTH);
public static decrypt(encrypted: string): string | null {
try {
const data = Buffer.from(encrypted, 'hex');
const salt = data.subarray(0, SALT_LENGTH);
const iv = data.subarray(SALT_LENGTH, SALT_LENGTH + IV_LENGTH);
const tag = data.subarray(SALT_LENGTH + IV_LENGTH, SALT_LENGTH + IV_LENGTH + TAG_LENGTH);
const encryptedValue = data.subarray(SALT_LENGTH + IV_LENGTH + TAG_LENGTH);
const key = getKey(salt);
const decipher = createDecipheriv(ALGORITHM, key, iv);
decipher.setAuthTag(tag);
const key = getKey(salt);
const decipher = createDecipheriv(ALGORITHM, key, iv);
decipher.setAuthTag(tag);
const decrypted = Buffer.concat([decipher.update(encryptedValue), decipher.final()]);
const decrypted = Buffer.concat([decipher.update(encryptedValue), decipher.final()]);
return decrypted.toString('utf8');
return decrypted.toString('utf8');
} catch (error) {
console.error('Decryption failed:', error);
return null;
}
}
public static encryptObject<T extends object>(obj: T): string {
@@ -50,8 +55,16 @@ export class CryptoService {
return this.encrypt(jsonString);
}
public static decryptObject<T extends object>(encrypted: string): T {
public static decryptObject<T extends object>(encrypted: string): T | null {
const decryptedString = this.decrypt(encrypted);
return JSON.parse(decryptedString) as T;
if (!decryptedString) {
return null;
}
try {
return JSON.parse(decryptedString) as T;
} catch (error) {
console.error('Failed to parse decrypted JSON:', error);
return null;
}
}
}

58
packages/backend/src/services/IngestionService.ts
View File

@@ -6,7 +6,7 @@ import type {
IngestionSource,
IngestionCredentials
} from '@open-archiver/types';
import { and, eq } from 'drizzle-orm';
import { and, desc, eq } from 'drizzle-orm';
import { CryptoService } from './CryptoService';
import { EmailProviderFactory } from './EmailProviderFactory';
import { ingestionQueue } from '../jobs/queues';
@@ -22,10 +22,16 @@ import { DatabaseService } from './DatabaseService';
export class IngestionService {
private static decryptSource(source: typeof ingestionSources.$inferSelect): IngestionSource {
private static decryptSource(source: typeof ingestionSources.$inferSelect): IngestionSource | null {
const decryptedCredentials = CryptoService.decryptObject<IngestionCredentials>(
source.credentials as string
);
if (!decryptedCredentials) {
logger.error({ sourceId: source.id }, 'Failed to decrypt ingestion source credentials.');
return null;
}
return { ...source, credentials: decryptedCredentials } as IngestionSource;
}
@@ -43,21 +49,29 @@ export class IngestionService {
const [newSource] = await db.insert(ingestionSources).values(valuesToInsert).returning();
const decryptedSource = this.decryptSource(newSource);
// Test the connection
const connector = EmailProviderFactory.createConnector(decryptedSource);
const isConnected = await connector.testConnection();
if (isConnected) {
return await this.update(decryptedSource.id, { status: 'auth_success' });
if (!decryptedSource) {
await this.delete(newSource.id);
throw new Error('Failed to process newly created ingestion source due to a decryption error.');
}
const connector = EmailProviderFactory.createConnector(decryptedSource);
return decryptedSource;
try {
await connector.testConnection();
// If connection succeeds, update status to auth_success, which triggers the initial import.
return await this.update(decryptedSource.id, { status: 'auth_success' });
} catch (error) {
// If connection fails, delete the newly created source and throw the error.
await this.delete(decryptedSource.id);
throw error;
}
}
public static async findAll(): Promise<IngestionSource[]> {
const sources = await db.select().from(ingestionSources);
return sources.map(this.decryptSource);
const sources = await db.select().from(ingestionSources).orderBy(desc(ingestionSources.createdAt));
return sources.flatMap(source => {
const decrypted = this.decryptSource(source);
return decrypted ? [decrypted] : [];
});
}
public static async findById(id: string): Promise<IngestionSource> {
@@ -65,7 +79,11 @@ export class IngestionService {
if (!source) {
throw new Error('Ingestion source not found');
}
return this.decryptSource(source);
const decryptedSource = this.decryptSource(source);
if (!decryptedSource) {
throw new Error('Failed to decrypt ingestion source credentials.');
}
return decryptedSource;
}
public static async update(
@@ -95,6 +113,10 @@ export class IngestionService {
const decryptedSource = this.decryptSource(updatedSource);
if (!decryptedSource) {
throw new Error('Failed to process updated ingestion source due to a decryption error.');
}
// If the status has changed to auth_success, trigger the initial import
if (
originalSource.status !== 'auth_success' &&
@@ -131,7 +153,15 @@ export class IngestionService {
.where(eq(ingestionSources.id, id))
.returning();
return this.decryptSource(deletedSource);
const decryptedSource = this.decryptSource(deletedSource);
if (!decryptedSource) {
// Even if decryption fails, we should confirm deletion.
// We might return a simpler object or just a success message.
// For now, we'll indicate the issue but still confirm deletion happened.
logger.warn({ sourceId: deletedSource.id }, 'Could not decrypt credentials of deleted source, but deletion was successful.');
return { ...deletedSource, credentials: null } as unknown as IngestionSource;
}
return decryptedSource;
}
public static async triggerInitialImport(id: string): Promise<void> {

4
packages/backend/src/services/UserService.ts
View File

@@ -3,7 +3,7 @@ import type { User } from '@open-archiver/types';
import type { IUserService } from './AuthService';
// This is a mock implementation of the IUserService.
// In a real application, this service would interact with a database.
// Later on, this service would interact with a database.
export class AdminUserService implements IUserService {
#users: User[] = [];
@@ -24,7 +24,7 @@ export class AdminUserService implements IUserService {
}
public async findByEmail(email: string): Promise<User | null> {
// In a real implementation, this would be a database query.
// once user service is ready, this would be a database query.
const user = this.#users.find(u => u.email === email);
return user || null;
}

154
packages/backend/src/services/ingestion-connectors/GoogleWorkspaceConnector.ts
View File

@@ -80,7 +80,7 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
return true;
} catch (error) {
logger.error({ err: error }, 'Failed to verify Google Workspace connection');
return false;
throw error;
}
}
@@ -165,41 +165,49 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
if (historyRecord.messagesAdded) {
for (const messageAdded of historyRecord.messagesAdded) {
if (messageAdded.message?.id) {
const msgResponse = await gmail.users.messages.get({
userId: 'me',
id: messageAdded.message.id,
format: 'RAW'
});
try {
const msgResponse = await gmail.users.messages.get({
userId: 'me',
id: messageAdded.message.id,
format: 'RAW'
});
if (msgResponse.data.raw) {
const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
const parsedEmail: ParsedMail = await simpleParser(rawEmail);
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
filename: attachment.filename || 'untitled',
contentType: attachment.contentType,
size: attachment.size,
content: attachment.content as Buffer
}));
const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
if (!addresses) return [];
const addressArray = Array.isArray(addresses) ? addresses : [addresses];
return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
};
yield {
id: msgResponse.data.id!,
userEmail: userEmail,
eml: rawEmail,
from: mapAddresses(parsedEmail.from),
to: mapAddresses(parsedEmail.to),
cc: mapAddresses(parsedEmail.cc),
bcc: mapAddresses(parsedEmail.bcc),
subject: parsedEmail.subject || '',
body: parsedEmail.text || '',
html: parsedEmail.html || '',
headers: parsedEmail.headers,
attachments,
receivedAt: parsedEmail.date || new Date(),
};
if (msgResponse.data.raw) {
const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
const parsedEmail: ParsedMail = await simpleParser(rawEmail);
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
filename: attachment.filename || 'untitled',
contentType: attachment.contentType,
size: attachment.size,
content: attachment.content as Buffer
}));
const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
if (!addresses) return [];
const addressArray = Array.isArray(addresses) ? addresses : [addresses];
return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
};
yield {
id: msgResponse.data.id!,
userEmail: userEmail,
eml: rawEmail,
from: mapAddresses(parsedEmail.from),
to: mapAddresses(parsedEmail.to),
cc: mapAddresses(parsedEmail.cc),
bcc: mapAddresses(parsedEmail.bcc),
subject: parsedEmail.subject || '',
body: parsedEmail.text || '',
html: parsedEmail.html || '',
headers: parsedEmail.headers,
attachments,
receivedAt: parsedEmail.date || new Date(),
};
}
} catch (error: any) {
if (error.code === 404) {
logger.warn({ messageId: messageAdded.message.id, userEmail }, 'Message not found, skipping.');
} else {
throw error;
}
}
}
}
@@ -229,41 +237,49 @@ export class GoogleWorkspaceConnector implements IEmailConnector {
for (const message of messages) {
if (message.id) {
const msgResponse = await gmail.users.messages.get({
userId: 'me',
id: message.id,
format: 'RAW'
});
try {
const msgResponse = await gmail.users.messages.get({
userId: 'me',
id: message.id,
format: 'RAW'
});
if (msgResponse.data.raw) {
const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
const parsedEmail: ParsedMail = await simpleParser(rawEmail);
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
filename: attachment.filename || 'untitled',
contentType: attachment.contentType,
size: attachment.size,
content: attachment.content as Buffer
}));
const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
if (!addresses) return [];
const addressArray = Array.isArray(addresses) ? addresses : [addresses];
return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
};
yield {
id: msgResponse.data.id!,
userEmail: userEmail,
eml: rawEmail,
from: mapAddresses(parsedEmail.from),
to: mapAddresses(parsedEmail.to),
cc: mapAddresses(parsedEmail.cc),
bcc: mapAddresses(parsedEmail.bcc),
subject: parsedEmail.subject || '',
body: parsedEmail.text || '',
html: parsedEmail.html || '',
headers: parsedEmail.headers,
attachments,
receivedAt: parsedEmail.date || new Date(),
};
if (msgResponse.data.raw) {
const rawEmail = Buffer.from(msgResponse.data.raw, 'base64url');
const parsedEmail: ParsedMail = await simpleParser(rawEmail);
const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
filename: attachment.filename || 'untitled',
contentType: attachment.contentType,
size: attachment.size,
content: attachment.content as Buffer
}));
const mapAddresses = (addresses: AddressObject | AddressObject[] | undefined): EmailAddress[] => {
if (!addresses) return [];
const addressArray = Array.isArray(addresses) ? addresses : [addresses];
return addressArray.flatMap(a => a.value.map(v => ({ name: v.name, address: v.address || '' })));
};
yield {
id: msgResponse.data.id!,
userEmail: userEmail,
eml: rawEmail,
from: mapAddresses(parsedEmail.from),
to: mapAddresses(parsedEmail.to),
cc: mapAddresses(parsedEmail.cc),
bcc: mapAddresses(parsedEmail.bcc),
subject: parsedEmail.subject || '',
body: parsedEmail.text || '',
html: parsedEmail.html || '',
headers: parsedEmail.headers,
attachments,
receivedAt: parsedEmail.date || new Date(),
};
}
} catch (error: any) {
if (error.code === 404) {
logger.warn({ messageId: message.id, userEmail }, 'Message not found during initial import, skipping.');
} else {
throw error;
}
}
}
}

7
packages/backend/src/services/ingestion-connectors/ImapConnector.ts
View File

@@ -38,9 +38,12 @@ export class ImapConnector implements IEmailConnector {
try {
await this.client.connect();
this.isConnected = true;
} catch (err) {
} catch (err: any) {
this.isConnected = false;
logger.error({ err }, 'IMAP connection failed');
if (err.responseText) {
throw new Error(`IMAP Connection Error: ${err.responseText}`);
}
throw err;
}
}
@@ -62,7 +65,7 @@ export class ImapConnector implements IEmailConnector {
return true;
} catch (error) {
logger.error({ error }, 'Failed to verify IMAP connection');
return false;
throw error;
}
}

2
packages/backend/src/services/ingestion-connectors/MicrosoftConnector.ts
View File

@@ -90,7 +90,7 @@ export class MicrosoftConnector implements IEmailConnector {
return true;
} catch (error) {
logger.error({ err: error }, 'Failed to verify Microsoft 365 connection');
return false;
throw error;
}
}

9
packages/frontend/package.json
View File

@@ -15,15 +15,16 @@
"lint": "prettier --check ."
},
"dependencies": {
"@iconify/svelte": "^5.0.1",
"@open-archiver/types": "workspace:*",
"@sveltejs/kit": "^2.16.0",
"bits-ui": "^2.8.10",
"clsx": "^2.1.1",
"d3-shape": "^3.2.0",
"jose": "^6.0.1",
"lucide-svelte": "^0.525.0",
"postal-mime": "^2.4.4",
"svelte-persisted-store": "^0.12.0",
"@sveltejs/kit": "^2.16.0",
"bits-ui": "^2.8.10",
"clsx": "^2.1.1",
"tailwind-merge": "^3.3.1",
"tailwind-variants": "^1.0.0"
},
@@ -37,11 +38,13 @@
"@types/d3-shape": "^3.1.7",
"dotenv": "^17.2.0",
"layerchart": "2.0.0-next.27",
"mode-watcher": "^1.1.0",
"prettier": "^3.4.2",
"prettier-plugin-svelte": "^3.3.3",
"prettier-plugin-tailwindcss": "^0.6.11",
"svelte": "^5.0.0",
"svelte-check": "^4.0.0",
"svelte-sonner": "^1.0.5",
"tailwindcss": "^4.0.0",
"tw-animate-css": "^1.3.5",
"typescript": "^5.0.0",

3
packages/frontend/src/lib/components/custom/Footer.svelte
View File

@@ -4,7 +4,8 @@
>
<div class="flex flex-col items-center gap-2">
<p class=" text-balance text-center text-xs font-medium leading-loose">
© {new Date().getFullYear()} Open Archiver. All rights reserved.
© {new Date().getFullYear()}
<a href="https://openarchiver.com/" target="_blank">Open Archiver</a>. All rights reserved.
</p>
</div>
</div>

123
packages/frontend/src/lib/components/custom/alert/Alerts.svelte Normal file
View File

@@ -0,0 +1,123 @@
<script lang="ts">
import type { AlertType } from './alert-state.svelte';
import { initialAlertState, setAlert } from './alert-state.svelte';
let { type, title, message, duration, show = false }: AlertType = $props();
import { fade, fly } from 'svelte/transition';
import { bounceIn } from 'svelte/easing';
import Icon from '@iconify/svelte';
let timeout: NodeJS.Timeout;
let styleConfig: {
icon: string;
color: string;
messageColor: string;
bgColor: string;
} = $state({
icon: 'heroicons-outline:check-circle',
color: 'text-green-800',
messageColor: 'text-green-700',
bgColor: 'text-green-50'
});
$effect(() => {
show;
if (show) {
timeout = scheduleHide();
}
});
$effect(() => {
type;
if (type === 'success') {
styleConfig = {
icon: 'heroicons-outline:check-circle',
color: 'text-green-600',
messageColor: 'text-green-500',
bgColor: 'bg-green-50'
};
} else if (type === 'error') {
styleConfig = {
icon: 'heroicons-outline:exclamation-circle',
color: 'text-yellow-600',
messageColor: 'text-yellow-600',
bgColor: 'bg-yellow-50'
};
} else if (type === 'warning') {
styleConfig = {
icon: 'heroicons-outline:exclamation',
color: 'text-yellow-600',
messageColor: 'text-yellow-600',
bgColor: 'bg-yellow-50'
};
}
});
function scheduleHide(): NodeJS.Timeout {
return setTimeout(() => {
setAlert(initialAlertState);
}, duration);
}
</script>
<!-- Global notification live region, render this permanently at the end of the document -->
{#if show}
<div
aria-live="assertive"
class="pointer-events-none fixed inset-0 flex px-4 py-6 items-start sm:p-6 z-999999"
in:fly={{ easing: bounceIn, x: 1000, duration: 500 }}
out:fade={{ duration: 100 }}
role="alert"
onmouseenter={() => {
clearTimeout(timeout);
}}
onmouseleave={() => {
timeout = scheduleHide();
}}
>
<div class="flex w-full flex-col items-center space-y-4 sm:items-end">
<!--
Notification panel, dynamically insert this into the live region when it needs to be displayed
Entering: "transform ease-out duration-300 transition"
From: "translate-y-2 opacity-0 sm:translate-y-0 sm:translate-x-2"
To: "translate-y-0 opacity-100 sm:translate-x-0"
Leaving: "transition ease-in duration-100"
From: "opacity-100"
To: "opacity-0"
-->
<div
class=" pointer-events-auto w-full max-w-sm overflow-hidden rounded-lg shadow-lg ring-1 ring-black/5 {styleConfig.bgColor} "
>
<div class="p-4">
<div class="flex items-start">
<div class="shrink-0">
<Icon icon={styleConfig.icon} class="size-6 {styleConfig.color}"></Icon>
</div>
<div class="ml-3 w-0 flex-1 pt-0.5">
<p class="text-sm font-medium {styleConfig.color}">{title}</p>
<p class="mt-1 text-sm {styleConfig.messageColor}">{message}</p>
</div>
<div class="ml-4 flex shrink-0">
<button
type="button"
class="inline-flex rounded-md {styleConfig.color} cursor-pointer"
onclick={() => {
setAlert(initialAlertState);
}}
>
<span class="sr-only">Close</span>
<svg
class="size-5"
viewBox="0 0 20 20"
fill="currentColor"
aria-hidden="true"
data-slot="icon"
>
<path
d="M6.28 5.22a.75.75 0 0 0-1.06 1.06L8.94 10l-3.72 3.72a.75.75 0 1 0 1.06 1.06L10 11.06l3.72 3.72a.75.75 0 1 0 1.06-1.06L11.06 10l3.72-3.72a.75.75 0 0 0-1.06-1.06L10 8.94 6.28 5.22Z"
/>
</svg>
</button>
</div>
</div>
</div>
</div>
</div>
</div>
{/if}

25
packages/frontend/src/lib/components/custom/alert/alert-state.svelte.ts Normal file
View File

@@ -0,0 +1,25 @@
export type AlertType = {
type: 'success' | 'warning' | 'error';
title: string;
message: string;
duration: number;
show: boolean;
};
export const initialAlertState: AlertType = {
type: 'success',
title: '',
message: '',
duration: 0,
show: false
};
let alertState = $state(initialAlertState);
export function setAlert(alert: AlertType) {
alertState = alert;
}
export function getAlert() {
return alertState;
}

29
packages/frontend/src/lib/components/ui/hover-card/hover-card-content.svelte Normal file
View File

@@ -0,0 +1,29 @@
<script lang="ts">
import { LinkPreview as HoverCardPrimitive } from "bits-ui";
import { cn } from "$lib/utils.js";
let {
ref = $bindable(null),
class: className,
align = "center",
sideOffset = 4,
portalProps,
...restProps
}: HoverCardPrimitive.ContentProps & {
portalProps?: HoverCardPrimitive.PortalProps;
} = $props();
</script>
<HoverCardPrimitive.Portal {...portalProps}>
<HoverCardPrimitive.Content
bind:ref
data-slot="hover-card-content"
{align}
{sideOffset}
class={cn(
"bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 outline-hidden z-50 mt-3 w-64 rounded-md border p-4 shadow-md outline-none",
className
)}
{...restProps}
/>
</HoverCardPrimitive.Portal>

7
packages/frontend/src/lib/components/ui/hover-card/hover-card-trigger.svelte Normal file
View File

@@ -0,0 +1,7 @@
<script lang="ts">
import { LinkPreview as HoverCardPrimitive } from "bits-ui";
let { ref = $bindable(null), ...restProps }: HoverCardPrimitive.TriggerProps = $props();
</script>
<HoverCardPrimitive.Trigger bind:ref data-slot="hover-card-trigger" {...restProps} />

14
packages/frontend/src/lib/components/ui/hover-card/index.ts Normal file
View File

@@ -0,0 +1,14 @@
import { LinkPreview as HoverCardPrimitive } from "bits-ui";
import Content from "./hover-card-content.svelte";
import Trigger from "./hover-card-trigger.svelte";
const Root = HoverCardPrimitive.Root;
export {
Root,
Content,
Trigger,
Root as HoverCard,
Content as HoverCardContent,
Trigger as HoverCardTrigger,
};

1
packages/frontend/src/lib/components/ui/sonner/index.ts Normal file
View File

@@ -0,0 +1 @@
export { default as Toaster } from "./sonner.svelte";

13
packages/frontend/src/lib/components/ui/sonner/sonner.svelte Normal file
View File

@@ -0,0 +1,13 @@
<script lang="ts">
import { Toaster as Sonner, type ToasterProps as SonnerProps } from "svelte-sonner";
import { mode } from "mode-watcher";
let { ...restProps }: SonnerProps = $props();
</script>
<Sonner
theme={mode.current}
class="toaster group"
style="--normal-bg: var(--color-popover); --normal-text: var(--color-popover-foreground); --normal-border: var(--color-border);"
{...restProps}
/>

3
packages/frontend/src/routes/+layout.svelte
View File

@@ -4,6 +4,8 @@
import { theme } from '$lib/stores/theme.store';
import { browser } from '$app/environment';
import Footer from '$lib/components/custom/Footer.svelte';
import { getAlert } from '$lib/components/custom/alert/alert-state.svelte';
import Alerts from '$lib/components/custom/alert/Alerts.svelte';
let { data, children } = $props();
@@ -21,6 +23,7 @@
});
</script>
<Alerts {...getAlert()}></Alerts>
<div class="flex min-h-screen flex-col">
<main class="flex-1">
{@render children()}

81
packages/frontend/src/routes/dashboard/ingestions/+page.svelte
View File

@@ -10,9 +10,10 @@
import { api } from '$lib/api.client';
import type { IngestionSource, CreateIngestionSourceDto } from '@open-archiver/types';
import Badge from '$lib/components/ui/badge/badge.svelte';
import { setAlert } from '$lib/components/custom/alert/alert-state.svelte';
import * as HoverCard from '$lib/components/ui/hover-card/index.js';
let { data }: { data: PageData } = $props();
let ingestionSources = $state(data.ingestionSources);
let isDialogOpen = $state(false);
let isDeleteDialogOpen = $state(false);
@@ -81,26 +82,48 @@
};
const handleFormSubmit = async (formData: CreateIngestionSourceDto) => {
if (selectedSource) {
// Update
const response = await api(`/ingestion-sources/${selectedSource.id}`, {
method: 'PUT',
body: JSON.stringify(formData)
try {
if (selectedSource) {
// Update
const response = await api(`/ingestion-sources/${selectedSource.id}`, {
method: 'PUT',
body: JSON.stringify(formData)
});
if (!response.ok) {
const errorData = await response.json();
throw new Error(errorData.message || 'Failed to update source.');
}
const updatedSource = await response.json();
ingestionSources = ingestionSources.map((s) =>
s.id === updatedSource.id ? updatedSource : s
);
} else {
// Create
const response = await api('/ingestion-sources', {
method: 'POST',
body: JSON.stringify(formData)
});
if (!response.ok) {
const errorData = await response.json();
throw new Error(errorData.message || 'Failed to create source.');
}
const newSource = await response.json();
ingestionSources = [...ingestionSources, newSource];
}
isDialogOpen = false;
} catch (error) {
let message = 'An unknown error occurred.';
if (error instanceof Error) {
message = error.message;
}
setAlert({
type: 'error',
title: 'Authentication Failed',
message,
duration: 5000,
show: true
});
const updatedSource = await response.json();
ingestionSources = ingestionSources.map((s) =>
s.id === updatedSource.id ? updatedSource : s
);
} else {
// Create
const response = await api('/ingestion-sources', {
method: 'POST',
body: JSON.stringify(formData)
});
const newSource = await response.json();
ingestionSources = [...ingestionSources, newSource];
}
isDialogOpen = false;
};
function getStatusClasses(status: IngestionSource['status']): string {
@@ -156,9 +179,21 @@
</Table.Cell>
<Table.Cell class="capitalize">{source.provider.split('_').join(' ')}</Table.Cell>
<Table.Cell class="min-w-24">
<Badge class="{getStatusClasses(source.status)} capitalize">
{source.status.split('_').join(' ')}
</Badge>
<HoverCard.Root>
<HoverCard.Trigger>
<Badge class="{getStatusClasses(source.status)} cursor-pointer capitalize">
{source.status.split('_').join(' ')}
</Badge>
</HoverCard.Trigger>
<HoverCard.Content class="{getStatusClasses(source.status)} ">
<div class="flex flex-col space-y-4 text-sm">
<p class=" font-mono">
<b>Last sync message:</b>
{source.lastSyncStatusMessage || 'Empty'}
</p>
</div>
</HoverCard.Content>
</HoverCard.Root>
</Table.Cell>
<Table.Cell>
<Switch
@@ -166,7 +201,7 @@
class="cursor-pointer"
checked={source.status !== 'paused'}
onCheckedChange={() => handleToggle(source)}
disabled={source.status !== 'active' && source.status !== 'paused'}
disabled={source.status === 'importing' || source.status === 'syncing'}
/>
</Table.Cell>
<Table.Cell>{new Date(source.createdAt).toLocaleDateString()}</Table.Cell>

39
packages/frontend/src/routes/signin/+page.svelte
View File

@@ -7,23 +7,28 @@
import { api } from '$lib/api.client';
import { authStore } from '$lib/stores/auth.store';
import type { LoginResponse } from '@open-archiver/types';
import { setAlert } from '$lib/components/custom/alert/alert-state.svelte';
let email = '';
let password = '';
let error: string | null = null;
let isLoading = false;
async function handleSubmit() {
isLoading = true;
error = null;
try {
const response = await api('/auth/login', {
method: 'POST',
body: JSON.stringify({ email, password })
});
if (!response.ok) {
const errorData = await response.json();
throw new Error(errorData.message || 'Failed to login');
let errorMessage = 'Failed to login';
try {
const errorData = await response.json();
errorMessage = errorData.message || errorMessage;
} catch (e) {
errorMessage = response.statusText;
}
throw new Error(errorMessage);
}
const loginData: LoginResponse = await response.json();
@@ -31,7 +36,13 @@
// Redirect to a protected page after login
goto('/dashboard');
} catch (e: any) {
error = e.message;
setAlert({
type: 'error',
title: 'Login Failed',
message: e.message,
duration: 5000,
show: true
});
} finally {
isLoading = false;
}
@@ -43,7 +54,19 @@
<meta name="description" content="Login to your Open Archiver account." />
</svelte:head>
<div class="flex min-h-screen items-center justify-center bg-gray-100 dark:bg-gray-900">
<div
class="flex min-h-screen flex-col items-center justify-center space-y-16 bg-gray-100 dark:bg-gray-900"
>
<div>
<a
href="https://openarchiver.com/"
target="_blank"
class="flex flex-row items-center gap-2 font-bold"
>
<img src="/logos/logo-sq.svg" alt="OpenArchiver Logo" class="h-16 w-16" />
<span class="text-2xl">Open Archiver</span>
</a>
</div>
<Card.Root class="w-full max-w-md">
<Card.Header class="space-y-1">
<Card.Title class="text-2xl">Login</Card.Title>
@@ -60,10 +83,6 @@
<Input id="password" type="password" bind:value={password} required />
</div>
{#if error}
<p class="mt-2 text-sm text-red-600">{error}</p>
{/if}
<Button type="submit" class=" w-full" disabled={isLoading}>
{isLoading ? 'Logging in...' : 'Login'}
</Button>

6
packages/types/src/ingestion.types.ts
View File

@@ -116,3 +116,9 @@ export type MailboxUser = {
primaryEmail: string;
displayName: string;
};
export type ProcessMailboxError = {
error: boolean;
message: string;
};

74
pnpm-lock.yaml generated
View File

@@ -151,6 +151,9 @@ importers:
packages/frontend:
dependencies:
'@iconify/svelte':
specifier: ^5.0.1
version: 5.0.1(svelte@5.35.5)
'@open-archiver/types':
specifier: workspace:*
version: link:../types
@@ -212,6 +215,9 @@ importers:
layerchart:
specifier: 2.0.0-next.27
version: 2.0.0-next.27(svelte@5.35.5)
mode-watcher:
specifier: ^1.1.0
version: 1.1.0(svelte@5.35.5)
prettier:
specifier: ^3.4.2
version: 3.6.2
@@ -227,6 +233,9 @@ importers:
svelte-check:
specifier: ^4.0.0
version: 4.2.2(picomatch@4.0.2)(svelte@5.35.5)(typescript@5.8.3)
svelte-sonner:
specifier: ^1.0.5
version: 1.0.5(svelte@5.35.5)
tailwindcss:
specifier: ^4.0.0
version: 4.1.11
@@ -1014,6 +1023,11 @@ packages:
'@iconify-json/simple-icons@1.2.44':
resolution: {integrity: sha512-CdWgSPygwDlDbKtDWjvi3NtUefnkoepXv90n3dQxJerqzD9kI+nEJOiWUBM+eOyMYQKtxBpLWFBrgeotF0IZKw==}
'@iconify/svelte@5.0.1':
resolution: {integrity: sha512-nR1ApafyeRDcsx6ytd+0mzaY0ASYfG5YBW9dN8rUWxwi792/HX401qAZEkXSjT0iHm2Dgua+PwXDs/3ttJhkqQ==}
peerDependencies:
svelte: '>4.0.0'
'@iconify/types@2.0.0':
resolution: {integrity: sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==}
@@ -1365,6 +1379,7 @@ packages:
'@smithy/middleware-endpoint@4.1.14':
resolution: {integrity: sha512-+BGLpK5D93gCcSEceaaYhUD/+OCGXM1IDaq/jKUQ+ujB0PTWlWN85noodKw/IPFZhIKFCNEe19PGd/reUMeLSQ==}
engines: {node: '>=18.0.0'}
deprecated: Please upgrade to @smithy/middleware-endpoint@4.1.15 or higher to fix a bug preventing the resolution of ENV and config file custom endpoints https://github.com/smithy-lang/smithy-typescript/issues/1645
'@smithy/middleware-retry@4.1.15':
resolution: {integrity: sha512-iKYUJpiyTQ33U2KlOZeUb0GwtzWR3C0soYcKuCnTmJrvt6XwTPQZhMfsjJZNw7PpQ3TU4Ati1qLSrkSJxnnSMQ==}
@@ -3357,6 +3372,11 @@ packages:
engines: {node: '>=10'}
hasBin: true
mode-watcher@1.1.0:
resolution: {integrity: sha512-mUT9RRGPDYenk59qJauN1rhsIMKBmWA3xMF+uRwE8MW/tjhaDSCCARqkSuDTq8vr4/2KcAxIGVjACxTjdk5C3g==}
peerDependencies:
svelte: ^5.27.0
mri@1.2.0:
resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==}
engines: {node: '>=4'}
@@ -3839,6 +3859,16 @@ packages:
resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==}
engines: {node: '>= 18'}
runed@0.23.4:
resolution: {integrity: sha512-9q8oUiBYeXIDLWNK5DfCWlkL0EW3oGbk845VdKlPeia28l751VpfesaB/+7pI6rnbx1I6rqoZ2fZxptOJLxILA==}
peerDependencies:
svelte: ^5.7.0
runed@0.25.0:
resolution: {integrity: sha512-7+ma4AG9FT2sWQEA0Egf6mb7PBT2vHyuHail1ie8ropfSjvZGtEAx8YTmUjv/APCsdRRxEVvArNjALk9zFSOrg==}
peerDependencies:
svelte: ^5.7.0
runed@0.28.0:
resolution: {integrity: sha512-k2xx7RuO9hWcdd9f+8JoBeqWtYrm5CALfgpkg2YDB80ds/QE4w0qqu34A7fqiAwiBBSBQOid7TLxwxVC27ymWQ==}
peerDependencies:
@@ -4098,6 +4128,17 @@ packages:
peerDependencies:
svelte: ^3.48.0 || ^4 || ^5
svelte-sonner@1.0.5:
resolution: {integrity: sha512-9dpGPFqKb/QWudYqGnEz93vuY+NgCEvyNvxoCLMVGw6sDN/3oVeKV1xiEirW2E1N3vJEyj5imSBNOGltQHA7mg==}
peerDependencies:
svelte: ^5.0.0
svelte-toolbelt@0.7.1:
resolution: {integrity: sha512-HcBOcR17Vx9bjaOceUvxkY3nGmbBmCBBbuWLLEWO6jtmWH8f/QoWmbyUfQZrpDINH39en1b8mptfPQT9VKQ1xQ==}
engines: {node: '>=18', pnpm: '>=8.7.0'}
peerDependencies:
svelte: ^5.0.0
svelte-toolbelt@0.9.3:
resolution: {integrity: sha512-HCSWxCtVmv+c6g1ACb8LTwHVbDqLKJvHpo6J8TaqwUme2hj9ATJCpjCPNISR1OCq2Q4U1KT41if9ON0isINQZw==}
engines: {node: '>=18', pnpm: '>=8.7.0'}
@@ -5386,6 +5427,11 @@ snapshots:
dependencies:
'@iconify/types': 2.0.0
'@iconify/svelte@5.0.1(svelte@5.35.5)':
dependencies:
'@iconify/types': 2.0.0
svelte: 5.35.5
'@iconify/types@2.0.0': {}
'@internationalized/date@3.8.2':
@@ -7985,6 +8031,12 @@ snapshots:
mkdirp@3.0.1: {}
mode-watcher@1.1.0(svelte@5.35.5):
dependencies:
runed: 0.25.0(svelte@5.35.5)
svelte: 5.35.5
svelte-toolbelt: 0.7.1(svelte@5.35.5)
mri@1.2.0: {}
mrmime@2.0.1: {}
@@ -8434,6 +8486,16 @@ snapshots:
transitivePeerDependencies:
- supports-color
runed@0.23.4(svelte@5.35.5):
dependencies:
esm-env: 1.2.2
svelte: 5.35.5
runed@0.25.0(svelte@5.35.5):
dependencies:
esm-env: 1.2.2
svelte: 5.35.5
runed@0.28.0(svelte@5.35.5):
dependencies:
esm-env: 1.2.2
@@ -8737,6 +8799,18 @@ snapshots:
dependencies:
svelte: 5.35.5
svelte-sonner@1.0.5(svelte@5.35.5):
dependencies:
runed: 0.28.0(svelte@5.35.5)
svelte: 5.35.5
svelte-toolbelt@0.7.1(svelte@5.35.5):
dependencies:
clsx: 2.1.1
runed: 0.23.4(svelte@5.35.5)
style-to-object: 1.0.9
svelte: 5.35.5
svelte-toolbelt@0.9.3(svelte@5.35.5):
dependencies:
clsx: 2.1.1