Role based access (#61)

* Format checked, contributing.md update

* Middleware setup

* IAP API, create user/roles in frontend

* RBAC using CASL library

* Switch to CASL, secure search, resource-level access control

* Remove inherent behavior, index userEmail, adding docs for IAM policies

* Format

---------

Co-authored-by: Wayne <5291640+ringoinca@users.noreply.github.com>

docs/services/iam-service.md

@@ -0,0 +1,289 @@
+# IAM Policies
+
+This document provides a guide to creating and managing IAM policies in Open Archiver. It is intended for developers and administrators who need to configure granular access control for users and roles.
+
+## Policy Structure
+
+IAM policies are defined as an array of JSON objects, where each object represents a single permission rule. The structure of a policy object is as follows:
+
+```json
+{
+	"action": "read" OR ["read", "create"],
+	"subject": "ingestion" OR ["ingestion", "dashboard"],
+	"conditions": {
+		"field_name": "value"
+	},
+	"inverted": false OR true,
+}
+```
+
+- `action`: The action(s) to be performed on the subject. Can be a single string or an array of strings.
+- `subject`: The resource(s) or entity on which the action is to be performed. Can be a single string or an array of strings.
+- `conditions`: (Optional) A set of conditions that must be met for the permission to be granted.
+- `inverted`: (Optional) When set to `true`, this inverts the rule, turning it from a "can" rule into a "cannot" rule. This is useful for creating exceptions to broader permissions.
+
+## Actions
+
+The following actions are available for use in IAM policies:
+
+- `manage`: A wildcard action that grants all permissions on a subject (`create`, `read`, `update`, `delete`, `search`, `sync`).
+- `create`: Allows the user to create a new resource.
+- `read`: Allows the user to view a resource.
+- `update`: Allows the user to modify an existing resource.
+- `delete`: Allows the user to delete a resource.
+- `search`: Allows the user to search for resources.
+- `sync`: Allows the user to synchronize a resource.
+
+## Subjects
+
+The following subjects are available for use in IAM policies:
+
+- `all`: A wildcard subject that represents all resources.
+- `archive`: Represents archived emails.
+- `ingestion`: Represents ingestion sources.
+- `settings`: Represents system settings.
+- `users`: Represents user accounts.
+- `roles`: Represents user roles.
+- `dashboard`: Represents the dashboard.
+
+## Advanced Conditions with MongoDB-Style Queries
+
+Conditions are the key to creating fine-grained access control rules. They are defined as a JSON object where each key represents a field on the subject, and the value defines the criteria for that field.
+
+All conditions within a single rule are implicitly joined with an **AND** logic. This means that for a permission to be granted, the resource must satisfy _all_ specified conditions.
+
+The power of this system comes from its use of a subset of [MongoDB's query language](https://www.mongodb.com/docs/manual/), which provides a flexible and expressive way to define complex rules. These rules are translated into native queries for both the PostgreSQL database (via Drizzle ORM) and the Meilisearch engine.
+
+### Supported Operators and Examples
+
+Here is a detailed breakdown of the supported operators with examples.
+
+#### `$eq` (Equal)
+
+This is the default operator. If you provide a simple key-value pair, it is treated as an equality check.
+
+```json
+// This rule...
+{ "status": "active" }
+
+// ...is equivalent to this:
+{ "status": { "$eq": "active" } }
+```
+
+**Use Case**: Grant access to an ingestion source only if its status is `active`.
+
+#### `$ne` (Not Equal)
+
+Matches documents where the field value is not equal to the specified value.
+
+```json
+{ "provider": { "$ne": "pst_import" } }
+```
+
+**Use Case**: Allow a user to see all ingestion sources except for PST imports.
+
+#### `$in` (In Array)
+
+Matches documents where the field value is one of the values in the specified array.
+
+```json
+{
+	"id": {
+		"$in": ["INGESTION_ID_1", "INGESTION_ID_2"]
+	}
+}
+```
+
+**Use Case**: Grant an auditor access to a specific list of ingestion sources.
+
+#### `$nin` (Not In Array)
+
+Matches documents where the field value is not one of the values in the specified array.
+
+```json
+{ "provider": { "$nin": ["pst_import", "eml_import"] } }
+```
+
+**Use Case**: Hide all manual import sources from a specific user role.
+
+#### `$lt` / `$lte` (Less Than / Less Than or Equal)
+
+Matches documents where the field value is less than (`$lt`) or less than or equal to (`$lte`) the specified value. This is useful for numeric or date-based comparisons.
+
+```json
+{ "sentAt": { "$lt": "2024-01-01T00:00:00.000Z" } }
+```
+
+#### `$gt` / `$gte` (Greater Than / Greater Than or Equal)
+
+Matches documents where the field value is greater than (`$gt`) or greater than or equal to (`$gte`) the specified value.
+
+```json
+{ "sentAt": { "$lt": "2024-01-01T00:00:00.000Z" } }
+```
+
+#### `$exists`
+
+Matches documents that have (or do not have) the specified field.
+
+```json
+// Grant access only if a 'lastSyncStatusMessage' exists
+{ "lastSyncStatusMessage": { "$exists": true } }
+```
+
+## Inverted Rules: Creating Exceptions with `cannot`
+
+By default, all rules are "can" rules, meaning they grant permissions. However, you can create a "cannot" rule by adding `"inverted": true` to a policy object. This is extremely useful for creating exceptions to broader permissions.
+
+A common pattern is to grant broad access and then use an inverted rule to carve out a specific restriction.
+
+**Use Case**: Grant a user access to all ingestion sources _except_ for one specific source.
+
+This is achieved with two rules:
+
+1.  A "can" rule that grants `read` access to the `ingestion` subject.
+2.  An inverted "cannot" rule that denies `read` access for the specific ingestion `id`.
+
+```json
+[
+	{
+		"action": "read",
+		"subject": "ingestion"
+	},
+	{
+		"inverted": true,
+		"action": "read",
+		"subject": "ingestion",
+		"conditions": {
+			"id": "SPECIFIC_INGESTION_ID_TO_EXCLUDE"
+		}
+	}
+]
+```
+
+## Policy Evaluation Logic
+
+The system evaluates policies by combining all relevant rules for a user. The logic is simple:
+
+- A user has permission if at least one `can` rule allows it.
+- A permission is denied if a `cannot` (`"inverted": true`) rule explicitly forbids it, even if a `can` rule allows it. `cannot` rules always take precedence.
+
+### Dynamic Policies with Placeholders
+
+To create dynamic policies that are specific to the current user, you can use the `${user.id}` placeholder in the `conditions` object. This placeholder will be replaced with the ID of the current user at runtime.
+
+## Special Permissions for User and Role Management
+
+It is important to note that while `read` access to `users` and `roles` can be granted granularly, any actions that modify these resources (`create`, `update`, `delete`) are restricted to Super Admins.
+
+A user must have the `{ "action": "manage", "subject": "all" }` permission (Typically a Super Admin role) to manage users and roles. This is a security measure to prevent unauthorized changes to user accounts and permissions.
+
+## Policy Examples
+
+Here are several examples based on the default roles in the system, demonstrating how to combine actions, subjects, and conditions to achieve specific access control scenarios.
+
+### Administrator
+
+This policy grants a user full access to all resources using wildcards.
+
+```json
+[
+	{
+		"action": "manage",
+		"subject": "all"
+	}
+]
+```
+
+### End-User
+
+This policy allows a user to view the dashboard, create new ingestion sources, and fully manage the ingestion sources they own.
+
+```json
+[
+	{
+		"action": "read",
+		"subject": "dashboard"
+	},
+	{
+		"action": "create",
+		"subject": "ingestion"
+	},
+	{
+		"action": "manage",
+		"subject": "ingestion",
+		"conditions": {
+			"userId": "${user.id}"
+		}
+	},
+	{
+		"action": "manage",
+		"subject": "archive",
+		"conditions": {
+			"ingestionSource.userId": "${user.id}" // also needs to give permission to archived emails created by the user
+		}
+	}
+]
+```
+
+### Global Read-Only Auditor
+
+This policy grants read and search access across most of the application's resources, making it suitable for an auditor who needs to view data without modifying it.
+
+```json
+[
+	{
+		"action": ["read", "search"],
+		"subject": ["ingestion", "archive", "dashboard", "users", "roles"]
+	}
+]
+```
+
+### Ingestion Admin
+
+This policy grants full control over all ingestion sources and archives, but no other resources.
+
+```json
+[
+	{
+		"action": "manage",
+		"subject": "ingestion"
+	}
+]
+```
+
+### Auditor for Specific Ingestion Sources
+
+This policy demonstrates how to grant access to a specific list of ingestion sources using the `$in` operator.
+
+```json
+[
+	{
+		"action": ["read", "search"],
+		"subject": "ingestion",
+		"conditions": {
+			"id": {
+				"$in": ["INGESTION_ID_1", "INGESTION_ID_2"]
+			}
+		}
+	}
+]
+```
+
+### Limit Access to a Specific Mailbox
+
+This policy grants a user access to a specific ingestion source, but only allows them to see emails belonging to a single user within that source.
+
+This is achieved by defining two specific `can` rules: The rule grants `read` and `search` access to the `archive` subject, but the `userEmail` must match.
+
+```json
+[
+	{
+		"action": ["read", "search"],
+		"subject": "archive",
+		"conditions": {
+			"userEmail": "user1@example.com"
+		}
+	}
+]
+```

packages/backend/src/api/controllers/auth.controller.ts

@@ -75,17 +75,15 @@ export class AuthController {
 
 	public status = async (req: Request, res: Response): Promise<Response> => {
 		try {
-			const users = await db
-				.select()
-				.from(schema.users);
+			const users = await db.select().from(schema.users);
 
 			/**
-			 * Check the situation where the only user has "Super Admin" role, but they don't actually have Super Admin permission because the role was set up in an earlier version, we need to change that "Super Admin" role to a the one used in the current version.
+			 * Check the situation where the only user has "Super Admin" role, but they don't actually have Super Admin permission because the role was set up in an earlier version, we need to change that "Super Admin" role to the one used in the current version.
 			 */
 			if (users.length === 1) {
-				const iamService = new IamService()
-				const userRoles = await iamService.getRolesForUser(users[0].id)
-				if (userRoles.some(r => r.name === 'Super Admin')) {
+				const iamService = new IamService();
+				const userRoles = await iamService.getRolesForUser(users[0].id);
+				if (userRoles.some((r) => r.name === 'Super Admin')) {
 					const authorizationService = new AuthorizationService();
 					const hasAdminPermission = await authorizationService.can(
 						users[0].id,
@@ -99,12 +97,18 @@ export class AuthController {
 								subject: 'all',
 							},
 						];
-						await db.update(schema.roles).set({ policies: suerAdminPolicies }).where(eq(schema.roles.name, 'Super Admin'))
+						await db
+							.update(schema.roles)
+							.set({
+								policies: suerAdminPolicies,
+								slug: 'predefined_super_admin',
+							})
+							.where(eq(schema.roles.name, 'Super Admin'));
 					}
 				}
 			}
 			// in case user uses older version with admin user variables, we will create the admin user using those variables.
-			const needsSetupUser = users.length === 0
+			const needsSetupUser = users.length === 0;
 			if (needsSetupUser && process.env.ADMIN_EMAIL && process.env.ADMIN_PASSWORD) {
 				await this.#userService.createAdminUser(
 					{

packages/backend/src/api/controllers/iam.controller.ts

@@ -3,6 +3,7 @@ import { IamService } from '../../services/IamService';
 import { PolicyValidator } from '../../iam-policy/policy-validator';
 import type { CaslPolicy } from '@open-archiver/types';
 import { logger } from '../../config/logger';
+import { config } from '../../config';
 
 export class IamController {
 	#iamService: IamService;
@@ -40,7 +41,10 @@ export class IamController {
 		}
 	};
 
-	public createRole = async (req: Request, res: Response): Promise<void> => {
+	public createRole = async (req: Request, res: Response) => {
+		if (config.app.isDemo) {
+			return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
+		}
 		const { name, policies } = req.body;
 
 		if (!name || !policies) {
@@ -48,15 +52,14 @@ export class IamController {
 			return;
 		}
 
-		for (const statement of policies) {
-			const { valid, reason } = PolicyValidator.isValid(statement as CaslPolicy);
-			if (!valid) {
-				res.status(400).json({ message: `Invalid policy statement: ${reason}` });
-				return;
-			}
-		}
-
 		try {
+			for (const statement of policies) {
+				const { valid, reason } = PolicyValidator.isValid(statement as CaslPolicy);
+				if (!valid) {
+					res.status(400).json({ message: `Invalid policy statement: ${reason}` });
+					return;
+				}
+			}
 			const role = await this.#iamService.createRole(name, policies);
 			res.status(201).json(role);
 		} catch (error) {
@@ -65,7 +68,10 @@ export class IamController {
 		}
 	};
 
-	public deleteRole = async (req: Request, res: Response): Promise<void> => {
+	public deleteRole = async (req: Request, res: Response) => {
+		if (config.app.isDemo) {
+			return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
+		}
 		const { id } = req.params;
 
 		try {
@@ -76,7 +82,10 @@ export class IamController {
 		}
 	};
 
-	public updateRole = async (req: Request, res: Response): Promise<void> => {
+	public updateRole = async (req: Request, res: Response) => {
+		if (config.app.isDemo) {
+			return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
+		}
 		const { id } = req.params;
 		const { name, policies } = req.body;
 
@@ -109,14 +118,14 @@ export class IamController {
 			await this.#iamService.createRole(
 				'End user',
 				[
-					{
-						action: 'create',
-						subject: 'ingestion',
-					},
 					{
 						action: 'read',
 						subject: 'dashboard',
 					},
+					{
+						action: 'create',
+						subject: 'ingestion',
+					},
 					{
 						action: 'manage',
 						subject: 'ingestion',
@@ -124,6 +133,13 @@ export class IamController {
 							userId: '${user.id}',
 						},
 					},
+					{
+						action: 'manage',
+						subject: 'archive',
+						conditions: {
+							'ingestionSource.userId': '${user.id}',
+						},
+					},
 				],
 				'predefined_end_user'
 			);

packages/backend/src/api/controllers/user.controller.ts

@@ -3,6 +3,7 @@ import { UserService } from '../../services/UserService';
 import * as schema from '../../database/schema';
 import { sql } from 'drizzle-orm';
 import { db } from '../../database';
+import { config } from '../../config';
 
 const userService = new UserService();
 
@@ -20,6 +21,9 @@ export const getUser = async (req: Request, res: Response) => {
 };
 
 export const createUser = async (req: Request, res: Response) => {
+	if (config.app.isDemo) {
+		return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
+	}
 	const { email, first_name, last_name, password, roleId } = req.body;
 
 	const newUser = await userService.createUser(
@@ -30,6 +34,9 @@ export const createUser = async (req: Request, res: Response) => {
 };
 
 export const updateUser = async (req: Request, res: Response) => {
+	if (config.app.isDemo) {
+		return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
+	}
 	const { email, first_name, last_name, roleId } = req.body;
 	const updatedUser = await userService.updateUser(
 		req.params.id,
@@ -43,6 +50,9 @@ export const updateUser = async (req: Request, res: Response) => {
 };
 
 export const deleteUser = async (req: Request, res: Response) => {
+	if (config.app.isDemo) {
+		return res.status(403).json({ message: 'This operation is not allowed in demo mode.' });
+	}
 	const userCountResult = await db.select({ count: sql<number>`count(*)` }).from(schema.users);
 	console.log('iusercount,', userCountResult[0].count);
 	const isOnlyUser = Number(userCountResult[0].count) === 1;

packages/backend/src/iam-policy/ability.ts

@@ -17,7 +17,17 @@ export type SubjectObject =
 	| InferSelectModel<typeof users>
 	| InferSelectModel<typeof roles>
 	| AppSubjects;
+
+// Function to create an ability instance from policies stored in the database
+export function createAbilityFor(policies: CaslPolicy[]) {
+	// We will not expand policies, if a role needs access to ingestion X and its archived emails, the policy should also grant access to archives belonging to ingestion X
+	// const allPolicies = expandPolicies(policies);
+
+	return createMongoAbility<AppAbility>(policies as AppRawRule[]);
+}
+
 /**
+ * @deprecated This function should not be used since we don't need the inheritable behavior anymore.
  * Translates conditions on an 'ingestion' subject to equivalent conditions on an 'archive' subject.
  * This is used to implement inherent permissions, where permission on an ingestion source
  * implies permission on the emails it has ingested.
@@ -59,6 +69,7 @@ function translateIngestionConditionsToArchive(
 }
 
 /**
+ * @deprecated This function should not be used since we don't need the inheritable behavior anymore.
  * Expands the given set of policies to include inherent permissions.
  * For example, a permission on an 'ingestion' source is expanded to grant
  * the same permission on 'archive' records related to that source.
@@ -105,10 +116,3 @@ function expandPolicies(policies: CaslPolicy[]): CaslPolicy[] {
 
 	return expandedPolicies;
 }
-
-// Function to create an ability instance from policies stored in the database
-export function createAbilityFor(policies: CaslPolicy[]) {
-	const allPolicies = expandPolicies(policies);
-
-	return createMongoAbility<AppAbility>(allPolicies as AppRawRule[]);
-}

packages/backend/src/services/IndexingService.ts

@@ -66,7 +66,11 @@ export class IndexingService {
 				.where(eq(emailAttachments.emailId, emailId));
 		}
 
-		const document = await this.createEmailDocument(email, emailAttachmentsResult, email.userEmail);
+		const document = await this.createEmailDocument(
+			email,
+			emailAttachmentsResult,
+			email.userEmail
+		);
 		await this.searchService.addDocuments('emails', [document], 'id');
 	}
 
@@ -95,7 +99,7 @@ export class IndexingService {
 			archivedEmailId,
 			email.userEmail || ''
 		);
-		console.log(document)
+		console.log(document);
 		await this.searchService.addDocuments('emails', [document], 'id');
 	}
 
@@ -107,7 +111,7 @@ export class IndexingService {
 		attachments: AttachmentsType,
 		ingestionSourceId: string,
 		archivedEmailId: string,
-		userEmail: string //the owner of the email inbox 
+		userEmail: string //the owner of the email inbox
 	): Promise<EmailDocument> {
 		const extractedAttachments = [];
 		for (const attachment of attachments) {
@@ -125,7 +129,7 @@ export class IndexingService {
 				//  skip attachment or fail the job
 			}
 		}
-		console.log('email.userEmail', userEmail)
+		console.log('email.userEmail', userEmail);
 		return {
 			id: archivedEmailId,
 			userEmail: userEmail,
@@ -147,7 +151,7 @@ export class IndexingService {
 	private async createEmailDocument(
 		email: typeof archivedEmails.$inferSelect,
 		attachments: Attachment[],
-		userEmail: string,//the owner of the email inbox 
+		userEmail: string //the owner of the email inbox
 	): Promise<EmailDocument> {
 		const attachmentContents = await this.extractAttachmentContents(attachments);
 
@@ -161,7 +165,7 @@ export class IndexingService {
 			'';
 
 		const recipients = email.recipients as DbRecipients;
-		console.log('email.userEmail', email.userEmail)
+		console.log('email.userEmail', email.userEmail);
 		return {
 			id: email.id,
 			userEmail: userEmail,

packages/backend/src/services/IngestionService.ts

@@ -408,7 +408,7 @@ export class IngestionService {
 				storageService
 			);
 			//assign userEmail
-			email.userEmail = userEmail
+			email.userEmail = userEmail;
 			await indexingService.indexByEmail(email, source.id, archivedEmail.id);
 		} catch (error) {
 			logger.error({

packages/backend/src/services/UserService.ts

@@ -145,7 +145,7 @@ export class UserService {
 			})
 			.returning();
 
-		const superAdminRole = await this.createAdminRole()
+		const superAdminRole = await this.createAdminRole();
 
 		await db.insert(schema.userRoles).values({
 			userId: newUser[0].id,
@@ -179,6 +179,6 @@ export class UserService {
 					.returning()
 			)[0];
 		}
-		return superAdminRole
+		return superAdminRole;
 	}
 }

packages/frontend/src/routes/dashboard/archived-emails/+page.server.ts

@@ -11,13 +11,17 @@ export const load: PageServerLoad = async (event) => {
 
 	const sourcesResponse = await api('/ingestion-sources', event);
 	const sourcesResponseText = await sourcesResponse.json();
+	let ingestionSources: IngestionSource[] = sourcesResponseText;
 	if (!sourcesResponse.ok) {
-		return error(
-			sourcesResponseText.status,
-			sourcesResponseText.message || 'Failed to load ingestion source.'
-		);
+		if (sourcesResponse.status === 403) {
+			ingestionSources = [];
+		} else {
+			return error(
+				sourcesResponse.status,
+				sourcesResponseText.message || 'Failed to load ingestion source.'
+			);
+		}
 	}
-	const ingestionSources: IngestionSource[] = sourcesResponseText;
 
 	let archivedEmails: PaginatedArchivedEmails = {
 		items: [],