From 82a83a71e4e9ffef8dc3292e2bc3c9b1976141cd Mon Sep 17 00:00:00 2001 From: Wayne <5291640+ringoinca@users.noreply.github.com> Date: Wed, 13 Aug 2025 21:55:22 +0300 Subject: [PATCH 1/2] BODY_SIZE_LIMIT fix, database url encode --- .env.example | 2 +- docs/user-guides/installation.md | 32 +++++++++++++----------- packages/backend/src/database/index.ts | 4 ++- packages/backend/src/database/migrate.ts | 4 ++- packages/backend/src/helpers/db.ts | 12 +++++++++ packages/frontend/svelte.config.js | 6 ++--- 6 files changed, 38 insertions(+), 22 deletions(-) create mode 100644 packages/backend/src/helpers/db.ts diff --git a/.env.example b/.env.example index ed0d4a0..a7a243d 100644 --- a/.env.example +++ b/.env.example @@ -34,7 +34,7 @@ REDIS_TLS_ENABLED=false # Choose your storage backend. Valid options are 'local' or 's3'. STORAGE_TYPE=local # The maximum request body size to accept in bytes including while streaming. The body size can also be specified with a unit suffix for kilobytes (K), megabytes (M), or gigabytes (G). For example, 512K or 1M. Defaults to 512kb. Or the value of Infinity if you don't want any upload limit. -FRONTEND_BODY_SIZE_LIMIT=100M +BODY_SIZE_LIMIT=100M # --- Local Storage Settings --- # The path inside the container where files will be stored. diff --git a/docs/user-guides/installation.md b/docs/user-guides/installation.md index 4bac769..7b26955 100644 --- a/docs/user-guides/installation.md +++ b/docs/user-guides/installation.md @@ -65,11 +65,12 @@ Here is a complete list of environment variables available for configuration: #### Application Settings -| Variable | Description | Default Value | -| --------------- | ---------------------------------- | ------------- | -| `NODE_ENV` | The application environment. | `development` | -| `PORT_BACKEND` | The port for the backend service. | `4000` | -| `PORT_FRONTEND` | The port for the frontend service. | `3000` | +| Variable | Description | Default Value | +| ---------------- | ----------------------------------------------------------------------------------------------------- | ------------- | +| `NODE_ENV` | The application environment. | `development` | +| `PORT_BACKEND` | The port for the backend service. | `4000` | +| `PORT_FRONTEND` | The port for the frontend service. | `3000` | +| `SYNC_FREQUENCY` | The frequency of continuous email syncing. See [cron syntax](https://crontab.guru/) for more details. | `* * * * *` | #### Docker Compose Service Configuration @@ -90,16 +91,17 @@ These variables are used by `docker-compose.yml` to configure the services. #### Storage Settings -| Variable | Description | Default Value | -| ------------------------------ | ------------------------------------------------------------------------------------- | ------------------------- | -| `STORAGE_TYPE` | The storage backend to use (`local` or `s3`). | `local` | -| `STORAGE_LOCAL_ROOT_PATH` | The root path for local file storage. | `/var/data/open-archiver` | -| `STORAGE_S3_ENDPOINT` | The endpoint for S3-compatible storage (required if `STORAGE_TYPE` is `s3`). | | -| `STORAGE_S3_BUCKET` | The bucket name for S3-compatible storage (required if `STORAGE_TYPE` is `s3`). | | -| `STORAGE_S3_ACCESS_KEY_ID` | The access key ID for S3-compatible storage (required if `STORAGE_TYPE` is `s3`). | | -| `STORAGE_S3_SECRET_ACCESS_KEY` | The secret access key for S3-compatible storage (required if `STORAGE_TYPE` is `s3`). | | -| `STORAGE_S3_REGION` | The region for S3-compatible storage (required if `STORAGE_TYPE` is `s3`). | | -| `STORAGE_S3_FORCE_PATH_STYLE` | Force path-style addressing for S3 (optional). | `false` | +| Variable | Description | Default Value | +| ------------------------------ | ----------------------------------------------------------------------------------------------------------- | ------------------------- | +| `STORAGE_TYPE` | The storage backend to use (`local` or `s3`). | `local` | +| `BODY_SIZE_LIMIT` | The maximum request body size for uploads. Can be a number in bytes or a string with a unit (e.g., `100M`). | `100M` | +| `STORAGE_LOCAL_ROOT_PATH` | The root path for local file storage. | `/var/data/open-archiver` | +| `STORAGE_S3_ENDPOINT` | The endpoint for S3-compatible storage (required if `STORAGE_TYPE` is `s3`). | | +| `STORAGE_S3_BUCKET` | The bucket name for S3-compatible storage (required if `STORAGE_TYPE` is `s3`). | | +| `STORAGE_S3_ACCESS_KEY_ID` | The access key ID for S3-compatible storage (required if `STORAGE_TYPE` is `s3`). | | +| `STORAGE_S3_SECRET_ACCESS_KEY` | The secret access key for S3-compatible storage (required if `STORAGE_TYPE` is `s3`). | | +| `STORAGE_S3_REGION` | The region for S3-compatible storage (required if `STORAGE_TYPE` is `s3`). | | +| `STORAGE_S3_FORCE_PATH_STYLE` | Force path-style addressing for S3 (optional). | `false` | #### Security & Authentication diff --git a/packages/backend/src/database/index.ts b/packages/backend/src/database/index.ts index aaa0b50..94744b1 100644 --- a/packages/backend/src/database/index.ts +++ b/packages/backend/src/database/index.ts @@ -3,10 +3,12 @@ import postgres from 'postgres'; import 'dotenv/config'; import * as schema from './schema'; +import { encodeDatabaseUrl } from '../helpers/db'; if (!process.env.DATABASE_URL) { throw new Error('DATABASE_URL is not set in the .env file'); } -const client = postgres(process.env.DATABASE_URL); +const connectionString = encodeDatabaseUrl(process.env.DATABASE_URL); +const client = postgres(connectionString); export const db = drizzle(client, { schema }); diff --git a/packages/backend/src/database/migrate.ts b/packages/backend/src/database/migrate.ts index d22dd66..610d960 100644 --- a/packages/backend/src/database/migrate.ts +++ b/packages/backend/src/database/migrate.ts @@ -2,6 +2,7 @@ import { migrate } from 'drizzle-orm/postgres-js/migrator'; import { drizzle } from 'drizzle-orm/postgres-js'; import postgres from 'postgres'; import { config } from 'dotenv'; +import { encodeDatabaseUrl } from '../helpers/db'; config(); @@ -10,7 +11,8 @@ const runMigrate = async () => { throw new Error('DATABASE_URL is not set in the .env file'); } - const connection = postgres(process.env.DATABASE_URL, { max: 1 }); + const connectionString = encodeDatabaseUrl(process.env.DATABASE_URL); + const connection = postgres(connectionString, { max: 1 }); const db = drizzle(connection); console.log('Running migrations...'); diff --git a/packages/backend/src/helpers/db.ts b/packages/backend/src/helpers/db.ts new file mode 100644 index 0000000..f57c50c --- /dev/null +++ b/packages/backend/src/helpers/db.ts @@ -0,0 +1,12 @@ +export const encodeDatabaseUrl = (databaseUrl: string): string => { + try { + const url = new URL(databaseUrl); + if (url.password) { + url.password = encodeURIComponent(url.password); + } + return url.toString(); + } catch (error) { + console.error("Invalid DATABASE_URL, please check your .env file.", error); + throw new Error("Invalid DATABASE_URL"); + } +}; diff --git a/packages/frontend/svelte.config.js b/packages/frontend/svelte.config.js index 4b4f698..0dbd5e5 100644 --- a/packages/frontend/svelte.config.js +++ b/packages/frontend/svelte.config.js @@ -1,15 +1,13 @@ import adapter from '@sveltejs/adapter-node'; import { vitePreprocess } from '@sveltejs/vite-plugin-svelte'; -import 'dotenv/config'; + /** @type {import('@sveltejs/kit').Config} */ const config = { // Consult https://svelte.dev/docs/kit/integrations // for more information about preprocessors preprocess: vitePreprocess(), kit: { - adapter: adapter({ - bodySizeLimit: process.env.FRONTEND_BODY_SIZE_LIMIT || '100M' - }) + adapter: adapter() } }; From 832e29bd92a7626c46cd4e2c71baafdc8bc71a46 Mon Sep 17 00:00:00 2001 From: Wayne <5291640+ringoinca@users.noreply.github.com> Date: Fri, 15 Aug 2025 13:45:58 +0300 Subject: [PATCH 2/2] Project prettier setup --- .prettierignore | 13 ++++++ .prettierrc | 15 +++++++ CODE_OF_CONDUCT.md | 30 +++++++------- CONTRIBUTING.md | 18 ++++---- package.json | 68 +++++++++++++++---------------- packages/frontend/.prettierignore | 4 ++ packages/frontend/.prettierrc | 5 +-- pnpm-lock.yaml | 11 ++--- 8 files changed, 97 insertions(+), 67 deletions(-) create mode 100644 .prettierignore create mode 100644 .prettierrc diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..5b190cb --- /dev/null +++ b/.prettierignore @@ -0,0 +1,13 @@ +# Ignore artifacts +dist +.svelte-kit +build +node_modules +pnpm-lock.yaml +meili_data/ + +## shadcn installs +components/ui/ + +# Ignore logs +*.log diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..7ebb855 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,15 @@ +{ + "useTabs": true, + "singleQuote": true, + "trailingComma": "none", + "printWidth": 100, + "plugins": ["prettier-plugin-svelte", "prettier-plugin-tailwindcss"], + "overrides": [ + { + "files": "*.svelte", + "options": { + "parser": "svelte" + } + } + ] +} diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 9f388d1..678766e 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -16,24 +16,24 @@ We pledge to act and interact in ways that are welcoming, open, and respectful. Examples of behavior that contributes to a positive environment for our community include: -- Demonstrating empathy and kindness toward other people -- Being respectful of differing opinions, viewpoints, and experiences -- Giving and gracefully accepting constructive feedback -- Accepting responsibility and apologizing to those affected by our mistakes, - and learning from the experience -- Focusing on what is best not just for us as individuals, but for the - overall community +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best not just for us as individuals, but for the + overall community Examples of unacceptable behavior include: -- The use of sexualized language or imagery, and sexual attention or - advances of any kind -- Trolling, insulting or derogatory comments, and personal or political attacks -- Public or private harassment -- Publishing others' private information, such as a physical or email - address, without their explicit permission -- Other conduct which could reasonably be considered inappropriate in a - professional setting +- The use of sexualized language or imagery, and sexual attention or + advances of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email + address, without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting ## Enforcement Responsibilities diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0f61263..edf78f0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,8 +6,8 @@ First off, thank you for considering contributing to Open Archiver! It's people Not sure where to start? You can: -- Look through the [open issues](https://github.com/LogicLabs-OU/OpenArchiver/issues) for bugs or feature requests. -- Check the issues labeled `good first issue` for tasks that are a good entry point into the codebase. +- Look through the [open issues](https://github.com/LogicLabs-OU/OpenArchiver/issues) for bugs or feature requests. +- Check the issues labeled `good first issue` for tasks that are a good entry point into the codebase. ## How to Contribute @@ -41,13 +41,13 @@ This project and everyone participating in it is governed by the [Open Archiver ### Git Commit Messages -- Use the present tense ("Add feature" not "Added feature"). -- Use the imperative mood ("Move cursor to..." not "Moves cursor to..."). -- Limit the first line to 72 characters or less. -- Reference issues and pull requests liberally after the first line. +- Use the present tense ("Add feature" not "Added feature"). +- Use the imperative mood ("Move cursor to..." not "Moves cursor to..."). +- Limit the first line to 72 characters or less. +- Reference issues and pull requests liberally after the first line. ### TypeScript Styleguide -- Follow the existing code style. -- Use TypeScript's strict mode. -- Avoid using `any` as a type. Define clear interfaces and types in the `packages/types` directory. +- Follow the existing code style. +- Use TypeScript's strict mode. +- Avoid using `any` as a type. Define clear interfaces and types in the `packages/types` directory. diff --git a/package.json b/package.json index 55fe99f..298a74a 100644 --- a/package.json +++ b/package.json @@ -1,36 +1,36 @@ { - "name": "open-archiver", - "private": true, - "scripts": { - "dev": "dotenv -- pnpm --filter \"./packages/*\" --parallel dev", - "build": "pnpm --filter \"./packages/*\" build", - "start": "dotenv -- pnpm --filter \"./packages/*\" --parallel start", - "start:workers": "dotenv -- concurrently \"pnpm --filter @open-archiver/backend start:ingestion-worker\" \"pnpm --filter @open-archiver/backend start:indexing-worker\" \"pnpm --filter @open-archiver/backend start:sync-scheduler\"", - "start:workers:dev": "dotenv -- concurrently \"pnpm --filter @open-archiver/backend start:ingestion-worker:dev\" \"pnpm --filter @open-archiver/backend start:indexing-worker:dev\" \"pnpm --filter @open-archiver/backend start:sync-scheduler:dev\"", - "db:generate": "dotenv -- pnpm --filter @open-archiver/backend db:generate", - "db:migrate": "dotenv -- pnpm --filter @open-archiver/backend db:migrate", - "db:migrate:dev": "dotenv -- pnpm --filter @open-archiver/backend db:migrate:dev", - "docker-start": "concurrently \"pnpm start:workers\" \"pnpm start\"", - "docs:dev": "vitepress dev docs --port 3009", - "docs:build": "vitepress build docs", - "docs:preview": "vitepress preview docs" - }, - "dependencies": { - "concurrently": "^9.2.0", - "dotenv-cli": "8.0.0" - }, - "devDependencies": { - "typescript": "5.8.3", - "vitepress": "^1.6.3" - }, - "packageManager": "pnpm@10.13.1", - "engines": { - "node": ">=22.0.0", - "pnpm": "10.13.1" - }, - "pnpm": { - "onlyBuiltDependencies": [ - "esbuild" - ] - } + "name": "open-archiver", + "private": true, + "scripts": { + "dev": "dotenv -- pnpm --filter \"./packages/*\" --parallel dev", + "build": "pnpm --filter \"./packages/*\" build", + "start": "dotenv -- pnpm --filter \"./packages/*\" --parallel start", + "start:workers": "dotenv -- concurrently \"pnpm --filter @open-archiver/backend start:ingestion-worker\" \"pnpm --filter @open-archiver/backend start:indexing-worker\" \"pnpm --filter @open-archiver/backend start:sync-scheduler\"", + "start:workers:dev": "dotenv -- concurrently \"pnpm --filter @open-archiver/backend start:ingestion-worker:dev\" \"pnpm --filter @open-archiver/backend start:indexing-worker:dev\" \"pnpm --filter @open-archiver/backend start:sync-scheduler:dev\"", + "db:generate": "dotenv -- pnpm --filter @open-archiver/backend db:generate", + "db:migrate": "dotenv -- pnpm --filter @open-archiver/backend db:migrate", + "db:migrate:dev": "dotenv -- pnpm --filter @open-archiver/backend db:migrate:dev", + "docker-start": "concurrently \"pnpm start:workers\" \"pnpm start\"", + "docs:dev": "vitepress dev docs --port 3009", + "docs:build": "vitepress build docs", + "docs:preview": "vitepress preview docs" + }, + "dependencies": { + "concurrently": "^9.2.0", + "dotenv-cli": "8.0.0" + }, + "devDependencies": { + "typescript": "5.8.3", + "vitepress": "^1.6.4" + }, + "packageManager": "pnpm@10.13.1", + "engines": { + "node": ">=22.0.0", + "pnpm": "10.13.1" + }, + "pnpm": { + "onlyBuiltDependencies": [ + "esbuild" + ] + } } diff --git a/packages/frontend/.prettierignore b/packages/frontend/.prettierignore index 7d74fe2..da0b565 100644 --- a/packages/frontend/.prettierignore +++ b/packages/frontend/.prettierignore @@ -5,5 +5,9 @@ yarn.lock bun.lock bun.lockb +## shadcn components +components/ui/ + + # Miscellaneous /static/ diff --git a/packages/frontend/.prettierrc b/packages/frontend/.prettierrc index b19073e..7ebb855 100644 --- a/packages/frontend/.prettierrc +++ b/packages/frontend/.prettierrc @@ -3,10 +3,7 @@ "singleQuote": true, "trailingComma": "none", "printWidth": 100, - "plugins": [ - "prettier-plugin-svelte", - "prettier-plugin-tailwindcss" - ], + "plugins": ["prettier-plugin-svelte", "prettier-plugin-tailwindcss"], "overrides": [ { "files": "*.svelte", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e158389..54c9ef5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -19,8 +19,8 @@ importers: specifier: 5.8.3 version: 5.8.3 vitepress: - specifier: ^1.6.3 - version: 1.6.3(@algolia/client-search@5.34.1)(@types/node@24.0.13)(axios@1.10.0)(lightningcss@1.30.1)(postcss@8.5.6)(search-insights@2.17.3)(typescript@5.8.3) + specifier: ^1.6.4 + version: 1.6.4(@algolia/client-search@5.34.1)(@types/node@24.0.13)(axios@1.10.0)(lightningcss@1.30.1)(postcss@8.5.6)(search-insights@2.17.3)(typescript@5.8.3) packages/backend: dependencies: @@ -3707,6 +3707,7 @@ packages: resolution: {integrity: sha512-Nkwo9qeCvqVH0ZgYRUfPyj6o4o7StvNIxMFECeiz4y0uMOVyqc5Y9hjsdFVxdYCeiUjjXLQXA8KIz0iJL3HM0w==} engines: {node: '>=20.18.0'} hasBin: true + bundledDependencies: [] peberminta@0.9.0: resolution: {integrity: sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==} @@ -4640,8 +4641,8 @@ packages: vite: optional: true - vitepress@1.6.3: - resolution: {integrity: sha512-fCkfdOk8yRZT8GD9BFqusW3+GggWYZ/rYncOfmgcDtP3ualNHCAg+Robxp2/6xfH1WwPHtGpPwv7mbA3qomtBw==} + vitepress@1.6.4: + resolution: {integrity: sha512-+2ym1/+0VVrbhNyRoFFesVvBvHAVMZMK0rw60E3X/5349M1GuVdKeazuksqopEdvkKwKGs21Q729jX81/bkBJg==} hasBin: true peerDependencies: markdown-it-mathjax3: ^4 @@ -9524,7 +9525,7 @@ snapshots: optionalDependencies: vite: 6.3.5(@types/node@24.0.13)(jiti@2.4.2)(lightningcss@1.30.1) - vitepress@1.6.3(@algolia/client-search@5.34.1)(@types/node@24.0.13)(axios@1.10.0)(lightningcss@1.30.1)(postcss@8.5.6)(search-insights@2.17.3)(typescript@5.8.3): + vitepress@1.6.4(@algolia/client-search@5.34.1)(@types/node@24.0.13)(axios@1.10.0)(lightningcss@1.30.1)(postcss@8.5.6)(search-insights@2.17.3)(typescript@5.8.3): dependencies: '@docsearch/css': 3.8.2 '@docsearch/js': 3.8.2(@algolia/client-search@5.34.1)(search-insights@2.17.3)