Compare commits

...

49 Commits

Author SHA1 Message Date
Rostislav Dugin
1b011bdcd4 Merge pull request #432 from databasus/develop
Develop
2026-03-13 18:51:50 +03:00
Rostislav Dugin
7e209ff537 REFACTOR (linters): Apply linters fixes 2026-03-13 18:50:57 +03:00
Rostislav Dugin
f712e3a437 FEATURE (linters): Introduce more strict linters 2026-03-13 18:03:38 +03:00
Rostislav Dugin
bcd7d8e1aa REFACTOR (formatters): Apply formatters auto fixes 2026-03-13 17:53:00 +03:00
Rostislav Dugin
880a7488e9 FEATURE (formatters): Add gofumpt and gci formatters 2026-03-13 17:50:07 +03:00
Rostislav Dugin
ca4d483f2c REFACTOR (golines): Apply golines fixes 2026-03-13 17:47:46 +03:00
Rostislav Dugin
1b511410a6 FEATURE (formatters): Fix config of golines 2026-03-13 17:47:29 +03:00
Rostislav Dugin
c8edff8046 FEATURE (golangci-lint): Upgrade golangci-lint to 2.11.3 in CI \ CD 2026-03-13 17:41:14 +03:00
Rostislav Dugin
f60e3d956b FEAUTRE (go): Upgrade Go version to 1.26.1 2026-03-13 17:37:39 +03:00
Rostislav Dugin
f2cb9022f2 FEATURE (agent): Setup agent directory, pre-commit and CI\CD workflow 2026-03-13 17:23:00 +03:00
Rostislav Dugin
4b3f36eea2 Merge pull request #429 from databasus/develop
FIX (readme): Add info about Anthropic and Open AI support via OSS pr…
2026-03-12 09:38:01 +03:00
Rostislav Dugin
460063e7a5 FIX (readme): Add info about Anthropic and Open AI support via OSS programs 2026-03-12 09:34:42 +03:00
Rostislav Dugin
a0f02b253e Merge pull request #427 from databasus/develop
FIX (retention): Fix GFS retention while hourly backups prevent daily…
2026-03-11 15:36:27 +03:00
Rostislav Dugin
812f11bc2f FIX (retention): Fix GFS retention while hourly backups prevent daily from cleanup 2026-03-11 15:35:53 +03:00
Rostislav Dugin
e796e3ddf0 Merge pull request #426 from databasus/develop
FIX (mysql): Detect supported compression levels
2026-03-11 12:53:35 +03:00
Rostislav Dugin
c96d3db337 FIX (mysql): Detect supported compression levels 2026-03-11 12:52:41 +03:00
Rostislav Dugin
ed6c3a2034 Merge pull request #425 from databasus/develop
Develop
2026-03-11 12:31:19 +03:00
Rostislav Dugin
05115047c3 FEATURE (version): Reload frontend if faced version mismatch with backend 2026-03-11 12:28:07 +03:00
Rostislav Dugin
446b96c6c0 FEATURE (arch): Add architecture to Databasus version in the bottom left of UI 2026-03-11 11:39:53 +03:00
Rostislav Dugin
36a0448da1 Merge pull request #420 from databasus/develop
FEATURE (email): Add skipping TLS for email notifier
2026-03-08 22:53:45 +03:00
Rostislav Dugin
8e392cfeab FEATURE (email): Add skipping TLS for email notifier 2026-03-08 22:48:28 +03:00
Rostislav Dugin
6683db1e52 Merge pull request #419 from databasus/develop
FIX (issues): Add DB version to issues template
2026-03-08 22:22:52 +03:00
Rostislav Dugin
703b883936 FIX (issues): Add DB version to issues template 2026-03-08 22:22:26 +03:00
Rostislav Dugin
e818bcff82 Merge pull request #415 from databasus/develop
Develop
2026-03-06 09:45:11 +03:00
Rostislav Dugin
b2f98f1332 FIX (mysql\mariadb): Increase max allowed packet size over restore for MySQL\MariaDB 2026-03-06 09:44:17 +03:00
Rostislav Dugin
230cc27ea6 FEATURE (backups): Add WAL API 2026-03-06 08:10:29 +03:00
Rostislav Dugin
cd197ff94b Merge pull request #410 from databasus/develop
FIX (readme): Update README
2026-03-01 10:43:47 +03:00
Rostislav Dugin
91f35a3e17 FIX (readme): Update README 2026-03-01 10:43:17 +03:00
Rostislav Dugin
30c2e2d156 Merge pull request #403 from databasus/develop
FIX (smtp): Add SMTP from field to env variables
2026-02-25 22:23:08 +03:00
Rostislav Dugin
ef7c5b45e6 FIX (smtp): Add SMTP from field to env variables 2026-02-25 22:13:04 +03:00
Rostislav Dugin
920c98e229 Merge pull request #397 from databasus/develop
FIX (migrations): Fix version of migrations tool goose
2026-02-22 23:43:55 +03:00
Rostislav Dugin
2a19a96aae FIX (migrations): Fix version of migrations tool goose 2026-02-22 23:43:23 +03:00
Rostislav Dugin
75aa2108d9 Merge pull request #396 from databasus/develop
FIX (email): Use current OS hostname instead of default localhost
2026-02-22 23:33:28 +03:00
Rostislav Dugin
0a0040839e FIX (email): Use current OS hostname instead of default localhost 2026-02-22 23:31:25 +03:00
Rostislav Dugin
ff4f795ece Merge pull request #394 from databasus/develop
FIX (nas): Add NAS share validation
2026-02-22 16:05:38 +03:00
Rostislav Dugin
dc05502580 FIX (nas): Add NAS share validation 2026-02-22 15:56:30 +03:00
Rostislav Dugin
1ca38f5583 Merge pull request #390 from databasus/develop
FEATURE (templates): Add PR template
2026-02-21 15:58:21 +03:00
Rostislav Dugin
40b3ff61c7 FEATURE (templates): Add PR template 2026-02-21 15:53:01 +03:00
Rostislav Dugin
e1b245a573 Merge pull request #389 from databasus/develop
Develop
2026-02-21 14:57:56 +03:00
Rostislav Dugin
fdf29b71f2 FIX (mongodb): Fix direct connection string parsing 2026-02-21 14:56:48 +03:00
Rostislav Dugin
49da981c21 Merge pull request #388 from databasus/main
Merge main into dev
2026-02-21 14:53:31 +03:00
Rostislav Dugin
9d611d3559 REFACTOR (mongodb): Refactor direct connection PR 2026-02-21 14:43:47 +03:00
ujstor
22cab53dab feature/mongodb-directConnection (#377)
FEATURE (mongodb): Add direct connection
2026-02-21 14:10:28 +03:00
Rostislav Dugin
d761c4156c Merge pull request #385 from databasus/develop
FIX (readme): Fix README typo
2026-02-20 17:17:45 +03:00
Rostislav Dugin
cbb8b82711 FIX (readme): Fix README typo 2026-02-20 17:01:44 +03:00
Rostislav Dugin
8e3d1e5bff Merge pull request #384 from databasus/develop
FIX (backups): Do not reload backups if request already in progress
2026-02-20 15:04:19 +03:00
Rostislav Dugin
349e7f0ee8 FIX (backups): Do not reload backups if request already in progress 2026-02-20 14:43:07 +03:00
Rostislav Dugin
3a274e135b Merge pull request #383 from databasus/develop
FEATURE (backups): Add GFS retention policy
2026-02-20 14:33:29 +03:00
Rostislav Dugin
61e937bc2a FEATURE (backups): Add GFS retention policy 2026-02-20 14:31:56 +03:00
255 changed files with 9130 additions and 1607 deletions

44
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View File

@@ -0,0 +1,44 @@
---
name: Bug Report
about: Report a bug or unexpected behavior in Databasus
labels: bug
---
## Databasus version (screenshot)
It is displayed in the bottom left corner of the Databasus UI. Please attach screenshot, not just version text
<!-- e.g. 1.4.2 -->
## Operating system and architecture
<!-- e.g. Ubuntu 22.04 x64, macOS 14 ARM, Windows 11 x64 -->
## Database type and version (optional, for DB-related bugs)
<!-- e.g. PostgreSQL 16 in Docker, MySQL 8.0 installed on server, MariaDB 11.4 in AWS Cloud -->
## Describe the bug (please write manually, do not ask AI to summarize)
**What happened:**
**What I expected:**
## Steps to reproduce
1.
2.
3.
## Have you asked AI how to solve the issue?
<!-- Using AI to diagnose issues before filing a bug report helps narrow down root causes. -->
- [ ] Claude Sonnet 4.6 or newer
- [ ] ChatGPT 5.2 or newer
- [ ] No
## Additional context / logs
<!-- Screenshots, error messages, relevant log output, etc. -->

View File

@@ -11,7 +11,7 @@ jobs:
lint-backend:
runs-on: self-hosted
container:
image: golang:1.24.9
image: golang:1.26.1
volumes:
- /runner-cache/go-pkg:/go/pkg/mod
- /runner-cache/go-build:/root/.cache/go-build
@@ -32,7 +32,7 @@ jobs:
- name: Install golangci-lint
run: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.7.2
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
- name: Install swag for swagger generation
@@ -86,6 +86,39 @@ jobs:
cd frontend
npm run build
lint-agent:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.26.1"
cache-dependency-path: agent/go.sum
- name: Download Go modules
run: |
cd agent
go mod download
- name: Install golangci-lint
run: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
- name: Run golangci-lint
run: |
cd agent
golangci-lint run
- name: Verify go mod tidy
run: |
cd agent
go mod tidy
git diff --exit-code go.mod go.sum || (echo "go mod tidy made changes, please run 'go mod tidy' and commit the changes" && exit 1)
test-frontend:
runs-on: ubuntu-latest
needs: [lint-frontend]
@@ -108,11 +141,34 @@ jobs:
cd frontend
npm run test
test-agent:
runs-on: ubuntu-latest
needs: [lint-agent]
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.26.1"
cache-dependency-path: agent/go.sum
- name: Download Go modules
run: |
cd agent
go mod download
- name: Run Go tests
run: |
cd agent
go test -count=1 -failfast ./internal/...
test-backend:
runs-on: self-hosted
needs: [lint-backend]
container:
image: golang:1.24.9
image: golang:1.26.1
options: --privileged -v /var/run/docker.sock:/var/run/docker.sock --add-host=host.docker.internal:host-gateway
volumes:
- /runner-cache/go-pkg:/go/pkg/mod
@@ -407,7 +463,7 @@ jobs:
- name: Run database migrations
run: |
cd backend
go install github.com/pressly/goose/v3/cmd/goose@latest
go install github.com/pressly/goose/v3/cmd/goose@v3.24.3
goose up
- name: Run Go tests
@@ -441,7 +497,7 @@ jobs:
runs-on: self-hosted
container:
image: node:20
needs: [test-backend, test-frontend]
needs: [test-backend, test-frontend, test-agent]
if: ${{ github.ref == 'refs/heads/main' && !contains(github.event.head_commit.message, '[skip-release]') }}
outputs:
should_release: ${{ steps.version_bump.outputs.should_release }}
@@ -534,7 +590,7 @@ jobs:
build-only:
runs-on: self-hosted
needs: [test-backend, test-frontend]
needs: [test-backend, test-frontend, test-agent]
if: ${{ github.ref == 'refs/heads/main' && contains(github.event.head_commit.message, '[skip-release]') }}
steps:
- name: Clean workspace

1
.gitignore vendored
View File

@@ -12,3 +12,4 @@ node_modules/
.DS_Store
/scripts
.vscode/settings.json
.claude

View File

@@ -41,3 +41,20 @@ repos:
language: system
files: ^backend/.*\.go$
pass_filenames: false
# Agent checks
- repo: local
hooks:
- id: agent-format-and-lint
name: Agent Format & Lint (golangci-lint)
entry: bash -c "cd agent && golangci-lint fmt ./internal/... ./cmd/... && golangci-lint run ./internal/... ./cmd/..."
language: system
files: ^agent/.*\.go$
pass_filenames: false
- id: agent-go-mod-tidy
name: Agent Go Mod Tidy
entry: bash -c "cd agent && go mod tidy"
language: system
files: ^agent/.*\.go$
pass_filenames: false

1
CLAUDE.md Normal file
View File

@@ -0,0 +1 @@
Look at @AGENTS.md

View File

@@ -22,7 +22,7 @@ RUN npm run build
# ========= BUILD BACKEND =========
# Backend build stage
FROM --platform=$BUILDPLATFORM golang:1.24.9 AS backend-build
FROM --platform=$BUILDPLATFORM golang:1.26.1 AS backend-build
# Make TARGET args available early so tools built here match the final image arch
ARG TARGETOS
@@ -66,13 +66,52 @@ RUN CGO_ENABLED=0 \
go build -o /app/main ./cmd/main.go
# ========= BUILD AGENT =========
# Builds the databasus-agent CLI binary for BOTH x86_64 and ARM64.
# Both architectures are always built because:
# - Databasus server runs on one arch (e.g. amd64)
# - The agent runs on remote PostgreSQL servers that may be on a
# different arch (e.g. arm64)
# - The backend serves the correct binary based on the agent's
# ?arch= query parameter
#
# We cross-compile from the build platform (no QEMU needed) because the
# agent is pure Go with zero C dependencies.
# CGO_ENABLED=0 produces fully static binaries — no glibc/musl dependency,
# so the agent runs on any Linux distro (Alpine, Debian, Ubuntu, RHEL, etc.).
# APP_VERSION is baked into the binary via -ldflags so the agent can
# compare its version against the server and auto-update when needed.
FROM --platform=$BUILDPLATFORM golang:1.26.1 AS agent-build
ARG APP_VERSION=dev
WORKDIR /agent
COPY agent/go.mod ./
RUN go mod download
COPY agent/ ./
# Build for x86_64 (amd64) — static binary, no glibc dependency
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
go build -ldflags "-X main.Version=${APP_VERSION}" \
-o /agent-binaries/databasus-agent-linux-amd64 ./cmd/main.go
# Build for ARM64 (arm64) — static binary, no glibc dependency
RUN CGO_ENABLED=0 GOOS=linux GOARCH=arm64 \
go build -ldflags "-X main.Version=${APP_VERSION}" \
-o /agent-binaries/databasus-agent-linux-arm64 ./cmd/main.go
# ========= RUNTIME =========
FROM debian:bookworm-slim
# Add version metadata to runtime image
ARG APP_VERSION=dev
ARG TARGETARCH
LABEL org.opencontainers.image.version=$APP_VERSION
ENV APP_VERSION=$APP_VERSION
ENV CONTAINER_ARCH=$TARGETARCH
# Set production mode for Docker containers
ENV ENV_MODE=production
@@ -218,6 +257,10 @@ COPY backend/migrations ./migrations
# Copy UI files
COPY --from=backend-build /app/ui/build ./ui/build
# Copy agent binaries (both architectures) — served by the backend
# at GET /api/v1/system/agent?arch=amd64|arm64
COPY --from=agent-build /agent-binaries ./agent-binaries
# Copy .env file (with fallback to .env.production.example)
COPY backend/.env* /app/
RUN if [ ! -f /app/.env ]; then \
@@ -269,7 +312,8 @@ window.__RUNTIME_CONFIG__ = {
GITHUB_CLIENT_ID: '\${GITHUB_CLIENT_ID:-}',
GOOGLE_CLIENT_ID: '\${GOOGLE_CLIENT_ID:-}',
IS_EMAIL_CONFIGURED: '\$IS_EMAIL_CONFIGURED',
CLOUDFLARE_TURNSTILE_SITE_KEY: '\${CLOUDFLARE_TURNSTILE_SITE_KEY:-}'
CLOUDFLARE_TURNSTILE_SITE_KEY: '\${CLOUDFLARE_TURNSTILE_SITE_KEY:-}',
CONTAINER_ARCH: '\${CONTAINER_ARCH:-unknown}'
};
JSEOF
@@ -394,6 +438,8 @@ fi
# Create database and set password for postgres user
echo "Setting up database and user..."
gosu postgres \$PG_BIN/psql -p 5437 -h localhost -d postgres << 'SQL'
# We use stub password, because internal DB is not exposed outside container
ALTER USER postgres WITH PASSWORD 'Q1234567';
SELECT 'CREATE DATABASE databasus OWNER postgres'
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'databasus')

View File

@@ -2,7 +2,7 @@
<img src="assets/logo.svg" alt="Databasus Logo" width="250"/>
<h3>Backup tool for PostgreSQL, MySQL and MongoDB</h3>
<p>Databasus is a free, open source and self-hosted tool to backup databases (with focus on PostgreSQL). Make backups with different storages (S3, Google Drive, FTP, etc.) and notifications about progress (Slack, Discord, Telegram, etc.). Previously known as Postgresus (see migration guide).</p>
<p>Databasus is a free, open source and self-hosted tool to backup databases (with focus on PostgreSQL). Make backups with different storages (S3, Google Drive, FTP, etc.) and notifications about progress (Slack, Discord, Telegram, etc.)</p>
<!-- Badges -->
[![PostgreSQL](https://img.shields.io/badge/PostgreSQL-336791?logo=postgresql&logoColor=white)](https://www.postgresql.org/)
@@ -41,7 +41,7 @@
- **PostgreSQL**: 12, 13, 14, 15, 16, 17 and 18
- **MySQL**: 5.7, 8 and 9
- **MariaDB**: 10 and 11
- **MariaDB**: 10, 11 and 12
- **MongoDB**: 4, 5, 6, 7 and 8
### 🔄 **Scheduled backups**
@@ -50,6 +50,13 @@
- **Precise timing**: run backups at specific times (e.g., 4 AM during low traffic)
- **Smart compression**: 4-8x space savings with balanced compression (~20% overhead)
### 🗑️ **Retention policies**
- **Time period**: Keep backups for a fixed duration (e.g., 7 days, 3 months, 1 year)
- **Count**: Keep a fixed number of the most recent backups (e.g., last 30)
- **GFS (Grandfather-Father-Son)**: Layered retention — keep hourly, daily, weekly, monthly and yearly backups independently for fine-grained long-term history (enterprises requirement)
- **Size limits**: Set per-backup and total storage size caps to control storage usage
### 🗄️ **Multiple storage destinations** <a href="https://databasus.com/storages">(view supported)</a>
- **Local storage**: Keep backups on your VPS/server
@@ -220,8 +227,9 @@ For more options (NodePort, TLS, HTTPRoute for Gateway API), see the [Helm chart
3. **Configure schedule**: Choose from hourly, daily, weekly, monthly or cron intervals
4. **Set database connection**: Enter your database credentials and connection details
5. **Choose storage**: Select where to store your backups (local, S3, Google Drive, etc.)
6. **Add notifications** (optional): Configure email, Telegram, Slack, or webhook notifications
7. **Save and start**: Databasus will validate settings and begin the backup schedule
6. **Configure retention policy**: Choose time period, count or GFS to control how long backups are kept
7. **Add notifications** (optional): Configure email, Telegram, Slack, or webhook notifications
8. **Save and start**: Databasus will validate settings and begin the backup schedule
### 🔑 Resetting password <a href="https://databasus.com/password">(docs)</a>
@@ -253,12 +261,17 @@ Also you can join our large community of developers, DBAs and DevOps engineers o
There have been questions about AI usage in project development in issues and discussions. As the project focuses on security, reliability and production usage, it's important to explain how AI is used in the development process.
First of all, we are proud to say that Databasus has been accepted into both [Claude for Open Source](https://claude.com/contact-sales/claude-for-oss) by Anthropic and [Codex for Open Source](https://developers.openai.com/codex/community/codex-for-oss/) by OpenAI in March 2026. For us it is one more signal that the project was recognized as important open-source software and was as critical infrastructure worth supporting independently by two of the world's leading AI companies. Read more at [databasus.com/faq](https://databasus.com/faq#oss-programs).
Despite of this, we have the following rules how AI is used in the development process:
AI is used as a helper for:
- verification of code quality and searching for vulnerabilities
- cleaning up and improving documentation, comments and code
- assistance during development
- double-checking PRs and commits after human review
- additional security analysis of PRs via Codex Security
AI is not used for:

1
agent/.env.example Normal file
View File

@@ -0,0 +1 @@
ENV_MODE=development

23
agent/.gitignore vendored Normal file
View File

@@ -0,0 +1,23 @@
main
.env
docker-compose.yml
pgdata
pgdata_test/
mysqldata/
mariadbdata/
main.exe
swagger/
swagger/*
swagger/docs.go
swagger/swagger.json
swagger/swagger.yaml
postgresus-backend.exe
databasus-backend.exe
ui/build/*
pgdata-for-restore/
temp/
cmd.exe
temp/
valkey-data/
victoria-logs-data/
databasus.json

41
agent/.golangci.yml Normal file
View File

@@ -0,0 +1,41 @@
version: "2"
run:
timeout: 5m
tests: false
concurrency: 4
linters:
default: standard
enable:
- funcorder
- bodyclose
- errorlint
- gocritic
- unconvert
- misspell
- errname
- noctx
- modernize
settings:
errcheck:
check-type-assertions: true
formatters:
enable:
- gofumpt
- golines
- gci
settings:
golines:
max-len: 120
gofumpt:
module-path: databasus-agent
extra-rules: true
gci:
sections:
- standard
- default
- localmodule

12
agent/Makefile Normal file
View File

@@ -0,0 +1,12 @@
# Usage: make run ARGS="start --pg-host localhost"
run:
go run cmd/main.go $(ARGS)
build:
CGO_ENABLED=0 go build -ldflags "-X main.Version=$(VERSION)" -o databasus-agent ./cmd/main.go
test:
go test -count=1 -failfast ./internal/...
lint:
golangci-lint fmt ./cmd/... ./internal/... && golangci-lint run ./cmd/... ./internal/...

174
agent/cmd/main.go Normal file
View File

@@ -0,0 +1,174 @@
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"strings"
"databasus-agent/internal/config"
"databasus-agent/internal/features/start"
"databasus-agent/internal/features/upgrade"
"databasus-agent/internal/logger"
)
var Version = "dev"
func main() {
if len(os.Args) < 2 {
printUsage()
os.Exit(1)
}
switch os.Args[1] {
case "start":
runStart(os.Args[2:])
case "stop":
runStop()
case "status":
runStatus()
case "restore":
runRestore(os.Args[2:])
case "version":
fmt.Println(Version)
default:
fmt.Fprintf(os.Stderr, "unknown command: %s\n", os.Args[1])
printUsage()
os.Exit(1)
}
}
func runStart(args []string) {
fs := flag.NewFlagSet("start", flag.ExitOnError)
isDebug := fs.Bool("debug", false, "Enable debug logging")
isSkipUpdate := fs.Bool("skip-update", false, "Skip auto-update check")
cfg := &config.Config{}
cfg.LoadFromJSONAndArgs(fs, args)
if err := cfg.SaveToJSON(); err != nil {
fmt.Fprintf(os.Stderr, "Failed to save config: %v\n", err)
}
logger.Init(*isDebug)
log := logger.GetLogger()
isDev := checkIsDevelopment()
runUpdateCheck(cfg.DatabasusHost, *isSkipUpdate, isDev, log)
if err := start.Run(cfg, log); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
func runStop() {
logger.Init(false)
logger.GetLogger().Info("stop: stub — not yet implemented")
}
func runStatus() {
logger.Init(false)
logger.GetLogger().Info("status: stub — not yet implemented")
}
func runRestore(args []string) {
fs := flag.NewFlagSet("restore", flag.ExitOnError)
targetDir := fs.String("target-dir", "", "Target pgdata directory")
backupID := fs.String("backup-id", "", "Full backup UUID (optional)")
targetTime := fs.String("target-time", "", "PITR target time in RFC3339 (optional)")
isYes := fs.Bool("yes", false, "Skip confirmation prompt")
isDebug := fs.Bool("debug", false, "Enable debug logging")
isSkipUpdate := fs.Bool("skip-update", false, "Skip auto-update check")
cfg := &config.Config{}
cfg.LoadFromJSONAndArgs(fs, args)
if err := cfg.SaveToJSON(); err != nil {
fmt.Fprintf(os.Stderr, "Failed to save config: %v\n", err)
}
logger.Init(*isDebug)
log := logger.GetLogger()
isDev := checkIsDevelopment()
runUpdateCheck(cfg.DatabasusHost, *isSkipUpdate, isDev, log)
log.Info("restore: stub — not yet implemented",
"targetDir", *targetDir,
"backupId", *backupID,
"targetTime", *targetTime,
"yes", *isYes,
)
}
func printUsage() {
fmt.Fprintln(os.Stderr, "Usage: databasus-agent <command> [flags]")
fmt.Fprintln(os.Stderr, "")
fmt.Fprintln(os.Stderr, "Commands:")
fmt.Fprintln(os.Stderr, " start Start the agent (WAL archiving + basebackups)")
fmt.Fprintln(os.Stderr, " stop Stop a running agent")
fmt.Fprintln(os.Stderr, " status Show agent status")
fmt.Fprintln(os.Stderr, " restore Restore a database from backup")
fmt.Fprintln(os.Stderr, " version Print agent version")
}
func runUpdateCheck(host string, isSkipUpdate, isDev bool, log interface {
Info(string, ...any)
Warn(string, ...any)
Error(string, ...any)
},
) {
if isSkipUpdate {
return
}
if host == "" {
return
}
if err := upgrade.CheckAndUpdate(host, Version, isDev, log); err != nil {
log.Error("Auto-update failed", "error", err)
os.Exit(1)
}
}
func checkIsDevelopment() bool {
dir, err := os.Getwd()
if err != nil {
return false
}
for range 3 {
if data, err := os.ReadFile(filepath.Join(dir, ".env")); err == nil {
return parseEnvMode(data)
}
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
return false
}
dir = filepath.Dir(dir)
}
return false
}
func parseEnvMode(data []byte) bool {
for line := range strings.SplitSeq(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
parts := strings.SplitN(line, "=", 2)
if len(parts) == 2 && strings.TrimSpace(parts[0]) == "ENV_MODE" {
return strings.TrimSpace(parts[1]) == "development"
}
}
return false
}

11
agent/go.mod Normal file
View File

@@ -0,0 +1,11 @@
module databasus-agent
go 1.26.1
require github.com/stretchr/testify v1.11.1
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

10
agent/go.sum Normal file
View File

@@ -0,0 +1,10 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -0,0 +1,136 @@
package config
import (
"encoding/json"
"flag"
"os"
"databasus-agent/internal/logger"
)
var log = logger.GetLogger()
const configFileName = "databasus.json"
type Config struct {
DatabasusHost string `json:"databasusHost"`
DbID string `json:"dbId"`
Token string `json:"token"`
flags parsedFlags
}
// LoadFromJSONAndArgs reads databasus.json into the struct
// and overrides JSON values with any explicitly provided CLI flags.
func (c *Config) LoadFromJSONAndArgs(fs *flag.FlagSet, args []string) {
c.loadFromJSON()
c.initSources()
c.flags.host = fs.String(
"databasus-host",
"",
"Databasus server URL (e.g. http://your-server:4005)",
)
c.flags.dbID = fs.String("db-id", "", "Database ID")
c.flags.token = fs.String("token", "", "Agent token")
if err := fs.Parse(args); err != nil {
os.Exit(1)
}
c.applyFlags()
log.Info("========= Loading config ============")
c.logConfigSources()
log.Info("========= Config has been loaded ====")
}
// SaveToJSON writes the current struct to databasus.json.
func (c *Config) SaveToJSON() error {
data, err := json.MarshalIndent(c, "", " ")
if err != nil {
return err
}
return os.WriteFile(configFileName, data, 0o644)
}
func (c *Config) loadFromJSON() {
data, err := os.ReadFile(configFileName)
if err != nil {
if os.IsNotExist(err) {
log.Info("No databasus.json found, will create on save")
return
}
log.Warn("Failed to read databasus.json", "error", err)
return
}
if err := json.Unmarshal(data, c); err != nil {
log.Warn("Failed to parse databasus.json", "error", err)
return
}
log.Info("Configuration loaded from " + configFileName)
}
func (c *Config) initSources() {
c.flags.sources = map[string]string{
"databasus-host": "not configured",
"db-id": "not configured",
"token": "not configured",
}
if c.DatabasusHost != "" {
c.flags.sources["databasus-host"] = configFileName
}
if c.DbID != "" {
c.flags.sources["db-id"] = configFileName
}
if c.Token != "" {
c.flags.sources["token"] = configFileName
}
}
func (c *Config) applyFlags() {
if c.flags.host != nil && *c.flags.host != "" {
c.DatabasusHost = *c.flags.host
c.flags.sources["databasus-host"] = "command line args"
}
if c.flags.dbID != nil && *c.flags.dbID != "" {
c.DbID = *c.flags.dbID
c.flags.sources["db-id"] = "command line args"
}
if c.flags.token != nil && *c.flags.token != "" {
c.Token = *c.flags.token
c.flags.sources["token"] = "command line args"
}
}
func (c *Config) logConfigSources() {
log.Info(
"databasus-host",
"value",
c.DatabasusHost,
"source",
c.flags.sources["databasus-host"],
)
log.Info("db-id", "value", c.DbID, "source", c.flags.sources["db-id"])
log.Info("token", "value", maskSensitive(c.Token), "source", c.flags.sources["token"])
}
func maskSensitive(value string) string {
if value == "" {
return "(not set)"
}
visibleLen := max(len(value)/4, 1)
return value[:visibleLen] + "***"
}

View File

@@ -0,0 +1,162 @@
package config
import (
"encoding/json"
"flag"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_LoadFromJSONAndArgs_ValuesLoadedFromJSON(t *testing.T) {
dir := setupTempDir(t)
writeConfigJSON(t, dir, Config{
DatabasusHost: "http://json-host:4005",
DbID: "json-db-id",
Token: "json-token",
})
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{})
assert.Equal(t, "http://json-host:4005", cfg.DatabasusHost)
assert.Equal(t, "json-db-id", cfg.DbID)
assert.Equal(t, "json-token", cfg.Token)
}
func Test_LoadFromJSONAndArgs_ValuesLoadedFromArgs_WhenNoJSON(t *testing.T) {
setupTempDir(t)
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{
"--databasus-host", "http://arg-host:4005",
"--db-id", "arg-db-id",
"--token", "arg-token",
})
assert.Equal(t, "http://arg-host:4005", cfg.DatabasusHost)
assert.Equal(t, "arg-db-id", cfg.DbID)
assert.Equal(t, "arg-token", cfg.Token)
}
func Test_LoadFromJSONAndArgs_ArgsOverrideJSON(t *testing.T) {
dir := setupTempDir(t)
writeConfigJSON(t, dir, Config{
DatabasusHost: "http://json-host:4005",
DbID: "json-db-id",
Token: "json-token",
})
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{
"--databasus-host", "http://arg-host:9999",
"--db-id", "arg-db-id-override",
"--token", "arg-token-override",
})
assert.Equal(t, "http://arg-host:9999", cfg.DatabasusHost)
assert.Equal(t, "arg-db-id-override", cfg.DbID)
assert.Equal(t, "arg-token-override", cfg.Token)
}
func Test_LoadFromJSONAndArgs_PartialArgsOverrideJSON(t *testing.T) {
dir := setupTempDir(t)
writeConfigJSON(t, dir, Config{
DatabasusHost: "http://json-host:4005",
DbID: "json-db-id",
Token: "json-token",
})
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{
"--databasus-host", "http://arg-host-only:4005",
})
assert.Equal(t, "http://arg-host-only:4005", cfg.DatabasusHost)
assert.Equal(t, "json-db-id", cfg.DbID)
assert.Equal(t, "json-token", cfg.Token)
}
func Test_SaveToJSON_ConfigSavedCorrectly(t *testing.T) {
setupTempDir(t)
cfg := &Config{
DatabasusHost: "http://save-host:4005",
DbID: "save-db-id",
Token: "save-token",
}
err := cfg.SaveToJSON()
require.NoError(t, err)
saved := readConfigJSON(t)
assert.Equal(t, "http://save-host:4005", saved.DatabasusHost)
assert.Equal(t, "save-db-id", saved.DbID)
assert.Equal(t, "save-token", saved.Token)
}
func Test_SaveToJSON_AfterArgsOverrideJSON_SavedFileContainsMergedValues(t *testing.T) {
dir := setupTempDir(t)
writeConfigJSON(t, dir, Config{
DatabasusHost: "http://json-host:4005",
DbID: "json-db-id",
Token: "json-token",
})
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{
"--databasus-host", "http://override-host:9999",
})
err := cfg.SaveToJSON()
require.NoError(t, err)
saved := readConfigJSON(t)
assert.Equal(t, "http://override-host:9999", saved.DatabasusHost)
assert.Equal(t, "json-db-id", saved.DbID)
assert.Equal(t, "json-token", saved.Token)
}
func setupTempDir(t *testing.T) string {
t.Helper()
origDir, err := os.Getwd()
require.NoError(t, err)
dir := t.TempDir()
require.NoError(t, os.Chdir(dir))
t.Cleanup(func() { os.Chdir(origDir) })
return dir
}
func writeConfigJSON(t *testing.T, dir string, cfg Config) {
t.Helper()
data, err := json.MarshalIndent(cfg, "", " ")
require.NoError(t, err)
require.NoError(t, os.WriteFile(dir+"/"+configFileName, data, 0o644))
}
func readConfigJSON(t *testing.T) Config {
t.Helper()
data, err := os.ReadFile(configFileName)
require.NoError(t, err)
var cfg Config
require.NoError(t, json.Unmarshal(data, &cfg))
return cfg
}

View File

@@ -0,0 +1,9 @@
package config
type parsedFlags struct {
host *string
dbID *string
token *string
sources map[string]string
}

View File

@@ -0,0 +1,37 @@
package start
import (
"errors"
"log/slog"
"databasus-agent/internal/config"
)
func Run(cfg *config.Config, log *slog.Logger) error {
if err := validateConfig(cfg); err != nil {
return err
}
log.Info("start: stub — not yet implemented",
"dbId", cfg.DbID,
"hasToken", cfg.Token != "",
)
return nil
}
func validateConfig(cfg *config.Config) error {
if cfg.DatabasusHost == "" {
return errors.New("argument databasus-host is required")
}
if cfg.DbID == "" {
return errors.New("argument db-id is required")
}
if cfg.Token == "" {
return errors.New("argument token is required")
}
return nil
}

View File

@@ -0,0 +1,159 @@
package upgrade
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"runtime"
"strings"
"syscall"
"time"
)
type Logger interface {
Info(msg string, args ...any)
Warn(msg string, args ...any)
Error(msg string, args ...any)
}
type versionResponse struct {
Version string `json:"version"`
}
func CheckAndUpdate(databasusHost, currentVersion string, isDev bool, log Logger) error {
if isDev {
log.Info("Skipping update check (development mode)")
return nil
}
serverVersion, err := fetchServerVersion(databasusHost, log)
if err != nil {
return nil
}
if serverVersion == currentVersion {
log.Info("Agent version is up to date", "version", currentVersion)
return nil
}
log.Info("Updating agent...", "current", currentVersion, "target", serverVersion)
selfPath, err := os.Executable()
if err != nil {
return fmt.Errorf("failed to determine executable path: %w", err)
}
tempPath := selfPath + ".update"
defer func() {
_ = os.Remove(tempPath)
}()
if err := downloadBinary(databasusHost, tempPath); err != nil {
return fmt.Errorf("failed to download update: %w", err)
}
if err := os.Chmod(tempPath, 0o755); err != nil {
return fmt.Errorf("failed to set permissions on update: %w", err)
}
if err := verifyBinary(tempPath, serverVersion); err != nil {
return fmt.Errorf("update verification failed: %w", err)
}
if err := os.Rename(tempPath, selfPath); err != nil {
return fmt.Errorf("failed to replace binary (try --skip-update if this persists): %w", err)
}
log.Info("Update complete, re-executing...")
return syscall.Exec(selfPath, os.Args, os.Environ())
}
func fetchServerVersion(host string, log Logger) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, host+"/api/v1/system/version", nil)
if err != nil {
return "", err
}
resp, err := client.Do(req)
if err != nil {
log.Warn("Could not reach server for update check, continuing", "error", err)
return "", err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
log.Warn(
"Server returned non-OK status for version check, continuing",
"status",
resp.StatusCode,
)
return "", fmt.Errorf("status %d", resp.StatusCode)
}
var ver versionResponse
if err := json.NewDecoder(resp.Body).Decode(&ver); err != nil {
log.Warn("Failed to parse server version response, continuing", "error", err)
return "", err
}
return ver.Version, nil
}
func downloadBinary(host, destPath string) error {
url := fmt.Sprintf("%s/api/v1/system/agent?arch=%s", host, runtime.GOARCH)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("server returned %d for agent download", resp.StatusCode)
}
f, err := os.Create(destPath)
if err != nil {
return err
}
defer func() { _ = f.Close() }()
_, err = io.Copy(f, resp.Body)
return err
}
func verifyBinary(binaryPath, expectedVersion string) error {
cmd := exec.CommandContext(context.Background(), binaryPath, "version")
output, err := cmd.Output()
if err != nil {
return fmt.Errorf("binary failed to execute: %w", err)
}
got := strings.TrimSpace(string(output))
if got != expectedVersion {
return fmt.Errorf("version mismatch: expected %q, got %q", expectedVersion, got)
}
return nil
}

View File

@@ -0,0 +1,47 @@
package logger
import (
"log/slog"
"os"
"sync"
"time"
)
var (
loggerInstance *slog.Logger
once sync.Once
)
func Init(isDebug bool) {
level := slog.LevelInfo
if isDebug {
level = slog.LevelDebug
}
once.Do(func() {
loggerInstance = slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
Level: level,
ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
if a.Key == slog.TimeKey {
a.Value = slog.StringValue(time.Now().Format("2006/01/02 15:04:05"))
}
if a.Key == slog.LevelKey {
return slog.Attr{}
}
return a
},
}))
loggerInstance.Info("Text structured logger initialized")
})
}
// GetLogger returns a singleton slog.Logger that logs to the console
func GetLogger() *slog.Logger {
if loggerInstance == nil {
Init(false)
}
return loggerInstance
}

View File

@@ -7,6 +7,16 @@ run:
linters:
default: standard
enable:
- funcorder
- bodyclose
- errorlint
- gocritic
- unconvert
- misspell
- errname
- noctx
- modernize
settings:
errcheck:
@@ -14,6 +24,18 @@ linters:
formatters:
enable:
- gofmt
- gofumpt
- golines
- goimports
- gci
settings:
golines:
max-len: 120
gofumpt:
module-path: databasus-backend
extra-rules: true
gci:
sections:
- standard
- default
- localmodule

View File

@@ -12,11 +12,18 @@ import (
"syscall"
"time"
"github.com/gin-contrib/cors"
"github.com/gin-contrib/gzip"
"github.com/gin-gonic/gin"
swaggerFiles "github.com/swaggo/files"
ginSwagger "github.com/swaggo/gin-swagger"
"databasus-backend/internal/config"
"databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/backups/backups"
"databasus-backend/internal/features/backups/backups/backuping"
backups_controllers "databasus-backend/internal/features/backups/backups/controllers"
backups_download "databasus-backend/internal/features/backups/backups/download"
backups_services "databasus-backend/internal/features/backups/backups/services"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/disk"
@@ -27,7 +34,9 @@ import (
"databasus-backend/internal/features/restores"
"databasus-backend/internal/features/restores/restoring"
"databasus-backend/internal/features/storages"
system_agent "databasus-backend/internal/features/system/agent"
system_healthcheck "databasus-backend/internal/features/system/healthcheck"
system_version "databasus-backend/internal/features/system/version"
task_cancellation "databasus-backend/internal/features/tasks/cancellation"
users_controllers "databasus-backend/internal/features/users/controllers"
users_middleware "databasus-backend/internal/features/users/middleware"
@@ -38,12 +47,6 @@ import (
files_utils "databasus-backend/internal/util/files"
"databasus-backend/internal/util/logger"
_ "databasus-backend/swagger" // swagger docs
"github.com/gin-contrib/cors"
"github.com/gin-contrib/gzip"
"github.com/gin-gonic/gin"
swaggerFiles "github.com/swaggo/files"
ginSwagger "github.com/swaggo/gin-swagger"
)
// @title Databasus Backend API
@@ -80,7 +83,6 @@ func main() {
config.GetEnv().TempFolder,
config.GetEnv().DataFolder,
})
if err != nil {
log.Error("Failed to ensure directories", "error", err)
os.Exit(1)
@@ -147,7 +149,7 @@ func handlePasswordReset(log *slog.Logger) {
resetPassword(*email, *newPassword, log)
}
func resetPassword(email string, newPassword string, log *slog.Logger) {
func resetPassword(email, newPassword string, log *slog.Logger) {
log.Info("Resetting password...")
userService := users_services.GetUserService()
@@ -209,7 +211,11 @@ func setUpRoutes(r *gin.Engine) {
userController := users_controllers.GetUserController()
userController.RegisterRoutes(v1)
system_healthcheck.GetHealthcheckController().RegisterRoutes(v1)
backups.GetBackupController().RegisterPublicRoutes(v1)
system_version.GetVersionController().RegisterRoutes(v1)
system_agent.GetAgentController().RegisterRoutes(v1)
backups_controllers.GetBackupController().RegisterPublicRoutes(v1)
backups_controllers.GetPostgresWalBackupController().RegisterRoutes(v1)
databases.GetDatabaseController().RegisterPublicRoutes(v1)
// Setup auth middleware
userService := users_services.GetUserService()
@@ -226,7 +232,7 @@ func setUpRoutes(r *gin.Engine) {
notifiers.GetNotifierController().RegisterRoutes(protected)
storages.GetStorageController().RegisterRoutes(protected)
databases.GetDatabaseController().RegisterRoutes(protected)
backups.GetBackupController().RegisterRoutes(protected)
backups_controllers.GetBackupController().RegisterRoutes(protected)
restores.GetRestoreController().RegisterRoutes(protected)
healthcheck_config.GetHealthcheckConfigController().RegisterRoutes(protected)
healthcheck_attempt.GetHealthcheckAttemptController().RegisterRoutes(protected)
@@ -238,7 +244,7 @@ func setUpRoutes(r *gin.Engine) {
func setUpDependencies() {
databases.SetupDependencies()
backups.SetupDependencies()
backups_services.SetupDependencies()
restores.SetupDependencies()
healthcheck_config.SetupDependencies()
audit_logs.SetupDependencies()
@@ -347,7 +353,9 @@ func generateSwaggerDocs(log *slog.Logger) {
return
}
cmd := exec.Command("swag", "init", "-d", currentDir, "-g", "cmd/main.go", "-o", "swagger")
cmd := exec.CommandContext(
context.Background(), "swag", "init", "-d", currentDir, "-g", "cmd/main.go", "-o", "swagger",
)
output, err := cmd.CombinedOutput()
if err != nil {
@@ -361,7 +369,7 @@ func generateSwaggerDocs(log *slog.Logger) {
func runMigrations(log *slog.Logger) {
log.Info("Running database migrations...")
cmd := exec.Command("goose", "-dir", "./migrations", "up")
cmd := exec.CommandContext(context.Background(), "goose", "-dir", "./migrations", "up")
cmd.Env = append(
os.Environ(),
"GOOSE_DRIVER=postgres",

View File

@@ -1,6 +1,6 @@
module databasus-backend
go 1.24.9
go 1.26.1
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0

View File

@@ -1,9 +1,6 @@
package config
import (
env_utils "databasus-backend/internal/util/env"
"databasus-backend/internal/util/logger"
"databasus-backend/internal/util/tools"
"os"
"path/filepath"
"strings"
@@ -11,6 +8,10 @@ import (
"github.com/ilyakaznacheev/cleanenv"
"github.com/joho/godotenv"
env_utils "databasus-backend/internal/util/env"
"databasus-backend/internal/util/logger"
"databasus-backend/internal/util/tools"
)
var log = logger.GetLogger()
@@ -29,7 +30,7 @@ type EnvVariables struct {
MongodbInstallDir string `env:"MONGODB_INSTALL_DIR"`
// Internal database
DatabaseDsn string `env:"DATABASE_DSN" required:"true"`
DatabaseDsn string `env:"DATABASE_DSN" required:"true"`
// Internal Valkey
ValkeyHost string `env:"VALKEY_HOST" required:"true"`
ValkeyPort string `env:"VALKEY_PORT" required:"true"`
@@ -124,6 +125,7 @@ type EnvVariables struct {
SMTPPort int `env:"SMTP_PORT"`
SMTPUser string `env:"SMTP_USER"`
SMTPPassword string `env:"SMTP_PASSWORD"`
SMTPFrom string `env:"SMTP_FROM"`
// Application URL (optional) - used for email links
DatabasusURL string `env:"DATABASUS_URL"`

View File

@@ -1,17 +1,17 @@
package audit_logs
import (
"databasus-backend/internal/storage"
"fmt"
"testing"
"time"
user_enums "databasus-backend/internal/features/users/enums"
users_testing "databasus-backend/internal/features/users/testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"gorm.io/gorm"
user_enums "databasus-backend/internal/features/users/enums"
users_testing "databasus-backend/internal/features/users/testing"
"databasus-backend/internal/storage"
)
func Test_CleanOldAuditLogs_DeletesLogsOlderThanOneYear(t *testing.T) {

View File

@@ -4,10 +4,10 @@ import (
"errors"
"net/http"
user_models "databasus-backend/internal/features/users/models"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
user_models "databasus-backend/internal/features/users/models"
)
type AuditLogController struct {

View File

@@ -6,15 +6,15 @@ import (
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
user_enums "databasus-backend/internal/features/users/enums"
users_middleware "databasus-backend/internal/features/users/middleware"
users_services "databasus-backend/internal/features/users/services"
users_testing "databasus-backend/internal/features/users/testing"
test_utils "databasus-backend/internal/util/testing"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
func Test_GetGlobalAuditLogs_WithDifferentUserRoles_EnforcesPermissionsCorrectly(t *testing.T) {

View File

@@ -8,14 +8,18 @@ import (
"databasus-backend/internal/util/logger"
)
var auditLogRepository = &AuditLogRepository{}
var auditLogService = &AuditLogService{
auditLogRepository,
logger.GetLogger(),
}
var (
auditLogRepository = &AuditLogRepository{}
auditLogService = &AuditLogService{
auditLogRepository,
logger.GetLogger(),
}
)
var auditLogController = &AuditLogController{
auditLogService,
}
var auditLogBackgroundService = &AuditLogBackgroundService{
auditLogService: auditLogService,
logger: logger.GetLogger(),

View File

@@ -1,10 +1,11 @@
package audit_logs
import (
"databasus-backend/internal/storage"
"time"
"github.com/google/uuid"
"databasus-backend/internal/storage"
)
type AuditLogRepository struct{}
@@ -21,7 +22,7 @@ func (r *AuditLogRepository) GetGlobal(
limit, offset int,
beforeDate *time.Time,
) ([]*AuditLogDTO, error) {
var auditLogs = make([]*AuditLogDTO, 0)
auditLogs := make([]*AuditLogDTO, 0)
sql := `
SELECT
@@ -37,7 +38,7 @@ func (r *AuditLogRepository) GetGlobal(
LEFT JOIN users u ON al.user_id = u.id
LEFT JOIN workspaces w ON al.workspace_id = w.id`
args := []interface{}{}
args := []any{}
if beforeDate != nil {
sql += " WHERE al.created_at < ?"
@@ -57,7 +58,7 @@ func (r *AuditLogRepository) GetByUser(
limit, offset int,
beforeDate *time.Time,
) ([]*AuditLogDTO, error) {
var auditLogs = make([]*AuditLogDTO, 0)
auditLogs := make([]*AuditLogDTO, 0)
sql := `
SELECT
@@ -74,7 +75,7 @@ func (r *AuditLogRepository) GetByUser(
LEFT JOIN workspaces w ON al.workspace_id = w.id
WHERE al.user_id = ?`
args := []interface{}{userID}
args := []any{userID}
if beforeDate != nil {
sql += " AND al.created_at < ?"
@@ -94,7 +95,7 @@ func (r *AuditLogRepository) GetByWorkspace(
limit, offset int,
beforeDate *time.Time,
) ([]*AuditLogDTO, error) {
var auditLogs = make([]*AuditLogDTO, 0)
auditLogs := make([]*AuditLogDTO, 0)
sql := `
SELECT
@@ -111,7 +112,7 @@ func (r *AuditLogRepository) GetByWorkspace(
LEFT JOIN workspaces w ON al.workspace_id = w.id
WHERE al.workspace_id = ?`
args := []interface{}{workspaceID}
args := []any{workspaceID}
if beforeDate != nil {
sql += " AND al.created_at < ?"

View File

@@ -4,10 +4,10 @@ import (
"log/slog"
"time"
"github.com/google/uuid"
user_enums "databasus-backend/internal/features/users/enums"
user_models "databasus-backend/internal/features/users/models"
"github.com/google/uuid"
)
type AuditLogService struct {

View File

@@ -4,11 +4,11 @@ import (
"testing"
"time"
user_enums "databasus-backend/internal/features/users/enums"
users_testing "databasus-backend/internal/features/users/testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
user_enums "databasus-backend/internal/features/users/enums"
users_testing "databasus-backend/internal/features/users/testing"
)
func Test_AuditLogs_WorkspaceSpecificLogs(t *testing.T) {

View File

@@ -5,6 +5,9 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
@@ -14,9 +17,6 @@ import (
users_testing "databasus-backend/internal/features/users/testing"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
cache_utils "databasus-backend/internal/util/cache"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
func Test_BackupExecuted_NotificationSent(t *testing.T) {

View File

@@ -18,7 +18,8 @@ import (
)
const (
cleanerTickerInterval = 1 * time.Minute
cleanerTickerInterval = 1 * time.Minute
recentBackupGracePeriod = 60 * time.Minute
)
type BackupCleaner struct {
@@ -51,8 +52,8 @@ func (c *BackupCleaner) Run(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
if err := c.cleanOldBackups(); err != nil {
c.logger.Error("Failed to clean old backups", "error", err)
if err := c.cleanByRetentionPolicy(); err != nil {
c.logger.Error("Failed to clean backups by retention policy", "error", err)
}
if err := c.cleanExceededBackups(); err != nil {
@@ -79,8 +80,7 @@ func (c *BackupCleaner) DeleteBackup(backup *backups_core.Backup) error {
return err
}
err = storage.DeleteFile(c.fieldEncryptor, backup.FileName)
if err != nil {
if err := storage.DeleteFile(c.fieldEncryptor, backup.FileName); err != nil {
// we do not return error here, because sometimes clean up performed
// before unavailable storage removal or change - therefore we should
// proceed even in case of error. It's possible that some S3 or
@@ -100,49 +100,30 @@ func (c *BackupCleaner) AddBackupRemoveListener(listener backups_core.BackupRemo
c.backupRemoveListeners = append(c.backupRemoveListeners, listener)
}
func (c *BackupCleaner) cleanOldBackups() error {
func (c *BackupCleaner) cleanByRetentionPolicy() error {
enabledBackupConfigs, err := c.backupConfigService.GetBackupConfigsWithEnabledBackups()
if err != nil {
return err
}
for _, backupConfig := range enabledBackupConfigs {
backupStorePeriod := backupConfig.StorePeriod
var cleanErr error
if backupStorePeriod == period.PeriodForever {
continue
switch backupConfig.RetentionPolicyType {
case backups_config.RetentionPolicyTypeCount:
cleanErr = c.cleanByCount(backupConfig)
case backups_config.RetentionPolicyTypeGFS:
cleanErr = c.cleanByGFS(backupConfig)
default:
cleanErr = c.cleanByTimePeriod(backupConfig)
}
storeDuration := backupStorePeriod.ToDuration()
dateBeforeBackupsShouldBeDeleted := time.Now().UTC().Add(-storeDuration)
oldBackups, err := c.backupRepository.FindBackupsBeforeDate(
backupConfig.DatabaseID,
dateBeforeBackupsShouldBeDeleted,
)
if err != nil {
if cleanErr != nil {
c.logger.Error(
"Failed to find old backups for database",
"databaseId",
backupConfig.DatabaseID,
"error",
err,
)
continue
}
for _, backup := range oldBackups {
if err := c.DeleteBackup(backup); err != nil {
c.logger.Error("Failed to delete old backup", "backupId", backup.ID, "error", err)
continue
}
c.logger.Info(
"Deleted old backup",
"backupId",
backup.ID,
"databaseId",
backupConfig.DatabaseID,
"Failed to clean backups by retention policy",
"databaseId", backupConfig.DatabaseID,
"policy", backupConfig.RetentionPolicyType,
"error", cleanErr,
)
}
}
@@ -179,6 +160,158 @@ func (c *BackupCleaner) cleanExceededBackups() error {
return nil
}
func (c *BackupCleaner) cleanByTimePeriod(backupConfig *backups_config.BackupConfig) error {
if backupConfig.RetentionTimePeriod == "" {
return nil
}
if backupConfig.RetentionTimePeriod == period.PeriodForever {
return nil
}
storeDuration := backupConfig.RetentionTimePeriod.ToDuration()
dateBeforeBackupsShouldBeDeleted := time.Now().UTC().Add(-storeDuration)
oldBackups, err := c.backupRepository.FindBackupsBeforeDate(
backupConfig.DatabaseID,
dateBeforeBackupsShouldBeDeleted,
)
if err != nil {
return fmt.Errorf(
"failed to find old backups for database %s: %w",
backupConfig.DatabaseID,
err,
)
}
for _, backup := range oldBackups {
if isRecentBackup(backup) {
continue
}
if err := c.DeleteBackup(backup); err != nil {
c.logger.Error("Failed to delete old backup", "backupId", backup.ID, "error", err)
continue
}
c.logger.Info(
"Deleted old backup",
"backupId", backup.ID,
"databaseId", backupConfig.DatabaseID,
)
}
return nil
}
func (c *BackupCleaner) cleanByCount(backupConfig *backups_config.BackupConfig) error {
if backupConfig.RetentionCount <= 0 {
return nil
}
completedBackups, err := c.backupRepository.FindByDatabaseIdAndStatus(
backupConfig.DatabaseID,
backups_core.BackupStatusCompleted,
)
if err != nil {
return fmt.Errorf(
"failed to find completed backups for database %s: %w",
backupConfig.DatabaseID,
err,
)
}
// completedBackups are ordered newest first; delete everything beyond position RetentionCount
if len(completedBackups) <= backupConfig.RetentionCount {
return nil
}
toDelete := completedBackups[backupConfig.RetentionCount:]
for _, backup := range toDelete {
if isRecentBackup(backup) {
continue
}
if err := c.DeleteBackup(backup); err != nil {
c.logger.Error(
"Failed to delete backup by count policy",
"backupId",
backup.ID,
"error",
err,
)
continue
}
c.logger.Info(
"Deleted backup by count policy",
"backupId", backup.ID,
"databaseId", backupConfig.DatabaseID,
"retentionCount", backupConfig.RetentionCount,
)
}
return nil
}
func (c *BackupCleaner) cleanByGFS(backupConfig *backups_config.BackupConfig) error {
if backupConfig.RetentionGfsHours <= 0 && backupConfig.RetentionGfsDays <= 0 &&
backupConfig.RetentionGfsWeeks <= 0 && backupConfig.RetentionGfsMonths <= 0 &&
backupConfig.RetentionGfsYears <= 0 {
return nil
}
completedBackups, err := c.backupRepository.FindByDatabaseIdAndStatus(
backupConfig.DatabaseID,
backups_core.BackupStatusCompleted,
)
if err != nil {
return fmt.Errorf(
"failed to find completed backups for database %s: %w",
backupConfig.DatabaseID,
err,
)
}
keepSet := buildGFSKeepSet(
completedBackups,
backupConfig.RetentionGfsHours,
backupConfig.RetentionGfsDays,
backupConfig.RetentionGfsWeeks,
backupConfig.RetentionGfsMonths,
backupConfig.RetentionGfsYears,
)
for _, backup := range completedBackups {
if keepSet[backup.ID] {
continue
}
if isRecentBackup(backup) {
continue
}
if err := c.DeleteBackup(backup); err != nil {
c.logger.Error(
"Failed to delete backup by GFS policy",
"backupId",
backup.ID,
"error",
err,
)
continue
}
c.logger.Info(
"Deleted backup by GFS policy",
"backupId", backup.ID,
"databaseId", backupConfig.DatabaseID,
)
}
return nil
}
func (c *BackupCleaner) cleanExceededBackupsForDatabase(
databaseID uuid.UUID,
limitperDbMB int64,
@@ -215,6 +348,21 @@ func (c *BackupCleaner) cleanExceededBackupsForDatabase(
}
backup := oldestBackups[0]
if isRecentBackup(backup) {
c.logger.Warn(
"Oldest backup is too recent to delete, stopping size cleanup",
"databaseId",
databaseID,
"backupId",
backup.ID,
"totalSizeMB",
backupsTotalSizeMB,
"limitMB",
limitperDbMB,
)
break
}
if err := c.DeleteBackup(backup); err != nil {
c.logger.Error(
"Failed to delete exceeded backup",
@@ -245,3 +393,128 @@ func (c *BackupCleaner) cleanExceededBackupsForDatabase(
return nil
}
func isRecentBackup(backup *backups_core.Backup) bool {
return time.Since(backup.CreatedAt) < recentBackupGracePeriod
}
// buildGFSKeepSet determines which backups to retain under the GFS rotation scheme.
// Backups must be sorted newest-first. A backup can fill multiple slots simultaneously
// (e.g. the newest backup of a year also fills the monthly, weekly, daily, and hourly slot).
func buildGFSKeepSet(
backups []*backups_core.Backup,
hours, days, weeks, months, years int,
) map[uuid.UUID]bool {
keep := make(map[uuid.UUID]bool)
if len(backups) == 0 {
return keep
}
hoursSeen := make(map[string]bool)
daysSeen := make(map[string]bool)
weeksSeen := make(map[string]bool)
monthsSeen := make(map[string]bool)
yearsSeen := make(map[string]bool)
hoursKept, daysKept, weeksKept, monthsKept, yearsKept := 0, 0, 0, 0, 0
// Compute per-level time-window cutoffs so higher-frequency slots
// cannot absorb backups that belong to lower-frequency levels.
ref := backups[0].CreatedAt
rawHourlyCutoff := ref.Add(-time.Duration(hours) * time.Hour)
rawDailyCutoff := ref.Add(-time.Duration(days) * 24 * time.Hour)
rawWeeklyCutoff := ref.Add(-time.Duration(weeks) * 7 * 24 * time.Hour)
rawMonthlyCutoff := ref.AddDate(0, -months, 0)
rawYearlyCutoff := ref.AddDate(-years, 0, 0)
// Hierarchical capping: each level's window cannot extend further back
// than the nearest active lower-frequency level's window.
yearlyCutoff := rawYearlyCutoff
monthlyCutoff := rawMonthlyCutoff
if years > 0 {
monthlyCutoff = laterOf(monthlyCutoff, yearlyCutoff)
}
weeklyCutoff := rawWeeklyCutoff
if months > 0 {
weeklyCutoff = laterOf(weeklyCutoff, monthlyCutoff)
} else if years > 0 {
weeklyCutoff = laterOf(weeklyCutoff, yearlyCutoff)
}
dailyCutoff := rawDailyCutoff
switch {
case weeks > 0:
dailyCutoff = laterOf(dailyCutoff, weeklyCutoff)
case months > 0:
dailyCutoff = laterOf(dailyCutoff, monthlyCutoff)
case years > 0:
dailyCutoff = laterOf(dailyCutoff, yearlyCutoff)
}
hourlyCutoff := rawHourlyCutoff
switch {
case days > 0:
hourlyCutoff = laterOf(hourlyCutoff, dailyCutoff)
case weeks > 0:
hourlyCutoff = laterOf(hourlyCutoff, weeklyCutoff)
case months > 0:
hourlyCutoff = laterOf(hourlyCutoff, monthlyCutoff)
case years > 0:
hourlyCutoff = laterOf(hourlyCutoff, yearlyCutoff)
}
for _, backup := range backups {
t := backup.CreatedAt
hourKey := t.Format("2006-01-02-15")
dayKey := t.Format("2006-01-02")
weekYear, week := t.ISOWeek()
weekKey := fmt.Sprintf("%d-%02d", weekYear, week)
monthKey := t.Format("2006-01")
yearKey := t.Format("2006")
if hours > 0 && hoursKept < hours && !hoursSeen[hourKey] && t.After(hourlyCutoff) {
keep[backup.ID] = true
hoursSeen[hourKey] = true
hoursKept++
}
if days > 0 && daysKept < days && !daysSeen[dayKey] && t.After(dailyCutoff) {
keep[backup.ID] = true
daysSeen[dayKey] = true
daysKept++
}
if weeks > 0 && weeksKept < weeks && !weeksSeen[weekKey] && t.After(weeklyCutoff) {
keep[backup.ID] = true
weeksSeen[weekKey] = true
weeksKept++
}
if months > 0 && monthsKept < months && !monthsSeen[monthKey] && t.After(monthlyCutoff) {
keep[backup.ID] = true
monthsSeen[monthKey] = true
monthsKept++
}
if years > 0 && yearsKept < years && !yearsSeen[yearKey] && t.After(yearlyCutoff) {
keep[backup.ID] = true
yearsSeen[yearKey] = true
yearsKept++
}
}
return keep
}
func laterOf(a, b time.Time) time.Time {
if a.After(b) {
return a
}
return b
}

File diff suppressed because it is too large Load Diff

View File

@@ -4,6 +4,9 @@ import (
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
@@ -15,12 +18,9 @@ import (
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
"databasus-backend/internal/storage"
"databasus-backend/internal/util/period"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {
func Test_CleanOldBackups_DeletesBackupsOlderThanRetentionTimePeriod(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
@@ -41,21 +41,20 @@ func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
// Create backup interval
interval := createTestInterval()
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
StorageID: &storage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
StorageID: &storage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
// Create backups with different ages
now := time.Now().UTC()
oldBackup1 := &backups_core.Backup{
ID: uuid.New(),
@@ -63,7 +62,7 @@ func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-10 * 24 * time.Hour), // 10 days old
CreatedAt: now.Add(-10 * 24 * time.Hour),
}
oldBackup2 := &backups_core.Backup{
ID: uuid.New(),
@@ -71,7 +70,7 @@ func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-8 * 24 * time.Hour), // 8 days old
CreatedAt: now.Add(-8 * 24 * time.Hour),
}
recentBackup := &backups_core.Backup{
ID: uuid.New(),
@@ -79,7 +78,7 @@ func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-3 * 24 * time.Hour), // 3 days old
CreatedAt: now.Add(-3 * 24 * time.Hour),
}
err = backupRepository.Save(oldBackup1)
@@ -89,19 +88,17 @@ func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {
err = backupRepository.Save(recentBackup)
assert.NoError(t, err)
// Run cleanup
cleaner := GetBackupCleaner()
err = cleaner.cleanOldBackups()
err = cleaner.cleanByRetentionPolicy()
assert.NoError(t, err)
// Verify old backups deleted, recent backup remains
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
assert.Equal(t, 1, len(remainingBackups))
assert.Equal(t, recentBackup.ID, remainingBackups[0].ID)
}
func Test_CleanOldBackups_SkipsDatabaseWithForeverStorePeriod(t *testing.T) {
func Test_CleanOldBackups_SkipsDatabaseWithForeverRetentionPeriod(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
@@ -122,38 +119,35 @@ func Test_CleanOldBackups_SkipsDatabaseWithForeverStorePeriod(t *testing.T) {
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
// Create backup interval
interval := createTestInterval()
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodForever,
StorageID: &storage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodForever,
StorageID: &storage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
// Create very old backup
oldBackup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: time.Now().UTC().Add(-365 * 24 * time.Hour), // 1 year old
CreatedAt: time.Now().UTC().Add(-365 * 24 * time.Hour),
}
err = backupRepository.Save(oldBackup)
assert.NoError(t, err)
// Run cleanup
cleaner := GetBackupCleaner()
err = cleaner.cleanOldBackups()
err = cleaner.cleanByRetentionPolicy()
assert.NoError(t, err)
// Verify backup still exists
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
assert.Equal(t, 1, len(remainingBackups))
@@ -181,22 +175,21 @@ func Test_CleanExceededBackups_WhenUnderLimit_NoBackupsDeleted(t *testing.T) {
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
// Create backup interval
interval := createTestInterval()
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodForever,
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodForever,
StorageID: &storage.ID,
MaxBackupsTotalSizeMB: 100, // 100 MB limit
MaxBackupsTotalSizeMB: 100,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
// Create 3 backups totaling 50MB (under limit)
for i := 0; i < 3; i++ {
backup := &backups_core.Backup{
ID: uuid.New(),
@@ -210,12 +203,10 @@ func Test_CleanExceededBackups_WhenUnderLimit_NoBackupsDeleted(t *testing.T) {
assert.NoError(t, err)
}
// Run cleanup
cleaner := GetBackupCleaner()
err = cleaner.cleanExceededBackups()
assert.NoError(t, err)
// Verify all backups remain
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
assert.Equal(t, 3, len(remainingBackups))
@@ -242,22 +233,21 @@ func Test_CleanExceededBackups_WhenOverLimit_DeletesOldestBackups(t *testing.T)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
// Create backup interval
interval := createTestInterval()
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodForever,
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodForever,
StorageID: &storage.ID,
MaxBackupsTotalSizeMB: 30, // 30 MB limit
MaxBackupsTotalSizeMB: 30,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
// Create 5 backups of 10MB each (total 50MB, over 30MB limit)
now := time.Now().UTC()
var backupIDs []uuid.UUID
for i := 0; i < 5; i++ {
@@ -267,33 +257,30 @@ func Test_CleanExceededBackups_WhenOverLimit_DeletesOldestBackups(t *testing.T)
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-time.Duration(4-i) * time.Hour), // Oldest first
CreatedAt: now.Add(-time.Duration(4-i) * time.Hour),
}
err = backupRepository.Save(backup)
assert.NoError(t, err)
backupIDs = append(backupIDs, backup.ID)
}
// Run cleanup
cleaner := GetBackupCleaner()
err = cleaner.cleanExceededBackups()
assert.NoError(t, err)
// Verify 2 oldest backups deleted, 3 newest remain
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
assert.Equal(t, 3, len(remainingBackups))
// Check that the newest 3 backups remain
remainingIDs := make(map[uuid.UUID]bool)
for _, backup := range remainingBackups {
remainingIDs[backup.ID] = true
}
assert.False(t, remainingIDs[backupIDs[0]]) // Oldest deleted
assert.False(t, remainingIDs[backupIDs[1]]) // 2nd oldest deleted
assert.True(t, remainingIDs[backupIDs[2]]) // 3rd remains
assert.True(t, remainingIDs[backupIDs[3]]) // 4th remains
assert.True(t, remainingIDs[backupIDs[4]]) // Newest remains
assert.False(t, remainingIDs[backupIDs[0]])
assert.False(t, remainingIDs[backupIDs[1]])
assert.True(t, remainingIDs[backupIDs[2]])
assert.True(t, remainingIDs[backupIDs[3]])
assert.True(t, remainingIDs[backupIDs[4]])
}
func Test_CleanExceededBackups_SkipsInProgressBackups(t *testing.T) {
@@ -317,15 +304,15 @@ func Test_CleanExceededBackups_SkipsInProgressBackups(t *testing.T) {
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
// Create backup interval
interval := createTestInterval()
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodForever,
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodForever,
StorageID: &storage.ID,
MaxBackupsTotalSizeMB: 50, // 50 MB limit
MaxBackupsTotalSizeMB: 50,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
@@ -334,7 +321,6 @@ func Test_CleanExceededBackups_SkipsInProgressBackups(t *testing.T) {
now := time.Now().UTC()
// Create 3 completed backups of 30MB each
completedBackups := make([]*backups_core.Backup, 3)
for i := 0; i < 3; i++ {
backup := &backups_core.Backup{
@@ -350,7 +336,6 @@ func Test_CleanExceededBackups_SkipsInProgressBackups(t *testing.T) {
completedBackups[i] = backup
}
// Create 1 in-progress backup (should be excluded from size calculation and deletion)
inProgressBackup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
@@ -362,19 +347,14 @@ func Test_CleanExceededBackups_SkipsInProgressBackups(t *testing.T) {
err = backupRepository.Save(inProgressBackup)
assert.NoError(t, err)
// Run cleanup
cleaner := GetBackupCleaner()
err = cleaner.cleanExceededBackups()
assert.NoError(t, err)
// Verify: only completed backups deleted, in-progress remains
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
// Should have in-progress + 1 completed (total 40MB completed + 10MB in-progress)
assert.GreaterOrEqual(t, len(remainingBackups), 2)
// Verify in-progress backup still exists
var inProgressFound bool
for _, backup := range remainingBackups {
if backup.ID == inProgressBackup.ID {
@@ -406,22 +386,21 @@ func Test_CleanExceededBackups_WithZeroLimit_SkipsDatabase(t *testing.T) {
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
// Create backup interval
interval := createTestInterval()
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodForever,
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodForever,
StorageID: &storage.ID,
MaxBackupsTotalSizeMB: 0, // No size limit
MaxBackupsTotalSizeMB: 0,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
// Create large backups
for i := 0; i < 10; i++ {
backup := &backups_core.Backup{
ID: uuid.New(),
@@ -435,12 +414,10 @@ func Test_CleanExceededBackups_WithZeroLimit_SkipsDatabase(t *testing.T) {
assert.NoError(t, err)
}
// Run cleanup
cleaner := GetBackupCleaner()
err = cleaner.cleanExceededBackups()
assert.NoError(t, err)
// Verify all backups remain
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
assert.Equal(t, 10, len(remainingBackups))
@@ -467,7 +444,6 @@ func Test_GetTotalSizeByDatabase_CalculatesCorrectly(t *testing.T) {
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
// Create completed backups
completedBackup1 := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
@@ -484,7 +460,6 @@ func Test_GetTotalSizeByDatabase_CalculatesCorrectly(t *testing.T) {
BackupSizeMb: 20.3,
CreatedAt: time.Now().UTC(),
}
// Create failed backup (should be included)
failedBackup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
@@ -493,7 +468,6 @@ func Test_GetTotalSizeByDatabase_CalculatesCorrectly(t *testing.T) {
BackupSizeMb: 5.2,
CreatedAt: time.Now().UTC(),
}
// Create in-progress backup (should be excluded)
inProgressBackup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
@@ -512,25 +486,215 @@ func Test_GetTotalSizeByDatabase_CalculatesCorrectly(t *testing.T) {
err = backupRepository.Save(inProgressBackup)
assert.NoError(t, err)
// Calculate total size
totalSize, err := backupRepository.GetTotalSizeByDatabase(database.ID)
assert.NoError(t, err)
// Should be 10.5 + 20.3 + 5.2 = 36.0 (excluding in-progress 100)
assert.InDelta(t, 36.0, totalSize, 0.1)
}
// Mock listener for testing
type mockBackupRemoveListener struct {
onBeforeBackupRemove func(*backups_core.Backup) error
}
func Test_CleanByCount_KeepsNewestNBackups_DeletesOlder(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
storage := storages.CreateTestStorage(workspace.ID)
notifier := notifiers.CreateTestNotifier(workspace.ID)
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
func (m *mockBackupRemoveListener) OnBeforeBackupRemove(backup *backups_core.Backup) error {
if m.onBeforeBackupRemove != nil {
return m.onBeforeBackupRemove(backup)
defer func() {
backups, _ := backupRepository.FindByDatabaseID(database.ID)
for _, backup := range backups {
backupRepository.DeleteByID(backup.ID)
}
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
interval := createTestInterval()
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: backups_config.RetentionPolicyTypeCount,
RetentionCount: 3,
StorageID: &storage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
now := time.Now().UTC()
var backupIDs []uuid.UUID
for i := 0; i < 5; i++ {
backup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(
-time.Duration(4-i) * time.Hour,
), // oldest first in loop, newest = i=4
}
err = backupRepository.Save(backup)
assert.NoError(t, err)
backupIDs = append(backupIDs, backup.ID)
}
return nil
cleaner := GetBackupCleaner()
err = cleaner.cleanByRetentionPolicy()
assert.NoError(t, err)
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
assert.Equal(t, 3, len(remainingBackups))
remainingIDs := make(map[uuid.UUID]bool)
for _, backup := range remainingBackups {
remainingIDs[backup.ID] = true
}
assert.False(t, remainingIDs[backupIDs[0]], "Oldest backup should be deleted")
assert.False(t, remainingIDs[backupIDs[1]], "2nd oldest backup should be deleted")
assert.True(t, remainingIDs[backupIDs[2]], "3rd backup should remain")
assert.True(t, remainingIDs[backupIDs[3]], "4th backup should remain")
assert.True(t, remainingIDs[backupIDs[4]], "Newest backup should remain")
}
func Test_CleanByCount_WhenUnderLimit_NoBackupsDeleted(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
storage := storages.CreateTestStorage(workspace.ID)
notifier := notifiers.CreateTestNotifier(workspace.ID)
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
defer func() {
backups, _ := backupRepository.FindByDatabaseID(database.ID)
for _, backup := range backups {
backupRepository.DeleteByID(backup.ID)
}
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
interval := createTestInterval()
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: backups_config.RetentionPolicyTypeCount,
RetentionCount: 10,
StorageID: &storage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
for i := 0; i < 5; i++ {
backup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: time.Now().UTC().Add(-time.Duration(i) * time.Hour),
}
err = backupRepository.Save(backup)
assert.NoError(t, err)
}
cleaner := GetBackupCleaner()
err = cleaner.cleanByRetentionPolicy()
assert.NoError(t, err)
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
assert.Equal(t, 5, len(remainingBackups))
}
func Test_CleanByCount_DoesNotDeleteInProgressBackups(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
storage := storages.CreateTestStorage(workspace.ID)
notifier := notifiers.CreateTestNotifier(workspace.ID)
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
defer func() {
backups, _ := backupRepository.FindByDatabaseID(database.ID)
for _, backup := range backups {
backupRepository.DeleteByID(backup.ID)
}
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
interval := createTestInterval()
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: backups_config.RetentionPolicyTypeCount,
RetentionCount: 2,
StorageID: &storage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
now := time.Now().UTC()
for i := 0; i < 3; i++ {
backup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-time.Duration(3-i) * time.Hour),
}
err = backupRepository.Save(backup)
assert.NoError(t, err)
}
inProgressBackup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusInProgress,
BackupSizeMb: 5,
CreatedAt: now,
}
err = backupRepository.Save(inProgressBackup)
assert.NoError(t, err)
cleaner := GetBackupCleaner()
err = cleaner.cleanByRetentionPolicy()
assert.NoError(t, err)
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
var inProgressFound bool
for _, backup := range remainingBackups {
if backup.ID == inProgressBackup.ID {
inProgressFound = true
}
}
assert.True(t, inProgressFound, "In-progress backup should not be deleted by count policy")
}
// Test_DeleteBackup_WhenStorageDeleteFails_BackupStillRemovedFromDatabase verifies resilience
@@ -579,6 +743,267 @@ func Test_DeleteBackup_WhenStorageDeleteFails_BackupStillRemovedFromDatabase(t *
assert.Nil(t, deletedBackup)
}
func Test_CleanByTimePeriod_SkipsRecentBackup_EvenIfOlderThanRetention(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
storage := storages.CreateTestStorage(workspace.ID)
notifier := notifiers.CreateTestNotifier(workspace.ID)
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
defer func() {
backups, _ := backupRepository.FindByDatabaseID(database.ID)
for _, backup := range backups {
backupRepository.DeleteByID(backup.ID)
}
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
interval := createTestInterval()
// Retention period is 1 day — any backup older than 1 day should be deleted.
// But the recent backup was created only 30 minutes ago and must be preserved.
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodDay,
StorageID: &storage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
now := time.Now().UTC()
oldBackup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-2 * 24 * time.Hour),
}
recentBackup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-30 * time.Minute),
}
err = backupRepository.Save(oldBackup)
assert.NoError(t, err)
err = backupRepository.Save(recentBackup)
assert.NoError(t, err)
cleaner := GetBackupCleaner()
err = cleaner.cleanByRetentionPolicy()
assert.NoError(t, err)
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
assert.Equal(t, 1, len(remainingBackups))
assert.Equal(t, recentBackup.ID, remainingBackups[0].ID)
}
func Test_CleanByCount_SkipsRecentBackup_EvenIfOverLimit(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
storage := storages.CreateTestStorage(workspace.ID)
notifier := notifiers.CreateTestNotifier(workspace.ID)
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
defer func() {
backups, _ := backupRepository.FindByDatabaseID(database.ID)
for _, backup := range backups {
backupRepository.DeleteByID(backup.ID)
}
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
interval := createTestInterval()
// Retention count is 2 — 4 backups exist so 2 should be deleted.
// The oldest backup in the "excess" tail was made 30 min ago — it must be preserved.
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: backups_config.RetentionPolicyTypeCount,
RetentionCount: 2,
StorageID: &storage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
now := time.Now().UTC()
oldBackup1 := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-5 * time.Hour),
}
oldBackup2 := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-3 * time.Hour),
}
// This backup is 3rd newest and would normally be deleted — but it is recent.
recentExcessBackup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-30 * time.Minute),
}
newestBackup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-10 * time.Minute),
}
for _, b := range []*backups_core.Backup{oldBackup1, oldBackup2, recentExcessBackup, newestBackup} {
err = backupRepository.Save(b)
assert.NoError(t, err)
}
cleaner := GetBackupCleaner()
err = cleaner.cleanByRetentionPolicy()
assert.NoError(t, err)
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
remainingIDs := make(map[uuid.UUID]bool)
for _, backup := range remainingBackups {
remainingIDs[backup.ID] = true
}
assert.False(t, remainingIDs[oldBackup1.ID], "Oldest non-recent backup should be deleted")
assert.False(t, remainingIDs[oldBackup2.ID], "2nd oldest non-recent backup should be deleted")
assert.True(
t,
remainingIDs[recentExcessBackup.ID],
"Recent backup must be preserved despite being over limit",
)
assert.True(t, remainingIDs[newestBackup.ID], "Newest backup should be preserved")
}
func Test_CleanExceededBackups_SkipsRecentBackup_WhenOverTotalSizeLimit(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
storage := storages.CreateTestStorage(workspace.ID)
notifier := notifiers.CreateTestNotifier(workspace.ID)
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
defer func() {
backups, _ := backupRepository.FindByDatabaseID(database.ID)
for _, backup := range backups {
backupRepository.DeleteByID(backup.ID)
}
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
interval := createTestInterval()
// Total size limit is 10 MB. We have two backups of 8 MB each (16 MB total).
// The oldest backup was created 30 minutes ago — within the grace period.
// The cleaner must stop and leave both backups intact.
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodForever,
StorageID: &storage.ID,
MaxBackupsTotalSizeMB: 10,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
now := time.Now().UTC()
olderRecentBackup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 8,
CreatedAt: now.Add(-30 * time.Minute),
}
newerRecentBackup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 8,
CreatedAt: now.Add(-10 * time.Minute),
}
err = backupRepository.Save(olderRecentBackup)
assert.NoError(t, err)
err = backupRepository.Save(newerRecentBackup)
assert.NoError(t, err)
cleaner := GetBackupCleaner()
err = cleaner.cleanExceededBackups()
assert.NoError(t, err)
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
assert.Equal(
t,
2,
len(remainingBackups),
"Both recent backups must be preserved even though total size exceeds limit",
)
}
// Mock listener for testing
type mockBackupRemoveListener struct {
onBeforeBackupRemove func(*backups_core.Backup) error
}
func (m *mockBackupRemoveListener) OnBeforeBackupRemove(backup *backups_core.Backup) error {
if m.onBeforeBackupRemove != nil {
return m.onBeforeBackupRemove(backup)
}
return nil
}
func createTestInterval() *intervals.Interval {
timeOfDay := "04:00"
interval := &intervals.Interval{

View File

@@ -6,15 +6,15 @@ import (
"sync/atomic"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/mock"
common "databasus-backend/internal/features/backups/backups/common"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/notifiers"
"databasus-backend/internal/features/storages"
"github.com/google/uuid"
"github.com/stretchr/testify/mock"
)
type MockNotificationSender struct {

View File

@@ -10,10 +10,10 @@ import (
"sync/atomic"
"time"
cache_utils "databasus-backend/internal/util/cache"
"github.com/google/uuid"
"github.com/valkey-io/valkey-go"
cache_utils "databasus-backend/internal/util/cache"
)
const (
@@ -415,7 +415,7 @@ func (r *BackupNodesRegistry) UnsubscribeNodeForBackupsAssignments() error {
return nil
}
func (r *BackupNodesRegistry) PublishBackupCompletion(nodeID uuid.UUID, backupID uuid.UUID) error {
func (r *BackupNodesRegistry) PublishBackupCompletion(nodeID, backupID uuid.UUID) error {
ctx := context.Background()
message := BackupCompletionMessage{
@@ -437,7 +437,7 @@ func (r *BackupNodesRegistry) PublishBackupCompletion(nodeID uuid.UUID, backupID
}
func (r *BackupNodesRegistry) SubscribeForBackupsCompletions(
handler func(nodeID uuid.UUID, backupID uuid.UUID),
handler func(nodeID, backupID uuid.UUID),
) error {
ctx := context.Background()

View File

@@ -9,11 +9,11 @@ import (
"testing"
"time"
cache_utils "databasus-backend/internal/util/cache"
"databasus-backend/internal/util/logger"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
cache_utils "databasus-backend/internal/util/cache"
"databasus-backend/internal/util/logger"
)
func Test_HearthbeatNodeInRegistry_RegistersNodeWithTTL(t *testing.T) {
@@ -903,7 +903,7 @@ func Test_SubscribeForBackupsCompletions_ReceivesCompletedBackups(t *testing.T)
receivedBackupID := make(chan uuid.UUID, 1)
receivedNodeID := make(chan uuid.UUID, 1)
handler := func(nodeID uuid.UUID, backupID uuid.UUID) {
handler := func(nodeID, backupID uuid.UUID) {
receivedNodeID <- nodeID
receivedBackupID <- backupID
}
@@ -940,7 +940,7 @@ func Test_SubscribeForBackupsCompletions_ParsesJsonCorrectly(t *testing.T) {
defer registry.UnsubscribeForBackupsCompletions()
receivedBackups := make(chan uuid.UUID, 2)
handler := func(nodeID uuid.UUID, backupID uuid.UUID) {
handler := func(nodeID, backupID uuid.UUID) {
receivedBackups <- backupID
}
@@ -969,7 +969,7 @@ func Test_SubscribeForBackupsCompletions_HandlesInvalidJson(t *testing.T) {
defer registry.UnsubscribeForBackupsCompletions()
receivedBackupID := make(chan uuid.UUID, 1)
handler := func(nodeID uuid.UUID, backupID uuid.UUID) {
handler := func(nodeID, backupID uuid.UUID) {
receivedBackupID <- backupID
}
@@ -997,7 +997,7 @@ func Test_UnsubscribeForBackupsCompletions_StopsReceivingMessages(t *testing.T)
backupID2 := uuid.New()
receivedBackupID := make(chan uuid.UUID, 2)
handler := func(nodeID uuid.UUID, backupID uuid.UUID) {
handler := func(nodeID, backupID uuid.UUID) {
receivedBackupID <- backupID
}
@@ -1032,7 +1032,7 @@ func Test_SubscribeForBackupsCompletions_WhenAlreadySubscribed_ReturnsError(t *t
registry := createTestRegistry()
defer registry.UnsubscribeForBackupsCompletions()
handler := func(nodeID uuid.UUID, backupID uuid.UUID) {}
handler := func(nodeID, backupID uuid.UUID) {}
err := registry.SubscribeForBackupsCompletions(handler)
assert.NoError(t, err)
@@ -1064,9 +1064,9 @@ func Test_MultipleSubscribers_EachReceivesCompletionMessages(t *testing.T) {
receivedBackups2 := make(chan uuid.UUID, 3)
receivedBackups3 := make(chan uuid.UUID, 3)
handler1 := func(nodeID uuid.UUID, backupID uuid.UUID) { receivedBackups1 <- backupID }
handler2 := func(nodeID uuid.UUID, backupID uuid.UUID) { receivedBackups2 <- backupID }
handler3 := func(nodeID uuid.UUID, backupID uuid.UUID) { receivedBackups3 <- backupID }
handler1 := func(nodeID, backupID uuid.UUID) { receivedBackups1 <- backupID }
handler2 := func(nodeID, backupID uuid.UUID) { receivedBackups2 <- backupID }
handler3 := func(nodeID, backupID uuid.UUID) { receivedBackups3 <- backupID }
err := registry1.SubscribeForBackupsCompletions(handler1)
assert.NoError(t, err)

View File

@@ -15,7 +15,6 @@ import (
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
task_cancellation "databasus-backend/internal/features/tasks/cancellation"
files_utils "databasus-backend/internal/util/files"
)
const (
@@ -171,13 +170,7 @@ func (s *BackupsScheduler) StartBackup(database *databases.Database, isCallNotif
timestamp := time.Now().UTC()
backup := &backups_core.Backup{
ID: backupID,
FileName: fmt.Sprintf(
"%s-%s-%s",
files_utils.SanitizeFilename(database.Name),
timestamp.Format("20060102-150405"),
backupID.String(),
),
ID: backupID,
DatabaseID: backupConfig.DatabaseID,
StorageID: *backupConfig.StorageID,
Status: backups_core.BackupStatusInProgress,
@@ -185,6 +178,8 @@ func (s *BackupsScheduler) StartBackup(database *databases.Database, isCallNotif
CreatedAt: timestamp,
}
backup.GenerateFilename(database.Name)
if err := s.backupRepository.Save(backup); err != nil {
s.logger.Error(
"Failed to save backup",
@@ -446,7 +441,7 @@ func (s *BackupsScheduler) calculateLeastBusyNode() (*uuid.UUID, error) {
return &bestNode.ID, nil
}
func (s *BackupsScheduler) onBackupCompleted(nodeID uuid.UUID, backupID uuid.UUID) {
func (s *BackupsScheduler) onBackupCompleted(nodeID, backupID uuid.UUID) {
// Verify this task is actually a backup (registry contains multiple task types)
_, err := s.backupRepository.FindByID(backupID)
if err != nil {

View File

@@ -1,6 +1,12 @@
package backuping
import (
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
@@ -12,11 +18,6 @@ import (
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
cache_utils "databasus-backend/internal/util/cache"
"databasus-backend/internal/util/period"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
func Test_RunPendingBackups_WhenLastBackupWasYesterday_CreatesNewBackup(t *testing.T) {
@@ -57,7 +58,8 @@ func Test_RunPendingBackups_WhenLastBackupWasYesterday_CreatesNewBackup(t *testi
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = true
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
@@ -126,7 +128,8 @@ func Test_RunPendingBackups_WhenLastBackupWasRecentlyCompleted_SkipsBackup(t *te
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = true
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
@@ -194,7 +197,8 @@ func Test_RunPendingBackups_WhenLastBackupFailedAndRetriesDisabled_SkipsBackup(t
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = true
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
backupConfig.IsRetryIfFailed = false
@@ -266,7 +270,8 @@ func Test_RunPendingBackups_WhenLastBackupFailedAndRetriesEnabled_CreatesNewBack
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = true
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
backupConfig.IsRetryIfFailed = true
@@ -339,7 +344,8 @@ func Test_RunPendingBackups_WhenFailedBackupsExceedMaxRetries_SkipsBackup(t *tes
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = true
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
backupConfig.IsRetryIfFailed = true
@@ -410,7 +416,8 @@ func Test_RunPendingBackups_WhenBackupsDisabled_SkipsBackup(t *testing.T) {
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = false
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
@@ -479,7 +486,8 @@ func Test_CheckDeadNodesAndFailBackups_WhenNodeDies_FailsBackupAndCleansUpRegist
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = true
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
@@ -582,7 +590,8 @@ func Test_OnBackupCompleted_WhenTaskIsNotBackup_SkipsProcessing(t *testing.T) {
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = true
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
@@ -759,7 +768,8 @@ func Test_FailBackupsInProgress_WhenSchedulerStarts_CancelsBackupsAndUpdatesStat
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = true
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
@@ -872,7 +882,8 @@ func Test_StartBackup_WhenBackupCompletes_DecrementsActiveTaskCount(t *testing.T
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = true
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
@@ -975,7 +986,8 @@ func Test_StartBackup_WhenBackupFails_DecrementsActiveTaskCount(t *testing.T) {
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = true
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
@@ -1069,7 +1081,8 @@ func Test_StartBackup_WhenBackupAlreadyInProgress_SkipsNewBackup(t *testing.T) {
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = true
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
@@ -1140,7 +1153,8 @@ func Test_RunPendingBackups_WhenLastBackupFailedWithIsSkipRetry_SkipsBackupEvenW
TimeOfDay: &timeOfDay,
}
backupConfig.IsBackupsEnabled = true
backupConfig.StorePeriod = period.PeriodWeek
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig.RetentionTimePeriod = period.PeriodWeek
backupConfig.Storage = storage
backupConfig.StorageID = &storage.ID
backupConfig.IsRetryIfFailed = true
@@ -1242,7 +1256,8 @@ func Test_StartBackup_When2BackupsStartedForDifferentDatabases_BothUseCasesAreCa
TimeOfDay: &timeOfDay,
}
backupConfig1.IsBackupsEnabled = true
backupConfig1.StorePeriod = period.PeriodWeek
backupConfig1.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig1.RetentionTimePeriod = period.PeriodWeek
backupConfig1.Storage = storage
backupConfig1.StorageID = &storage.ID
@@ -1259,7 +1274,8 @@ func Test_StartBackup_When2BackupsStartedForDifferentDatabases_BothUseCasesAreCa
TimeOfDay: &timeOfDay,
}
backupConfig2.IsBackupsEnabled = true
backupConfig2.StorePeriod = period.PeriodWeek
backupConfig2.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
backupConfig2.RetentionTimePeriod = period.PeriodWeek
backupConfig2.Storage = storage
backupConfig2.StorageID = &storage.ID

View File

@@ -8,6 +8,9 @@ import (
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
backups_core "databasus-backend/internal/features/backups/backups/core"
"databasus-backend/internal/features/backups/backups/usecases"
backups_config "databasus-backend/internal/features/backups/config"
@@ -19,9 +22,6 @@ import (
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/logger"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
)
func CreateTestRouter() *gin.Engine {

View File

@@ -1,10 +1,11 @@
package common
import (
backups_config "databasus-backend/internal/features/backups/config"
"errors"
"github.com/google/uuid"
backups_config "databasus-backend/internal/features/backups/config"
)
type BackupMetadata struct {

View File

@@ -7,6 +7,10 @@ type CountingWriter struct {
BytesWritten int64
}
func NewCountingWriter(writer io.Writer) *CountingWriter {
return &CountingWriter{Writer: writer}
}
func (cw *CountingWriter) Write(p []byte) (n int, err error) {
n, err = cw.Writer.Write(p)
cw.BytesWritten += int64(n)
@@ -16,7 +20,3 @@ func (cw *CountingWriter) Write(p []byte) (n int, err error) {
func (cw *CountingWriter) GetBytesWritten() int64 {
return cw.BytesWritten
}
func NewCountingWriter(writer io.Writer) *CountingWriter {
return &CountingWriter{Writer: writer}
}

View File

@@ -1,12 +1,8 @@
package backups
package backups_controllers
import (
"context"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_download "databasus-backend/internal/features/backups/backups/download"
"databasus-backend/internal/features/databases"
users_middleware "databasus-backend/internal/features/users/middleware"
files_utils "databasus-backend/internal/util/files"
"errors"
"fmt"
"io"
"net/http"
@@ -14,10 +10,18 @@ import (
"github.com/gin-gonic/gin"
"github.com/google/uuid"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_download "databasus-backend/internal/features/backups/backups/download"
backups_dto "databasus-backend/internal/features/backups/backups/dto"
backups_services "databasus-backend/internal/features/backups/backups/services"
"databasus-backend/internal/features/databases"
users_middleware "databasus-backend/internal/features/users/middleware"
files_utils "databasus-backend/internal/util/files"
)
type BackupController struct {
backupService *BackupService
backupService *backups_services.BackupService
}
func (c *BackupController) RegisterRoutes(router *gin.RouterGroup) {
@@ -42,7 +46,7 @@ func (c *BackupController) RegisterPublicRoutes(router *gin.RouterGroup) {
// @Param database_id query string true "Database ID"
// @Param limit query int false "Number of items per page" default(10)
// @Param offset query int false "Offset for pagination" default(0)
// @Success 200 {object} GetBackupsResponse
// @Success 200 {object} backups_dto.GetBackupsResponse
// @Failure 400
// @Failure 401
// @Failure 500
@@ -54,7 +58,7 @@ func (c *BackupController) GetBackups(ctx *gin.Context) {
return
}
var request GetBackupsRequest
var request backups_dto.GetBackupsRequest
if err := ctx.ShouldBindQuery(&request); err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
@@ -81,7 +85,7 @@ func (c *BackupController) GetBackups(ctx *gin.Context) {
// @Tags backups
// @Accept json
// @Produce json
// @Param request body MakeBackupRequest true "Backup creation data"
// @Param request body backups_dto.MakeBackupRequest true "Backup creation data"
// @Success 200 {object} map[string]string
// @Failure 400
// @Failure 401
@@ -94,7 +98,7 @@ func (c *BackupController) MakeBackup(ctx *gin.Context) {
return
}
var request MakeBackupRequest
var request backups_dto.MakeBackupRequest
if err := ctx.ShouldBindJSON(&request); err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
@@ -195,7 +199,7 @@ func (c *BackupController) GenerateDownloadToken(ctx *gin.Context) {
response, err := c.backupService.GenerateDownloadToken(user, id)
if err != nil {
if err == backups_download.ErrDownloadAlreadyInProgress {
if errors.Is(err, backups_download.ErrDownloadAlreadyInProgress) {
ctx.JSON(
http.StatusConflict,
gin.H{
@@ -246,7 +250,7 @@ func (c *BackupController) GetFile(ctx *gin.Context) {
downloadToken, rateLimiter, err := c.backupService.ValidateDownloadToken(token)
if err != nil {
if err == backups_download.ErrDownloadAlreadyInProgress {
if errors.Is(err, backups_download.ErrDownloadAlreadyInProgress) {
ctx.JSON(
http.StatusConflict,
gin.H{
@@ -310,10 +314,6 @@ func (c *BackupController) GetFile(ctx *gin.Context) {
c.backupService.WriteAuditLogForDownload(downloadToken.UserID, backup, database)
}
type MakeBackupRequest struct {
DatabaseID uuid.UUID `json:"database_id" binding:"required"`
}
func (c *BackupController) generateBackupFilename(
backup *backups_core.Backup,
database *databases.Database,

View File

@@ -1,4 +1,4 @@
package backups
package backups_controllers
import (
"context"
@@ -24,11 +24,14 @@ import (
backups_common "databasus-backend/internal/features/backups/backups/common"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_download "databasus-backend/internal/features/backups/backups/download"
backups_dto "databasus-backend/internal/features/backups/backups/dto"
backups_services "databasus-backend/internal/features/backups/backups/services"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/databases/databases/postgresql"
"databasus-backend/internal/features/storages"
local_storage "databasus-backend/internal/features/storages/models/local"
task_cancellation "databasus-backend/internal/features/tasks/cancellation"
users_dto "databasus-backend/internal/features/users/dto"
users_enums "databasus-backend/internal/features/users/enums"
users_services "databasus-backend/internal/features/users/services"
@@ -119,7 +122,7 @@ func Test_GetBackups_PermissionsEnforced(t *testing.T) {
)
if tt.expectSuccess {
var response GetBackupsResponse
var response backups_dto.GetBackupsResponse
err := json.Unmarshal(testResp.Body, &response)
assert.NoError(t, err)
assert.GreaterOrEqual(t, len(response.Backups), 1)
@@ -214,7 +217,7 @@ func Test_CreateBackup_PermissionsEnforced(t *testing.T) {
testUserToken = nonMember.Token
}
request := MakeBackupRequest{DatabaseID: database.ID}
request := backups_dto.MakeBackupRequest{DatabaseID: database.ID}
testResp := test_utils.MakePostRequest(
t,
router,
@@ -245,7 +248,7 @@ func Test_CreateBackup_AuditLogWritten(t *testing.T) {
database := createTestDatabase("Test Database", workspace.ID, owner.Token, router)
enableBackupForDatabase(database.ID)
request := MakeBackupRequest{DatabaseID: database.ID}
request := backups_dto.MakeBackupRequest{DatabaseID: database.ID}
test_utils.MakePostRequest(
t,
router,
@@ -373,7 +376,7 @@ func Test_DeleteBackup_PermissionsEnforced(t *testing.T) {
ownerUser, err := userService.GetUserFromToken(owner.Token)
assert.NoError(t, err)
response, err := GetBackupService().GetBackups(ownerUser, database.ID, 10, 0)
response, err := backups_services.GetBackupService().GetBackups(ownerUser, database.ID, 10, 0)
assert.NoError(t, err)
assert.Equal(t, 0, len(response.Backups))
}
@@ -999,7 +1002,7 @@ func Test_CancelBackup_InProgressBackup_SuccessfullyCancelled(t *testing.T) {
assert.NoError(t, err)
// Register a cancellable context for the backup
GetBackupService().taskCancelManager.RegisterTask(backup.ID, func() {})
task_cancellation.GetTaskCancelManager().RegisterTask(backup.ID, func() {})
resp := test_utils.MakePostRequest(
t,
@@ -1091,7 +1094,7 @@ func Test_ConcurrentDownloadPrevention(t *testing.T) {
time.Sleep(50 * time.Millisecond)
service := GetBackupService()
service := backups_services.GetBackupService()
if !service.IsDownloadInProgress(owner.UserID) {
t.Log("Warning: First download completed before we could test concurrency")
<-downloadComplete
@@ -1192,7 +1195,7 @@ func Test_GenerateDownloadToken_BlockedWhenDownloadInProgress(t *testing.T) {
time.Sleep(50 * time.Millisecond)
service := GetBackupService()
service := backups_services.GetBackupService()
if !service.IsDownloadInProgress(owner.UserID) {
t.Log("Warning: First download completed before we could test token generation blocking")
<-downloadComplete
@@ -1268,7 +1271,7 @@ func Test_MakeBackup_VerifyBackupAndMetadataFilesExistInStorage(t *testing.T) {
initialBackups, err := backupRepo.FindByDatabaseID(database.ID)
assert.NoError(t, err)
request := MakeBackupRequest{DatabaseID: database.ID}
request := backups_dto.MakeBackupRequest{DatabaseID: database.ID}
test_utils.MakePostRequest(
t,
router,
@@ -1502,7 +1505,7 @@ func createTestBackup(
}
func createExpiredDownloadToken(backupID, userID uuid.UUID) string {
tokenService := GetBackupService().downloadTokenService
tokenService := backups_download.GetDownloadTokenService()
token, err := tokenService.Generate(backupID, userID)
if err != nil {
panic(fmt.Sprintf("Failed to generate download token: %v", err))
@@ -1843,7 +1846,7 @@ func Test_DeleteBackup_RemovesBackupAndMetadataFilesFromDisk(t *testing.T) {
initialBackups, err := backupRepo.FindByDatabaseID(database.ID)
assert.NoError(t, err)
request := MakeBackupRequest{DatabaseID: database.ID}
request := backups_dto.MakeBackupRequest{DatabaseID: database.ID}
test_utils.MakePostRequest(
t,
router,

View File

@@ -0,0 +1,23 @@
package backups_controllers
import (
backups_services "databasus-backend/internal/features/backups/backups/services"
"databasus-backend/internal/features/databases"
)
var backupController = &BackupController{
backups_services.GetBackupService(),
}
func GetBackupController() *BackupController {
return backupController
}
var postgresWalBackupController = &PostgreWalBackupController{
databases.GetDatabaseService(),
backups_services.GetWalService(),
}
func GetPostgresWalBackupController() *PostgreWalBackupController {
return postgresWalBackupController
}

View File

@@ -0,0 +1,291 @@
package backups_controllers
import (
"io"
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_dto "databasus-backend/internal/features/backups/backups/dto"
backups_services "databasus-backend/internal/features/backups/backups/services"
"databasus-backend/internal/features/databases"
)
// PostgreWalBackupController handles WAL backup endpoints used by the databasus-cli agent.
// Authentication is via a plain agent token in the Authorization header (no Bearer prefix).
type PostgreWalBackupController struct {
databaseService *databases.DatabaseService
walService *backups_services.PostgreWalBackupService
}
func (c *PostgreWalBackupController) RegisterRoutes(router *gin.RouterGroup) {
walRoutes := router.Group("/backups/postgres/wal")
walRoutes.GET("/next-full-backup-time", c.GetNextFullBackupTime)
walRoutes.POST("/error", c.ReportError)
walRoutes.POST("/upload", c.Upload)
walRoutes.GET("/restore/plan", c.GetRestorePlan)
walRoutes.GET("/restore/download", c.DownloadBackupFile)
}
// GetNextFullBackupTime
// @Summary Get next full backup time
// @Description Returns the next scheduled full basebackup time for the authenticated database
// @Tags backups-wal
// @Produce json
// @Security AgentToken
// @Success 200 {object} backups_dto.GetNextFullBackupTimeResponse
// @Failure 401 {object} map[string]string
// @Failure 500 {object} map[string]string
// @Router /backups/postgres/wal/next-full-backup-time [get]
func (c *PostgreWalBackupController) GetNextFullBackupTime(ctx *gin.Context) {
database, err := c.getDatabase(ctx)
if err != nil {
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})
return
}
response, err := c.walService.GetNextFullBackupTime(database)
if err != nil {
ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
ctx.JSON(http.StatusOK, response)
}
// ReportError
// @Summary Report agent error
// @Description Records a fatal error from the agent against the database record and marks it as errored
// @Tags backups-wal
// @Accept json
// @Security AgentToken
// @Param request body backups_dto.ReportErrorRequest true "Error details"
// @Success 200
// @Failure 400 {object} map[string]string
// @Failure 401 {object} map[string]string
// @Failure 500 {object} map[string]string
// @Router /backups/postgres/wal/error [post]
func (c *PostgreWalBackupController) ReportError(ctx *gin.Context) {
database, err := c.getDatabase(ctx)
if err != nil {
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})
return
}
var request backups_dto.ReportErrorRequest
if err := ctx.ShouldBindJSON(&request); err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
if err := c.walService.ReportError(database, request.Error); err != nil {
ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
ctx.Status(http.StatusOK)
}
// Upload
// @Summary Stream upload a basebackup or WAL segment
// @Description Accepts a zstd-compressed binary stream and stores it in the database's configured storage.
// The server generates the storage filename; agents do not control the destination path.
// For WAL segment uploads the server validates the WAL chain and returns 409 if a gap is detected
// or 400 if no full backup exists yet (agent should trigger a full basebackup in both cases).
// @Tags backups-wal
// @Accept application/octet-stream
// @Produce json
// @Security AgentToken
// @Param X-Upload-Type header string true "Upload type" Enums(basebackup, wal)
// @Param X-Wal-Segment-Name header string false "24-hex WAL segment identifier (required for wal uploads, e.g. 0000000100000001000000AB)"
// @Param X-Wal-Segment-Size header int false "WAL segment size in bytes reported by the PostgreSQL instance (default: 16777216)"
// @Param fullBackupWalStartSegment query string false "First WAL segment needed to make the basebackup consistent (required for basebackup uploads)"
// @Param fullBackupWalStopSegment query string false "Last WAL segment included in the basebackup (required for basebackup uploads)"
// @Success 204
// @Failure 400 {object} backups_dto.UploadGapResponse "No full backup exists (error: no_full_backup)"
// @Failure 401 {object} map[string]string
// @Failure 409 {object} backups_dto.UploadGapResponse "WAL chain gap detected (error: gap_detected)"
// @Failure 500 {object} map[string]string
// @Router /backups/postgres/wal/upload [post]
func (c *PostgreWalBackupController) Upload(ctx *gin.Context) {
database, err := c.getDatabase(ctx)
if err != nil {
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})
return
}
uploadType := backups_core.PgWalUploadType(ctx.GetHeader("X-Upload-Type"))
if uploadType != backups_core.PgWalUploadTypeBasebackup &&
uploadType != backups_core.PgWalUploadTypeWal {
ctx.JSON(
http.StatusBadRequest,
gin.H{"error": "X-Upload-Type must be 'basebackup' or 'wal'"},
)
return
}
walSegmentName := ""
if uploadType == backups_core.PgWalUploadTypeWal {
walSegmentName = ctx.GetHeader("X-Wal-Segment-Name")
if walSegmentName == "" {
ctx.JSON(
http.StatusBadRequest,
gin.H{"error": "X-Wal-Segment-Name is required for wal uploads"},
)
return
}
}
if uploadType == backups_core.PgWalUploadTypeBasebackup {
if ctx.Query("fullBackupWalStartSegment") == "" ||
ctx.Query("fullBackupWalStopSegment") == "" {
ctx.JSON(
http.StatusBadRequest,
gin.H{
"error": "fullBackupWalStartSegment and fullBackupWalStopSegment are required for basebackup uploads",
},
)
return
}
}
walSegmentSizeBytes := int64(0)
if raw := ctx.GetHeader("X-Wal-Segment-Size"); raw != "" {
parsed, parseErr := strconv.ParseInt(raw, 10, 64)
if parseErr != nil || parsed <= 0 {
ctx.JSON(
http.StatusBadRequest,
gin.H{"error": "X-Wal-Segment-Size must be a positive integer"},
)
return
}
walSegmentSizeBytes = parsed
}
gapResp, uploadErr := c.walService.UploadWal(
ctx.Request.Context(),
database,
uploadType,
walSegmentName,
ctx.Query("fullBackupWalStartSegment"),
ctx.Query("fullBackupWalStopSegment"),
walSegmentSizeBytes,
ctx.Request.Body,
)
if uploadErr != nil {
ctx.JSON(http.StatusInternalServerError, gin.H{"error": uploadErr.Error()})
return
}
if gapResp != nil {
if gapResp.Error == "no_full_backup" {
ctx.JSON(http.StatusBadRequest, gapResp)
return
}
ctx.JSON(http.StatusConflict, gapResp)
return
}
ctx.Status(http.StatusNoContent)
}
// GetRestorePlan
// @Summary Get restore plan
// @Description Resolves the full backup and all required WAL segments needed for recovery. Validates the WAL chain is continuous.
// @Tags backups-wal
// @Produce json
// @Security AgentToken
// @Param backupId query string false "UUID of a specific full backup to restore from; defaults to the most recent"
// @Success 200 {object} backups_dto.GetRestorePlanResponse
// @Failure 400 {object} map[string]string "Broken WAL chain or no backups available"
// @Failure 401 {object} map[string]string
// @Failure 500 {object} map[string]string
// @Router /backups/postgres/wal/restore/plan [get]
func (c *PostgreWalBackupController) GetRestorePlan(ctx *gin.Context) {
database, err := c.getDatabase(ctx)
if err != nil {
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})
return
}
var backupID *uuid.UUID
if raw := ctx.Query("backupId"); raw != "" {
parsed, parseErr := uuid.Parse(raw)
if parseErr != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": "invalid backupId format"})
return
}
backupID = &parsed
}
response, planErr, err := c.walService.GetRestorePlan(database, backupID)
if err != nil {
ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if planErr != nil {
ctx.JSON(http.StatusBadRequest, planErr)
return
}
ctx.JSON(http.StatusOK, response)
}
// DownloadBackupFile
// @Summary Download a backup or WAL segment file for restore
// @Description Retrieves the backup file by ID (validated against the authenticated database), decrypts it server-side if encrypted, and streams the zstd-compressed result to the agent
// @Tags backups-wal
// @Produce application/octet-stream
// @Security AgentToken
// @Param backupId query string true "Backup ID from the restore plan response"
// @Success 200 {file} file
// @Failure 400 {object} map[string]string
// @Failure 401 {object} map[string]string
// @Router /backups/postgres/wal/restore/download [get]
func (c *PostgreWalBackupController) DownloadBackupFile(ctx *gin.Context) {
database, err := c.getDatabase(ctx)
if err != nil {
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})
return
}
backupIDRaw := ctx.Query("backupId")
if backupIDRaw == "" {
ctx.JSON(http.StatusBadRequest, gin.H{"error": "backupId is required"})
return
}
backupID, err := uuid.Parse(backupIDRaw)
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": "invalid backupId format"})
return
}
reader, err := c.walService.DownloadBackupFile(database, backupID)
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
defer func() { _ = reader.Close() }()
ctx.Header("Content-Type", "application/octet-stream")
ctx.Status(http.StatusOK)
_, _ = io.Copy(ctx.Writer, reader)
}
func (c *PostgreWalBackupController) getDatabase(
ctx *gin.Context,
) (*databases.Database, error) {
token := ctx.GetHeader("Authorization")
return c.databaseService.GetDatabaseByAgentToken(token)
}

View File

@@ -1,17 +1,17 @@
package backups
package backups_controllers
import (
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
workspaces_controllers "databasus-backend/internal/features/workspaces/controllers"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
)
func CreateTestRouter() *gin.Engine {
@@ -41,7 +41,7 @@ func WaitForBackupCompletion(
deadline := time.Now().UTC().Add(timeout)
for time.Now().UTC().Before(deadline) {
backups, err := backupRepository.FindByDatabaseID(databaseID)
backups, err := backups_core.GetBackupRepository().FindByDatabaseID(databaseID)
if err != nil {
t.Logf("WaitForBackupCompletion: error finding backups: %v", err)
time.Sleep(50 * time.Millisecond)

View File

@@ -0,0 +1,7 @@
package backups_core
var backupRepository = &BackupRepository{}
func GetBackupRepository() *BackupRepository {
return backupRepository
}

View File

@@ -8,3 +8,10 @@ const (
BackupStatusFailed BackupStatus = "FAILED"
BackupStatusCanceled BackupStatus = "CANCELED"
)
type PgWalUploadType string
const (
PgWalUploadTypeBasebackup PgWalUploadType = "basebackup"
PgWalUploadTypeWal PgWalUploadType = "wal"
)

View File

@@ -1,10 +1,20 @@
package backups_core
import (
backups_config "databasus-backend/internal/features/backups/config"
"fmt"
"time"
"github.com/google/uuid"
backups_config "databasus-backend/internal/features/backups/config"
files_utils "databasus-backend/internal/util/files"
)
type PgWalBackupType string
const (
PgWalBackupTypeFullBackup PgWalBackupType = "PG_FULL_BACKUP"
PgWalBackupTypeWalSegment PgWalBackupType = "PG_WAL_SEGMENT"
)
type Backup struct {
@@ -26,5 +36,23 @@ type Backup struct {
EncryptionIV *string `json:"-" gorm:"column:encryption_iv"`
Encryption backups_config.BackupEncryption `json:"encryption" gorm:"column:encryption;type:text;not null;default:'NONE'"`
// Postgres WAL backup specific fields
PgWalBackupType *PgWalBackupType `json:"pgWalBackupType" gorm:"column:pg_wal_backup_type;type:text"`
PgFullBackupWalStartSegmentName *string `json:"pgFullBackupWalStartSegmentName" gorm:"column:pg_wal_start_segment;type:text"`
PgFullBackupWalStopSegmentName *string `json:"pgFullBackupWalStopSegmentName" gorm:"column:pg_wal_stop_segment;type:text"`
PgVersion *string `json:"pgVersion" gorm:"column:pg_version;type:text"`
PgWalSegmentName *string `json:"pgWalSegmentName" gorm:"column:pg_wal_segment_name;type:text"`
CreatedAt time.Time `json:"createdAt" gorm:"column:created_at"`
}
func (b *Backup) GenerateFilename(dbName string) {
timestamp := time.Now().UTC()
b.FileName = fmt.Sprintf(
"%s-%s-%s",
files_utils.SanitizeFilename(dbName),
timestamp.Format("20060102-150405"),
b.ID.String(),
)
}

View File

@@ -1,13 +1,13 @@
package backups_core
import (
"databasus-backend/internal/storage"
"errors"
"time"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/storage"
)
type BackupRepository struct{}
@@ -88,7 +88,7 @@ func (r *BackupRepository) FindLastByDatabaseID(databaseID uuid.UUID) (*Backup,
Where("database_id = ?", databaseID).
Order("created_at DESC").
First(&backup).Error; err != nil {
if err == gorm.ErrRecordNotFound {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
@@ -245,3 +245,134 @@ func (r *BackupRepository) FindOldestByDatabaseExcludingInProgress(
return backups, nil
}
func (r *BackupRepository) FindCompletedFullWalBackupByID(
databaseID uuid.UUID,
backupID uuid.UUID,
) (*Backup, error) {
var backup Backup
err := storage.
GetDb().
Where(
"database_id = ? AND id = ? AND pg_wal_backup_type = ? AND status = ?",
databaseID,
backupID,
PgWalBackupTypeFullBackup,
BackupStatusCompleted,
).
First(&backup).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err
}
return &backup, nil
}
func (r *BackupRepository) FindCompletedWalSegmentsAfter(
databaseID uuid.UUID,
afterSegmentName string,
) ([]*Backup, error) {
var backups []*Backup
err := storage.
GetDb().
Where(
"database_id = ? AND pg_wal_backup_type = ? AND pg_wal_segment_name >= ? AND status = ?",
databaseID,
PgWalBackupTypeWalSegment,
afterSegmentName,
BackupStatusCompleted,
).
Order("pg_wal_segment_name ASC").
Find(&backups).Error
if err != nil {
return nil, err
}
return backups, nil
}
func (r *BackupRepository) FindLastCompletedFullWalBackupByDatabaseID(
databaseID uuid.UUID,
) (*Backup, error) {
var backup Backup
err := storage.
GetDb().
Where(
"database_id = ? AND pg_wal_backup_type = ? AND status = ?",
databaseID,
PgWalBackupTypeFullBackup,
BackupStatusCompleted,
).
Order("created_at DESC").
First(&backup).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err
}
return &backup, nil
}
func (r *BackupRepository) FindWalSegmentByName(
databaseID uuid.UUID,
segmentName string,
) (*Backup, error) {
var backup Backup
err := storage.
GetDb().
Where(
"database_id = ? AND pg_wal_backup_type = ? AND pg_wal_segment_name = ?",
databaseID,
PgWalBackupTypeWalSegment,
segmentName,
).
First(&backup).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err
}
return &backup, nil
}
func (r *BackupRepository) FindLastWalSegmentAfter(
databaseID uuid.UUID,
afterSegmentName string,
) (*Backup, error) {
var backup Backup
err := storage.
GetDb().
Where(
"database_id = ? AND pg_wal_backup_type = ? AND pg_wal_segment_name > ? AND status = ?",
databaseID,
PgWalBackupTypeWalSegment,
afterSegmentName,
BackupStatusCompleted,
).
Order("pg_wal_segment_name DESC").
First(&backup).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err
}
return &backup, nil
}

View File

@@ -13,9 +13,11 @@ var downloadTokenRepository = &DownloadTokenRepository{}
var downloadTracker = NewDownloadTracker(cache_utils.GetValkeyClient())
var bandwidthManager *BandwidthManager
var downloadTokenService *DownloadTokenService
var downloadTokenBackgroundService *DownloadTokenBackgroundService
var (
bandwidthManager *BandwidthManager
downloadTokenService *DownloadTokenService
downloadTokenBackgroundService *DownloadTokenBackgroundService
)
func init() {
env := config.GetEnv()

View File

@@ -66,9 +66,7 @@ func (rl *RateLimiter) Wait(bytes int64) {
tokensNeeded := float64(bytes) - rl.availableTokens
waitTime := time.Duration(tokensNeeded/float64(rl.bytesPerSecond)*1000) * time.Millisecond
if waitTime < time.Millisecond {
waitTime = time.Millisecond
}
waitTime = max(waitTime, time.Millisecond)
rl.mu.Unlock()
time.Sleep(waitTime)

View File

@@ -2,12 +2,14 @@ package backups_download
import (
"crypto/rand"
"databasus-backend/internal/storage"
"encoding/base64"
"errors"
"time"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/storage"
)
type DownloadTokenRepository struct{}
@@ -28,9 +30,8 @@ func (r *DownloadTokenRepository) FindByToken(token string) (*DownloadToken, err
err := storage.GetDb().
Where("token = ?", token).
First(&downloadToken).Error
if err != nil {
if err == gorm.ErrRecordNotFound {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err

View File

@@ -1,12 +1,13 @@
package backups_download
import (
cache_utils "databasus-backend/internal/util/cache"
"errors"
"time"
"github.com/google/uuid"
"github.com/valkey-io/valkey-go"
cache_utils "databasus-backend/internal/util/cache"
)
const (
@@ -16,9 +17,7 @@ const (
downloadHeartbeatDelay = 3 * time.Second
)
var (
ErrDownloadAlreadyInProgress = errors.New("download already in progress for this user")
)
var ErrDownloadAlreadyInProgress = errors.New("download already in progress for this user")
type DownloadTracker struct {
cache *cache_utils.CacheUtil[string]

View File

@@ -1,29 +0,0 @@
package backups
import (
backups_core "databasus-backend/internal/features/backups/backups/core"
"databasus-backend/internal/features/backups/backups/encryption"
"io"
)
type GetBackupsRequest struct {
DatabaseID string `form:"database_id" binding:"required"`
Limit int `form:"limit"`
Offset int `form:"offset"`
}
type GetBackupsResponse struct {
Backups []*backups_core.Backup `json:"backups"`
Total int64 `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
type DecryptionReaderCloser struct {
*encryption.DecryptionReader
BaseReader io.ReadCloser
}
func (r *DecryptionReaderCloser) Close() error {
return r.BaseReader.Close()
}

View File

@@ -0,0 +1,79 @@
package backups_dto
import (
"io"
"time"
"github.com/google/uuid"
backups_core "databasus-backend/internal/features/backups/backups/core"
"databasus-backend/internal/features/backups/backups/encryption"
)
type GetBackupsRequest struct {
DatabaseID string `form:"database_id" binding:"required"`
Limit int `form:"limit"`
Offset int `form:"offset"`
}
type GetBackupsResponse struct {
Backups []*backups_core.Backup `json:"backups"`
Total int64 `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
type DecryptionReaderCloser struct {
*encryption.DecryptionReader
BaseReader io.ReadCloser
}
func (r *DecryptionReaderCloser) Close() error {
return r.BaseReader.Close()
}
type MakeBackupRequest struct {
DatabaseID uuid.UUID `json:"database_id" binding:"required"`
}
type GetNextFullBackupTimeResponse struct {
NextFullBackupTime *time.Time `json:"nextFullBackupTime"`
}
type ReportErrorRequest struct {
Error string `json:"error" binding:"required"`
}
type UploadGapResponse struct {
Error string `json:"error"`
ExpectedSegmentName string `json:"expectedSegmentName"`
ReceivedSegmentName string `json:"receivedSegmentName"`
}
type RestorePlanFullBackup struct {
BackupID uuid.UUID `json:"id"`
FullBackupWalStartSegment string `json:"fullBackupWalStartSegment"`
FullBackupWalStopSegment string `json:"fullBackupWalStopSegment"`
PgVersion string `json:"pgVersion"`
CreatedAt time.Time `json:"createdAt"`
SizeBytes int64 `json:"sizeBytes"`
}
type RestorePlanWalSegment struct {
BackupID uuid.UUID `json:"backupId"`
SegmentName string `json:"segmentName"`
SizeBytes int64 `json:"sizeBytes"`
}
type GetRestorePlanErrorResponse struct {
Error string `json:"error"`
Message string `json:"message"`
LastContiguousSegment string `json:"lastContiguousSegment,omitempty"`
}
type GetRestorePlanResponse struct {
FullBackup RestorePlanFullBackup `json:"fullBackup"`
WalSegments []RestorePlanWalSegment `json:"walSegments"`
TotalSizeBytes int64 `json:"totalSizeBytes"`
LatestAvailableSegment string `json:"latestAvailableSegment"`
}

View File

@@ -4,6 +4,7 @@ import (
"crypto/aes"
"crypto/cipher"
"encoding/binary"
"errors"
"fmt"
"io"
@@ -69,7 +70,7 @@ func NewDecryptionReader(
func (r *DecryptionReader) Read(p []byte) (n int, err error) {
for len(r.buffer) < len(p) && !r.eof {
if err := r.readAndDecryptChunk(); err != nil {
if err == io.EOF {
if errors.Is(err, io.EOF) {
r.eof = true
break
}

View File

@@ -0,0 +1,45 @@
package encryption
import (
"encoding/base64"
"fmt"
"io"
"github.com/google/uuid"
)
// EncryptionSetup holds the result of setting up encryption for a backup stream.
type EncryptionSetup struct {
Writer *EncryptionWriter
SaltBase64 string
NonceBase64 string
}
// SetupEncryptionWriter generates salt/nonce, creates an EncryptionWriter, and
// returns the base64-encoded salt and nonce for storage on the backup record.
func SetupEncryptionWriter(
baseWriter io.Writer,
masterKey string,
backupID uuid.UUID,
) (*EncryptionSetup, error) {
salt, err := GenerateSalt()
if err != nil {
return nil, fmt.Errorf("failed to generate salt: %w", err)
}
nonce, err := GenerateNonce()
if err != nil {
return nil, fmt.Errorf("failed to generate nonce: %w", err)
}
encWriter, err := NewEncryptionWriter(baseWriter, masterKey, backupID, salt, nonce)
if err != nil {
return nil, fmt.Errorf("failed to create encryption writer: %w", err)
}
return &EncryptionSetup{
Writer: encWriter,
SaltBase64: base64.StdEncoding.EncodeToString(salt),
NonceBase64: base64.StdEncoding.EncodeToString(nonce),
}, nil
}

View File

@@ -1,4 +1,4 @@
package backups
package backups_services
import (
"sync"
@@ -20,14 +20,12 @@ import (
"databasus-backend/internal/util/logger"
)
var backupRepository = &backups_core.BackupRepository{}
var taskCancelManager = task_cancellation.GetTaskCancelManager()
var backupService = &BackupService{
databases.GetDatabaseService(),
storages.GetStorageService(),
backupRepository,
backups_core.GetBackupRepository(),
notifiers.GetNotifierService(),
notifiers.GetNotifierService(),
backups_config.GetBackupConfigService(),
@@ -44,16 +42,21 @@ var backupService = &BackupService{
backuping.GetBackupCleaner(),
}
var backupController = &BackupController{
backupService: backupService,
}
func GetBackupService() *BackupService {
return backupService
}
func GetBackupController() *BackupController {
return backupController
var walService = &PostgreWalBackupService{
backups_config.GetBackupConfigService(),
backups_core.GetBackupRepository(),
encryption.GetFieldEncryptor(),
encryption_secrets.GetSecretKeyService(),
logger.GetLogger(),
backupService,
}
func GetWalService() *PostgreWalBackupService {
return walService
}
var (

View File

@@ -0,0 +1,613 @@
package backups_services
import (
"context"
"fmt"
"io"
"log/slog"
"time"
"github.com/google/uuid"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_dto "databasus-backend/internal/features/backups/backups/dto"
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/databases/databases/postgresql"
encryption_secrets "databasus-backend/internal/features/encryption/secrets"
util_encryption "databasus-backend/internal/util/encryption"
util_wal "databasus-backend/internal/util/wal"
)
// PostgreWalBackupService handles WAL segment and basebackup uploads from the databasus-cli agent.
type PostgreWalBackupService struct {
backupConfigService *backups_config.BackupConfigService
backupRepository *backups_core.BackupRepository
fieldEncryptor util_encryption.FieldEncryptor
secretKeyService *encryption_secrets.SecretKeyService
logger *slog.Logger
backupService *BackupService
}
// UploadWal accepts a streaming WAL segment or basebackup upload from the agent.
// For WAL segments it validates the WAL chain before accepting. Returns an UploadGapResponse
// (409) when the chain is broken so the agent knows to trigger a full basebackup.
func (s *PostgreWalBackupService) UploadWal(
ctx context.Context,
database *databases.Database,
uploadType backups_core.PgWalUploadType,
walSegmentName string,
fullBackupWalStartSegment string,
fullBackupWalStopSegment string,
walSegmentSizeBytes int64,
body io.Reader,
) (*backups_dto.UploadGapResponse, error) {
if err := s.validateWalBackupType(database); err != nil {
return nil, err
}
if uploadType == backups_core.PgWalUploadTypeBasebackup {
if fullBackupWalStartSegment == "" || fullBackupWalStopSegment == "" {
return nil, fmt.Errorf(
"fullBackupWalStartSegment and fullBackupWalStopSegment are required for basebackup uploads",
)
}
}
backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)
if err != nil {
return nil, fmt.Errorf("failed to get backup config: %w", err)
}
if backupConfig.Storage == nil {
return nil, fmt.Errorf("no storage configured for database %s", database.ID)
}
if uploadType == backups_core.PgWalUploadTypeWal {
// Idempotency: check before chain validation so a successful re-upload is
// not misidentified as a gap.
existing, err := s.backupRepository.FindWalSegmentByName(database.ID, walSegmentName)
if err != nil {
return nil, fmt.Errorf("failed to check for duplicate WAL segment: %w", err)
}
if existing != nil {
return nil, nil
}
gapResp, err := s.validateWalChain(database.ID, walSegmentName, walSegmentSizeBytes)
if err != nil {
return nil, err
}
if gapResp != nil {
return gapResp, nil
}
}
backup := s.createBackupRecord(
database.ID,
backupConfig.Storage.ID,
uploadType,
database.Name,
walSegmentName,
fullBackupWalStartSegment,
fullBackupWalStopSegment,
backupConfig.Encryption,
)
if err := s.backupRepository.Save(backup); err != nil {
return nil, fmt.Errorf("failed to create backup record: %w", err)
}
sizeBytes, streamErr := s.streamToStorage(ctx, backup, backupConfig, body)
if streamErr != nil {
errMsg := streamErr.Error()
s.markFailed(backup, errMsg)
return nil, fmt.Errorf("upload failed: %w", streamErr)
}
s.markCompleted(backup, sizeBytes)
return nil, nil
}
func (s *PostgreWalBackupService) GetRestorePlan(
database *databases.Database,
backupID *uuid.UUID,
) (*backups_dto.GetRestorePlanResponse, *backups_dto.GetRestorePlanErrorResponse, error) {
if err := s.validateWalBackupType(database); err != nil {
return nil, nil, err
}
fullBackup, err := s.resolveFullBackup(database.ID, backupID)
if err != nil {
return nil, nil, err
}
if fullBackup == nil {
msg := "no full backups available for this database"
if backupID != nil {
msg = fmt.Sprintf("full backup %s not found or not completed", backupID)
}
return nil, &backups_dto.GetRestorePlanErrorResponse{
Error: "no_backups",
Message: msg,
}, nil
}
startSegment := ""
if fullBackup.PgFullBackupWalStartSegmentName != nil {
startSegment = *fullBackup.PgFullBackupWalStartSegmentName
}
walSegments, err := s.backupRepository.FindCompletedWalSegmentsAfter(database.ID, startSegment)
if err != nil {
return nil, nil, fmt.Errorf("failed to query WAL segments: %w", err)
}
chainErr := s.validateRestoreWalChain(fullBackup, walSegments)
if chainErr != nil {
return nil, chainErr, nil
}
fullBackupSizeBytes := int64(fullBackup.BackupSizeMb * 1024 * 1024)
pgVersion := ""
if fullBackup.PgVersion != nil {
pgVersion = *fullBackup.PgVersion
}
stopSegment := ""
if fullBackup.PgFullBackupWalStopSegmentName != nil {
stopSegment = *fullBackup.PgFullBackupWalStopSegmentName
}
response := &backups_dto.GetRestorePlanResponse{
FullBackup: backups_dto.RestorePlanFullBackup{
BackupID: fullBackup.ID,
FullBackupWalStartSegment: startSegment,
FullBackupWalStopSegment: stopSegment,
PgVersion: pgVersion,
CreatedAt: fullBackup.CreatedAt,
SizeBytes: fullBackupSizeBytes,
},
TotalSizeBytes: fullBackupSizeBytes,
}
for _, seg := range walSegments {
segName := ""
if seg.PgWalSegmentName != nil {
segName = *seg.PgWalSegmentName
}
segSizeBytes := int64(seg.BackupSizeMb * 1024 * 1024)
response.WalSegments = append(response.WalSegments, backups_dto.RestorePlanWalSegment{
BackupID: seg.ID,
SegmentName: segName,
SizeBytes: segSizeBytes,
})
response.TotalSizeBytes += segSizeBytes
response.LatestAvailableSegment = segName
}
return response, nil, nil
}
// DownloadBackupFile returns a reader for a backup file belonging to the given database.
// Decryption is handled transparently if the backup is encrypted.
func (s *PostgreWalBackupService) DownloadBackupFile(
database *databases.Database,
backupID uuid.UUID,
) (io.ReadCloser, error) {
if err := s.validateWalBackupType(database); err != nil {
return nil, err
}
backup, err := s.backupRepository.FindByID(backupID)
if err != nil {
return nil, fmt.Errorf("backup not found: %w", err)
}
if backup.DatabaseID != database.ID {
return nil, fmt.Errorf("backup does not belong to this database")
}
if backup.Status != backups_core.BackupStatusCompleted {
return nil, fmt.Errorf("backup is not completed")
}
return s.backupService.GetBackupReader(backupID)
}
func (s *PostgreWalBackupService) GetNextFullBackupTime(
database *databases.Database,
) (*backups_dto.GetNextFullBackupTimeResponse, error) {
if err := s.validateWalBackupType(database); err != nil {
return nil, err
}
backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)
if err != nil {
return nil, fmt.Errorf("failed to get backup config: %w", err)
}
if backupConfig.BackupInterval == nil {
return nil, fmt.Errorf("no backup interval configured for database %s", database.ID)
}
lastFullBackup, err := s.backupRepository.FindLastCompletedFullWalBackupByDatabaseID(
database.ID,
)
if err != nil {
return nil, fmt.Errorf("failed to query last full backup: %w", err)
}
var lastBackupTime *time.Time
if lastFullBackup != nil {
lastBackupTime = &lastFullBackup.CreatedAt
}
now := time.Now().UTC()
nextTime := backupConfig.BackupInterval.NextTriggerTime(now, lastBackupTime)
return &backups_dto.GetNextFullBackupTimeResponse{
NextFullBackupTime: nextTime,
}, nil
}
// ReportError creates a FAILED backup record with the agent's error message.
func (s *PostgreWalBackupService) ReportError(
database *databases.Database,
errorMsg string,
) error {
if err := s.validateWalBackupType(database); err != nil {
return err
}
backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)
if err != nil {
return fmt.Errorf("failed to get backup config: %w", err)
}
if backupConfig.Storage == nil {
return fmt.Errorf("no storage configured for database %s", database.ID)
}
now := time.Now().UTC()
backup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: backupConfig.Storage.ID,
Status: backups_core.BackupStatusFailed,
FailMessage: &errorMsg,
Encryption: backupConfig.Encryption,
CreatedAt: now,
}
backup.GenerateFilename(database.Name)
if err := s.backupRepository.Save(backup); err != nil {
return fmt.Errorf("failed to save error backup record: %w", err)
}
return nil
}
func (s *PostgreWalBackupService) validateWalChain(
databaseID uuid.UUID,
incomingSegment string,
walSegmentSizeBytes int64,
) (*backups_dto.UploadGapResponse, error) {
fullBackup, err := s.backupRepository.FindLastCompletedFullWalBackupByDatabaseID(databaseID)
if err != nil {
return nil, fmt.Errorf("failed to query full backup: %w", err)
}
// No full backup exists yet: cannot accept WAL segments without a chain anchor.
if fullBackup == nil || fullBackup.PgFullBackupWalStopSegmentName == nil {
return &backups_dto.UploadGapResponse{
Error: "no_full_backup",
ExpectedSegmentName: "",
ReceivedSegmentName: incomingSegment,
}, nil
}
stopSegment := *fullBackup.PgFullBackupWalStopSegmentName
lastWal, err := s.backupRepository.FindLastWalSegmentAfter(databaseID, stopSegment)
if err != nil {
return nil, fmt.Errorf("failed to query last WAL segment: %w", err)
}
walCalculator := util_wal.NewWalCalculator(walSegmentSizeBytes)
var chainTail string
if lastWal != nil && lastWal.PgWalSegmentName != nil {
chainTail = *lastWal.PgWalSegmentName
} else {
chainTail = stopSegment
}
expectedNext, err := walCalculator.NextSegment(chainTail)
if err != nil {
return nil, fmt.Errorf("WAL arithmetic failed for %q: %w", chainTail, err)
}
if incomingSegment != expectedNext {
return &backups_dto.UploadGapResponse{
Error: "gap_detected",
ExpectedSegmentName: expectedNext,
ReceivedSegmentName: incomingSegment,
}, nil
}
return nil, nil
}
func (s *PostgreWalBackupService) createBackupRecord(
databaseID uuid.UUID,
storageID uuid.UUID,
uploadType backups_core.PgWalUploadType,
dbName string,
walSegmentName string,
fullBackupWalStartSegment string,
fullBackupWalStopSegment string,
encryption backups_config.BackupEncryption,
) *backups_core.Backup {
now := time.Now().UTC()
backup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: databaseID,
StorageID: storageID,
Status: backups_core.BackupStatusInProgress,
Encryption: encryption,
CreatedAt: now,
}
backup.GenerateFilename(dbName)
if uploadType == backups_core.PgWalUploadTypeBasebackup {
walBackupType := backups_core.PgWalBackupTypeFullBackup
backup.PgWalBackupType = &walBackupType
if fullBackupWalStartSegment != "" {
backup.PgFullBackupWalStartSegmentName = &fullBackupWalStartSegment
}
if fullBackupWalStopSegment != "" {
backup.PgFullBackupWalStopSegmentName = &fullBackupWalStopSegment
}
} else {
walBackupType := backups_core.PgWalBackupTypeWalSegment
backup.PgWalBackupType = &walBackupType
backup.PgWalSegmentName = &walSegmentName
}
return backup
}
func (s *PostgreWalBackupService) streamToStorage(
ctx context.Context,
backup *backups_core.Backup,
backupConfig *backups_config.BackupConfig,
body io.Reader,
) (int64, error) {
if backupConfig.Encryption == backups_config.BackupEncryptionEncrypted {
return s.streamEncrypted(ctx, backup, backupConfig, body, backup.FileName)
}
return s.streamDirect(ctx, backupConfig, body, backup.FileName)
}
func (s *PostgreWalBackupService) streamDirect(
ctx context.Context,
backupConfig *backups_config.BackupConfig,
body io.Reader,
fileName string,
) (int64, error) {
cr := &countingReader{r: body}
if err := backupConfig.Storage.SaveFile(ctx, s.fieldEncryptor, s.logger, fileName, cr); err != nil {
return 0, err
}
return cr.n, nil
}
func (s *PostgreWalBackupService) streamEncrypted(
ctx context.Context,
backup *backups_core.Backup,
backupConfig *backups_config.BackupConfig,
body io.Reader,
fileName string,
) (int64, error) {
masterKey, err := s.secretKeyService.GetSecretKey()
if err != nil {
return 0, fmt.Errorf("failed to get master encryption key: %w", err)
}
pipeReader, pipeWriter := io.Pipe()
encryptionSetup, err := backup_encryption.SetupEncryptionWriter(
pipeWriter,
masterKey,
backup.ID,
)
if err != nil {
_ = pipeWriter.Close()
return 0, err
}
copyErrCh := make(chan error, 1)
go func() {
_, copyErr := io.Copy(encryptionSetup.Writer, body)
if copyErr != nil {
_ = encryptionSetup.Writer.Close()
_ = pipeWriter.CloseWithError(copyErr)
copyErrCh <- copyErr
return
}
if closeErr := encryptionSetup.Writer.Close(); closeErr != nil {
_ = pipeWriter.CloseWithError(closeErr)
copyErrCh <- closeErr
return
}
copyErrCh <- pipeWriter.Close()
}()
cr := &countingReader{r: pipeReader}
saveErr := backupConfig.Storage.SaveFile(ctx, s.fieldEncryptor, s.logger, fileName, cr)
copyErr := <-copyErrCh
if copyErr != nil {
return 0, copyErr
}
if saveErr != nil {
return 0, saveErr
}
backup.EncryptionSalt = &encryptionSetup.SaltBase64
backup.EncryptionIV = &encryptionSetup.NonceBase64
return cr.n, nil
}
func (s *PostgreWalBackupService) markCompleted(backup *backups_core.Backup, sizeBytes int64) {
backup.Status = backups_core.BackupStatusCompleted
backup.BackupSizeMb = float64(sizeBytes) / (1024 * 1024)
if err := s.backupRepository.Save(backup); err != nil {
s.logger.Error(
"failed to mark WAL backup as completed",
"backupId",
backup.ID,
"error",
err,
)
}
}
func (s *PostgreWalBackupService) markFailed(backup *backups_core.Backup, errMsg string) {
backup.Status = backups_core.BackupStatusFailed
backup.FailMessage = &errMsg
if err := s.backupRepository.Save(backup); err != nil {
s.logger.Error("failed to mark WAL backup as failed", "backupId", backup.ID, "error", err)
}
}
func (s *PostgreWalBackupService) resolveFullBackup(
databaseID uuid.UUID,
backupID *uuid.UUID,
) (*backups_core.Backup, error) {
if backupID != nil {
return s.backupRepository.FindCompletedFullWalBackupByID(databaseID, *backupID)
}
return s.backupRepository.FindLastCompletedFullWalBackupByDatabaseID(databaseID)
}
func (s *PostgreWalBackupService) validateRestoreWalChain(
fullBackup *backups_core.Backup,
walSegments []*backups_core.Backup,
) *backups_dto.GetRestorePlanErrorResponse {
if len(walSegments) == 0 {
return nil
}
stopSegment := ""
if fullBackup.PgFullBackupWalStopSegmentName != nil {
stopSegment = *fullBackup.PgFullBackupWalStopSegmentName
}
walCalculator := util_wal.NewWalCalculator(0)
expectedNext, err := walCalculator.NextSegment(stopSegment)
if err != nil {
return nil
}
for _, seg := range walSegments {
segName := ""
if seg.PgWalSegmentName != nil {
segName = *seg.PgWalSegmentName
}
cmp, cmpErr := walCalculator.Compare(segName, stopSegment)
if cmpErr != nil {
return nil
}
// Skip segments that are <= stopSegment (they are part of the basebackup range)
if cmp <= 0 {
continue
}
if segName != expectedNext {
lastContiguous := stopSegment
// Walk back to find the segment before the gap
for _, prev := range walSegments {
prevName := ""
if prev.PgWalSegmentName != nil {
prevName = *prev.PgWalSegmentName
}
prevCmp, _ := walCalculator.Compare(prevName, stopSegment)
if prevCmp <= 0 {
continue
}
if prevName == segName {
break
}
lastContiguous = prevName
}
return &backups_dto.GetRestorePlanErrorResponse{
Error: "wal_chain_broken",
Message: fmt.Sprintf(
"WAL chain has a gap after segment %s. Recovery is only possible up to this segment.",
lastContiguous,
),
LastContiguousSegment: lastContiguous,
}
}
expectedNext, err = walCalculator.NextSegment(segName)
if err != nil {
return nil
}
}
return nil
}
func (s *PostgreWalBackupService) validateWalBackupType(database *databases.Database) error {
if database.Postgresql == nil ||
database.Postgresql.BackupType != postgresql.PostgresBackupTypeWalV1 {
return fmt.Errorf("database %s is not configured for WAL backups", database.ID)
}
return nil
}
type countingReader struct {
r io.Reader
n int64
}
func (cr *countingReader) Read(p []byte) (n int, err error) {
n, err = cr.r.Read(p)
cr.n += int64(n)
return n, err
}

View File

@@ -1,4 +1,4 @@
package backups
package backups_services
import (
"encoding/base64"
@@ -7,10 +7,13 @@ import (
"io"
"log/slog"
"github.com/google/uuid"
audit_logs "databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/backups/backups/backuping"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_download "databasus-backend/internal/features/backups/backups/download"
backups_dto "databasus-backend/internal/features/backups/backups/dto"
"databasus-backend/internal/features/backups/backups/encryption"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
@@ -22,8 +25,6 @@ import (
workspaces_services "databasus-backend/internal/features/workspaces/services"
util_encryption "databasus-backend/internal/util/encryption"
files_utils "databasus-backend/internal/util/files"
"github.com/google/uuid"
)
type BackupService struct {
@@ -108,7 +109,7 @@ func (s *BackupService) GetBackups(
user *users_models.User,
databaseID uuid.UUID,
limit, offset int,
) (*GetBackupsResponse, error) {
) (*backups_dto.GetBackupsResponse, error) {
database, err := s.databaseService.GetDatabaseByID(databaseID)
if err != nil {
return nil, err
@@ -143,7 +144,7 @@ func (s *BackupService) GetBackups(
return nil, err
}
return &GetBackupsResponse{
return &backups_dto.GetBackupsResponse{
Backups: backups,
Total: total,
Limit: limit,
@@ -274,7 +275,7 @@ func (s *BackupService) GetBackupFile(
database.WorkspaceID,
)
reader, err := s.getBackupReader(backupID)
reader, err := s.GetBackupReader(backupID)
if err != nil {
return nil, nil, nil, err
}
@@ -282,39 +283,9 @@ func (s *BackupService) GetBackupFile(
return reader, backup, database, nil
}
func (s *BackupService) deleteDbBackups(databaseID uuid.UUID) error {
dbBackupsInProgress, err := s.backupRepository.FindByDatabaseIdAndStatus(
databaseID,
backups_core.BackupStatusInProgress,
)
if err != nil {
return err
}
if len(dbBackupsInProgress) > 0 {
return errors.New("backup is in progress, storage cannot be removed")
}
dbBackups, err := s.backupRepository.FindByDatabaseID(
databaseID,
)
if err != nil {
return err
}
for _, dbBackup := range dbBackups {
err := s.backupCleaner.DeleteBackup(dbBackup)
if err != nil {
return err
}
}
return nil
}
// GetBackupReader returns a reader for the backup file
// If encrypted, wraps with DecryptionReader
func (s *BackupService) getBackupReader(backupID uuid.UUID) (io.ReadCloser, error) {
// GetBackupReader returns a reader for the backup file.
// If encrypted, wraps with DecryptionReader.
func (s *BackupService) GetBackupReader(backupID uuid.UUID) (io.ReadCloser, error) {
backup, err := s.backupRepository.FindByID(backupID)
if err != nil {
return nil, fmt.Errorf("failed to find backup: %w", err)
@@ -394,7 +365,7 @@ func (s *BackupService) getBackupReader(backupID uuid.UUID) (io.ReadCloser, erro
s.logger.Info("Returning encrypted backup with decryption", "backupId", backupID)
return &DecryptionReaderCloser{
return &backups_dto.DecryptionReaderCloser{
DecryptionReader: decryptionReader,
BaseReader: fileReader,
}, nil
@@ -465,7 +436,7 @@ func (s *BackupService) GetBackupFileWithoutAuth(
return nil, nil, nil, err
}
reader, err := s.getBackupReader(backupID)
reader, err := s.GetBackupReader(backupID)
if err != nil {
return nil, nil, nil, err
}
@@ -501,6 +472,36 @@ func (s *BackupService) UnregisterDownload(userID uuid.UUID) {
s.downloadTokenService.UnregisterDownload(userID)
}
func (s *BackupService) deleteDbBackups(databaseID uuid.UUID) error {
dbBackupsInProgress, err := s.backupRepository.FindByDatabaseIdAndStatus(
databaseID,
backups_core.BackupStatusInProgress,
)
if err != nil {
return err
}
if len(dbBackupsInProgress) > 0 {
return errors.New("backup is in progress, storage cannot be removed")
}
dbBackups, err := s.backupRepository.FindByDatabaseID(
databaseID,
)
if err != nil {
return err
}
for _, dbBackup := range dbBackups {
err := s.backupCleaner.DeleteBackup(dbBackup)
if err != nil {
return err
}
}
return nil
}
func (s *BackupService) generateBackupFilename(
backup *backups_core.Backup,
database *databases.Database,

View File

@@ -2,7 +2,6 @@ package usecases_mariadb
import (
"context"
"encoding/base64"
"errors"
"fmt"
"io"
@@ -123,6 +122,10 @@ func (uc *CreateMariadbBackupUsecase) buildMariadbDumpArgs(
args = append(args, "--compress")
if !config.GetEnv().IsCloud {
args = append(args, "--max-allowed-packet=1G")
}
if mdb.IsHttps {
args = append(args, "--ssl")
args = append(args, "--skip-ssl-verify-server-cert")
@@ -276,10 +279,10 @@ func (uc *CreateMariadbBackupUsecase) createTempMyCnfFile(
password string,
) (string, error) {
tempFolder := config.GetEnv().TempFolder
if err := os.MkdirAll(tempFolder, 0700); err != nil {
if err := os.MkdirAll(tempFolder, 0o700); err != nil {
return "", fmt.Errorf("failed to ensure temp folder exists: %w", err)
}
if err := os.Chmod(tempFolder, 0700); err != nil {
if err := os.Chmod(tempFolder, 0o700); err != nil {
return "", fmt.Errorf("failed to set temp folder permissions: %w", err)
}
@@ -288,7 +291,7 @@ func (uc *CreateMariadbBackupUsecase) createTempMyCnfFile(
return "", fmt.Errorf("failed to create temp directory: %w", err)
}
if err := os.Chmod(tempDir, 0700); err != nil {
if err := os.Chmod(tempDir, 0o700); err != nil {
_ = os.RemoveAll(tempDir)
return "", fmt.Errorf("failed to set temp directory permissions: %w", err)
}
@@ -308,7 +311,7 @@ port=%d
content += "ssl=false\n"
}
err = os.WriteFile(myCnfFile, []byte(content), 0600)
err = os.WriteFile(myCnfFile, []byte(content), 0o600)
if err != nil {
_ = os.RemoveAll(tempDir)
return "", fmt.Errorf("failed to write .my.cnf: %w", err)
@@ -437,40 +440,22 @@ func (uc *CreateMariadbBackupUsecase) setupBackupEncryption(
return storageWriter, nil, metadata, nil
}
salt, err := backup_encryption.GenerateSalt()
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to generate salt: %w", err)
}
nonce, err := backup_encryption.GenerateNonce()
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to generate nonce: %w", err)
}
masterKey, err := uc.secretKeyService.GetSecretKey()
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to get master key: %w", err)
}
encWriter, err := backup_encryption.NewEncryptionWriter(
storageWriter,
masterKey,
backupID,
salt,
nonce,
)
encSetup, err := backup_encryption.SetupEncryptionWriter(storageWriter, masterKey, backupID)
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to create encrypting writer: %w", err)
return nil, nil, metadata, err
}
saltBase64 := base64.StdEncoding.EncodeToString(salt)
nonceBase64 := base64.StdEncoding.EncodeToString(nonce)
metadata.EncryptionSalt = &saltBase64
metadata.EncryptionIV = &nonceBase64
metadata.EncryptionSalt = &encSetup.SaltBase64
metadata.EncryptionIV = &encSetup.NonceBase64
metadata.Encryption = backups_config.BackupEncryptionEncrypted
uc.logger.Info("Encryption enabled for backup", "backupId", backupID)
return encWriter, encWriter, metadata, nil
return encSetup.Writer, encSetup.Writer, metadata, nil
}
func (uc *CreateMariadbBackupUsecase) cleanupOnCancellation(
@@ -563,8 +548,8 @@ func (uc *CreateMariadbBackupUsecase) buildMariadbDumpErrorMessage(
stderrStr,
)
exitErr, ok := waitErr.(*exec.ExitError)
if !ok {
var exitErr *exec.ExitError
if !errors.As(waitErr, &exitErr) {
return errors.New(errorMsg)
}

View File

@@ -2,7 +2,6 @@ package usecases_mongodb
import (
"context"
"encoding/base64"
"errors"
"fmt"
"io"
@@ -277,41 +276,21 @@ func (uc *CreateMongodbBackupUsecase) setupBackupEncryption(
return storageWriter, nil, backupMetadata, nil
}
salt, err := backup_encryption.GenerateSalt()
if err != nil {
return nil, nil, backupMetadata, fmt.Errorf("failed to generate salt: %w", err)
}
nonce, err := backup_encryption.GenerateNonce()
if err != nil {
return nil, nil, backupMetadata, fmt.Errorf("failed to generate nonce: %w", err)
}
masterKey, err := uc.secretKeyService.GetSecretKey()
if err != nil {
return nil, nil, backupMetadata, fmt.Errorf("failed to get master key: %w", err)
}
encryptionWriter, err := backup_encryption.NewEncryptionWriter(
storageWriter,
masterKey,
backupID,
salt,
nonce,
)
encSetup, err := backup_encryption.SetupEncryptionWriter(storageWriter, masterKey, backupID)
if err != nil {
return nil, nil, backupMetadata, fmt.Errorf("failed to create encryption writer: %w", err)
return nil, nil, backupMetadata, err
}
saltBase64 := base64.StdEncoding.EncodeToString(salt)
nonceBase64 := base64.StdEncoding.EncodeToString(nonce)
backupMetadata.BackupID = backupID
backupMetadata.Encryption = backups_config.BackupEncryptionEncrypted
backupMetadata.EncryptionSalt = &saltBase64
backupMetadata.EncryptionIV = &nonceBase64
backupMetadata.EncryptionSalt = &encSetup.SaltBase64
backupMetadata.EncryptionIV = &encSetup.NonceBase64
return encryptionWriter, encryptionWriter, backupMetadata, nil
return encSetup.Writer, encSetup.Writer, backupMetadata, nil
}
func (uc *CreateMongodbBackupUsecase) copyWithShutdownCheck(

View File

@@ -2,7 +2,6 @@ package usecases_mysql
import (
"context"
"encoding/base64"
"errors"
"fmt"
"io"
@@ -119,7 +118,11 @@ func (uc *CreateMysqlBackupUsecase) buildMysqldumpArgs(my *mysqltypes.MysqlDatab
args = append(args, "--events")
}
args = append(args, uc.getNetworkCompressionArgs(my.Version)...)
args = append(args, uc.getNetworkCompressionArgs(my)...)
if !config.GetEnv().IsCloud {
args = append(args, "--max-allowed-packet=1G")
}
if my.IsHttps {
args = append(args, "--ssl-mode=REQUIRED")
@@ -132,15 +135,21 @@ func (uc *CreateMysqlBackupUsecase) buildMysqldumpArgs(my *mysqltypes.MysqlDatab
return args
}
func (uc *CreateMysqlBackupUsecase) getNetworkCompressionArgs(version tools.MysqlVersion) []string {
func (uc *CreateMysqlBackupUsecase) getNetworkCompressionArgs(
my *mysqltypes.MysqlDatabase,
) []string {
const zstdCompressionLevel = 5
switch version {
switch my.Version {
case tools.MysqlVersion80, tools.MysqlVersion84, tools.MysqlVersion9:
return []string{
"--compression-algorithms=zstd",
fmt.Sprintf("--zstd-compression-level=%d", zstdCompressionLevel),
if my.IsZstdSupported {
return []string{
"--compression-algorithms=zstd",
fmt.Sprintf("--zstd-compression-level=%d", zstdCompressionLevel),
}
}
return []string{"--compress"}
case tools.MysqlVersion57:
return []string{"--compress"}
default:
@@ -289,10 +298,10 @@ func (uc *CreateMysqlBackupUsecase) createTempMyCnfFile(
password string,
) (string, error) {
tempFolder := config.GetEnv().TempFolder
if err := os.MkdirAll(tempFolder, 0700); err != nil {
if err := os.MkdirAll(tempFolder, 0o700); err != nil {
return "", fmt.Errorf("failed to ensure temp folder exists: %w", err)
}
if err := os.Chmod(tempFolder, 0700); err != nil {
if err := os.Chmod(tempFolder, 0o700); err != nil {
return "", fmt.Errorf("failed to set temp folder permissions: %w", err)
}
@@ -301,7 +310,7 @@ func (uc *CreateMysqlBackupUsecase) createTempMyCnfFile(
return "", fmt.Errorf("failed to create temp directory: %w", err)
}
if err := os.Chmod(tempDir, 0700); err != nil {
if err := os.Chmod(tempDir, 0o700); err != nil {
_ = os.RemoveAll(tempDir)
return "", fmt.Errorf("failed to set temp directory permissions: %w", err)
}
@@ -319,7 +328,7 @@ port=%d
content += "ssl-mode=REQUIRED\n"
}
err = os.WriteFile(myCnfFile, []byte(content), 0600)
err = os.WriteFile(myCnfFile, []byte(content), 0o600)
if err != nil {
_ = os.RemoveAll(tempDir)
return "", fmt.Errorf("failed to write .my.cnf: %w", err)
@@ -448,40 +457,22 @@ func (uc *CreateMysqlBackupUsecase) setupBackupEncryption(
return storageWriter, nil, metadata, nil
}
salt, err := backup_encryption.GenerateSalt()
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to generate salt: %w", err)
}
nonce, err := backup_encryption.GenerateNonce()
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to generate nonce: %w", err)
}
masterKey, err := uc.secretKeyService.GetSecretKey()
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to get master key: %w", err)
}
encWriter, err := backup_encryption.NewEncryptionWriter(
storageWriter,
masterKey,
backupID,
salt,
nonce,
)
encSetup, err := backup_encryption.SetupEncryptionWriter(storageWriter, masterKey, backupID)
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to create encrypting writer: %w", err)
return nil, nil, metadata, err
}
saltBase64 := base64.StdEncoding.EncodeToString(salt)
nonceBase64 := base64.StdEncoding.EncodeToString(nonce)
metadata.EncryptionSalt = &saltBase64
metadata.EncryptionIV = &nonceBase64
metadata.EncryptionSalt = &encSetup.SaltBase64
metadata.EncryptionIV = &encSetup.NonceBase64
metadata.Encryption = backups_config.BackupEncryptionEncrypted
uc.logger.Info("Encryption enabled for backup", "backupId", backupID)
return encWriter, encWriter, metadata, nil
return encSetup.Writer, encSetup.Writer, metadata, nil
}
func (uc *CreateMysqlBackupUsecase) cleanupOnCancellation(
@@ -574,8 +565,8 @@ func (uc *CreateMysqlBackupUsecase) buildMysqldumpErrorMessage(
stderrStr,
)
exitErr, ok := waitErr.(*exec.ExitError)
if !ok {
var exitErr *exec.ExitError
if !errors.As(waitErr, &exitErr) {
return errors.New(errorMsg)
}
@@ -604,6 +595,15 @@ func (uc *CreateMysqlBackupUsecase) handleConnectionErrors(stderrStr string) err
)
}
if containsIgnoreCase(stderrStr, "compression algorithm") ||
containsIgnoreCase(stderrStr, "2066") {
return fmt.Errorf(
"MySQL connection failed due to unsupported compression algorithm. "+
"Try re-saving the database connection to re-detect compression support. stderr: %s",
stderrStr,
)
}
if containsIgnoreCase(stderrStr, "unknown database") {
return fmt.Errorf(
"MySQL database does not exist. stderr: %s",

View File

@@ -2,7 +2,6 @@ package usecases_postgresql
import (
"context"
"encoding/base64"
"errors"
"fmt"
"io"
@@ -14,6 +13,8 @@ import (
"strings"
"time"
"github.com/google/uuid"
"databasus-backend/internal/config"
common "databasus-backend/internal/features/backups/backups/common"
backups_core "databasus-backend/internal/features/backups/backups/core"
@@ -25,8 +26,6 @@ import (
"databasus-backend/internal/features/storages"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
"github.com/google/uuid"
)
const (
@@ -492,40 +491,22 @@ func (uc *CreatePostgresqlBackupUsecase) setupBackupEncryption(
return storageWriter, nil, metadata, nil
}
salt, err := backup_encryption.GenerateSalt()
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to generate salt: %w", err)
}
nonce, err := backup_encryption.GenerateNonce()
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to generate nonce: %w", err)
}
masterKey, err := uc.secretKeyService.GetSecretKey()
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to get master key: %w", err)
}
encWriter, err := backup_encryption.NewEncryptionWriter(
storageWriter,
masterKey,
backupID,
salt,
nonce,
)
encSetup, err := backup_encryption.SetupEncryptionWriter(storageWriter, masterKey, backupID)
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to create encrypting writer: %w", err)
return nil, nil, metadata, err
}
saltBase64 := base64.StdEncoding.EncodeToString(salt)
nonceBase64 := base64.StdEncoding.EncodeToString(nonce)
metadata.EncryptionSalt = &saltBase64
metadata.EncryptionIV = &nonceBase64
metadata.EncryptionSalt = &encSetup.SaltBase64
metadata.EncryptionIV = &encSetup.NonceBase64
metadata.Encryption = backups_config.BackupEncryptionEncrypted
uc.logger.Info("Encryption enabled for backup", "backupId", backupID)
return encWriter, encWriter, metadata, nil
return encSetup.Writer, encSetup.Writer, metadata, nil
}
func (uc *CreatePostgresqlBackupUsecase) cleanupOnCancellation(
@@ -614,8 +595,8 @@ func (uc *CreatePostgresqlBackupUsecase) buildPgDumpErrorMessage(
stderrStr := string(stderrOutput)
errorMsg := fmt.Sprintf("%s failed: %v stderr: %s", filepath.Base(pgBin), waitErr, stderrStr)
exitErr, ok := waitErr.(*exec.ExitError)
if !ok {
var exitErr *exec.ExitError
if !errors.As(waitErr, &exitErr) {
return errors.New(errorMsg)
}
@@ -767,10 +748,10 @@ func (uc *CreatePostgresqlBackupUsecase) createTempPgpassFile(
)
tempFolder := config.GetEnv().TempFolder
if err := os.MkdirAll(tempFolder, 0700); err != nil {
if err := os.MkdirAll(tempFolder, 0o700); err != nil {
return "", fmt.Errorf("failed to ensure temp folder exists: %w", err)
}
if err := os.Chmod(tempFolder, 0700); err != nil {
if err := os.Chmod(tempFolder, 0o700); err != nil {
return "", fmt.Errorf("failed to set temp folder permissions: %w", err)
}
@@ -779,13 +760,13 @@ func (uc *CreatePostgresqlBackupUsecase) createTempPgpassFile(
return "", fmt.Errorf("failed to create temporary directory: %w", err)
}
if err := os.Chmod(tempDir, 0700); err != nil {
if err := os.Chmod(tempDir, 0o700); err != nil {
_ = os.RemoveAll(tempDir)
return "", fmt.Errorf("failed to set temporary directory permissions: %w", err)
}
pgpassFile := filepath.Join(tempDir, ".pgpass")
err = os.WriteFile(pgpassFile, []byte(pgpassContent), 0600)
err = os.WriteFile(pgpassFile, []byte(pgpassContent), 0o600)
if err != nil {
_ = os.RemoveAll(tempDir)
return "", fmt.Errorf("failed to write temporary .pgpass file: %w", err)

View File

@@ -4,10 +4,10 @@ import (
"errors"
"net/http"
users_middleware "databasus-backend/internal/features/users/middleware"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
users_middleware "databasus-backend/internal/features/users/middleware"
)
type BackupConfigController struct {

View File

@@ -118,9 +118,10 @@ func Test_SaveBackupConfig_PermissionsEnforced(t *testing.T) {
timeOfDay := "04:00"
request := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -146,7 +147,7 @@ func Test_SaveBackupConfig_PermissionsEnforced(t *testing.T) {
if tt.expectSuccess {
assert.Equal(t, database.ID, response.DatabaseID)
assert.True(t, response.IsBackupsEnabled)
assert.Equal(t, period.PeriodWeek, response.StorePeriod)
assert.Equal(t, period.PeriodWeek, response.RetentionTimePeriod)
} else {
assert.Contains(t, string(testResp.Body), "insufficient permissions")
}
@@ -170,9 +171,10 @@ func Test_SaveBackupConfig_WhenUserIsNotWorkspaceMember_ReturnsForbidden(t *test
timeOfDay := "04:00"
request := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -337,7 +339,7 @@ func Test_GetBackupConfigByDbID_ReturnsDefaultConfigForNewDatabase(t *testing.T)
assert.Equal(t, database.ID, response.DatabaseID)
assert.False(t, response.IsBackupsEnabled)
assert.Equal(t, plan.MaxStoragePeriod, response.StorePeriod)
assert.Equal(t, plan.MaxStoragePeriod, response.RetentionTimePeriod)
assert.Equal(t, plan.MaxBackupSizeMB, response.MaxBackupSizeMB)
assert.Equal(t, plan.MaxBackupsTotalSizeMB, response.MaxBackupsTotalSizeMB)
assert.True(t, response.IsRetryIfFailed)
@@ -411,9 +413,10 @@ func Test_SaveBackupConfig_WhenPlanLimitsAreAdjusted_ValidationEnforced(t *testi
// Test 1: Try to save backup config with exceeded backup size limit
timeOfDay := "04:00"
backupConfigExceededSize := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -440,9 +443,10 @@ func Test_SaveBackupConfig_WhenPlanLimitsAreAdjusted_ValidationEnforced(t *testi
// Test 2: Try to save backup config with exceeded total size limit
backupConfigExceededTotal := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -469,9 +473,10 @@ func Test_SaveBackupConfig_WhenPlanLimitsAreAdjusted_ValidationEnforced(t *testi
// Test 3: Try to save backup config with exceeded storage period limit
backupConfigExceededPeriod := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodYear, // Exceeds limit of Month
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodYear, // Exceeds limit of Month
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -498,9 +503,10 @@ func Test_SaveBackupConfig_WhenPlanLimitsAreAdjusted_ValidationEnforced(t *testi
// Test 4: Save backup config within all limits - should succeed
backupConfigValid := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek, // Within Month limit
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek, // Within Month limit
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -529,7 +535,7 @@ func Test_SaveBackupConfig_WhenPlanLimitsAreAdjusted_ValidationEnforced(t *testi
assert.Equal(t, database.ID, responseValid.DatabaseID)
assert.Equal(t, int64(80), responseValid.MaxBackupSizeMB)
assert.Equal(t, int64(800), responseValid.MaxBackupsTotalSizeMB)
assert.Equal(t, period.PeriodWeek, responseValid.StorePeriod)
assert.Equal(t, period.PeriodWeek, responseValid.RetentionTimePeriod)
}
func Test_IsStorageUsing_PermissionsEnforced(t *testing.T) {
@@ -618,9 +624,10 @@ func Test_SaveBackupConfig_WithEncryptionNone_ConfigSaved(t *testing.T) {
timeOfDay := "04:00"
request := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -662,9 +669,10 @@ func Test_SaveBackupConfig_WithEncryptionEncrypted_ConfigSaved(t *testing.T) {
timeOfDay := "04:00"
request := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -959,9 +967,10 @@ func Test_TransferDatabase_ToNewStorage_DatabaseTransferd(t *testing.T) {
timeOfDay := "04:00"
backupConfigRequest := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -1045,9 +1054,10 @@ func Test_TransferDatabase_WithExistingStorage_DatabaseAndStorageTransferd(t *te
timeOfDay := "04:00"
backupConfigRequest := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -1142,9 +1152,10 @@ func Test_TransferDatabase_StorageHasOtherDBs_CannotTransfer(t *testing.T) {
timeOfDay := "04:00"
backupConfigRequest1 := BackupConfig{
DatabaseID: database1.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database1.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -1168,9 +1179,10 @@ func Test_TransferDatabase_StorageHasOtherDBs_CannotTransfer(t *testing.T) {
)
backupConfigRequest2 := BackupConfig{
DatabaseID: database2.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database2.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -1244,9 +1256,10 @@ func Test_TransferDatabase_WithNotifiers_NotifiersTransferred(t *testing.T) {
timeOfDay := "04:00"
backupConfigRequest := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -1364,9 +1377,10 @@ func Test_TransferDatabase_NotifierHasOtherDBs_NotifierSkipped(t *testing.T) {
timeOfDay := "04:00"
backupConfigRequest := BackupConfig{
DatabaseID: database1.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database1.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -1486,9 +1500,10 @@ func Test_TransferDatabase_WithMultipleNotifiers_OnlyExclusiveOnesTransferred(t
timeOfDay := "04:00"
backupConfigRequest := BackupConfig{
DatabaseID: database1.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database1.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -1585,9 +1600,10 @@ func Test_TransferDatabase_WithTargetNotifiers_NotifiersAssigned(t *testing.T) {
timeOfDay := "04:00"
backupConfigRequest := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -1665,9 +1681,10 @@ func Test_TransferDatabase_TargetNotifierFromDifferentWorkspace_ReturnsBadReques
timeOfDay := "04:00"
backupConfigRequest := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -1730,9 +1747,10 @@ func Test_TransferDatabase_TargetStorageFromDifferentWorkspace_ReturnsBadRequest
timeOfDay := "04:00"
backupConfigRequest := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -1789,9 +1807,10 @@ func Test_SaveBackupConfig_WithSystemStorage_CanBeUsedByAnyDatabase(t *testing.T
timeOfDay := "04:00"
backupConfigWithRegularStorage := BackupConfig{
DatabaseID: databaseA.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: databaseA.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -1840,9 +1859,10 @@ func Test_SaveBackupConfig_WithSystemStorage_CanBeUsedByAnyDatabase(t *testing.T
assert.True(t, savedSystemStorage.IsSystem)
backupConfigWithSystemStorage := BackupConfig{
DatabaseID: databaseA.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: databaseA.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,

View File

@@ -12,16 +12,19 @@ import (
"databasus-backend/internal/util/logger"
)
var backupConfigRepository = &BackupConfigRepository{}
var backupConfigService = &BackupConfigService{
backupConfigRepository,
databases.GetDatabaseService(),
storages.GetStorageService(),
notifiers.GetNotifierService(),
workspaces_services.GetWorkspaceService(),
plans.GetDatabasePlanService(),
nil,
}
var (
backupConfigRepository = &BackupConfigRepository{}
backupConfigService = &BackupConfigService{
backupConfigRepository,
databases.GetDatabaseService(),
storages.GetStorageService(),
notifiers.GetNotifierService(),
workspaces_services.GetWorkspaceService(),
plans.GetDatabasePlanService(),
nil,
}
)
var backupConfigController = &BackupConfigController{
backupConfigService,
}

View File

@@ -13,3 +13,11 @@ const (
BackupEncryptionNone BackupEncryption = "NONE"
BackupEncryptionEncrypted BackupEncryption = "ENCRYPTED"
)
type RetentionPolicyType string
const (
RetentionPolicyTypeTimePeriod RetentionPolicyType = "TIME_PERIOD"
RetentionPolicyTypeCount RetentionPolicyType = "COUNT"
RetentionPolicyTypeGFS RetentionPolicyType = "GFS"
)

View File

@@ -1,16 +1,17 @@
package backups_config
import (
"databasus-backend/internal/config"
"databasus-backend/internal/features/intervals"
plans "databasus-backend/internal/features/plan"
"databasus-backend/internal/features/storages"
"databasus-backend/internal/util/period"
"errors"
"strings"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/config"
"databasus-backend/internal/features/intervals"
plans "databasus-backend/internal/features/plan"
"databasus-backend/internal/features/storages"
"databasus-backend/internal/util/period"
)
type BackupConfig struct {
@@ -18,7 +19,15 @@ type BackupConfig struct {
IsBackupsEnabled bool `json:"isBackupsEnabled" gorm:"column:is_backups_enabled;type:boolean;not null"`
StorePeriod period.Period `json:"storePeriod" gorm:"column:store_period;type:text;not null"`
RetentionPolicyType RetentionPolicyType `json:"retentionPolicyType" gorm:"column:retention_policy_type;type:text;not null;default:'TIME_PERIOD'"`
RetentionTimePeriod period.TimePeriod `json:"retentionTimePeriod" gorm:"column:retention_time_period;type:text;not null;default:''"`
RetentionCount int `json:"retentionCount" gorm:"column:retention_count;type:int;not null;default:0"`
RetentionGfsHours int `json:"retentionGfsHours" gorm:"column:retention_gfs_hours;type:int;not null;default:0"`
RetentionGfsDays int `json:"retentionGfsDays" gorm:"column:retention_gfs_days;type:int;not null;default:0"`
RetentionGfsWeeks int `json:"retentionGfsWeeks" gorm:"column:retention_gfs_weeks;type:int;not null;default:0"`
RetentionGfsMonths int `json:"retentionGfsMonths" gorm:"column:retention_gfs_months;type:int;not null;default:0"`
RetentionGfsYears int `json:"retentionGfsYears" gorm:"column:retention_gfs_years;type:int;not null;default:0"`
BackupIntervalID uuid.UUID `json:"backupIntervalId" gorm:"column:backup_interval_id;type:uuid;not null"`
BackupInterval *intervals.Interval `json:"backupInterval,omitempty" gorm:"foreignKey:BackupIntervalID"`
@@ -35,7 +44,7 @@ type BackupConfig struct {
Encryption BackupEncryption `json:"encryption" gorm:"column:encryption;type:text;not null;default:'NONE'"`
// MaxBackupSizeMB limits individual backup size. 0 = unlimited.
MaxBackupSizeMB int64 `json:"maxBackupSizeMb" gorm:"column:max_backup_size_mb;type:int;not null"`
MaxBackupSizeMB int64 `json:"maxBackupSizeMb" gorm:"column:max_backup_size_mb;type:int;not null"`
// MaxBackupsTotalSizeMB limits total size of all backups. 0 = unlimited.
MaxBackupsTotalSizeMB int64 `json:"maxBackupsTotalSizeMb" gorm:"column:max_backups_total_size_mb;type:int;not null"`
}
@@ -78,13 +87,12 @@ func (b *BackupConfig) AfterFind(tx *gorm.DB) error {
}
func (b *BackupConfig) Validate(plan *plans.DatabasePlan) error {
// Backup interval is required either as ID or as object
if b.BackupIntervalID == uuid.Nil && b.BackupInterval == nil {
return errors.New("backup interval is required")
}
if b.StorePeriod == "" {
return errors.New("store period is required")
if err := b.validateRetentionPolicy(plan); err != nil {
return err
}
if b.IsRetryIfFailed && b.MaxFailedTriesCount <= 0 {
@@ -110,22 +118,12 @@ func (b *BackupConfig) Validate(plan *plans.DatabasePlan) error {
return errors.New("max backups total size must be non-negative")
}
// Validate against plan limits
// Check storage period limit
if plan.MaxStoragePeriod != period.PeriodForever {
if b.StorePeriod.CompareTo(plan.MaxStoragePeriod) > 0 {
return errors.New("storage period exceeds plan limit")
}
}
// Check max backup size limit (0 in plan means unlimited)
if plan.MaxBackupSizeMB > 0 {
if b.MaxBackupSizeMB == 0 || b.MaxBackupSizeMB > plan.MaxBackupSizeMB {
return errors.New("max backup size exceeds plan limit")
}
}
// Check max total backups size limit (0 in plan means unlimited)
if plan.MaxBackupsTotalSizeMB > 0 {
if b.MaxBackupsTotalSizeMB == 0 ||
b.MaxBackupsTotalSizeMB > plan.MaxBackupsTotalSizeMB {
@@ -140,7 +138,14 @@ func (b *BackupConfig) Copy(newDatabaseID uuid.UUID) *BackupConfig {
return &BackupConfig{
DatabaseID: newDatabaseID,
IsBackupsEnabled: b.IsBackupsEnabled,
StorePeriod: b.StorePeriod,
RetentionPolicyType: b.RetentionPolicyType,
RetentionTimePeriod: b.RetentionTimePeriod,
RetentionCount: b.RetentionCount,
RetentionGfsHours: b.RetentionGfsHours,
RetentionGfsDays: b.RetentionGfsDays,
RetentionGfsWeeks: b.RetentionGfsWeeks,
RetentionGfsMonths: b.RetentionGfsMonths,
RetentionGfsYears: b.RetentionGfsYears,
BackupIntervalID: uuid.Nil,
BackupInterval: b.BackupInterval.Copy(),
StorageID: b.StorageID,
@@ -152,3 +157,34 @@ func (b *BackupConfig) Copy(newDatabaseID uuid.UUID) *BackupConfig {
MaxBackupsTotalSizeMB: b.MaxBackupsTotalSizeMB,
}
}
func (b *BackupConfig) validateRetentionPolicy(plan *plans.DatabasePlan) error {
switch b.RetentionPolicyType {
case RetentionPolicyTypeTimePeriod, "":
if b.RetentionTimePeriod == "" {
return errors.New("retention time period is required")
}
if plan.MaxStoragePeriod != period.PeriodForever {
if b.RetentionTimePeriod.CompareTo(plan.MaxStoragePeriod) > 0 {
return errors.New("storage period exceeds plan limit")
}
}
case RetentionPolicyTypeCount:
if b.RetentionCount <= 0 {
return errors.New("retention count must be greater than 0")
}
case RetentionPolicyTypeGFS:
if b.RetentionGfsHours <= 0 && b.RetentionGfsDays <= 0 && b.RetentionGfsWeeks <= 0 &&
b.RetentionGfsMonths <= 0 && b.RetentionGfsYears <= 0 {
return errors.New("at least one GFS retention field must be greater than 0")
}
default:
return errors.New("invalid retention policy type")
}
return nil
}

View File

@@ -3,17 +3,17 @@ package backups_config
import (
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"databasus-backend/internal/features/intervals"
plans "databasus-backend/internal/features/plan"
"databasus-backend/internal/util/period"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
func Test_Validate_WhenStoragePeriodIsWeekAndPlanAllowsMonth_ValidationPasses(t *testing.T) {
func Test_Validate_WhenRetentionTimePeriodIsWeekAndPlanAllowsMonth_ValidationPasses(t *testing.T) {
config := createValidBackupConfig()
config.StorePeriod = period.PeriodWeek
config.RetentionTimePeriod = period.PeriodWeek
plan := createUnlimitedPlan()
plan.MaxStoragePeriod = period.PeriodMonth
@@ -22,9 +22,9 @@ func Test_Validate_WhenStoragePeriodIsWeekAndPlanAllowsMonth_ValidationPasses(t
assert.NoError(t, err)
}
func Test_Validate_WhenStoragePeriodIsYearAndPlanAllowsMonth_ValidationFails(t *testing.T) {
func Test_Validate_WhenRetentionTimePeriodIsYearAndPlanAllowsMonth_ValidationFails(t *testing.T) {
config := createValidBackupConfig()
config.StorePeriod = period.PeriodYear
config.RetentionTimePeriod = period.PeriodYear
plan := createUnlimitedPlan()
plan.MaxStoragePeriod = period.PeriodMonth
@@ -33,9 +33,11 @@ func Test_Validate_WhenStoragePeriodIsYearAndPlanAllowsMonth_ValidationFails(t *
assert.EqualError(t, err, "storage period exceeds plan limit")
}
func Test_Validate_WhenStoragePeriodIsForeverAndPlanAllowsForever_ValidationPasses(t *testing.T) {
func Test_Validate_WhenRetentionTimePeriodIsForeverAndPlanAllowsForever_ValidationPasses(
t *testing.T,
) {
config := createValidBackupConfig()
config.StorePeriod = period.PeriodForever
config.RetentionTimePeriod = period.PeriodForever
plan := createUnlimitedPlan()
plan.MaxStoragePeriod = period.PeriodForever
@@ -44,9 +46,9 @@ func Test_Validate_WhenStoragePeriodIsForeverAndPlanAllowsForever_ValidationPass
assert.NoError(t, err)
}
func Test_Validate_WhenStoragePeriodIsForeverAndPlanAllowsYear_ValidationFails(t *testing.T) {
func Test_Validate_WhenRetentionTimePeriodIsForeverAndPlanAllowsYear_ValidationFails(t *testing.T) {
config := createValidBackupConfig()
config.StorePeriod = period.PeriodForever
config.RetentionTimePeriod = period.PeriodForever
plan := createUnlimitedPlan()
plan.MaxStoragePeriod = period.PeriodYear
@@ -55,9 +57,9 @@ func Test_Validate_WhenStoragePeriodIsForeverAndPlanAllowsYear_ValidationFails(t
assert.EqualError(t, err, "storage period exceeds plan limit")
}
func Test_Validate_WhenStoragePeriodEqualsExactPlanLimit_ValidationPasses(t *testing.T) {
func Test_Validate_WhenRetentionTimePeriodEqualsExactPlanLimit_ValidationPasses(t *testing.T) {
config := createValidBackupConfig()
config.StorePeriod = period.PeriodMonth
config.RetentionTimePeriod = period.PeriodMonth
plan := createUnlimitedPlan()
plan.MaxStoragePeriod = period.PeriodMonth
@@ -178,7 +180,7 @@ func Test_Validate_WhenTotalSizeEqualsExactPlanLimit_ValidationPasses(t *testing
func Test_Validate_WhenAllLimitsAreUnlimitedInPlan_AnyConfigurationPasses(t *testing.T) {
config := createValidBackupConfig()
config.StorePeriod = period.PeriodForever
config.RetentionTimePeriod = period.PeriodForever
config.MaxBackupSizeMB = 0
config.MaxBackupsTotalSizeMB = 0
@@ -190,7 +192,7 @@ func Test_Validate_WhenAllLimitsAreUnlimitedInPlan_AnyConfigurationPasses(t *tes
func Test_Validate_WhenMultipleLimitsExceeded_ValidationFailsWithFirstError(t *testing.T) {
config := createValidBackupConfig()
config.StorePeriod = period.PeriodYear
config.RetentionTimePeriod = period.PeriodYear
config.MaxBackupSizeMB = 500
config.MaxBackupsTotalSizeMB = 5000
@@ -249,14 +251,14 @@ func Test_Validate_WhenEncryptionIsInvalid_ValidationFailsRegardlessOfPlan(t *te
assert.EqualError(t, err, "encryption must be NONE or ENCRYPTED")
}
func Test_Validate_WhenStoragePeriodIsEmpty_ValidationFails(t *testing.T) {
func Test_Validate_WhenRetentionTimePeriodIsEmpty_ValidationFails(t *testing.T) {
config := createValidBackupConfig()
config.StorePeriod = ""
config.RetentionTimePeriod = ""
plan := createUnlimitedPlan()
err := config.Validate(plan)
assert.EqualError(t, err, "store period is required")
assert.EqualError(t, err, "retention time period is required")
}
func Test_Validate_WhenMaxBackupSizeIsNegative_ValidationFails(t *testing.T) {
@@ -282,8 +284,8 @@ func Test_Validate_WhenMaxTotalSizeIsNegative_ValidationFails(t *testing.T) {
func Test_Validate_WhenPlanLimitsAreAtBoundary_ValidationWorks(t *testing.T) {
tests := []struct {
name string
configPeriod period.Period
planPeriod period.Period
configPeriod period.TimePeriod
planPeriod period.TimePeriod
configSize int64
planSize int64
configTotal int64
@@ -345,7 +347,7 @@ func Test_Validate_WhenPlanLimitsAreAtBoundary_ValidationWorks(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config := createValidBackupConfig()
config.StorePeriod = tt.configPeriod
config.RetentionTimePeriod = tt.configPeriod
config.MaxBackupSizeMB = tt.configSize
config.MaxBackupsTotalSizeMB = tt.configTotal
@@ -364,12 +366,96 @@ func Test_Validate_WhenPlanLimitsAreAtBoundary_ValidationWorks(t *testing.T) {
}
}
func Test_Validate_WhenPolicyTypeIsCount_RequiresPositiveCount(t *testing.T) {
config := createValidBackupConfig()
config.RetentionPolicyType = RetentionPolicyTypeCount
config.RetentionCount = 0
plan := createUnlimitedPlan()
err := config.Validate(plan)
assert.EqualError(t, err, "retention count must be greater than 0")
}
func Test_Validate_WhenPolicyTypeIsCount_WithPositiveCount_ValidationPasses(t *testing.T) {
config := createValidBackupConfig()
config.RetentionPolicyType = RetentionPolicyTypeCount
config.RetentionCount = 10
plan := createUnlimitedPlan()
err := config.Validate(plan)
assert.NoError(t, err)
}
func Test_Validate_WhenPolicyTypeIsGFS_RequiresAtLeastOneField(t *testing.T) {
config := createValidBackupConfig()
config.RetentionPolicyType = RetentionPolicyTypeGFS
config.RetentionGfsDays = 0
config.RetentionGfsWeeks = 0
config.RetentionGfsMonths = 0
config.RetentionGfsYears = 0
plan := createUnlimitedPlan()
err := config.Validate(plan)
assert.EqualError(t, err, "at least one GFS retention field must be greater than 0")
}
func Test_Validate_WhenPolicyTypeIsGFS_WithOnlyHours_ValidationPasses(t *testing.T) {
config := createValidBackupConfig()
config.RetentionPolicyType = RetentionPolicyTypeGFS
config.RetentionGfsHours = 24
plan := createUnlimitedPlan()
err := config.Validate(plan)
assert.NoError(t, err)
}
func Test_Validate_WhenPolicyTypeIsGFS_WithOnlyDays_ValidationPasses(t *testing.T) {
config := createValidBackupConfig()
config.RetentionPolicyType = RetentionPolicyTypeGFS
config.RetentionGfsDays = 7
plan := createUnlimitedPlan()
err := config.Validate(plan)
assert.NoError(t, err)
}
func Test_Validate_WhenPolicyTypeIsGFS_WithAllFields_ValidationPasses(t *testing.T) {
config := createValidBackupConfig()
config.RetentionPolicyType = RetentionPolicyTypeGFS
config.RetentionGfsHours = 24
config.RetentionGfsDays = 7
config.RetentionGfsWeeks = 4
config.RetentionGfsMonths = 12
config.RetentionGfsYears = 3
plan := createUnlimitedPlan()
err := config.Validate(plan)
assert.NoError(t, err)
}
func Test_Validate_WhenPolicyTypeIsInvalid_ValidationFails(t *testing.T) {
config := createValidBackupConfig()
config.RetentionPolicyType = "INVALID"
plan := createUnlimitedPlan()
err := config.Validate(plan)
assert.EqualError(t, err, "invalid retention policy type")
}
func createValidBackupConfig() *BackupConfig {
intervalID := uuid.New()
return &BackupConfig{
DatabaseID: uuid.New(),
IsBackupsEnabled: true,
StorePeriod: period.PeriodMonth,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodMonth,
BackupIntervalID: intervalID,
BackupInterval: &intervals.Interval{ID: intervalID},
SendNotificationsOn: []BackupNotificationType{},

View File

@@ -1,11 +1,12 @@
package backups_config
import (
"databasus-backend/internal/storage"
"errors"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/storage"
)
type BackupConfigRepository struct{}
@@ -47,7 +48,6 @@ func (r *BackupConfigRepository) Save(
return nil
})
if err != nil {
return nil, err
}

View File

@@ -3,6 +3,8 @@ package backups_config
import (
"errors"
"github.com/google/uuid"
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/intervals"
"databasus-backend/internal/features/notifiers"
@@ -10,8 +12,6 @@ import (
"databasus-backend/internal/features/storages"
users_models "databasus-backend/internal/features/users/models"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"github.com/google/uuid"
)
type BackupConfigService struct {
@@ -214,38 +214,6 @@ func (s *BackupConfigService) CreateDisabledBackupConfig(databaseID uuid.UUID) e
return s.initializeDefaultConfig(databaseID)
}
func (s *BackupConfigService) initializeDefaultConfig(
databaseID uuid.UUID,
) error {
plan, err := s.databasePlanService.GetDatabasePlan(databaseID)
if err != nil {
return err
}
timeOfDay := "04:00"
_, err = s.backupConfigRepository.Save(&BackupConfig{
DatabaseID: databaseID,
IsBackupsEnabled: false,
StorePeriod: plan.MaxStoragePeriod,
MaxBackupSizeMB: plan.MaxBackupSizeMB,
MaxBackupsTotalSizeMB: plan.MaxBackupsTotalSizeMB,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
},
SendNotificationsOn: []BackupNotificationType{
NotificationBackupFailed,
NotificationBackupSuccess,
},
IsRetryIfFailed: true,
MaxFailedTriesCount: 3,
Encryption: BackupEncryptionNone,
})
return err
}
func (s *BackupConfigService) TransferDatabaseToWorkspace(
user *users_models.User,
databaseID uuid.UUID,
@@ -289,7 +257,8 @@ func (s *BackupConfigService) TransferDatabaseToWorkspace(
s.transferNotifiers(user, database, request.TargetWorkspaceID)
}
if request.IsTransferWithStorage {
switch {
case request.IsTransferWithStorage:
if backupConfig.StorageID == nil {
return ErrDatabaseHasNoStorage
}
@@ -314,7 +283,7 @@ func (s *BackupConfigService) TransferDatabaseToWorkspace(
if err != nil {
return err
}
} else if request.TargetStorageID != nil {
case request.TargetStorageID != nil:
targetStorage, err := s.storageService.GetStorageByID(*request.TargetStorageID)
if err != nil {
return err
@@ -331,7 +300,7 @@ func (s *BackupConfigService) TransferDatabaseToWorkspace(
if err != nil {
return err
}
} else {
default:
return ErrTargetStorageNotSpecified
}
@@ -350,6 +319,39 @@ func (s *BackupConfigService) TransferDatabaseToWorkspace(
return nil
}
func (s *BackupConfigService) initializeDefaultConfig(
databaseID uuid.UUID,
) error {
plan, err := s.databasePlanService.GetDatabasePlan(databaseID)
if err != nil {
return err
}
timeOfDay := "04:00"
_, err = s.backupConfigRepository.Save(&BackupConfig{
DatabaseID: databaseID,
IsBackupsEnabled: false,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: plan.MaxStoragePeriod,
MaxBackupSizeMB: plan.MaxBackupSizeMB,
MaxBackupsTotalSizeMB: plan.MaxBackupsTotalSizeMB,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
},
SendNotificationsOn: []BackupNotificationType{
NotificationBackupFailed,
NotificationBackupSuccess,
},
IsRetryIfFailed: true,
MaxFailedTriesCount: 3,
Encryption: BackupEncryptionNone,
})
return err
}
func (s *BackupConfigService) transferNotifiers(
user *users_models.User,
database *databases.Database,

View File

@@ -35,9 +35,10 @@ func Test_AttachStorageFromSameWorkspace_SuccessfullyAttached(t *testing.T) {
timeOfDay := "04:00"
request := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -87,9 +88,10 @@ func Test_AttachStorageFromDifferentWorkspace_ReturnsForbidden(t *testing.T) {
timeOfDay := "04:00"
request := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -131,9 +133,10 @@ func Test_DeleteStorageWithAttachedDatabases_CannotDelete(t *testing.T) {
timeOfDay := "04:00"
request := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
@@ -191,9 +194,10 @@ func Test_TransferStorageWithAttachedDatabase_CannotTransfer(t *testing.T) {
timeOfDay := "04:00"
request := BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodWeek,
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodWeek,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,

View File

@@ -1,11 +1,11 @@
package backups_config
import (
"github.com/google/uuid"
"databasus-backend/internal/features/intervals"
"databasus-backend/internal/features/storages"
"databasus-backend/internal/util/period"
"github.com/google/uuid"
)
func EnableBackupsForTestDatabase(
@@ -15,9 +15,10 @@ func EnableBackupsForTestDatabase(
timeOfDay := "16:00"
backupConfig := &BackupConfig{
DatabaseID: databaseID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodDay,
DatabaseID: databaseID,
IsBackupsEnabled: true,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: period.PeriodDay,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,

View File

@@ -1,13 +1,14 @@
package databases
import (
users_middleware "databasus-backend/internal/features/users/middleware"
users_services "databasus-backend/internal/features/users/services"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"net/http"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
users_middleware "databasus-backend/internal/features/users/middleware"
users_services "databasus-backend/internal/features/users/services"
workspaces_services "databasus-backend/internal/features/workspaces/services"
)
type DatabaseController struct {
@@ -29,6 +30,11 @@ func (c *DatabaseController) RegisterRoutes(router *gin.RouterGroup) {
router.GET("/databases/notifier/:id/databases-count", c.CountDatabasesByNotifier)
router.POST("/databases/is-readonly", c.IsUserReadOnly)
router.POST("/databases/create-readonly-user", c.CreateReadOnlyUser)
router.POST("/databases/:id/regenerate-token", c.RegenerateAgentToken)
}
func (c *DatabaseController) RegisterPublicRoutes(router *gin.RouterGroup) {
router.POST("/databases/verify-token", c.VerifyAgentToken)
}
// CreateDatabase
@@ -438,3 +444,61 @@ func (c *DatabaseController) CreateReadOnlyUser(ctx *gin.Context) {
Password: password,
})
}
// RegenerateAgentToken
// @Summary Regenerate agent token for a database
// @Description Generate a new agent token for the database. The token is returned once and stored as a hash.
// @Tags databases
// @Produce json
// @Security BearerAuth
// @Param id path string true "Database ID"
// @Success 200 {object} map[string]string
// @Failure 400 {object} map[string]string
// @Failure 401 {object} map[string]string
// @Router /databases/{id}/regenerate-token [post]
func (c *DatabaseController) RegenerateAgentToken(ctx *gin.Context) {
user, ok := users_middleware.GetUserFromContext(ctx)
if !ok {
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "User not authenticated"})
return
}
id, err := uuid.Parse(ctx.Param("id"))
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": "invalid database ID"})
return
}
token, err := c.databaseService.RegenerateAgentToken(user, id)
if err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
ctx.JSON(http.StatusOK, gin.H{"token": token})
}
// VerifyAgentToken
// @Summary Verify agent token
// @Description Verify that a given agent token is valid for any database
// @Tags databases
// @Accept json
// @Produce json
// @Param request body VerifyAgentTokenRequest true "Token to verify"
// @Success 200 {object} map[string]string
// @Failure 401 {object} map[string]string
// @Router /databases/verify-token [post]
func (c *DatabaseController) VerifyAgentToken(ctx *gin.Context) {
var request VerifyAgentTokenRequest
if err := ctx.ShouldBindJSON(&request); err != nil {
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
if err := c.databaseService.VerifyAgentToken(request.Token); err != nil {
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "invalid token"})
return
}
ctx.JSON(http.StatusOK, gin.H{"message": "token is valid"})
}

View File

@@ -13,10 +13,13 @@ import (
"github.com/stretchr/testify/assert"
"databasus-backend/internal/config"
"databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/databases/databases/mariadb"
"databasus-backend/internal/features/databases/databases/mongodb"
"databasus-backend/internal/features/databases/databases/postgresql"
users_enums "databasus-backend/internal/features/users/enums"
users_middleware "databasus-backend/internal/features/users/middleware"
users_services "databasus-backend/internal/features/users/services"
users_testing "databasus-backend/internal/features/users/testing"
workspaces_controllers "databasus-backend/internal/features/workspaces/controllers"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
@@ -144,6 +147,66 @@ func Test_CreateDatabase_WhenUserIsNotWorkspaceMember_ReturnsForbidden(t *testin
assert.Contains(t, string(testResp.Body), "insufficient permissions")
}
func Test_CreateDatabase_WalV1Type_NoConnectionFieldsRequired(t *testing.T) {
router := createTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
defer workspaces_testing.RemoveTestWorkspace(workspace, router)
request := Database{
Name: "Test WAL Database",
WorkspaceID: &workspace.ID,
Type: DatabaseTypePostgres,
Postgresql: &postgresql.PostgresqlDatabase{
BackupType: postgresql.PostgresBackupTypeWalV1,
CpuCount: 1,
},
}
var response Database
test_utils.MakePostRequestAndUnmarshal(
t,
router,
"/api/v1/databases/create",
"Bearer "+owner.Token,
request,
http.StatusCreated,
&response,
)
defer RemoveTestDatabase(&response)
assert.Equal(t, "Test WAL Database", response.Name)
assert.NotEqual(t, uuid.Nil, response.ID)
}
func Test_CreateDatabase_PgDumpType_ConnectionFieldsRequired(t *testing.T) {
router := createTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
defer workspaces_testing.RemoveTestWorkspace(workspace, router)
request := Database{
Name: "Test PG_DUMP Database",
WorkspaceID: &workspace.ID,
Type: DatabaseTypePostgres,
Postgresql: &postgresql.PostgresqlDatabase{
BackupType: postgresql.PostgresBackupTypePgDump,
CpuCount: 1,
},
}
testResp := test_utils.MakePostRequest(
t,
router,
"/api/v1/databases/create",
"Bearer "+owner.Token,
request,
http.StatusBadRequest,
)
assert.Contains(t, string(testResp.Body), "host is required")
}
func Test_UpdateDatabase_PermissionsEnforced(t *testing.T) {
tests := []struct {
name string
@@ -256,6 +319,52 @@ func Test_UpdateDatabase_WhenUserIsNotWorkspaceMember_ReturnsForbidden(t *testin
assert.Contains(t, string(testResp.Body), "insufficient permissions")
}
func Test_UpdateDatabase_WhenDatabaseTypeChanged_ReturnsBadRequest(t *testing.T) {
router := createTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
defer workspaces_testing.RemoveTestWorkspace(workspace, router)
database := createTestDatabaseViaAPI("Test Database", workspace.ID, owner.Token, router)
defer RemoveTestDatabase(database)
database.Type = DatabaseTypeMysql
testResp := test_utils.MakePostRequest(
t,
router,
"/api/v1/databases/update",
"Bearer "+owner.Token,
database,
http.StatusBadRequest,
)
assert.Contains(t, string(testResp.Body), "database type cannot be changed")
}
func Test_UpdateDatabase_WhenBackupTypeChanged_ReturnsBadRequest(t *testing.T) {
router := createTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
defer workspaces_testing.RemoveTestWorkspace(workspace, router)
database := createTestDatabaseViaAPI("Test Database", workspace.ID, owner.Token, router)
defer RemoveTestDatabase(database)
database.Postgresql.BackupType = postgresql.PostgresBackupTypeWalV1
testResp := test_utils.MakePostRequest(
t,
router,
"/api/v1/databases/update",
"Bearer "+owner.Token,
database,
http.StatusBadRequest,
)
assert.Contains(t, string(testResp.Body), "backup type cannot be changed")
}
func Test_DeleteDatabase_PermissionsEnforced(t *testing.T) {
tests := []struct {
name string
@@ -753,7 +862,7 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
name string
databaseType DatabaseType
createDatabase func(workspaceID uuid.UUID) *Database
updateDatabase func(workspaceID uuid.UUID, databaseID uuid.UUID) *Database
updateDatabase func(workspaceID, databaseID uuid.UUID) *Database
verifySensitiveData func(t *testing.T, database *Database)
verifyHiddenData func(t *testing.T, database *Database)
}{
@@ -769,7 +878,7 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
Postgresql: pgConfig,
}
},
updateDatabase: func(workspaceID uuid.UUID, databaseID uuid.UUID) *Database {
updateDatabase: func(workspaceID, databaseID uuid.UUID) *Database {
pgConfig := getTestPostgresConfig()
pgConfig.Password = ""
return &Database{
@@ -805,7 +914,7 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
Mariadb: mariaConfig,
}
},
updateDatabase: func(workspaceID uuid.UUID, databaseID uuid.UUID) *Database {
updateDatabase: func(workspaceID, databaseID uuid.UUID) *Database {
mariaConfig := getTestMariadbConfig()
mariaConfig.Password = ""
return &Database{
@@ -841,7 +950,7 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
Mongodb: mongoConfig,
}
},
updateDatabase: func(workspaceID uuid.UUID, databaseID uuid.UUID) *Database {
updateDatabase: func(workspaceID, databaseID uuid.UUID) *Database {
mongoConfig := getTestMongodbConfig()
mongoConfig.Password = ""
return &Database{
@@ -1050,6 +1159,87 @@ func Test_TestConnection_PermissionsEnforced(t *testing.T) {
}
}
func Test_RegenerateAgentToken_ReturnsToken(t *testing.T) {
router := createTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
defer workspaces_testing.RemoveTestWorkspace(workspace, router)
database := createTestDatabaseViaAPI("Test Database", workspace.ID, owner.Token, router)
defer RemoveTestDatabase(database)
var response map[string]string
test_utils.MakePostRequestAndUnmarshal(
t,
router,
"/api/v1/databases/"+database.ID.String()+"/regenerate-token",
"Bearer "+owner.Token,
nil,
http.StatusOK,
&response,
)
assert.NotEmpty(t, response["token"])
assert.Len(t, response["token"], 32)
var updatedDatabase Database
test_utils.MakeGetRequestAndUnmarshal(
t,
router,
"/api/v1/databases/"+database.ID.String(),
"Bearer "+owner.Token,
http.StatusOK,
&updatedDatabase,
)
assert.True(t, updatedDatabase.IsAgentTokenGenerated)
}
func Test_VerifyAgentToken_WithValidToken_Succeeds(t *testing.T) {
router := createTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
defer workspaces_testing.RemoveTestWorkspace(workspace, router)
database := createTestDatabaseViaAPI("Test Database", workspace.ID, owner.Token, router)
defer RemoveTestDatabase(database)
var regenerateResponse map[string]string
test_utils.MakePostRequestAndUnmarshal(
t,
router,
"/api/v1/databases/"+database.ID.String()+"/regenerate-token",
"Bearer "+owner.Token,
nil,
http.StatusOK,
&regenerateResponse,
)
token := regenerateResponse["token"]
assert.NotEmpty(t, token)
w := workspaces_testing.MakeAPIRequest(
router,
"POST",
"/api/v1/databases/verify-token",
"",
VerifyAgentTokenRequest{Token: token},
)
assert.Equal(t, http.StatusOK, w.Code)
}
func Test_VerifyAgentToken_WithInvalidToken_Returns401(t *testing.T) {
router := createTestRouter()
w := workspaces_testing.MakeAPIRequest(
router,
"POST",
"/api/v1/databases/verify-token",
"",
VerifyAgentTokenRequest{Token: "invalidtoken00000000000000000000"},
)
assert.Equal(t, http.StatusUnauthorized, w.Code)
}
func createTestDatabaseViaAPI(
name string,
workspaceID uuid.UUID,
@@ -1101,11 +1291,20 @@ func createTestDatabaseViaAPI(
}
func createTestRouter() *gin.Engine {
router := workspaces_testing.CreateTestRouter(
workspaces_controllers.GetWorkspaceController(),
workspaces_controllers.GetMembershipController(),
GetDatabaseController(),
)
gin.SetMode(gin.TestMode)
router := gin.New()
v1 := router.Group("/api/v1")
protected := v1.Group("").Use(users_middleware.AuthMiddleware(users_services.GetUserService()))
workspaces_controllers.GetWorkspaceController().RegisterRoutes(protected.(*gin.RouterGroup))
workspaces_controllers.GetMembershipController().RegisterRoutes(protected.(*gin.RouterGroup))
GetDatabaseController().RegisterRoutes(protected.(*gin.RouterGroup))
GetDatabaseController().RegisterPublicRoutes(v1)
audit_logs.SetupDependencies()
return router
}
@@ -1118,13 +1317,14 @@ func getTestPostgresConfig() *postgresql.PostgresqlDatabase {
testDbName := "testdb"
return &postgresql.PostgresqlDatabase{
Version: tools.PostgresqlVersion16,
Host: config.GetEnv().TestLocalhost,
Port: port,
Username: "testuser",
Password: "testpassword",
Database: &testDbName,
CpuCount: 1,
BackupType: postgresql.PostgresBackupTypePgDump,
Version: tools.PostgresqlVersion16,
Host: config.GetEnv().TestLocalhost,
Port: port,
Username: "testuser",
Password: "testpassword",
Database: &testDbName,
CpuCount: 1,
}
}

View File

@@ -12,11 +12,11 @@ import (
"strings"
"time"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
"github.com/go-sql-driver/mysql"
"github.com/google/uuid"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
)
type MariadbDatabase struct {
@@ -391,7 +391,7 @@ func (m *MariadbDatabase) HasPrivilege(priv string) bool {
}
func HasPrivilege(privileges, priv string) bool {
for _, p := range strings.Split(privileges, ",") {
for p := range strings.SplitSeq(privileges, ",") {
if strings.TrimSpace(p) == priv {
return true
}
@@ -399,7 +399,7 @@ func HasPrivilege(privileges, priv string) bool {
return false
}
func (m *MariadbDatabase) buildDSN(password string, database string) string {
func (m *MariadbDatabase) buildDSN(password, database string) string {
tlsConfig := "false"
if m.IsHttps {

View File

@@ -10,13 +10,13 @@ import (
"strings"
"time"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
"github.com/google/uuid"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
)
type MongodbDatabase struct {
@@ -25,15 +25,16 @@ type MongodbDatabase struct {
Version tools.MongodbVersion `json:"version" gorm:"type:text;not null"`
Host string `json:"host" gorm:"type:text;not null"`
Port *int `json:"port" gorm:"type:int"`
Username string `json:"username" gorm:"type:text;not null"`
Password string `json:"password" gorm:"type:text;not null"`
Database string `json:"database" gorm:"type:text;not null"`
AuthDatabase string `json:"authDatabase" gorm:"type:text;not null;default:'admin'"`
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
IsSrv bool `json:"isSrv" gorm:"column:is_srv;type:boolean;not null;default:false"`
CpuCount int `json:"cpuCount" gorm:"column:cpu_count;type:int;not null;default:1"`
Host string `json:"host" gorm:"type:text;not null"`
Port *int `json:"port" gorm:"type:int"`
Username string `json:"username" gorm:"type:text;not null"`
Password string `json:"password" gorm:"type:text;not null"`
Database string `json:"database" gorm:"type:text;not null"`
AuthDatabase string `json:"authDatabase" gorm:"type:text;not null;default:'admin'"`
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
IsSrv bool `json:"isSrv" gorm:"column:is_srv;type:boolean;not null;default:false"`
IsDirectConnection bool `json:"isDirectConnection" gorm:"column:is_direct_connection;type:boolean;not null;default:false"`
CpuCount int `json:"cpuCount" gorm:"column:cpu_count;type:int;not null;default:1"`
}
func (m *MongodbDatabase) TableName() string {
@@ -132,6 +133,7 @@ func (m *MongodbDatabase) Update(incoming *MongodbDatabase) {
m.AuthDatabase = incoming.AuthDatabase
m.IsHttps = incoming.IsHttps
m.IsSrv = incoming.IsSrv
m.IsDirectConnection = incoming.IsDirectConnection
m.CpuCount = incoming.CpuCount
if incoming.Password != "" {
@@ -432,7 +434,6 @@ func (m *MongodbDatabase) CreateReadOnlyUser(
},
}},
}).Err()
if err != nil {
if attempt < maxRetries-1 {
continue
@@ -450,6 +451,48 @@ func (m *MongodbDatabase) CreateReadOnlyUser(
return "", "", errors.New("failed to generate unique username after 3 attempts")
}
// BuildMongodumpURI builds a URI suitable for mongodump (without database in path)
func (m *MongodbDatabase) BuildMongodumpURI(password string) string {
authDB := m.AuthDatabase
if authDB == "" {
authDB = "admin"
}
extraParams := ""
if m.IsHttps {
extraParams += "&tls=true&tlsInsecure=true"
}
if m.IsDirectConnection {
extraParams += "&directConnection=true"
}
if m.IsSrv {
return fmt.Sprintf(
"mongodb+srv://%s:%s@%s/?authSource=%s&connectTimeoutMS=15000%s",
url.QueryEscape(m.Username),
url.QueryEscape(password),
m.Host,
authDB,
extraParams,
)
}
port := 27017
if m.Port != nil {
port = *m.Port
}
return fmt.Sprintf(
"mongodb://%s:%s@%s:%d/?authSource=%s&connectTimeoutMS=15000%s",
url.QueryEscape(m.Username),
url.QueryEscape(password),
m.Host,
port,
authDB,
extraParams,
)
}
// buildConnectionURI builds a MongoDB connection URI
func (m *MongodbDatabase) buildConnectionURI(password string) string {
authDB := m.AuthDatabase
@@ -457,9 +500,12 @@ func (m *MongodbDatabase) buildConnectionURI(password string) string {
authDB = "admin"
}
tlsParams := ""
extraParams := ""
if m.IsHttps {
tlsParams = "&tls=true&tlsInsecure=true"
extraParams += "&tls=true&tlsInsecure=true"
}
if m.IsDirectConnection {
extraParams += "&directConnection=true"
}
if m.IsSrv {
@@ -470,7 +516,7 @@ func (m *MongodbDatabase) buildConnectionURI(password string) string {
m.Host,
m.Database,
authDB,
tlsParams,
extraParams,
)
}
@@ -487,46 +533,7 @@ func (m *MongodbDatabase) buildConnectionURI(password string) string {
port,
m.Database,
authDB,
tlsParams,
)
}
// BuildMongodumpURI builds a URI suitable for mongodump (without database in path)
func (m *MongodbDatabase) BuildMongodumpURI(password string) string {
authDB := m.AuthDatabase
if authDB == "" {
authDB = "admin"
}
tlsParams := ""
if m.IsHttps {
tlsParams = "&tls=true&tlsInsecure=true"
}
if m.IsSrv {
return fmt.Sprintf(
"mongodb+srv://%s:%s@%s/?authSource=%s&connectTimeoutMS=15000%s",
url.QueryEscape(m.Username),
url.QueryEscape(password),
m.Host,
authDB,
tlsParams,
)
}
port := 27017
if m.Port != nil {
port = *m.Port
}
return fmt.Sprintf(
"mongodb://%s:%s@%s:%d/?authSource=%s&connectTimeoutMS=15000%s",
url.QueryEscape(m.Username),
url.QueryEscape(password),
m.Host,
port,
authDB,
tlsParams,
extraParams,
)
}

View File

@@ -631,6 +631,89 @@ func Test_Validate_SrvConnection_AllowsNullPort(t *testing.T) {
assert.NoError(t, err)
}
func Test_BuildConnectionURI_WithDirectConnection_ReturnsCorrectUri(t *testing.T) {
port := 27017
model := &MongodbDatabase{
Host: "mongo.example.local",
Port: &port,
Username: "testuser",
Password: "testpass123",
Database: "mydb",
AuthDatabase: "admin",
IsHttps: false,
IsSrv: false,
IsDirectConnection: true,
}
uri := model.buildConnectionURI("testpass123")
assert.Contains(t, uri, "mongodb://")
assert.Contains(t, uri, "directConnection=true")
assert.Contains(t, uri, "mongo.example.local:27017")
assert.Contains(t, uri, "authSource=admin")
}
func Test_BuildConnectionURI_WithoutDirectConnection_OmitsParam(t *testing.T) {
port := 27017
model := &MongodbDatabase{
Host: "localhost",
Port: &port,
Username: "testuser",
Password: "testpass123",
Database: "mydb",
AuthDatabase: "admin",
IsHttps: false,
IsSrv: false,
IsDirectConnection: false,
}
uri := model.buildConnectionURI("testpass123")
assert.NotContains(t, uri, "directConnection")
}
func Test_BuildMongodumpURI_WithDirectConnection_ReturnsCorrectUri(t *testing.T) {
port := 27017
model := &MongodbDatabase{
Host: "mongo.example.local",
Port: &port,
Username: "testuser",
Password: "testpass123",
Database: "mydb",
AuthDatabase: "admin",
IsHttps: false,
IsSrv: false,
IsDirectConnection: true,
}
uri := model.BuildMongodumpURI("testpass123")
assert.Contains(t, uri, "mongodb://")
assert.Contains(t, uri, "directConnection=true")
assert.NotContains(t, uri, "/mydb")
}
func Test_BuildConnectionURI_WithDirectConnectionAndTls_ReturnsBothParams(t *testing.T) {
port := 27017
model := &MongodbDatabase{
Host: "mongo.example.local",
Port: &port,
Username: "testuser",
Password: "testpass123",
Database: "mydb",
AuthDatabase: "admin",
IsHttps: true,
IsSrv: false,
IsDirectConnection: true,
}
uri := model.buildConnectionURI("testpass123")
assert.Contains(t, uri, "directConnection=true")
assert.Contains(t, uri, "tls=true")
assert.Contains(t, uri, "tlsInsecure=true")
}
func Test_Validate_StandardConnection_RequiresPort(t *testing.T) {
model := &MongodbDatabase{
Host: "localhost",

View File

@@ -12,11 +12,11 @@ import (
"strings"
"time"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
"github.com/go-sql-driver/mysql"
"github.com/google/uuid"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
)
type MysqlDatabase struct {
@@ -25,13 +25,14 @@ type MysqlDatabase struct {
Version tools.MysqlVersion `json:"version" gorm:"type:text;not null"`
Host string `json:"host" gorm:"type:text;not null"`
Port int `json:"port" gorm:"type:int;not null"`
Username string `json:"username" gorm:"type:text;not null"`
Password string `json:"password" gorm:"type:text;not null"`
Database *string `json:"database" gorm:"type:text"`
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
Privileges string `json:"privileges" gorm:"column:privileges;type:text;not null;default:''"`
Host string `json:"host" gorm:"type:text;not null"`
Port int `json:"port" gorm:"type:int;not null"`
Username string `json:"username" gorm:"type:text;not null"`
Password string `json:"password" gorm:"type:text;not null"`
Database *string `json:"database" gorm:"type:text"`
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
Privileges string `json:"privileges" gorm:"column:privileges;type:text;not null;default:''"`
IsZstdSupported bool `json:"isZstdSupported" gorm:"column:is_zstd_supported;type:boolean;not null;default:true"`
}
func (m *MysqlDatabase) TableName() string {
@@ -102,6 +103,7 @@ func (m *MysqlDatabase) TestConnection(
return err
}
m.Privileges = privileges
m.IsZstdSupported = detectZstdSupport(ctx, db)
if err := checkBackupPermissions(m.Privileges); err != nil {
return err
@@ -125,6 +127,7 @@ func (m *MysqlDatabase) Update(incoming *MysqlDatabase) {
m.Database = incoming.Database
m.IsHttps = incoming.IsHttps
m.Privileges = incoming.Privileges
m.IsZstdSupported = incoming.IsZstdSupported
if incoming.Password != "" {
m.Password = incoming.Password
@@ -185,6 +188,7 @@ func (m *MysqlDatabase) PopulateDbData(
return err
}
m.Privileges = privileges
m.IsZstdSupported = detectZstdSupport(ctx, db)
return nil
}
@@ -223,6 +227,7 @@ func (m *MysqlDatabase) PopulateVersion(
return err
}
m.Version = detectedVersion
m.IsZstdSupported = detectZstdSupport(ctx, db)
return nil
}
@@ -398,7 +403,7 @@ func HasPrivilege(privileges, priv string) bool {
return false
}
func (m *MysqlDatabase) buildDSN(password string, database string) string {
func (m *MysqlDatabase) buildDSN(password, database string) string {
tlsConfig := "false"
allowCleartext := ""
@@ -575,6 +580,22 @@ func checkBackupPermissions(privileges string) error {
return nil
}
// detectZstdSupport checks if the MySQL server supports zstd network compression.
// The protocol_compression_algorithms variable was introduced in MySQL 8.0.18.
// Managed MySQL providers (e.g. PlanetScale) may not support zstd even on 8.0+.
func detectZstdSupport(ctx context.Context, db *sql.DB) bool {
var varName, value string
err := db.QueryRowContext(ctx,
"SHOW VARIABLES LIKE 'protocol_compression_algorithms'",
).Scan(&varName, &value)
if err != nil {
return false
}
return strings.Contains(strings.ToLower(value), "zstd")
}
func decryptPasswordIfNeeded(
password string,
encryptor encryption.FieldEncryptor,

View File

@@ -177,6 +177,38 @@ func Test_TestConnection_SufficientPermissions_Success(t *testing.T) {
}
}
func Test_TestConnection_DetectsZstdSupport(t *testing.T) {
env := config.GetEnv()
cases := []struct {
name string
version tools.MysqlVersion
port string
isExpectZstd bool
}{
{"MySQL 5.7", tools.MysqlVersion57, env.TestMysql57Port, false},
{"MySQL 8.0", tools.MysqlVersion80, env.TestMysql80Port, true},
{"MySQL 8.4", tools.MysqlVersion84, env.TestMysql84Port, true},
{"MySQL 9", tools.MysqlVersion9, env.TestMysql90Port, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
container := connectToMysqlContainer(t, tc.port, tc.version)
defer container.DB.Close()
mysqlModel := createMysqlModel(container)
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
err := mysqlModel.TestConnection(logger, nil, uuid.New())
assert.NoError(t, err)
assert.Equal(t, tc.isExpectZstd, mysqlModel.IsZstdSupported,
"IsZstdSupported mismatch for %s", tc.name)
})
}
}
func Test_IsUserReadOnly_AdminUser_ReturnsFalse(t *testing.T) {
env := config.GetEnv()
cases := []struct {

View File

@@ -2,8 +2,6 @@ package postgresql
import (
"context"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
"errors"
"fmt"
"log/slog"
@@ -15,6 +13,17 @@ import (
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"gorm.io/gorm"
"databasus-backend/internal/config"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
)
type PostgresBackupType string
const (
PostgresBackupTypePgDump PostgresBackupType = "PG_DUMP"
PostgresBackupTypeWalV1 PostgresBackupType = "WAL_V1"
)
type PostgresqlDatabase struct {
@@ -24,11 +33,13 @@ type PostgresqlDatabase struct {
Version tools.PostgresqlVersion `json:"version" gorm:"type:text;not null"`
// connection data
Host string `json:"host" gorm:"type:text;not null"`
Port int `json:"port" gorm:"type:int;not null"`
Username string `json:"username" gorm:"type:text;not null"`
Password string `json:"password" gorm:"type:text;not null"`
BackupType PostgresBackupType `json:"backupType" gorm:"column:backup_type;type:text;not null;default:'PG_DUMP'"`
// connection data — required for PG_DUMP, optional for WAL_V1
Host string `json:"host" gorm:"type:text"`
Port int `json:"port" gorm:"type:int"`
Username string `json:"username" gorm:"type:text"`
Password string `json:"password" gorm:"type:text"`
Database *string `json:"database" gorm:"type:text"`
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
@@ -66,20 +77,30 @@ func (p *PostgresqlDatabase) AfterFind(_ *gorm.DB) error {
}
func (p *PostgresqlDatabase) Validate() error {
if p.Host == "" {
return errors.New("host is required")
if p.BackupType == "" {
p.BackupType = PostgresBackupTypePgDump
}
if p.Port == 0 {
return errors.New("port is required")
if p.BackupType == PostgresBackupTypePgDump && config.GetEnv().IsCloud {
return errors.New("PG_DUMP backup type is not supported in cloud mode")
}
if p.Username == "" {
return errors.New("username is required")
}
if p.BackupType == PostgresBackupTypePgDump {
if p.Host == "" {
return errors.New("host is required")
}
if p.Password == "" {
return errors.New("password is required")
if p.Port == 0 {
return errors.New("port is required")
}
if p.Username == "" {
return errors.New("username is required")
}
if p.Password == "" {
return errors.New("password is required")
}
}
if p.CpuCount <= 0 {
@@ -90,7 +111,7 @@ func (p *PostgresqlDatabase) Validate() error {
// Databasus runs an internal PostgreSQL instance that should not be backed up through the UI
// because it would expose internal metadata to non-system administrators.
// To properly backup Databasus, see: https://databasus.com/faq#backup-databasus
if p.Database != nil && *p.Database != "" {
if p.BackupType == PostgresBackupTypePgDump && p.Database != nil && *p.Database != "" {
localhostHosts := []string{
"localhost",
"127.0.0.1",
@@ -130,6 +151,10 @@ func (p *PostgresqlDatabase) TestConnection(
encryptor encryption.FieldEncryptor,
databaseID uuid.UUID,
) error {
if p.BackupType == PostgresBackupTypeWalV1 {
return errors.New("test connection is not supported for WAL backup type")
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
@@ -144,7 +169,21 @@ func (p *PostgresqlDatabase) HideSensitiveData() {
p.Password = ""
}
func (p *PostgresqlDatabase) ValidateUpdate(old *PostgresqlDatabase) error {
// BackupType cannot be changed after creation — the full backup structure
// (WAL hierarchy, storage files, cleanup logic) is built around
// the type chosen at creation time. Automatically migrating this state is
// error-prone; it is safer for the user to create a new database and
// remove the old one.
if old.BackupType != p.BackupType {
return errors.New("backup type cannot be changed; create a new database instead")
}
return nil
}
func (p *PostgresqlDatabase) Update(incoming *PostgresqlDatabase) {
p.BackupType = incoming.BackupType
p.Version = incoming.Version
p.Host = incoming.Host
p.Port = incoming.Port
@@ -181,6 +220,10 @@ func (p *PostgresqlDatabase) PopulateDbData(
encryptor encryption.FieldEncryptor,
databaseID uuid.UUID,
) error {
if p.BackupType == PostgresBackupTypeWalV1 {
return nil
}
return p.PopulateVersion(logger, encryptor, databaseID)
}
@@ -243,6 +286,10 @@ func (p *PostgresqlDatabase) IsUserReadOnly(
encryptor encryption.FieldEncryptor,
databaseID uuid.UUID,
) (bool, []string, error) {
if p.BackupType == PostgresBackupTypeWalV1 {
return false, nil, errors.New("read-only check is not supported for WAL backup type")
}
password, err := decryptPasswordIfNeeded(p.Password, encryptor, databaseID)
if err != nil {
return false, nil, fmt.Errorf("failed to decrypt password: %w", err)
@@ -415,6 +462,10 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
encryptor encryption.FieldEncryptor,
databaseID uuid.UUID,
) (string, string, error) {
if p.BackupType == PostgresBackupTypeWalV1 {
return "", "", errors.New("read-only user creation is not supported for WAL backup type")
}
password, err := decryptPasswordIfNeeded(p.Password, encryptor, databaseID)
if err != nil {
return "", "", fmt.Errorf("failed to decrypt password: %w", err)
@@ -1062,7 +1113,7 @@ func checkBackupPermissions(
}
// buildConnectionStringForDB builds connection string for specific database
func buildConnectionStringForDB(p *PostgresqlDatabase, dbName string, password string) string {
func buildConnectionStringForDB(p *PostgresqlDatabase, dbName, password string) string {
sslMode := "disable"
if p.IsHttps {
sslMode = "require"
@@ -1102,8 +1153,8 @@ func isSupabaseConnection(host, username string) bool {
}
func extractSupabaseProjectID(username string) string {
if idx := strings.Index(username, "."); idx != -1 {
return username[idx+1:]
if _, after, found := strings.Cut(username, "."); found {
return after
}
return ""
}

View File

@@ -9,3 +9,7 @@ type IsReadOnlyResponse struct {
IsReadOnly bool `json:"isReadOnly"`
Privileges []string `json:"privileges"`
}
type VerifyAgentTokenRequest struct {
Token string `json:"token" binding:"required"`
}

View File

@@ -1,10 +1,11 @@
package databases
import (
"databasus-backend/internal/util/encryption"
"log/slog"
"github.com/google/uuid"
"databasus-backend/internal/util/encryption"
)
type DatabaseValidator interface {

View File

@@ -2,17 +2,18 @@ package databases
import (
"context"
"errors"
"log/slog"
"time"
"github.com/google/uuid"
"databasus-backend/internal/features/databases/databases/mariadb"
"databasus-backend/internal/features/databases/databases/mongodb"
"databasus-backend/internal/features/databases/databases/mysql"
"databasus-backend/internal/features/databases/databases/postgresql"
"databasus-backend/internal/features/notifiers"
"databasus-backend/internal/util/encryption"
"errors"
"log/slog"
"time"
"github.com/google/uuid"
)
type Database struct {
@@ -37,6 +38,9 @@ type Database struct {
LastBackupErrorMessage *string `json:"lastBackupErrorMessage,omitempty" gorm:"column:last_backup_error_message;type:text"`
HealthStatus *HealthStatus `json:"healthStatus" gorm:"column:health_status;type:text;not null"`
AgentToken *string `json:"-" gorm:"column:agent_token;type:text"`
IsAgentTokenGenerated bool `json:"isAgentTokenGenerated" gorm:"column:is_agent_token_generated;not null;default:false"`
}
func (d *Database) Validate() error {
@@ -71,8 +75,19 @@ func (d *Database) Validate() error {
}
func (d *Database) ValidateUpdate(old, new Database) error {
// Database type cannot be changed after creation — the entire backup
// structure (storage files, schedulers, WAL hierarchy, etc.) is tied to
// the type at creation time. Recreating that state automatically is
// error-prone; it is safer for the user to create a new database and
// remove the old one.
if old.Type != new.Type {
return errors.New("database type is not allowed to change")
return errors.New("database type cannot be changed; create a new database instead")
}
if old.Type == DatabaseTypePostgres && old.Postgresql != nil && new.Postgresql != nil {
if err := new.Postgresql.ValidateUpdate(old.Postgresql); err != nil {
return err
}
}
return nil

View File

@@ -1,15 +1,16 @@
package databases
import (
"errors"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/features/databases/databases/mariadb"
"databasus-backend/internal/features/databases/databases/mongodb"
"databasus-backend/internal/features/databases/databases/mysql"
"databasus-backend/internal/features/databases/databases/postgresql"
"databasus-backend/internal/storage"
"errors"
"github.com/google/uuid"
"gorm.io/gorm"
)
type DatabaseRepository struct{}
@@ -120,7 +121,6 @@ func (r *DatabaseRepository) Save(database *Database) (*Database, error) {
return nil
})
if err != nil {
return nil, err
}
@@ -244,6 +244,18 @@ func (r *DatabaseRepository) GetAllDatabases() ([]*Database, error) {
return databases, nil
}
func (r *DatabaseRepository) FindByAgentTokenHash(hash string) (*Database, error) {
var database Database
if err := storage.GetDb().
Where("agent_token = ?", hash).
First(&database).Error; err != nil {
return nil, err
}
return &database, nil
}
func (r *DatabaseRepository) GetDatabasesIDsByNotifierID(
notifierID uuid.UUID,
) ([]uuid.UUID, error) {

View File

@@ -2,11 +2,15 @@ package databases
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"log/slog"
"strings"
"time"
"github.com/google/uuid"
"databasus-backend/internal/config"
audit_logs "databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/databases/databases/mariadb"
@@ -17,8 +21,6 @@ import (
users_models "databasus-backend/internal/features/users/models"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"databasus-backend/internal/util/encryption"
"github.com/google/uuid"
)
type DatabaseService struct {
@@ -87,21 +89,8 @@ func (s *DatabaseService) CreateDatabase(
return nil, fmt.Errorf("failed to auto-detect database data: %w", err)
}
if config.GetEnv().IsCloud {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
isReadOnly, permissions, err := database.IsUserReadOnly(ctx, s.logger, s.fieldEncryptor)
if err != nil {
return nil, fmt.Errorf("failed to verify user permissions: %w", err)
}
if !isReadOnly {
return nil, fmt.Errorf(
"in cloud mode, only read-only database users are allowed (user has permissions: %v)",
permissions,
)
}
if err := s.verifyReadOnlyUserIfNeeded(database); err != nil {
return nil, err
}
if err := database.EncryptSensitiveFields(s.fieldEncryptor); err != nil {
@@ -171,25 +160,8 @@ func (s *DatabaseService) UpdateDatabase(
return fmt.Errorf("failed to auto-detect database data: %w", err)
}
if config.GetEnv().IsCloud {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
isReadOnly, permissions, err := existingDatabase.IsUserReadOnly(
ctx,
s.logger,
s.fieldEncryptor,
)
if err != nil {
return fmt.Errorf("failed to verify user permissions: %w", err)
}
if !isReadOnly {
return fmt.Errorf(
"in cloud mode, only read-only database users are allowed (user has permissions: %v)",
permissions,
)
}
if err := s.verifyReadOnlyUserIfNeeded(existingDatabase); err != nil {
return err
}
oldName := existingDatabase.Name
@@ -485,6 +457,7 @@ func (s *DatabaseService) CopyDatabase(
newDatabase.Postgresql = &postgresql.PostgresqlDatabase{
ID: uuid.Nil,
DatabaseID: nil,
BackupType: existingDatabase.Postgresql.BackupType,
Version: existingDatabase.Postgresql.Version,
Host: existingDatabase.Postgresql.Host,
Port: existingDatabase.Postgresql.Port,
@@ -638,6 +611,71 @@ func (s *DatabaseService) SetHealthStatus(
return nil
}
func (s *DatabaseService) RegenerateAgentToken(
user *users_models.User,
databaseID uuid.UUID,
) (string, error) {
database, err := s.dbRepository.FindByID(databaseID)
if err != nil {
return "", err
}
if database.WorkspaceID == nil {
return "", errors.New("cannot regenerate token for database without workspace")
}
canManage, err := s.workspaceService.CanUserManageDBs(*database.WorkspaceID, user)
if err != nil {
return "", err
}
if !canManage {
return "", errors.New(
"insufficient permissions to regenerate agent token for this database",
)
}
plainToken := strings.ReplaceAll(uuid.New().String(), "-", "")
tokenHash := hashAgentToken(plainToken)
database.AgentToken = &tokenHash
database.IsAgentTokenGenerated = true
_, err = s.dbRepository.Save(database)
if err != nil {
return "", err
}
s.auditLogService.WriteAuditLog(
fmt.Sprintf("Agent token regenerated for database: %s", database.Name),
&user.ID,
database.WorkspaceID,
)
return plainToken, nil
}
func (s *DatabaseService) VerifyAgentToken(token string) error {
hash := hashAgentToken(token)
_, err := s.dbRepository.FindByAgentTokenHash(hash)
if err != nil {
return errors.New("invalid token")
}
return nil
}
func (s *DatabaseService) GetDatabaseByAgentToken(token string) (*Database, error) {
hash := hashAgentToken(token)
partial, err := s.dbRepository.FindByAgentTokenHash(hash)
if err != nil {
return nil, errors.New("invalid agent token")
}
return s.dbRepository.FindByID(partial.ID)
}
func (s *DatabaseService) OnBeforeWorkspaceDeletion(workspaceID uuid.UUID) error {
databases, err := s.dbRepository.FindByWorkspaceID(workspaceID)
if err != nil {
@@ -809,3 +847,36 @@ func (s *DatabaseService) CreateReadOnlyUser(
return username, password, nil
}
func (s *DatabaseService) verifyReadOnlyUserIfNeeded(database *Database) error {
if !config.GetEnv().IsCloud {
return nil
}
if database.Postgresql != nil &&
database.Postgresql.BackupType == postgresql.PostgresBackupTypeWalV1 {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
isReadOnly, permissions, err := database.IsUserReadOnly(ctx, s.logger, s.fieldEncryptor)
if err != nil {
return fmt.Errorf("failed to verify user permissions: %w", err)
}
if !isReadOnly {
return fmt.Errorf(
"in cloud mode, only read-only database users are allowed (user has permissions: %v)",
permissions,
)
}
return nil
}
func hashAgentToken(token string) string {
hash := sha256.Sum256([]byte(token))
return fmt.Sprintf("%x", hash)
}

View File

@@ -4,6 +4,8 @@ import (
"fmt"
"strconv"
"github.com/google/uuid"
"databasus-backend/internal/config"
"databasus-backend/internal/features/databases/databases/mariadb"
"databasus-backend/internal/features/databases/databases/mongodb"
@@ -12,8 +14,6 @@ import (
"databasus-backend/internal/features/storages"
"databasus-backend/internal/storage"
"databasus-backend/internal/util/tools"
"github.com/google/uuid"
)
func GetTestPostgresConfig() *postgresql.PostgresqlDatabase {

View File

@@ -1,12 +1,13 @@
package disk
import (
"databasus-backend/internal/config"
"fmt"
"path/filepath"
"runtime"
"github.com/shirou/gopsutil/v4/disk"
"databasus-backend/internal/config"
)
type DiskService struct{}

View File

@@ -5,8 +5,10 @@ import (
"databasus-backend/internal/util/logger"
)
var env = config.GetEnv()
var log = logger.GetLogger()
var (
env = config.GetEnv()
log = logger.GetLogger()
)
var emailSMTPSender = &EmailSMTPSender{
log,
@@ -14,6 +16,7 @@ var emailSMTPSender = &EmailSMTPSender{
env.SMTPPort,
env.SMTPUser,
env.SMTPPassword,
env.SMTPFrom,
env.SMTPHost != "" && env.SMTPPort != 0,
}

View File

@@ -1,6 +1,7 @@
package email
import (
"context"
"crypto/tls"
"fmt"
"log/slog"
@@ -24,6 +25,7 @@ type EmailSMTPSender struct {
smtpPort int
smtpUser string
smtpPassword string
smtpFrom string
isConfigured bool
}
@@ -33,7 +35,10 @@ func (s *EmailSMTPSender) SendEmail(to, subject, body string) error {
return nil
}
from := s.smtpUser
from := s.smtpFrom
if from == "" {
from = s.smtpUser
}
if from == "" {
from = "noreply@" + s.smtpHost
}
@@ -110,7 +115,7 @@ func (s *EmailSMTPSender) createImplicitTLSClient() (*smtp.Client, func(), error
tlsConfig := &tls.Config{ServerName: s.smtpHost}
dialer := &net.Dialer{Timeout: DefaultTimeout}
conn, err := tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
conn, err := (&tls.Dialer{NetDialer: dialer, Config: tlsConfig}).DialContext(context.Background(), "tcp", addr)
if err != nil {
return nil, nil, fmt.Errorf("failed to connect to SMTP server: %w", err)
}

View File

@@ -5,12 +5,12 @@ import (
"fmt"
"os"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/config"
user_models "databasus-backend/internal/features/users/models"
"databasus-backend/internal/storage"
"github.com/google/uuid"
"gorm.io/gorm"
)
type SecretKeyService struct {
@@ -33,7 +33,7 @@ func (s *SecretKeyService) MigrateKeyFromDbToFileIfExist() error {
}
secretKeyPath := config.GetEnv().SecretKeyPath
if err := os.WriteFile(secretKeyPath, []byte(secretKey.Secret), 0600); err != nil {
if err := os.WriteFile(secretKeyPath, []byte(secretKey.Secret), 0o600); err != nil {
return fmt.Errorf("failed to write secret key to file: %w", err)
}
@@ -54,7 +54,7 @@ func (s *SecretKeyService) GetSecretKey() (string, error) {
if err != nil {
if os.IsNotExist(err) {
newKey := s.generateNewSecretKey()
if err := os.WriteFile(secretKeyPath, []byte(newKey), 0600); err != nil {
if err := os.WriteFile(secretKeyPath, []byte(newKey), 0o600); err != nil {
return "", fmt.Errorf("failed to write new secret key: %w", err)
}
s.cachedKey = &newKey

View File

@@ -1,9 +1,6 @@
package healthcheck_attempt
import (
"databasus-backend/internal/features/databases"
healthcheck_config "databasus-backend/internal/features/healthcheck/config"
"databasus-backend/internal/util/logger"
"errors"
"fmt"
"log/slog"
@@ -11,6 +8,10 @@ import (
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/features/databases"
healthcheck_config "databasus-backend/internal/features/healthcheck/config"
"databasus-backend/internal/util/logger"
)
type CheckDatabaseHealthUseCase struct {
@@ -251,5 +252,4 @@ func (uc *CheckDatabaseHealthUseCase) sendDbStatusNotification(
messageBody,
)
}
}

View File

@@ -6,6 +6,9 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"databasus-backend/internal/features/databases"
healthcheck_config "databasus-backend/internal/features/healthcheck/config"
"databasus-backend/internal/features/notifiers"
@@ -13,9 +16,6 @@ import (
users_enums "databasus-backend/internal/features/users/enums"
users_testing "databasus-backend/internal/features/users/testing"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
func Test_CheckDatabaseHealthUseCase(t *testing.T) {

View File

@@ -1,12 +1,13 @@
package healthcheck_attempt
import (
users_middleware "databasus-backend/internal/features/users/middleware"
"net/http"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
users_middleware "databasus-backend/internal/features/users/middleware"
)
type HealthcheckAttemptController struct {

View File

@@ -11,12 +11,14 @@ import (
"databasus-backend/internal/util/logger"
)
var healthcheckAttemptRepository = &HealthcheckAttemptRepository{}
var healthcheckAttemptService = &HealthcheckAttemptService{
healthcheckAttemptRepository,
databases.GetDatabaseService(),
workspaces_services.GetWorkspaceService(),
}
var (
healthcheckAttemptRepository = &HealthcheckAttemptRepository{}
healthcheckAttemptService = &HealthcheckAttemptService{
healthcheckAttemptRepository,
databases.GetDatabaseService(),
workspaces_services.GetWorkspaceService(),
}
)
var checkDatabaseHealthUseCase = &CheckDatabaseHealthUseCase{
healthcheckAttemptRepository,
@@ -31,6 +33,7 @@ var healthcheckAttemptBackgroundService = &HealthcheckAttemptBackgroundService{
runOnce: sync.Once{},
hasRun: atomic.Bool{},
}
var healthcheckAttemptController = &HealthcheckAttemptController{
healthcheckAttemptService,
}

Some files were not shown because too many files have changed in this diff Show More