mirror of
https://github.com/databasus/databasus.git
synced 2026-04-06 00:32:03 +02:00
Compare commits
63 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a0f02b253e | ||
|
|
812f11bc2f | ||
|
|
e796e3ddf0 | ||
|
|
c96d3db337 | ||
|
|
ed6c3a2034 | ||
|
|
05115047c3 | ||
|
|
446b96c6c0 | ||
|
|
36a0448da1 | ||
|
|
8e392cfeab | ||
|
|
6683db1e52 | ||
|
|
703b883936 | ||
|
|
e818bcff82 | ||
|
|
b2f98f1332 | ||
|
|
230cc27ea6 | ||
|
|
cd197ff94b | ||
|
|
91f35a3e17 | ||
|
|
30c2e2d156 | ||
|
|
ef7c5b45e6 | ||
|
|
920c98e229 | ||
|
|
2a19a96aae | ||
|
|
75aa2108d9 | ||
|
|
0a0040839e | ||
|
|
ff4f795ece | ||
|
|
dc05502580 | ||
|
|
1ca38f5583 | ||
|
|
40b3ff61c7 | ||
|
|
e1b245a573 | ||
|
|
fdf29b71f2 | ||
|
|
49da981c21 | ||
|
|
9d611d3559 | ||
|
|
22cab53dab | ||
|
|
d761c4156c | ||
|
|
cbb8b82711 | ||
|
|
8e3d1e5bff | ||
|
|
349e7f0ee8 | ||
|
|
3a274e135b | ||
|
|
61e937bc2a | ||
|
|
f67919fe1a | ||
|
|
91ee5966d8 | ||
|
|
d77d7d69a3 | ||
|
|
fc88b730d5 | ||
|
|
1f1d80245f | ||
|
|
16a29cf458 | ||
|
|
43e04500ac | ||
|
|
cee3022f85 | ||
|
|
f46d92c480 | ||
|
|
10677238d7 | ||
|
|
2553203fcf | ||
|
|
7b05bd8000 | ||
|
|
8d45728f73 | ||
|
|
c70ad82c95 | ||
|
|
e4bc34d319 | ||
|
|
257ae85da7 | ||
|
|
b42c820bb2 | ||
|
|
da5c13fb11 | ||
|
|
35180360e5 | ||
|
|
e4f6cd7a5d | ||
|
|
d7b8e6d56a | ||
|
|
6016f23fb2 | ||
|
|
e7c4ee8f6f | ||
|
|
a75702a01b | ||
|
|
81a21eb907 | ||
|
|
33d6bf0147 |
44
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
44
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
@@ -0,0 +1,44 @@
|
||||
---
|
||||
name: Bug Report
|
||||
about: Report a bug or unexpected behavior in Databasus
|
||||
labels: bug
|
||||
---
|
||||
|
||||
## Databasus version (screenshot)
|
||||
|
||||
It is displayed in the bottom left corner of the Databasus UI. Please attach screenshot, not just version text
|
||||
|
||||
<!-- e.g. 1.4.2 -->
|
||||
|
||||
## Operating system and architecture
|
||||
|
||||
<!-- e.g. Ubuntu 22.04 x64, macOS 14 ARM, Windows 11 x64 -->
|
||||
|
||||
## Database type and version (optional, for DB-related bugs)
|
||||
|
||||
<!-- e.g. PostgreSQL 16 in Docker, MySQL 8.0 installed on server, MariaDB 11.4 in AWS Cloud -->
|
||||
|
||||
## Describe the bug (please write manually, do not ask AI to summarize)
|
||||
|
||||
**What happened:**
|
||||
|
||||
**What I expected:**
|
||||
|
||||
## Steps to reproduce
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
## Have you asked AI how to solve the issue?
|
||||
|
||||
<!-- Using AI to diagnose issues before filing a bug report helps narrow down root causes. -->
|
||||
|
||||
- [ ] Claude Sonnet 4.6 or newer
|
||||
- [ ] ChatGPT 5.2 or newer
|
||||
- [ ] No
|
||||
|
||||
|
||||
## Additional context / logs
|
||||
|
||||
<!-- Screenshots, error messages, relevant log output, etc. -->
|
||||
2
.github/workflows/ci-release.yml
vendored
2
.github/workflows/ci-release.yml
vendored
@@ -407,7 +407,7 @@ jobs:
|
||||
- name: Run database migrations
|
||||
run: |
|
||||
cd backend
|
||||
go install github.com/pressly/goose/v3/cmd/goose@latest
|
||||
go install github.com/pressly/goose/v3/cmd/goose@v3.24.3
|
||||
goose up
|
||||
|
||||
- name: Run Go tests
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -12,3 +12,4 @@ node_modules/
|
||||
.DS_Store
|
||||
/scripts
|
||||
.vscode/settings.json
|
||||
.claude
|
||||
256
AGENTS.md
256
AGENTS.md
@@ -1,35 +1,37 @@
|
||||
# Agent Rules and Guidelines
|
||||
|
||||
This document contains all coding standards, conventions and best practices recommended for the Databasus project.
|
||||
This document contains all coding standards, conventions and best practices recommended for the TgTaps project.
|
||||
This is NOT a strict set of rules, but a set of recommendations to help you write better code.
|
||||
|
||||
---
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Engineering Philosophy](#engineering-philosophy)
|
||||
- [Backend Guidelines](#backend-guidelines)
|
||||
- [Code Style](#code-style)
|
||||
- [Engineering philosophy](#engineering-philosophy)
|
||||
- [Backend guidelines](#backend-guidelines)
|
||||
- [Code style](#code-style)
|
||||
- [Boolean naming](#boolean-naming)
|
||||
- [Add reasonable new lines between logical statements](#add-reasonable-new-lines-between-logical-statements)
|
||||
- [Comments](#comments)
|
||||
- [Controllers](#controllers)
|
||||
- [Dependency Injection (DI)](#dependency-injection-di)
|
||||
- [Dependency injection (DI)](#dependency-injection-di)
|
||||
- [Migrations](#migrations)
|
||||
- [Refactoring](#refactoring)
|
||||
- [Testing](#testing)
|
||||
- [Time Handling](#time-handling)
|
||||
- [CRUD Examples](#crud-examples)
|
||||
- [Frontend Guidelines](#frontend-guidelines)
|
||||
- [React Component Structure](#react-component-structure)
|
||||
- [Time handling](#time-handling)
|
||||
- [CRUD examples](#crud-examples)
|
||||
- [Frontend guidelines](#frontend-guidelines)
|
||||
- [React component structure](#react-component-structure)
|
||||
|
||||
---
|
||||
|
||||
## Engineering Philosophy
|
||||
## Engineering philosophy
|
||||
|
||||
**Think like a skeptical senior engineer and code reviewer. Don't just do what was asked—also think about what should have been asked.**
|
||||
|
||||
⚠️ **Balance vigilance with pragmatism:** Catch real issues, not theoretical ones. Don't let perfect be the enemy of good.
|
||||
|
||||
### Task Context Assessment:
|
||||
### Task context assessment:
|
||||
|
||||
**First, assess the task scope:**
|
||||
|
||||
@@ -38,7 +40,7 @@ This is NOT a strict set of rules, but a set of recommendations to help you writ
|
||||
- **Complex** (architecture, security, performance-critical): Full analysis required
|
||||
- **Unclear** (ambiguous requirements): Always clarify assumptions first
|
||||
|
||||
### For Non-Trivial Tasks:
|
||||
### For non-trivial tasks:
|
||||
|
||||
1. **Restate the objective and list assumptions** (explicit + implicit)
|
||||
- If any assumption is shaky, call it out clearly
|
||||
@@ -71,7 +73,7 @@ This is NOT a strict set of rules, but a set of recommendations to help you writ
|
||||
- Patch the answer accordingly
|
||||
- Verify edge cases are handled
|
||||
|
||||
### Application Guidelines:
|
||||
### Application guidelines:
|
||||
|
||||
**Scale your response to the task:**
|
||||
|
||||
@@ -84,9 +86,9 @@ This is NOT a strict set of rules, but a set of recommendations to help you writ
|
||||
|
||||
---
|
||||
|
||||
## Backend Guidelines
|
||||
## Backend guidelines
|
||||
|
||||
### Code Style
|
||||
### Code style
|
||||
|
||||
**Always place private methods to the bottom of file**
|
||||
|
||||
@@ -94,7 +96,7 @@ This rule applies to ALL Go files including tests, services, controllers, reposi
|
||||
|
||||
In Go, exported (public) functions/methods start with uppercase letters, while unexported (private) ones start with lowercase letters.
|
||||
|
||||
#### Structure Order:
|
||||
#### Structure order:
|
||||
|
||||
1. Type definitions and constants
|
||||
2. Public methods/functions (uppercase)
|
||||
@@ -227,7 +229,7 @@ func (c *ProjectController) extractProjectID(ctx *gin.Context) uuid.UUID {
|
||||
}
|
||||
```
|
||||
|
||||
#### Key Points:
|
||||
#### Key points:
|
||||
|
||||
- **Exported/Public** = starts with uppercase letter (CreateUser, GetProject)
|
||||
- **Unexported/Private** = starts with lowercase letter (validateUser, handleError)
|
||||
@@ -237,13 +239,13 @@ func (c *ProjectController) extractProjectID(ctx *gin.Context) uuid.UUID {
|
||||
|
||||
---
|
||||
|
||||
### Boolean Naming
|
||||
### Boolean naming
|
||||
|
||||
**Always prefix boolean variables with verbs like `is`, `has`, `was`, `should`, `can`, etc.**
|
||||
|
||||
This makes the code more readable and clearly indicates that the variable represents a true/false state.
|
||||
|
||||
#### Good Examples:
|
||||
#### Good examples:
|
||||
|
||||
```go
|
||||
type User struct {
|
||||
@@ -265,7 +267,7 @@ wasCompleted := false
|
||||
hasPermission := checkPermissions()
|
||||
```
|
||||
|
||||
#### Bad Examples:
|
||||
#### Bad examples:
|
||||
|
||||
```go
|
||||
type User struct {
|
||||
@@ -286,7 +288,7 @@ completed := false // Should be: wasCompleted
|
||||
permission := true // Should be: hasPermission
|
||||
```
|
||||
|
||||
#### Common Boolean Prefixes:
|
||||
#### Common boolean prefixes:
|
||||
|
||||
- **is** - current state (IsActive, IsValid, IsEnabled)
|
||||
- **has** - possession or presence (HasAccess, HasPermission, HasError)
|
||||
@@ -297,6 +299,167 @@ permission := true // Should be: hasPermission
|
||||
|
||||
---
|
||||
|
||||
### Add reasonable new lines between logical statements
|
||||
|
||||
**Add blank lines between logical blocks to improve code readability.**
|
||||
|
||||
Separate different logical operations within a function with blank lines. This makes the code flow clearer and helps identify distinct steps in the logic.
|
||||
|
||||
#### Guidelines:
|
||||
|
||||
- Add blank line before final `return` statement
|
||||
- Add blank line after variable declarations before using them
|
||||
- Add blank line between error handling and subsequent logic
|
||||
- Add blank line between different logical operations
|
||||
|
||||
#### Bad example (without spacing):
|
||||
|
||||
```go
|
||||
func (t *Task) BeforeSave(tx *gorm.DB) error {
|
||||
if len(t.Messages) > 0 {
|
||||
messagesBytes, err := json.Marshal(t.Messages)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t.MessagesJSON = string(messagesBytes)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Task) AfterFind(tx *gorm.DB) error {
|
||||
if t.MessagesJSON != "" {
|
||||
var messages []onewin_dto.TaskCompletionMessage
|
||||
if err := json.Unmarshal([]byte(t.MessagesJSON), &messages); err != nil {
|
||||
return err
|
||||
}
|
||||
t.Messages = messages
|
||||
}
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
#### Good example (with proper spacing):
|
||||
|
||||
```go
|
||||
func (t *Task) BeforeSave(tx *gorm.DB) error {
|
||||
if len(t.Messages) > 0 {
|
||||
messagesBytes, err := json.Marshal(t.Messages)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
t.MessagesJSON = string(messagesBytes)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Task) AfterFind(tx *gorm.DB) error {
|
||||
if t.MessagesJSON != "" {
|
||||
var messages []onewin_dto.TaskCompletionMessage
|
||||
if err := json.Unmarshal([]byte(t.MessagesJSON), &messages); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
t.Messages = messages
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
#### More examples:
|
||||
|
||||
**Service method with multiple operations:**
|
||||
|
||||
```go
|
||||
func (s *UserService) CreateUser(request *CreateUserRequest) (*User, error) {
|
||||
// Validate input
|
||||
if err := s.validateUserRequest(request); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create user entity
|
||||
user := &User{
|
||||
ID: uuid.New(),
|
||||
Name: request.Name,
|
||||
Email: request.Email,
|
||||
}
|
||||
|
||||
// Save to database
|
||||
if err := s.repository.Create(user); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Send notification
|
||||
s.notificationService.SendWelcomeEmail(user.Email)
|
||||
|
||||
return user, nil
|
||||
}
|
||||
```
|
||||
|
||||
**Repository method with query building:**
|
||||
|
||||
```go
|
||||
func (r *Repository) GetFiltered(filters *Filters) ([]*Entity, error) {
|
||||
query := storage.GetDb().Model(&Entity{})
|
||||
|
||||
if filters.Status != "" {
|
||||
query = query.Where("status = ?", filters.Status)
|
||||
}
|
||||
|
||||
if filters.CreatedAfter != nil {
|
||||
query = query.Where("created_at > ?", filters.CreatedAfter)
|
||||
}
|
||||
|
||||
var entities []*Entity
|
||||
if err := query.Find(&entities).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return entities, nil
|
||||
}
|
||||
```
|
||||
|
||||
**Repository method with error handling:**
|
||||
|
||||
Bad (without spacing):
|
||||
|
||||
```go
|
||||
func (r *Repository) FindById(id uuid.UUID) (*models.Task, error) {
|
||||
var task models.Task
|
||||
result := storage.GetDb().Where("id = ?", id).First(&task)
|
||||
if result.Error != nil {
|
||||
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
|
||||
return nil, errors.New("task not found")
|
||||
}
|
||||
return nil, result.Error
|
||||
}
|
||||
return &task, nil
|
||||
}
|
||||
```
|
||||
|
||||
Good (with proper spacing):
|
||||
|
||||
```go
|
||||
func (r *Repository) FindById(id uuid.UUID) (*models.Task, error) {
|
||||
var task models.Task
|
||||
|
||||
result := storage.GetDb().Where("id = ?", id).First(&task)
|
||||
if result.Error != nil {
|
||||
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
|
||||
return nil, errors.New("task not found")
|
||||
}
|
||||
|
||||
return nil, result.Error
|
||||
}
|
||||
|
||||
return &task, nil
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Comments
|
||||
|
||||
#### Guidelines
|
||||
@@ -305,13 +468,14 @@ permission := true // Should be: hasPermission
|
||||
2. **Functions and variables should have meaningful names** - Code should be self-documenting
|
||||
3. **Comments for unclear code only** - Only add comments when code logic isn't immediately clear
|
||||
|
||||
#### Key Principles:
|
||||
#### Key principles:
|
||||
|
||||
- **Code should tell a story** - Use descriptive variable and function names
|
||||
- **Comments explain WHY, not WHAT** - The code shows what happens, comments explain business logic or complex decisions
|
||||
- **Prefer refactoring over commenting** - If code needs explaining, consider making it clearer instead
|
||||
- **API documentation is required** - Swagger comments for all HTTP endpoints are mandatory
|
||||
- **Complex algorithms deserve comments** - Mathematical formulas, business rules, or non-obvious optimizations
|
||||
- **Do not write summary sections in .md files unless directly requested** - Avoid adding "Summary" or "Conclusion" sections at the end of documentation files unless the user explicitly asks for them
|
||||
|
||||
#### Example of useless comments:
|
||||
|
||||
@@ -343,7 +507,7 @@ func CreateValidLogItems(count int, uniqueID string) []logs_receiving.LogItemReq
|
||||
|
||||
### Controllers
|
||||
|
||||
#### Controller Guidelines:
|
||||
#### Controller guidelines:
|
||||
|
||||
1. **When we write controller:**
|
||||
- We combine all routes to single controller
|
||||
@@ -475,7 +639,7 @@ func (c *AuditLogController) GetUserAuditLogs(ctx *gin.Context) {
|
||||
|
||||
---
|
||||
|
||||
### Dependency Injection (DI)
|
||||
### Dependency injection (DI)
|
||||
|
||||
For DI files use **implicit fields declaration styles** (especially for controllers, services, repositories, use cases, etc., not simple data structures).
|
||||
|
||||
@@ -503,7 +667,7 @@ var orderController = &OrderController{
|
||||
|
||||
**This is needed to avoid forgetting to update DI style when we add new dependency.**
|
||||
|
||||
#### Force Such Usage
|
||||
#### Force such usage
|
||||
|
||||
Please force such usage if file look like this (see some services\controllers\repos definitions and getters):
|
||||
|
||||
@@ -549,13 +713,13 @@ func GetOrderRepository() *repositories.OrderRepository {
|
||||
}
|
||||
```
|
||||
|
||||
#### SetupDependencies() Pattern
|
||||
#### SetupDependencies() pattern
|
||||
|
||||
**All `SetupDependencies()` functions must use sync.Once to ensure idempotent execution.**
|
||||
|
||||
This pattern allows `SetupDependencies()` to be safely called multiple times (especially in tests) while ensuring the actual setup logic executes only once.
|
||||
|
||||
**Implementation Pattern:**
|
||||
**Implementation pattern:**
|
||||
|
||||
```go
|
||||
package feature
|
||||
@@ -588,7 +752,7 @@ func SetupDependencies() {
|
||||
}
|
||||
```
|
||||
|
||||
**Why This Pattern:**
|
||||
**Why this pattern:**
|
||||
|
||||
- **Tests can call multiple times**: Test setup often calls `SetupDependencies()` multiple times without issues
|
||||
- **Thread-safe**: Works correctly with concurrent calls (nanoseconds or seconds apart)
|
||||
@@ -604,13 +768,13 @@ func SetupDependencies() {
|
||||
|
||||
---
|
||||
|
||||
### Background Services
|
||||
### Background services
|
||||
|
||||
**All background service `Run()` methods must panic if called multiple times to prevent corrupted states.**
|
||||
|
||||
Background services run infinite loops and must never be started twice on the same instance. Multiple calls indicate a serious bug that would cause duplicate goroutines, resource leaks, and data corruption.
|
||||
|
||||
**Implementation Pattern:**
|
||||
**Implementation pattern:**
|
||||
|
||||
```go
|
||||
package feature
|
||||
@@ -654,14 +818,14 @@ func (s *BackgroundService) Run(ctx context.Context) {
|
||||
}
|
||||
```
|
||||
|
||||
**Why Panic Instead of Warning:**
|
||||
**Why panic instead of warning:**
|
||||
|
||||
- **Prevents corruption**: Multiple `Run()` calls would create duplicate goroutines consuming resources
|
||||
- **Fails fast**: Catches critical bugs immediately in tests and production
|
||||
- **Clear indication**: Panic clearly indicates a serious programming error
|
||||
- **Applies everywhere**: Same protection in tests and production
|
||||
|
||||
**When This Applies:**
|
||||
**When this applies:**
|
||||
|
||||
- All background services with infinite loops
|
||||
- Registry services (BackupNodesRegistry, RestoreNodesRegistry)
|
||||
@@ -727,14 +891,14 @@ You can shortify, make more readable, improve code quality, etc. Common logic ca
|
||||
|
||||
**After writing tests, always launch them and verify that they pass.**
|
||||
|
||||
#### Test Naming Format
|
||||
#### Test naming format
|
||||
|
||||
Use these naming patterns:
|
||||
|
||||
- `Test_WhatWeDo_WhatWeExpect`
|
||||
- `Test_WhatWeDo_WhichConditions_WhatWeExpect`
|
||||
|
||||
#### Examples from Real Codebase:
|
||||
#### Examples from real codebase:
|
||||
|
||||
- `Test_CreateApiKey_WhenUserIsProjectOwner_ApiKeyCreated`
|
||||
- `Test_UpdateProject_WhenUserIsProjectAdmin_ProjectUpdated`
|
||||
@@ -742,22 +906,22 @@ Use these naming patterns:
|
||||
- `Test_GetProjectAuditLogs_WithDifferentUserRoles_EnforcesPermissionsCorrectly`
|
||||
- `Test_ProjectLifecycleE2E_CompletesSuccessfully`
|
||||
|
||||
#### Testing Philosophy
|
||||
#### Testing philosophy
|
||||
|
||||
**Prefer Controllers Over Unit Tests:**
|
||||
**Prefer controllers over unit tests:**
|
||||
|
||||
- Test through HTTP endpoints via controllers whenever possible
|
||||
- Avoid testing repositories, services in isolation - test via API instead
|
||||
- Only use unit tests for complex model logic when no API exists
|
||||
- Name test files `controller_test.go` or `service_test.go`, not `integration_test.go`
|
||||
|
||||
**Extract Common Logic to Testing Utilities:**
|
||||
**Extract common logic to testing utilities:**
|
||||
|
||||
- Create `testing.go` or `testing/testing.go` files for shared test utilities
|
||||
- Extract router creation, user setup, models creation helpers (in API, not just structs creation)
|
||||
- Reuse common patterns across different test files
|
||||
|
||||
**Refactor Existing Tests:**
|
||||
**Refactor existing tests:**
|
||||
|
||||
- When working with existing tests, always look for opportunities to refactor and improve
|
||||
- Extract repetitive setup code to common utilities
|
||||
@@ -766,7 +930,7 @@ Use these naming patterns:
|
||||
- Consolidate similar test patterns across different test files
|
||||
- Make tests more readable and maintainable for other developers
|
||||
|
||||
**Clean Up Test Data:**
|
||||
**Clean up test data:**
|
||||
|
||||
- If the feature supports cleanup operations (DELETE endpoints, cleanup methods), use them in tests
|
||||
- Clean up resources after test execution to avoid test data pollution
|
||||
@@ -803,7 +967,7 @@ func Test_BackupLifecycle_CreateAndDelete(t *testing.T) {
|
||||
}
|
||||
```
|
||||
|
||||
#### Testing Utilities Structure
|
||||
#### Testing utilities structure
|
||||
|
||||
**Create `testing.go` or `testing/testing.go` files with common utilities:**
|
||||
|
||||
@@ -839,7 +1003,7 @@ func AddMemberToProject(project *projects_models.Project, member *users_dto.Sign
|
||||
}
|
||||
```
|
||||
|
||||
#### Controller Test Examples
|
||||
#### Controller test examples
|
||||
|
||||
**Permission-based testing:**
|
||||
|
||||
@@ -906,7 +1070,7 @@ func Test_ProjectLifecycleE2E_CompletesSuccessfully(t *testing.T) {
|
||||
|
||||
---
|
||||
|
||||
### Time Handling
|
||||
### Time handling
|
||||
|
||||
**Always use `time.Now().UTC()` instead of `time.Now()`**
|
||||
|
||||
@@ -914,7 +1078,7 @@ This ensures consistent timezone handling across the application.
|
||||
|
||||
---
|
||||
|
||||
### CRUD Examples
|
||||
### CRUD examples
|
||||
|
||||
This is an example of complete CRUD implementation structure:
|
||||
|
||||
@@ -1578,9 +1742,9 @@ func createTimedLog(db *gorm.DB, userID *uuid.UUID, message string, createdAt ti
|
||||
|
||||
---
|
||||
|
||||
## Frontend Guidelines
|
||||
## Frontend guidelines
|
||||
|
||||
### React Component Structure
|
||||
### React component structure
|
||||
|
||||
Write React components with the following structure:
|
||||
|
||||
@@ -1614,7 +1778,7 @@ export const ReactComponent = ({ someValue }: Props): JSX.Element => {
|
||||
}
|
||||
```
|
||||
|
||||
#### Structure Order:
|
||||
#### Structure order:
|
||||
|
||||
1. **Props interface** - Define component props
|
||||
2. **Helper functions** (outside component) - Pure utility functions
|
||||
|
||||
10
Dockerfile
10
Dockerfile
@@ -71,8 +71,10 @@ FROM debian:bookworm-slim
|
||||
|
||||
# Add version metadata to runtime image
|
||||
ARG APP_VERSION=dev
|
||||
ARG TARGETARCH
|
||||
LABEL org.opencontainers.image.version=$APP_VERSION
|
||||
ENV APP_VERSION=$APP_VERSION
|
||||
ENV CONTAINER_ARCH=$TARGETARCH
|
||||
|
||||
# Set production mode for Docker containers
|
||||
ENV ENV_MODE=production
|
||||
@@ -268,7 +270,9 @@ window.__RUNTIME_CONFIG__ = {
|
||||
IS_CLOUD: '\${IS_CLOUD:-false}',
|
||||
GITHUB_CLIENT_ID: '\${GITHUB_CLIENT_ID:-}',
|
||||
GOOGLE_CLIENT_ID: '\${GOOGLE_CLIENT_ID:-}',
|
||||
IS_EMAIL_CONFIGURED: '\$IS_EMAIL_CONFIGURED'
|
||||
IS_EMAIL_CONFIGURED: '\$IS_EMAIL_CONFIGURED',
|
||||
CLOUDFLARE_TURNSTILE_SITE_KEY: '\${CLOUDFLARE_TURNSTILE_SITE_KEY:-}',
|
||||
CONTAINER_ARCH: '\${CONTAINER_ARCH:-unknown}'
|
||||
};
|
||||
JSEOF
|
||||
|
||||
@@ -431,6 +435,8 @@ fi
|
||||
exec ./main
|
||||
EOF
|
||||
|
||||
LABEL org.opencontainers.image.source="https://github.com/databasus/databasus"
|
||||
|
||||
RUN chmod +x /app/start.sh
|
||||
|
||||
EXPOSE 4005
|
||||
@@ -439,4 +445,4 @@ EXPOSE 4005
|
||||
VOLUME ["/databasus-data"]
|
||||
|
||||
ENTRYPOINT ["/app/start.sh"]
|
||||
CMD []
|
||||
CMD []
|
||||
|
||||
64
README.md
64
README.md
@@ -2,7 +2,7 @@
|
||||
<img src="assets/logo.svg" alt="Databasus Logo" width="250"/>
|
||||
|
||||
<h3>Backup tool for PostgreSQL, MySQL and MongoDB</h3>
|
||||
<p>Databasus is a free, open source and self-hosted tool to backup databases (with focus on PostgreSQL). Make backups with different storages (S3, Google Drive, FTP, etc.) and notifications about progress (Slack, Discord, Telegram, etc.). Previously known as Postgresus (see migration guide).</p>
|
||||
<p>Databasus is a free, open source and self-hosted tool to backup databases (with focus on PostgreSQL). Make backups with different storages (S3, Google Drive, FTP, etc.) and notifications about progress (Slack, Discord, Telegram, etc.)</p>
|
||||
|
||||
<!-- Badges -->
|
||||
[](https://www.postgresql.org/)
|
||||
@@ -11,7 +11,7 @@
|
||||
[](https://www.mongodb.com/)
|
||||
<br />
|
||||
[](LICENSE)
|
||||
[](https://hub.docker.com/r/rostislavdugin/postgresus)
|
||||
[](https://hub.docker.com/r/databasus/databasus)
|
||||
[](https://github.com/databasus/databasus)
|
||||
[](https://github.com/databasus/databasus)
|
||||
[](https://github.com/databasus/databasus)
|
||||
@@ -31,8 +31,6 @@
|
||||
<img src="assets/dashboard-dark.svg" alt="Databasus Dark Dashboard" width="800" style="margin-bottom: 10px;"/>
|
||||
|
||||
<img src="assets/dashboard.svg" alt="Databasus Dashboard" width="800"/>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
@@ -43,7 +41,7 @@
|
||||
|
||||
- **PostgreSQL**: 12, 13, 14, 15, 16, 17 and 18
|
||||
- **MySQL**: 5.7, 8 and 9
|
||||
- **MariaDB**: 10 and 11
|
||||
- **MariaDB**: 10, 11 and 12
|
||||
- **MongoDB**: 4, 5, 6, 7 and 8
|
||||
|
||||
### 🔄 **Scheduled backups**
|
||||
@@ -52,6 +50,13 @@
|
||||
- **Precise timing**: run backups at specific times (e.g., 4 AM during low traffic)
|
||||
- **Smart compression**: 4-8x space savings with balanced compression (~20% overhead)
|
||||
|
||||
### 🗑️ **Retention policies**
|
||||
|
||||
- **Time period**: Keep backups for a fixed duration (e.g., 7 days, 3 months, 1 year)
|
||||
- **Count**: Keep a fixed number of the most recent backups (e.g., last 30)
|
||||
- **GFS (Grandfather-Father-Son)**: Layered retention — keep hourly, daily, weekly, monthly and yearly backups independently for fine-grained long-term history (enterprises requirement)
|
||||
- **Size limits**: Set per-backup and total storage size caps to control storage usage
|
||||
|
||||
### 🗄️ **Multiple storage destinations** <a href="https://databasus.com/storages">(view supported)</a>
|
||||
|
||||
- **Local storage**: Keep backups on your VPS/server
|
||||
@@ -71,6 +76,8 @@
|
||||
- **Encryption for secrets**: Any sensitive data is encrypted and never exposed, even in logs or error messages
|
||||
- **Read-only user**: Databasus uses a read-only user by default for backups and never stores anything that can modify your data
|
||||
|
||||
It is also important for Databasus that you are able to decrypt and restore backups from storages (local, S3, etc.) without Databasus itself. To do so, read our guide on [how to recover directly from storage](https://databasus.com/how-to-recover-without-databasus). We avoid "vendor lock-in" even to open source tool!
|
||||
|
||||
### 👥 **Suitable for teams** <a href="https://databasus.com/access-management">(docs)</a>
|
||||
|
||||
- **Workspaces**: Group databases, notifiers and storages for different projects or teams
|
||||
@@ -220,8 +227,9 @@ For more options (NodePort, TLS, HTTPRoute for Gateway API), see the [Helm chart
|
||||
3. **Configure schedule**: Choose from hourly, daily, weekly, monthly or cron intervals
|
||||
4. **Set database connection**: Enter your database credentials and connection details
|
||||
5. **Choose storage**: Select where to store your backups (local, S3, Google Drive, etc.)
|
||||
6. **Add notifications** (optional): Configure email, Telegram, Slack, or webhook notifications
|
||||
7. **Save and start**: Databasus will validate settings and begin the backup schedule
|
||||
6. **Configure retention policy**: Choose time period, count or GFS to control how long backups are kept
|
||||
7. **Add notifications** (optional): Configure email, Telegram, Slack, or webhook notifications
|
||||
8. **Save and start**: Databasus will validate settings and begin the backup schedule
|
||||
|
||||
### 🔑 Resetting password <a href="https://databasus.com/password">(docs)</a>
|
||||
|
||||
@@ -233,56 +241,22 @@ docker exec -it databasus ./main --new-password="YourNewSecurePassword123" --ema
|
||||
|
||||
Replace `admin` with the actual email address of the user whose password you want to reset.
|
||||
|
||||
### 💾 Backuping Databasus itself
|
||||
|
||||
After installation, it is also recommended to <a href="https://databasus.com/faq/#backup-databasus">backup your Databasus itself</a> or, at least, to copy secret key used for encryption (30 seconds is needed). So you are able to restore from your encrypted backups if you lose access to the server with Databasus or it is corrupted.
|
||||
|
||||
---
|
||||
|
||||
## 📝 License
|
||||
|
||||
This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENSE) file for details
|
||||
|
||||
---
|
||||
|
||||
## 🤝 Contributing
|
||||
|
||||
Contributions are welcome! Read the <a href="https://databasus.com/contribute">contributing guide</a> for more details, priorities and rules. If you want to contribute but don't know where to start, message me on Telegram [@rostislav_dugin](https://t.me/rostislav_dugin)
|
||||
|
||||
Also you can join our large community of developers, DBAs and DevOps engineers on Telegram [@databasus_community](https://t.me/databasus_community).
|
||||
|
||||
--
|
||||
|
||||
## 📖 Migration guide
|
||||
|
||||
Databasus is the new name for Postgresus. You can stay with latest version of Postgresus if you wish. If you want to migrate - follow installation steps for Databasus itself.
|
||||
|
||||
Just renaming an image is not enough as Postgresus and Databasus use different data folders and internal database naming.
|
||||
|
||||
You can put a new Databasus image with updated volume near the old Postgresus and run it (stop Postgresus before):
|
||||
|
||||
```
|
||||
services:
|
||||
databasus:
|
||||
container_name: databasus
|
||||
image: databasus/databasus:latest
|
||||
ports:
|
||||
- "4005:4005"
|
||||
volumes:
|
||||
- ./databasus-data:/databasus-data
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
Then manually move databases from Postgresus to Databasus.
|
||||
|
||||
### Why was Postgresus renamed to Databasus?
|
||||
|
||||
Databasus has been developed since 2023. It was internal tool to backup production and home projects databases. In start of 2025 it was released as open source project on GitHub. By the end of 2025 it became popular and the time for renaming has come in December 2025.
|
||||
|
||||
It was an important step for the project to grow. Actually, there are a couple of reasons:
|
||||
|
||||
1. Postgresus is no longer a little tool that just adds UI for pg_dump for little projects. It became a tool both for individual users, DevOps, DBAs, teams, companies and even large enterprises. Tens of thousands of users use Postgresus every day. Postgresus grew into a reliable backup management tool. Initial positioning is no longer suitable: the project is not just a UI wrapper, it's a solid backup management system now (despite it's still easy to use).
|
||||
|
||||
2. New databases are supported: although the primary focus is PostgreSQL (with 100% support in the most efficient way) and always will be, Databasus added support for MySQL, MariaDB and MongoDB. Later more databases will be supported.
|
||||
|
||||
3. Trademark issue: "postgres" is a trademark of PostgreSQL Inc. and cannot be used in the project name. So for safety and legal reasons, we had to rename the project.
|
||||
|
||||
## AI disclaimer
|
||||
|
||||
There have been questions about AI usage in project development in issues and discussions. As the project focuses on security, reliability and production usage, it's important to explain how AI is used in the development process.
|
||||
|
||||
@@ -11,6 +11,9 @@ VICTORIA_LOGS_PASSWORD=devpassword
|
||||
# tests
|
||||
TEST_LOCALHOST=localhost
|
||||
IS_SKIP_EXTERNAL_RESOURCES_TESTS=false
|
||||
# cloudflare turnstile
|
||||
CLOUDFLARE_TURNSTILE_SITE_KEY=
|
||||
CLOUDFLARE_TURNSTILE_SECRET_KEY=
|
||||
# db
|
||||
DATABASE_DSN=host=dev-db user=postgres password=Q1234567 dbname=databasus port=5437 sslmode=disable
|
||||
DATABASE_URL=postgres://postgres:Q1234567@dev-db:5437/databasus?sslmode=disable
|
||||
|
||||
@@ -14,9 +14,10 @@ import (
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
"databasus-backend/internal/features/audit_logs"
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
"databasus-backend/internal/features/backups/backups/backuping"
|
||||
backups_controllers "databasus-backend/internal/features/backups/backups/controllers"
|
||||
backups_download "databasus-backend/internal/features/backups/backups/download"
|
||||
backups_services "databasus-backend/internal/features/backups/backups/services"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/disk"
|
||||
@@ -28,6 +29,7 @@ import (
|
||||
"databasus-backend/internal/features/restores/restoring"
|
||||
"databasus-backend/internal/features/storages"
|
||||
system_healthcheck "databasus-backend/internal/features/system/healthcheck"
|
||||
system_version "databasus-backend/internal/features/system/version"
|
||||
task_cancellation "databasus-backend/internal/features/tasks/cancellation"
|
||||
users_controllers "databasus-backend/internal/features/users/controllers"
|
||||
users_middleware "databasus-backend/internal/features/users/middleware"
|
||||
@@ -209,7 +211,10 @@ func setUpRoutes(r *gin.Engine) {
|
||||
userController := users_controllers.GetUserController()
|
||||
userController.RegisterRoutes(v1)
|
||||
system_healthcheck.GetHealthcheckController().RegisterRoutes(v1)
|
||||
backups.GetBackupController().RegisterPublicRoutes(v1)
|
||||
system_version.GetVersionController().RegisterRoutes(v1)
|
||||
backups_controllers.GetBackupController().RegisterPublicRoutes(v1)
|
||||
backups_controllers.GetPostgresWalBackupController().RegisterRoutes(v1)
|
||||
databases.GetDatabaseController().RegisterPublicRoutes(v1)
|
||||
|
||||
// Setup auth middleware
|
||||
userService := users_services.GetUserService()
|
||||
@@ -226,7 +231,7 @@ func setUpRoutes(r *gin.Engine) {
|
||||
notifiers.GetNotifierController().RegisterRoutes(protected)
|
||||
storages.GetStorageController().RegisterRoutes(protected)
|
||||
databases.GetDatabaseController().RegisterRoutes(protected)
|
||||
backups.GetBackupController().RegisterRoutes(protected)
|
||||
backups_controllers.GetBackupController().RegisterRoutes(protected)
|
||||
restores.GetRestoreController().RegisterRoutes(protected)
|
||||
healthcheck_config.GetHealthcheckConfigController().RegisterRoutes(protected)
|
||||
healthcheck_attempt.GetHealthcheckAttemptController().RegisterRoutes(protected)
|
||||
@@ -238,7 +243,7 @@ func setUpRoutes(r *gin.Engine) {
|
||||
|
||||
func setUpDependencies() {
|
||||
databases.SetupDependencies()
|
||||
backups.SetupDependencies()
|
||||
backups_services.SetupDependencies()
|
||||
restores.SetupDependencies()
|
||||
healthcheck_config.SetupDependencies()
|
||||
audit_logs.SetupDependencies()
|
||||
|
||||
@@ -104,6 +104,10 @@ type EnvVariables struct {
|
||||
GoogleClientID string `env:"GOOGLE_CLIENT_ID"`
|
||||
GoogleClientSecret string `env:"GOOGLE_CLIENT_SECRET"`
|
||||
|
||||
// Cloudflare Turnstile
|
||||
CloudflareTurnstileSecretKey string `env:"CLOUDFLARE_TURNSTILE_SECRET_KEY"`
|
||||
CloudflareTurnstileSiteKey string `env:"CLOUDFLARE_TURNSTILE_SITE_KEY"`
|
||||
|
||||
// testing Telegram
|
||||
TestTelegramBotToken string `env:"TEST_TELEGRAM_BOT_TOKEN"`
|
||||
TestTelegramChatID string `env:"TEST_TELEGRAM_CHAT_ID"`
|
||||
@@ -120,6 +124,7 @@ type EnvVariables struct {
|
||||
SMTPPort int `env:"SMTP_PORT"`
|
||||
SMTPUser string `env:"SMTP_USER"`
|
||||
SMTPPassword string `env:"SMTP_PASSWORD"`
|
||||
SMTPFrom string `env:"SMTP_FROM"`
|
||||
|
||||
// Application URL (optional) - used for email links
|
||||
DatabasusURL string `env:"DATABASUS_URL"`
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package backuping
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
@@ -196,7 +198,7 @@ func (n *BackuperNode) MakeBackup(backupID uuid.UUID, isCallNotifier bool) {
|
||||
|
||||
backupMetadata, err := n.createBackupUseCase.Execute(
|
||||
ctx,
|
||||
backup.ID,
|
||||
backup,
|
||||
backupConfig,
|
||||
database,
|
||||
storage,
|
||||
@@ -263,7 +265,7 @@ func (n *BackuperNode) MakeBackup(backupID uuid.UUID, isCallNotifier bool) {
|
||||
// Delete partial backup from storage
|
||||
storage, storageErr := n.storageService.GetStorageByID(backup.StorageID)
|
||||
if storageErr == nil {
|
||||
if deleteErr := storage.DeleteFile(n.fieldEncryptor, backup.ID); deleteErr != nil {
|
||||
if deleteErr := storage.DeleteFile(n.fieldEncryptor, backup.FileName); deleteErr != nil {
|
||||
n.logger.Error(
|
||||
"Failed to delete partial backup file",
|
||||
"backupId",
|
||||
@@ -311,6 +313,13 @@ func (n *BackuperNode) MakeBackup(backupID uuid.UUID, isCallNotifier bool) {
|
||||
|
||||
// Update backup with encryption metadata if provided
|
||||
if backupMetadata != nil {
|
||||
backupMetadata.BackupID = backup.ID
|
||||
|
||||
if err := backupMetadata.Validate(); err != nil {
|
||||
n.logger.Error("Failed to validate backup metadata", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
backup.EncryptionSalt = backupMetadata.EncryptionSalt
|
||||
backup.EncryptionIV = backupMetadata.EncryptionIV
|
||||
backup.Encryption = backupMetadata.Encryption
|
||||
@@ -321,6 +330,39 @@ func (n *BackuperNode) MakeBackup(backupID uuid.UUID, isCallNotifier bool) {
|
||||
return
|
||||
}
|
||||
|
||||
// Save metadata file to storage
|
||||
if backupMetadata != nil {
|
||||
metadataJSON, err := json.Marshal(backupMetadata)
|
||||
if err != nil {
|
||||
n.logger.Error("Failed to marshal backup metadata to JSON",
|
||||
"backupId", backup.ID,
|
||||
"error", err,
|
||||
)
|
||||
} else {
|
||||
metadataReader := bytes.NewReader(metadataJSON)
|
||||
metadataFileName := backup.FileName + ".metadata"
|
||||
|
||||
if err := storage.SaveFile(
|
||||
context.Background(),
|
||||
n.fieldEncryptor,
|
||||
n.logger,
|
||||
metadataFileName,
|
||||
metadataReader,
|
||||
); err != nil {
|
||||
n.logger.Error("Failed to save backup metadata file to storage",
|
||||
"backupId", backup.ID,
|
||||
"fileName", metadataFileName,
|
||||
"error", err,
|
||||
)
|
||||
} else {
|
||||
n.logger.Info("Backup metadata file saved successfully",
|
||||
"backupId", backup.ID,
|
||||
"fileName", metadataFileName,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update database last backup time
|
||||
now := time.Now().UTC()
|
||||
if updateErr := n.databaseService.SetLastBackupTime(databaseID, now); updateErr != nil {
|
||||
|
||||
@@ -18,7 +18,8 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
cleanerTickerInterval = 1 * time.Minute
|
||||
cleanerTickerInterval = 1 * time.Minute
|
||||
recentBackupGracePeriod = 60 * time.Minute
|
||||
)
|
||||
|
||||
type BackupCleaner struct {
|
||||
@@ -51,8 +52,8 @@ func (c *BackupCleaner) Run(ctx context.Context) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := c.cleanOldBackups(); err != nil {
|
||||
c.logger.Error("Failed to clean old backups", "error", err)
|
||||
if err := c.cleanByRetentionPolicy(); err != nil {
|
||||
c.logger.Error("Failed to clean backups by retention policy", "error", err)
|
||||
}
|
||||
|
||||
if err := c.cleanExceededBackups(); err != nil {
|
||||
@@ -79,8 +80,7 @@ func (c *BackupCleaner) DeleteBackup(backup *backups_core.Backup) error {
|
||||
return err
|
||||
}
|
||||
|
||||
err = storage.DeleteFile(c.fieldEncryptor, backup.ID)
|
||||
if err != nil {
|
||||
if err := storage.DeleteFile(c.fieldEncryptor, backup.FileName); err != nil {
|
||||
// we do not return error here, because sometimes clean up performed
|
||||
// before unavailable storage removal or change - therefore we should
|
||||
// proceed even in case of error. It's possible that some S3 or
|
||||
@@ -88,6 +88,11 @@ func (c *BackupCleaner) DeleteBackup(backup *backups_core.Backup) error {
|
||||
c.logger.Error("Failed to delete backup file", "error", err)
|
||||
}
|
||||
|
||||
metadataFileName := backup.FileName + ".metadata"
|
||||
if err := storage.DeleteFile(c.fieldEncryptor, metadataFileName); err != nil {
|
||||
c.logger.Error("Failed to delete backup metadata file", "error", err)
|
||||
}
|
||||
|
||||
return c.backupRepository.DeleteByID(backup.ID)
|
||||
}
|
||||
|
||||
@@ -95,49 +100,30 @@ func (c *BackupCleaner) AddBackupRemoveListener(listener backups_core.BackupRemo
|
||||
c.backupRemoveListeners = append(c.backupRemoveListeners, listener)
|
||||
}
|
||||
|
||||
func (c *BackupCleaner) cleanOldBackups() error {
|
||||
func (c *BackupCleaner) cleanByRetentionPolicy() error {
|
||||
enabledBackupConfigs, err := c.backupConfigService.GetBackupConfigsWithEnabledBackups()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, backupConfig := range enabledBackupConfigs {
|
||||
backupStorePeriod := backupConfig.StorePeriod
|
||||
var cleanErr error
|
||||
|
||||
if backupStorePeriod == period.PeriodForever {
|
||||
continue
|
||||
switch backupConfig.RetentionPolicyType {
|
||||
case backups_config.RetentionPolicyTypeCount:
|
||||
cleanErr = c.cleanByCount(backupConfig)
|
||||
case backups_config.RetentionPolicyTypeGFS:
|
||||
cleanErr = c.cleanByGFS(backupConfig)
|
||||
default:
|
||||
cleanErr = c.cleanByTimePeriod(backupConfig)
|
||||
}
|
||||
|
||||
storeDuration := backupStorePeriod.ToDuration()
|
||||
dateBeforeBackupsShouldBeDeleted := time.Now().UTC().Add(-storeDuration)
|
||||
|
||||
oldBackups, err := c.backupRepository.FindBackupsBeforeDate(
|
||||
backupConfig.DatabaseID,
|
||||
dateBeforeBackupsShouldBeDeleted,
|
||||
)
|
||||
if err != nil {
|
||||
if cleanErr != nil {
|
||||
c.logger.Error(
|
||||
"Failed to find old backups for database",
|
||||
"databaseId",
|
||||
backupConfig.DatabaseID,
|
||||
"error",
|
||||
err,
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, backup := range oldBackups {
|
||||
if err := c.DeleteBackup(backup); err != nil {
|
||||
c.logger.Error("Failed to delete old backup", "backupId", backup.ID, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
c.logger.Info(
|
||||
"Deleted old backup",
|
||||
"backupId",
|
||||
backup.ID,
|
||||
"databaseId",
|
||||
backupConfig.DatabaseID,
|
||||
"Failed to clean backups by retention policy",
|
||||
"databaseId", backupConfig.DatabaseID,
|
||||
"policy", backupConfig.RetentionPolicyType,
|
||||
"error", cleanErr,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -174,6 +160,158 @@ func (c *BackupCleaner) cleanExceededBackups() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *BackupCleaner) cleanByTimePeriod(backupConfig *backups_config.BackupConfig) error {
|
||||
if backupConfig.RetentionTimePeriod == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if backupConfig.RetentionTimePeriod == period.PeriodForever {
|
||||
return nil
|
||||
}
|
||||
|
||||
storeDuration := backupConfig.RetentionTimePeriod.ToDuration()
|
||||
dateBeforeBackupsShouldBeDeleted := time.Now().UTC().Add(-storeDuration)
|
||||
|
||||
oldBackups, err := c.backupRepository.FindBackupsBeforeDate(
|
||||
backupConfig.DatabaseID,
|
||||
dateBeforeBackupsShouldBeDeleted,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"failed to find old backups for database %s: %w",
|
||||
backupConfig.DatabaseID,
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
for _, backup := range oldBackups {
|
||||
if isRecentBackup(backup) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := c.DeleteBackup(backup); err != nil {
|
||||
c.logger.Error("Failed to delete old backup", "backupId", backup.ID, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
c.logger.Info(
|
||||
"Deleted old backup",
|
||||
"backupId", backup.ID,
|
||||
"databaseId", backupConfig.DatabaseID,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *BackupCleaner) cleanByCount(backupConfig *backups_config.BackupConfig) error {
|
||||
if backupConfig.RetentionCount <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
completedBackups, err := c.backupRepository.FindByDatabaseIdAndStatus(
|
||||
backupConfig.DatabaseID,
|
||||
backups_core.BackupStatusCompleted,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"failed to find completed backups for database %s: %w",
|
||||
backupConfig.DatabaseID,
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
// completedBackups are ordered newest first; delete everything beyond position RetentionCount
|
||||
if len(completedBackups) <= backupConfig.RetentionCount {
|
||||
return nil
|
||||
}
|
||||
|
||||
toDelete := completedBackups[backupConfig.RetentionCount:]
|
||||
for _, backup := range toDelete {
|
||||
if isRecentBackup(backup) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := c.DeleteBackup(backup); err != nil {
|
||||
c.logger.Error(
|
||||
"Failed to delete backup by count policy",
|
||||
"backupId",
|
||||
backup.ID,
|
||||
"error",
|
||||
err,
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
c.logger.Info(
|
||||
"Deleted backup by count policy",
|
||||
"backupId", backup.ID,
|
||||
"databaseId", backupConfig.DatabaseID,
|
||||
"retentionCount", backupConfig.RetentionCount,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *BackupCleaner) cleanByGFS(backupConfig *backups_config.BackupConfig) error {
|
||||
if backupConfig.RetentionGfsHours <= 0 && backupConfig.RetentionGfsDays <= 0 &&
|
||||
backupConfig.RetentionGfsWeeks <= 0 && backupConfig.RetentionGfsMonths <= 0 &&
|
||||
backupConfig.RetentionGfsYears <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
completedBackups, err := c.backupRepository.FindByDatabaseIdAndStatus(
|
||||
backupConfig.DatabaseID,
|
||||
backups_core.BackupStatusCompleted,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"failed to find completed backups for database %s: %w",
|
||||
backupConfig.DatabaseID,
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
keepSet := buildGFSKeepSet(
|
||||
completedBackups,
|
||||
backupConfig.RetentionGfsHours,
|
||||
backupConfig.RetentionGfsDays,
|
||||
backupConfig.RetentionGfsWeeks,
|
||||
backupConfig.RetentionGfsMonths,
|
||||
backupConfig.RetentionGfsYears,
|
||||
)
|
||||
|
||||
for _, backup := range completedBackups {
|
||||
if keepSet[backup.ID] {
|
||||
continue
|
||||
}
|
||||
|
||||
if isRecentBackup(backup) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := c.DeleteBackup(backup); err != nil {
|
||||
c.logger.Error(
|
||||
"Failed to delete backup by GFS policy",
|
||||
"backupId",
|
||||
backup.ID,
|
||||
"error",
|
||||
err,
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
c.logger.Info(
|
||||
"Deleted backup by GFS policy",
|
||||
"backupId", backup.ID,
|
||||
"databaseId", backupConfig.DatabaseID,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *BackupCleaner) cleanExceededBackupsForDatabase(
|
||||
databaseID uuid.UUID,
|
||||
limitperDbMB int64,
|
||||
@@ -210,6 +348,21 @@ func (c *BackupCleaner) cleanExceededBackupsForDatabase(
|
||||
}
|
||||
|
||||
backup := oldestBackups[0]
|
||||
if isRecentBackup(backup) {
|
||||
c.logger.Warn(
|
||||
"Oldest backup is too recent to delete, stopping size cleanup",
|
||||
"databaseId",
|
||||
databaseID,
|
||||
"backupId",
|
||||
backup.ID,
|
||||
"totalSizeMB",
|
||||
backupsTotalSizeMB,
|
||||
"limitMB",
|
||||
limitperDbMB,
|
||||
)
|
||||
break
|
||||
}
|
||||
|
||||
if err := c.DeleteBackup(backup); err != nil {
|
||||
c.logger.Error(
|
||||
"Failed to delete exceeded backup",
|
||||
@@ -240,3 +393,126 @@ func (c *BackupCleaner) cleanExceededBackupsForDatabase(
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isRecentBackup(backup *backups_core.Backup) bool {
|
||||
return time.Since(backup.CreatedAt) < recentBackupGracePeriod
|
||||
}
|
||||
|
||||
// buildGFSKeepSet determines which backups to retain under the GFS rotation scheme.
|
||||
// Backups must be sorted newest-first. A backup can fill multiple slots simultaneously
|
||||
// (e.g. the newest backup of a year also fills the monthly, weekly, daily, and hourly slot).
|
||||
func buildGFSKeepSet(
|
||||
backups []*backups_core.Backup,
|
||||
hours, days, weeks, months, years int,
|
||||
) map[uuid.UUID]bool {
|
||||
keep := make(map[uuid.UUID]bool)
|
||||
|
||||
if len(backups) == 0 {
|
||||
return keep
|
||||
}
|
||||
|
||||
hoursSeen := make(map[string]bool)
|
||||
daysSeen := make(map[string]bool)
|
||||
weeksSeen := make(map[string]bool)
|
||||
monthsSeen := make(map[string]bool)
|
||||
yearsSeen := make(map[string]bool)
|
||||
|
||||
hoursKept, daysKept, weeksKept, monthsKept, yearsKept := 0, 0, 0, 0, 0
|
||||
|
||||
// Compute per-level time-window cutoffs so higher-frequency slots
|
||||
// cannot absorb backups that belong to lower-frequency levels.
|
||||
ref := backups[0].CreatedAt
|
||||
|
||||
rawHourlyCutoff := ref.Add(-time.Duration(hours) * time.Hour)
|
||||
rawDailyCutoff := ref.Add(-time.Duration(days) * 24 * time.Hour)
|
||||
rawWeeklyCutoff := ref.Add(-time.Duration(weeks) * 7 * 24 * time.Hour)
|
||||
rawMonthlyCutoff := ref.AddDate(0, -months, 0)
|
||||
rawYearlyCutoff := ref.AddDate(-years, 0, 0)
|
||||
|
||||
// Hierarchical capping: each level's window cannot extend further back
|
||||
// than the nearest active lower-frequency level's window.
|
||||
yearlyCutoff := rawYearlyCutoff
|
||||
|
||||
monthlyCutoff := rawMonthlyCutoff
|
||||
if years > 0 {
|
||||
monthlyCutoff = laterOf(monthlyCutoff, yearlyCutoff)
|
||||
}
|
||||
|
||||
weeklyCutoff := rawWeeklyCutoff
|
||||
if months > 0 {
|
||||
weeklyCutoff = laterOf(weeklyCutoff, monthlyCutoff)
|
||||
} else if years > 0 {
|
||||
weeklyCutoff = laterOf(weeklyCutoff, yearlyCutoff)
|
||||
}
|
||||
|
||||
dailyCutoff := rawDailyCutoff
|
||||
if weeks > 0 {
|
||||
dailyCutoff = laterOf(dailyCutoff, weeklyCutoff)
|
||||
} else if months > 0 {
|
||||
dailyCutoff = laterOf(dailyCutoff, monthlyCutoff)
|
||||
} else if years > 0 {
|
||||
dailyCutoff = laterOf(dailyCutoff, yearlyCutoff)
|
||||
}
|
||||
|
||||
hourlyCutoff := rawHourlyCutoff
|
||||
if days > 0 {
|
||||
hourlyCutoff = laterOf(hourlyCutoff, dailyCutoff)
|
||||
} else if weeks > 0 {
|
||||
hourlyCutoff = laterOf(hourlyCutoff, weeklyCutoff)
|
||||
} else if months > 0 {
|
||||
hourlyCutoff = laterOf(hourlyCutoff, monthlyCutoff)
|
||||
} else if years > 0 {
|
||||
hourlyCutoff = laterOf(hourlyCutoff, yearlyCutoff)
|
||||
}
|
||||
|
||||
for _, backup := range backups {
|
||||
t := backup.CreatedAt
|
||||
|
||||
hourKey := t.Format("2006-01-02-15")
|
||||
dayKey := t.Format("2006-01-02")
|
||||
weekYear, week := t.ISOWeek()
|
||||
weekKey := fmt.Sprintf("%d-%02d", weekYear, week)
|
||||
monthKey := t.Format("2006-01")
|
||||
yearKey := t.Format("2006")
|
||||
|
||||
if hours > 0 && hoursKept < hours && !hoursSeen[hourKey] && t.After(hourlyCutoff) {
|
||||
keep[backup.ID] = true
|
||||
hoursSeen[hourKey] = true
|
||||
hoursKept++
|
||||
}
|
||||
|
||||
if days > 0 && daysKept < days && !daysSeen[dayKey] && t.After(dailyCutoff) {
|
||||
keep[backup.ID] = true
|
||||
daysSeen[dayKey] = true
|
||||
daysKept++
|
||||
}
|
||||
|
||||
if weeks > 0 && weeksKept < weeks && !weeksSeen[weekKey] && t.After(weeklyCutoff) {
|
||||
keep[backup.ID] = true
|
||||
weeksSeen[weekKey] = true
|
||||
weeksKept++
|
||||
}
|
||||
|
||||
if months > 0 && monthsKept < months && !monthsSeen[monthKey] && t.After(monthlyCutoff) {
|
||||
keep[backup.ID] = true
|
||||
monthsSeen[monthKey] = true
|
||||
monthsKept++
|
||||
}
|
||||
|
||||
if years > 0 && yearsKept < years && !yearsSeen[yearKey] && t.After(yearlyCutoff) {
|
||||
keep[backup.ID] = true
|
||||
yearsSeen[yearKey] = true
|
||||
yearsKept++
|
||||
}
|
||||
}
|
||||
|
||||
return keep
|
||||
}
|
||||
|
||||
func laterOf(a, b time.Time) time.Time {
|
||||
if a.After(b) {
|
||||
return a
|
||||
}
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -20,7 +20,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {
|
||||
func Test_CleanOldBackups_DeletesBackupsOlderThanRetentionTimePeriod(t *testing.T) {
|
||||
router := CreateTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
@@ -41,21 +41,20 @@ func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
// Create backup interval
|
||||
interval := createTestInterval()
|
||||
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
StorageID: &storage.ID,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
StorageID: &storage.ID,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
}
|
||||
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Create backups with different ages
|
||||
now := time.Now().UTC()
|
||||
oldBackup1 := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
@@ -63,7 +62,7 @@ func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: now.Add(-10 * 24 * time.Hour), // 10 days old
|
||||
CreatedAt: now.Add(-10 * 24 * time.Hour),
|
||||
}
|
||||
oldBackup2 := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
@@ -71,7 +70,7 @@ func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: now.Add(-8 * 24 * time.Hour), // 8 days old
|
||||
CreatedAt: now.Add(-8 * 24 * time.Hour),
|
||||
}
|
||||
recentBackup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
@@ -79,7 +78,7 @@ func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: now.Add(-3 * 24 * time.Hour), // 3 days old
|
||||
CreatedAt: now.Add(-3 * 24 * time.Hour),
|
||||
}
|
||||
|
||||
err = backupRepository.Save(oldBackup1)
|
||||
@@ -89,19 +88,17 @@ func Test_CleanOldBackups_DeletesBackupsOlderThanStorePeriod(t *testing.T) {
|
||||
err = backupRepository.Save(recentBackup)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Run cleanup
|
||||
cleaner := GetBackupCleaner()
|
||||
err = cleaner.cleanOldBackups()
|
||||
err = cleaner.cleanByRetentionPolicy()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify old backups deleted, recent backup remains
|
||||
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(remainingBackups))
|
||||
assert.Equal(t, recentBackup.ID, remainingBackups[0].ID)
|
||||
}
|
||||
|
||||
func Test_CleanOldBackups_SkipsDatabaseWithForeverStorePeriod(t *testing.T) {
|
||||
func Test_CleanOldBackups_SkipsDatabaseWithForeverRetentionPeriod(t *testing.T) {
|
||||
router := CreateTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
@@ -122,38 +119,35 @@ func Test_CleanOldBackups_SkipsDatabaseWithForeverStorePeriod(t *testing.T) {
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
// Create backup interval
|
||||
interval := createTestInterval()
|
||||
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodForever,
|
||||
StorageID: &storage.ID,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodForever,
|
||||
StorageID: &storage.ID,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
}
|
||||
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Create very old backup
|
||||
oldBackup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: time.Now().UTC().Add(-365 * 24 * time.Hour), // 1 year old
|
||||
CreatedAt: time.Now().UTC().Add(-365 * 24 * time.Hour),
|
||||
}
|
||||
err = backupRepository.Save(oldBackup)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Run cleanup
|
||||
cleaner := GetBackupCleaner()
|
||||
err = cleaner.cleanOldBackups()
|
||||
err = cleaner.cleanByRetentionPolicy()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify backup still exists
|
||||
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(remainingBackups))
|
||||
@@ -181,22 +175,21 @@ func Test_CleanExceededBackups_WhenUnderLimit_NoBackupsDeleted(t *testing.T) {
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
// Create backup interval
|
||||
interval := createTestInterval()
|
||||
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodForever,
|
||||
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodForever,
|
||||
StorageID: &storage.ID,
|
||||
MaxBackupsTotalSizeMB: 100, // 100 MB limit
|
||||
MaxBackupsTotalSizeMB: 100,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
}
|
||||
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Create 3 backups totaling 50MB (under limit)
|
||||
for i := 0; i < 3; i++ {
|
||||
backup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
@@ -210,12 +203,10 @@ func Test_CleanExceededBackups_WhenUnderLimit_NoBackupsDeleted(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// Run cleanup
|
||||
cleaner := GetBackupCleaner()
|
||||
err = cleaner.cleanExceededBackups()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify all backups remain
|
||||
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, len(remainingBackups))
|
||||
@@ -242,22 +233,21 @@ func Test_CleanExceededBackups_WhenOverLimit_DeletesOldestBackups(t *testing.T)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
// Create backup interval
|
||||
interval := createTestInterval()
|
||||
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodForever,
|
||||
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodForever,
|
||||
StorageID: &storage.ID,
|
||||
MaxBackupsTotalSizeMB: 30, // 30 MB limit
|
||||
MaxBackupsTotalSizeMB: 30,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
}
|
||||
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Create 5 backups of 10MB each (total 50MB, over 30MB limit)
|
||||
now := time.Now().UTC()
|
||||
var backupIDs []uuid.UUID
|
||||
for i := 0; i < 5; i++ {
|
||||
@@ -267,33 +257,30 @@ func Test_CleanExceededBackups_WhenOverLimit_DeletesOldestBackups(t *testing.T)
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: now.Add(-time.Duration(4-i) * time.Hour), // Oldest first
|
||||
CreatedAt: now.Add(-time.Duration(4-i) * time.Hour),
|
||||
}
|
||||
err = backupRepository.Save(backup)
|
||||
assert.NoError(t, err)
|
||||
backupIDs = append(backupIDs, backup.ID)
|
||||
}
|
||||
|
||||
// Run cleanup
|
||||
cleaner := GetBackupCleaner()
|
||||
err = cleaner.cleanExceededBackups()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify 2 oldest backups deleted, 3 newest remain
|
||||
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, len(remainingBackups))
|
||||
|
||||
// Check that the newest 3 backups remain
|
||||
remainingIDs := make(map[uuid.UUID]bool)
|
||||
for _, backup := range remainingBackups {
|
||||
remainingIDs[backup.ID] = true
|
||||
}
|
||||
assert.False(t, remainingIDs[backupIDs[0]]) // Oldest deleted
|
||||
assert.False(t, remainingIDs[backupIDs[1]]) // 2nd oldest deleted
|
||||
assert.True(t, remainingIDs[backupIDs[2]]) // 3rd remains
|
||||
assert.True(t, remainingIDs[backupIDs[3]]) // 4th remains
|
||||
assert.True(t, remainingIDs[backupIDs[4]]) // Newest remains
|
||||
assert.False(t, remainingIDs[backupIDs[0]])
|
||||
assert.False(t, remainingIDs[backupIDs[1]])
|
||||
assert.True(t, remainingIDs[backupIDs[2]])
|
||||
assert.True(t, remainingIDs[backupIDs[3]])
|
||||
assert.True(t, remainingIDs[backupIDs[4]])
|
||||
}
|
||||
|
||||
func Test_CleanExceededBackups_SkipsInProgressBackups(t *testing.T) {
|
||||
@@ -317,15 +304,15 @@ func Test_CleanExceededBackups_SkipsInProgressBackups(t *testing.T) {
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
// Create backup interval
|
||||
interval := createTestInterval()
|
||||
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodForever,
|
||||
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodForever,
|
||||
StorageID: &storage.ID,
|
||||
MaxBackupsTotalSizeMB: 50, // 50 MB limit
|
||||
MaxBackupsTotalSizeMB: 50,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
}
|
||||
@@ -334,7 +321,6 @@ func Test_CleanExceededBackups_SkipsInProgressBackups(t *testing.T) {
|
||||
|
||||
now := time.Now().UTC()
|
||||
|
||||
// Create 3 completed backups of 30MB each
|
||||
completedBackups := make([]*backups_core.Backup, 3)
|
||||
for i := 0; i < 3; i++ {
|
||||
backup := &backups_core.Backup{
|
||||
@@ -350,7 +336,6 @@ func Test_CleanExceededBackups_SkipsInProgressBackups(t *testing.T) {
|
||||
completedBackups[i] = backup
|
||||
}
|
||||
|
||||
// Create 1 in-progress backup (should be excluded from size calculation and deletion)
|
||||
inProgressBackup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
@@ -362,19 +347,14 @@ func Test_CleanExceededBackups_SkipsInProgressBackups(t *testing.T) {
|
||||
err = backupRepository.Save(inProgressBackup)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Run cleanup
|
||||
cleaner := GetBackupCleaner()
|
||||
err = cleaner.cleanExceededBackups()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify: only completed backups deleted, in-progress remains
|
||||
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Should have in-progress + 1 completed (total 40MB completed + 10MB in-progress)
|
||||
assert.GreaterOrEqual(t, len(remainingBackups), 2)
|
||||
|
||||
// Verify in-progress backup still exists
|
||||
var inProgressFound bool
|
||||
for _, backup := range remainingBackups {
|
||||
if backup.ID == inProgressBackup.ID {
|
||||
@@ -406,22 +386,21 @@ func Test_CleanExceededBackups_WithZeroLimit_SkipsDatabase(t *testing.T) {
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
// Create backup interval
|
||||
interval := createTestInterval()
|
||||
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodForever,
|
||||
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodForever,
|
||||
StorageID: &storage.ID,
|
||||
MaxBackupsTotalSizeMB: 0, // No size limit
|
||||
MaxBackupsTotalSizeMB: 0,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
}
|
||||
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Create large backups
|
||||
for i := 0; i < 10; i++ {
|
||||
backup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
@@ -435,12 +414,10 @@ func Test_CleanExceededBackups_WithZeroLimit_SkipsDatabase(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// Run cleanup
|
||||
cleaner := GetBackupCleaner()
|
||||
err = cleaner.cleanExceededBackups()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify all backups remain
|
||||
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 10, len(remainingBackups))
|
||||
@@ -467,7 +444,6 @@ func Test_GetTotalSizeByDatabase_CalculatesCorrectly(t *testing.T) {
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
// Create completed backups
|
||||
completedBackup1 := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
@@ -484,7 +460,6 @@ func Test_GetTotalSizeByDatabase_CalculatesCorrectly(t *testing.T) {
|
||||
BackupSizeMb: 20.3,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
}
|
||||
// Create failed backup (should be included)
|
||||
failedBackup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
@@ -493,7 +468,6 @@ func Test_GetTotalSizeByDatabase_CalculatesCorrectly(t *testing.T) {
|
||||
BackupSizeMb: 5.2,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
}
|
||||
// Create in-progress backup (should be excluded)
|
||||
inProgressBackup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
@@ -512,25 +486,215 @@ func Test_GetTotalSizeByDatabase_CalculatesCorrectly(t *testing.T) {
|
||||
err = backupRepository.Save(inProgressBackup)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Calculate total size
|
||||
totalSize, err := backupRepository.GetTotalSizeByDatabase(database.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Should be 10.5 + 20.3 + 5.2 = 36.0 (excluding in-progress 100)
|
||||
assert.InDelta(t, 36.0, totalSize, 0.1)
|
||||
}
|
||||
|
||||
// Mock listener for testing
|
||||
type mockBackupRemoveListener struct {
|
||||
onBeforeBackupRemove func(*backups_core.Backup) error
|
||||
}
|
||||
func Test_CleanByCount_KeepsNewestNBackups_DeletesOlder(t *testing.T) {
|
||||
router := CreateTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
storage := storages.CreateTestStorage(workspace.ID)
|
||||
notifier := notifiers.CreateTestNotifier(workspace.ID)
|
||||
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
|
||||
|
||||
func (m *mockBackupRemoveListener) OnBeforeBackupRemove(backup *backups_core.Backup) error {
|
||||
if m.onBeforeBackupRemove != nil {
|
||||
return m.onBeforeBackupRemove(backup)
|
||||
defer func() {
|
||||
backups, _ := backupRepository.FindByDatabaseID(database.ID)
|
||||
for _, backup := range backups {
|
||||
backupRepository.DeleteByID(backup.ID)
|
||||
}
|
||||
|
||||
databases.RemoveTestDatabase(database)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
notifiers.RemoveTestNotifier(notifier)
|
||||
storages.RemoveTestStorage(storage.ID)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
interval := createTestInterval()
|
||||
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: backups_config.RetentionPolicyTypeCount,
|
||||
RetentionCount: 3,
|
||||
StorageID: &storage.ID,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
}
|
||||
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
now := time.Now().UTC()
|
||||
var backupIDs []uuid.UUID
|
||||
for i := 0; i < 5; i++ {
|
||||
backup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: now.Add(
|
||||
-time.Duration(4-i) * time.Hour,
|
||||
), // oldest first in loop, newest = i=4
|
||||
}
|
||||
err = backupRepository.Save(backup)
|
||||
assert.NoError(t, err)
|
||||
backupIDs = append(backupIDs, backup.ID)
|
||||
}
|
||||
|
||||
return nil
|
||||
cleaner := GetBackupCleaner()
|
||||
err = cleaner.cleanByRetentionPolicy()
|
||||
assert.NoError(t, err)
|
||||
|
||||
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 3, len(remainingBackups))
|
||||
|
||||
remainingIDs := make(map[uuid.UUID]bool)
|
||||
for _, backup := range remainingBackups {
|
||||
remainingIDs[backup.ID] = true
|
||||
}
|
||||
assert.False(t, remainingIDs[backupIDs[0]], "Oldest backup should be deleted")
|
||||
assert.False(t, remainingIDs[backupIDs[1]], "2nd oldest backup should be deleted")
|
||||
assert.True(t, remainingIDs[backupIDs[2]], "3rd backup should remain")
|
||||
assert.True(t, remainingIDs[backupIDs[3]], "4th backup should remain")
|
||||
assert.True(t, remainingIDs[backupIDs[4]], "Newest backup should remain")
|
||||
}
|
||||
|
||||
func Test_CleanByCount_WhenUnderLimit_NoBackupsDeleted(t *testing.T) {
|
||||
router := CreateTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
storage := storages.CreateTestStorage(workspace.ID)
|
||||
notifier := notifiers.CreateTestNotifier(workspace.ID)
|
||||
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
|
||||
|
||||
defer func() {
|
||||
backups, _ := backupRepository.FindByDatabaseID(database.ID)
|
||||
for _, backup := range backups {
|
||||
backupRepository.DeleteByID(backup.ID)
|
||||
}
|
||||
|
||||
databases.RemoveTestDatabase(database)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
notifiers.RemoveTestNotifier(notifier)
|
||||
storages.RemoveTestStorage(storage.ID)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
interval := createTestInterval()
|
||||
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: backups_config.RetentionPolicyTypeCount,
|
||||
RetentionCount: 10,
|
||||
StorageID: &storage.ID,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
}
|
||||
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
backup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: time.Now().UTC().Add(-time.Duration(i) * time.Hour),
|
||||
}
|
||||
err = backupRepository.Save(backup)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
cleaner := GetBackupCleaner()
|
||||
err = cleaner.cleanByRetentionPolicy()
|
||||
assert.NoError(t, err)
|
||||
|
||||
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 5, len(remainingBackups))
|
||||
}
|
||||
|
||||
func Test_CleanByCount_DoesNotDeleteInProgressBackups(t *testing.T) {
|
||||
router := CreateTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
storage := storages.CreateTestStorage(workspace.ID)
|
||||
notifier := notifiers.CreateTestNotifier(workspace.ID)
|
||||
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
|
||||
|
||||
defer func() {
|
||||
backups, _ := backupRepository.FindByDatabaseID(database.ID)
|
||||
for _, backup := range backups {
|
||||
backupRepository.DeleteByID(backup.ID)
|
||||
}
|
||||
|
||||
databases.RemoveTestDatabase(database)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
notifiers.RemoveTestNotifier(notifier)
|
||||
storages.RemoveTestStorage(storage.ID)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
interval := createTestInterval()
|
||||
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: backups_config.RetentionPolicyTypeCount,
|
||||
RetentionCount: 2,
|
||||
StorageID: &storage.ID,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
}
|
||||
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
now := time.Now().UTC()
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
backup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: now.Add(-time.Duration(3-i) * time.Hour),
|
||||
}
|
||||
err = backupRepository.Save(backup)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
inProgressBackup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusInProgress,
|
||||
BackupSizeMb: 5,
|
||||
CreatedAt: now,
|
||||
}
|
||||
err = backupRepository.Save(inProgressBackup)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cleaner := GetBackupCleaner()
|
||||
err = cleaner.cleanByRetentionPolicy()
|
||||
assert.NoError(t, err)
|
||||
|
||||
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
var inProgressFound bool
|
||||
for _, backup := range remainingBackups {
|
||||
if backup.ID == inProgressBackup.ID {
|
||||
inProgressFound = true
|
||||
}
|
||||
}
|
||||
assert.True(t, inProgressFound, "In-progress backup should not be deleted by count policy")
|
||||
}
|
||||
|
||||
// Test_DeleteBackup_WhenStorageDeleteFails_BackupStillRemovedFromDatabase verifies resilience
|
||||
@@ -579,6 +743,267 @@ func Test_DeleteBackup_WhenStorageDeleteFails_BackupStillRemovedFromDatabase(t *
|
||||
assert.Nil(t, deletedBackup)
|
||||
}
|
||||
|
||||
func Test_CleanByTimePeriod_SkipsRecentBackup_EvenIfOlderThanRetention(t *testing.T) {
|
||||
router := CreateTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
storage := storages.CreateTestStorage(workspace.ID)
|
||||
notifier := notifiers.CreateTestNotifier(workspace.ID)
|
||||
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
|
||||
|
||||
defer func() {
|
||||
backups, _ := backupRepository.FindByDatabaseID(database.ID)
|
||||
for _, backup := range backups {
|
||||
backupRepository.DeleteByID(backup.ID)
|
||||
}
|
||||
|
||||
databases.RemoveTestDatabase(database)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
notifiers.RemoveTestNotifier(notifier)
|
||||
storages.RemoveTestStorage(storage.ID)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
interval := createTestInterval()
|
||||
|
||||
// Retention period is 1 day — any backup older than 1 day should be deleted.
|
||||
// But the recent backup was created only 30 minutes ago and must be preserved.
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodDay,
|
||||
StorageID: &storage.ID,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
}
|
||||
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
now := time.Now().UTC()
|
||||
|
||||
oldBackup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: now.Add(-2 * 24 * time.Hour),
|
||||
}
|
||||
recentBackup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: now.Add(-30 * time.Minute),
|
||||
}
|
||||
|
||||
err = backupRepository.Save(oldBackup)
|
||||
assert.NoError(t, err)
|
||||
err = backupRepository.Save(recentBackup)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cleaner := GetBackupCleaner()
|
||||
err = cleaner.cleanByRetentionPolicy()
|
||||
assert.NoError(t, err)
|
||||
|
||||
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(remainingBackups))
|
||||
assert.Equal(t, recentBackup.ID, remainingBackups[0].ID)
|
||||
}
|
||||
|
||||
func Test_CleanByCount_SkipsRecentBackup_EvenIfOverLimit(t *testing.T) {
|
||||
router := CreateTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
storage := storages.CreateTestStorage(workspace.ID)
|
||||
notifier := notifiers.CreateTestNotifier(workspace.ID)
|
||||
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
|
||||
|
||||
defer func() {
|
||||
backups, _ := backupRepository.FindByDatabaseID(database.ID)
|
||||
for _, backup := range backups {
|
||||
backupRepository.DeleteByID(backup.ID)
|
||||
}
|
||||
|
||||
databases.RemoveTestDatabase(database)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
notifiers.RemoveTestNotifier(notifier)
|
||||
storages.RemoveTestStorage(storage.ID)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
interval := createTestInterval()
|
||||
|
||||
// Retention count is 2 — 4 backups exist so 2 should be deleted.
|
||||
// The oldest backup in the "excess" tail was made 30 min ago — it must be preserved.
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: backups_config.RetentionPolicyTypeCount,
|
||||
RetentionCount: 2,
|
||||
StorageID: &storage.ID,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
}
|
||||
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
now := time.Now().UTC()
|
||||
|
||||
oldBackup1 := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: now.Add(-5 * time.Hour),
|
||||
}
|
||||
oldBackup2 := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: now.Add(-3 * time.Hour),
|
||||
}
|
||||
// This backup is 3rd newest and would normally be deleted — but it is recent.
|
||||
recentExcessBackup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: now.Add(-30 * time.Minute),
|
||||
}
|
||||
newestBackup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10,
|
||||
CreatedAt: now.Add(-10 * time.Minute),
|
||||
}
|
||||
|
||||
for _, b := range []*backups_core.Backup{oldBackup1, oldBackup2, recentExcessBackup, newestBackup} {
|
||||
err = backupRepository.Save(b)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
cleaner := GetBackupCleaner()
|
||||
err = cleaner.cleanByRetentionPolicy()
|
||||
assert.NoError(t, err)
|
||||
|
||||
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
remainingIDs := make(map[uuid.UUID]bool)
|
||||
for _, backup := range remainingBackups {
|
||||
remainingIDs[backup.ID] = true
|
||||
}
|
||||
|
||||
assert.False(t, remainingIDs[oldBackup1.ID], "Oldest non-recent backup should be deleted")
|
||||
assert.False(t, remainingIDs[oldBackup2.ID], "2nd oldest non-recent backup should be deleted")
|
||||
assert.True(
|
||||
t,
|
||||
remainingIDs[recentExcessBackup.ID],
|
||||
"Recent backup must be preserved despite being over limit",
|
||||
)
|
||||
assert.True(t, remainingIDs[newestBackup.ID], "Newest backup should be preserved")
|
||||
}
|
||||
|
||||
func Test_CleanExceededBackups_SkipsRecentBackup_WhenOverTotalSizeLimit(t *testing.T) {
|
||||
router := CreateTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
storage := storages.CreateTestStorage(workspace.ID)
|
||||
notifier := notifiers.CreateTestNotifier(workspace.ID)
|
||||
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
|
||||
|
||||
defer func() {
|
||||
backups, _ := backupRepository.FindByDatabaseID(database.ID)
|
||||
for _, backup := range backups {
|
||||
backupRepository.DeleteByID(backup.ID)
|
||||
}
|
||||
|
||||
databases.RemoveTestDatabase(database)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
notifiers.RemoveTestNotifier(notifier)
|
||||
storages.RemoveTestStorage(storage.ID)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
interval := createTestInterval()
|
||||
|
||||
// Total size limit is 10 MB. We have two backups of 8 MB each (16 MB total).
|
||||
// The oldest backup was created 30 minutes ago — within the grace period.
|
||||
// The cleaner must stop and leave both backups intact.
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: backups_config.RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodForever,
|
||||
StorageID: &storage.ID,
|
||||
MaxBackupsTotalSizeMB: 10,
|
||||
BackupIntervalID: interval.ID,
|
||||
BackupInterval: interval,
|
||||
}
|
||||
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
now := time.Now().UTC()
|
||||
|
||||
olderRecentBackup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 8,
|
||||
CreatedAt: now.Add(-30 * time.Minute),
|
||||
}
|
||||
newerRecentBackup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 8,
|
||||
CreatedAt: now.Add(-10 * time.Minute),
|
||||
}
|
||||
|
||||
err = backupRepository.Save(olderRecentBackup)
|
||||
assert.NoError(t, err)
|
||||
err = backupRepository.Save(newerRecentBackup)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cleaner := GetBackupCleaner()
|
||||
err = cleaner.cleanExceededBackups()
|
||||
assert.NoError(t, err)
|
||||
|
||||
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(
|
||||
t,
|
||||
2,
|
||||
len(remainingBackups),
|
||||
"Both recent backups must be preserved even though total size exceeds limit",
|
||||
)
|
||||
}
|
||||
|
||||
// Mock listener for testing
|
||||
type mockBackupRemoveListener struct {
|
||||
onBeforeBackupRemove func(*backups_core.Backup) error
|
||||
}
|
||||
|
||||
func (m *mockBackupRemoveListener) OnBeforeBackupRemove(backup *backups_core.Backup) error {
|
||||
if m.onBeforeBackupRemove != nil {
|
||||
return m.onBeforeBackupRemove(backup)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createTestInterval() *intervals.Interval {
|
||||
timeOfDay := "04:00"
|
||||
interval := &intervals.Interval{
|
||||
|
||||
@@ -25,24 +25,24 @@ var backupRepository = &backups_core.BackupRepository{}
|
||||
var taskCancelManager = tasks_cancellation.GetTaskCancelManager()
|
||||
|
||||
var backupCleaner = &BackupCleaner{
|
||||
backupRepository: backupRepository,
|
||||
storageService: storages.GetStorageService(),
|
||||
backupConfigService: backups_config.GetBackupConfigService(),
|
||||
fieldEncryptor: encryption.GetFieldEncryptor(),
|
||||
logger: logger.GetLogger(),
|
||||
backupRemoveListeners: []backups_core.BackupRemoveListener{},
|
||||
runOnce: sync.Once{},
|
||||
hasRun: atomic.Bool{},
|
||||
backupRepository,
|
||||
storages.GetStorageService(),
|
||||
backups_config.GetBackupConfigService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
logger.GetLogger(),
|
||||
[]backups_core.BackupRemoveListener{},
|
||||
sync.Once{},
|
||||
atomic.Bool{},
|
||||
}
|
||||
|
||||
var backupNodesRegistry = &BackupNodesRegistry{
|
||||
client: cache_utils.GetValkeyClient(),
|
||||
logger: logger.GetLogger(),
|
||||
timeout: cache_utils.DefaultCacheTimeout,
|
||||
pubsubBackups: cache_utils.NewPubSubManager(),
|
||||
pubsubCompletions: cache_utils.NewPubSubManager(),
|
||||
runOnce: sync.Once{},
|
||||
hasRun: atomic.Bool{},
|
||||
cache_utils.GetValkeyClient(),
|
||||
logger.GetLogger(),
|
||||
cache_utils.DefaultCacheTimeout,
|
||||
cache_utils.NewPubSubManager(),
|
||||
cache_utils.NewPubSubManager(),
|
||||
sync.Once{},
|
||||
atomic.Bool{},
|
||||
}
|
||||
|
||||
func getNodeID() uuid.UUID {
|
||||
@@ -50,34 +50,35 @@ func getNodeID() uuid.UUID {
|
||||
}
|
||||
|
||||
var backuperNode = &BackuperNode{
|
||||
databaseService: databases.GetDatabaseService(),
|
||||
fieldEncryptor: encryption.GetFieldEncryptor(),
|
||||
workspaceService: workspaces_services.GetWorkspaceService(),
|
||||
backupRepository: backupRepository,
|
||||
backupConfigService: backups_config.GetBackupConfigService(),
|
||||
storageService: storages.GetStorageService(),
|
||||
notificationSender: notifiers.GetNotifierService(),
|
||||
backupCancelManager: taskCancelManager,
|
||||
backupNodesRegistry: backupNodesRegistry,
|
||||
logger: logger.GetLogger(),
|
||||
createBackupUseCase: usecases.GetCreateBackupUsecase(),
|
||||
nodeID: getNodeID(),
|
||||
lastHeartbeat: time.Time{},
|
||||
runOnce: sync.Once{},
|
||||
hasRun: atomic.Bool{},
|
||||
databases.GetDatabaseService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
workspaces_services.GetWorkspaceService(),
|
||||
backupRepository,
|
||||
backups_config.GetBackupConfigService(),
|
||||
storages.GetStorageService(),
|
||||
notifiers.GetNotifierService(),
|
||||
taskCancelManager,
|
||||
backupNodesRegistry,
|
||||
logger.GetLogger(),
|
||||
usecases.GetCreateBackupUsecase(),
|
||||
getNodeID(),
|
||||
time.Time{},
|
||||
sync.Once{},
|
||||
atomic.Bool{},
|
||||
}
|
||||
|
||||
var backupsScheduler = &BackupsScheduler{
|
||||
backupRepository: backupRepository,
|
||||
backupConfigService: backups_config.GetBackupConfigService(),
|
||||
taskCancelManager: taskCancelManager,
|
||||
backupNodesRegistry: backupNodesRegistry,
|
||||
lastBackupTime: time.Now().UTC(),
|
||||
logger: logger.GetLogger(),
|
||||
backupToNodeRelations: make(map[uuid.UUID]BackupToNodeRelation),
|
||||
backuperNode: backuperNode,
|
||||
runOnce: sync.Once{},
|
||||
hasRun: atomic.Bool{},
|
||||
backupRepository,
|
||||
backups_config.GetBackupConfigService(),
|
||||
taskCancelManager,
|
||||
backupNodesRegistry,
|
||||
databases.GetDatabaseService(),
|
||||
time.Now().UTC(),
|
||||
logger.GetLogger(),
|
||||
make(map[uuid.UUID]BackupToNodeRelation),
|
||||
backuperNode,
|
||||
sync.Once{},
|
||||
atomic.Bool{},
|
||||
}
|
||||
|
||||
func GetBackupsScheduler() *BackupsScheduler {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
common "databasus-backend/internal/features/backups/backups/common"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/notifiers"
|
||||
@@ -32,7 +33,7 @@ type CreateFailedBackupUsecase struct{}
|
||||
|
||||
func (uc *CreateFailedBackupUsecase) Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
database *databases.Database,
|
||||
storage *storages.Storage,
|
||||
@@ -46,7 +47,7 @@ type CreateSuccessBackupUsecase struct{}
|
||||
|
||||
func (uc *CreateSuccessBackupUsecase) Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
database *databases.Database,
|
||||
storage *storages.Storage,
|
||||
@@ -65,7 +66,7 @@ type CreateLargeBackupUsecase struct{}
|
||||
|
||||
func (uc *CreateLargeBackupUsecase) Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
database *databases.Database,
|
||||
storage *storages.Storage,
|
||||
@@ -84,7 +85,7 @@ type CreateProgressiveBackupUsecase struct{}
|
||||
|
||||
func (uc *CreateProgressiveBackupUsecase) Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
database *databases.Database,
|
||||
storage *storages.Storage,
|
||||
@@ -124,7 +125,7 @@ type CreateMediumBackupUsecase struct{}
|
||||
|
||||
func (uc *CreateMediumBackupUsecase) Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
database *databases.Database,
|
||||
storage *storages.Storage,
|
||||
@@ -152,7 +153,7 @@ func NewMockTrackingBackupUsecase() *MockTrackingBackupUsecase {
|
||||
|
||||
func (m *MockTrackingBackupUsecase) Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
database *databases.Database,
|
||||
storage *storages.Storage,
|
||||
@@ -162,7 +163,7 @@ func (m *MockTrackingBackupUsecase) Execute(
|
||||
|
||||
// Send backup ID to channel (non-blocking)
|
||||
select {
|
||||
case m.calledBackupIDs <- backupID:
|
||||
case m.calledBackupIDs <- backup.ID:
|
||||
default:
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"databasus-backend/internal/config"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
task_cancellation "databasus-backend/internal/features/tasks/cancellation"
|
||||
)
|
||||
|
||||
@@ -27,6 +28,7 @@ type BackupsScheduler struct {
|
||||
backupConfigService *backups_config.BackupConfigService
|
||||
taskCancelManager *task_cancellation.TaskCancelManager
|
||||
backupNodesRegistry *BackupNodesRegistry
|
||||
databaseService *databases.DatabaseService
|
||||
|
||||
lastBackupTime time.Time
|
||||
logger *slog.Logger
|
||||
@@ -113,28 +115,28 @@ func (s *BackupsScheduler) IsBackupNodesAvailable() bool {
|
||||
return len(nodes) > 0
|
||||
}
|
||||
|
||||
func (s *BackupsScheduler) StartBackup(databaseID uuid.UUID, isCallNotifier bool) {
|
||||
backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(databaseID)
|
||||
func (s *BackupsScheduler) StartBackup(database *databases.Database, isCallNotifier bool) {
|
||||
backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to get backup config by database ID", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
if backupConfig.StorageID == nil {
|
||||
s.logger.Error("Backup config storage ID is nil", "databaseId", databaseID)
|
||||
s.logger.Error("Backup config storage ID is nil", "databaseId", database.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// Check for existing in-progress backups
|
||||
inProgressBackups, err := s.backupRepository.FindByDatabaseIdAndStatus(
|
||||
databaseID,
|
||||
database.ID,
|
||||
backups_core.BackupStatusInProgress,
|
||||
)
|
||||
if err != nil {
|
||||
s.logger.Error(
|
||||
"Failed to check for in-progress backups",
|
||||
"databaseId",
|
||||
databaseID,
|
||||
database.ID,
|
||||
"error",
|
||||
err,
|
||||
)
|
||||
@@ -145,7 +147,7 @@ func (s *BackupsScheduler) StartBackup(databaseID uuid.UUID, isCallNotifier bool
|
||||
s.logger.Warn(
|
||||
"Backup already in progress for database, skipping new backup",
|
||||
"databaseId",
|
||||
databaseID,
|
||||
database.ID,
|
||||
"existingBackupId",
|
||||
inProgressBackups[0].ID,
|
||||
)
|
||||
@@ -164,15 +166,20 @@ func (s *BackupsScheduler) StartBackup(databaseID uuid.UUID, isCallNotifier bool
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("make backup")
|
||||
backupID := uuid.New()
|
||||
timestamp := time.Now().UTC()
|
||||
|
||||
backup := &backups_core.Backup{
|
||||
ID: backupID,
|
||||
DatabaseID: backupConfig.DatabaseID,
|
||||
StorageID: *backupConfig.StorageID,
|
||||
Status: backups_core.BackupStatusInProgress,
|
||||
BackupSizeMb: 0,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
CreatedAt: timestamp,
|
||||
}
|
||||
|
||||
backup.GenerateFilename(database.Name)
|
||||
|
||||
if err := s.backupRepository.Save(backup); err != nil {
|
||||
s.logger.Error(
|
||||
"Failed to save backup",
|
||||
@@ -224,8 +231,8 @@ func (s *BackupsScheduler) StartBackup(databaseID uuid.UUID, isCallNotifier bool
|
||||
s.backupToNodeRelations[*leastBusyNodeID] = relation
|
||||
} else {
|
||||
s.backupToNodeRelations[*leastBusyNodeID] = BackupToNodeRelation{
|
||||
NodeID: *leastBusyNodeID,
|
||||
BackupsIDs: []uuid.UUID{backup.ID},
|
||||
*leastBusyNodeID,
|
||||
[]uuid.UUID{backup.ID},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -329,7 +336,13 @@ func (s *BackupsScheduler) runPendingBackups() error {
|
||||
backupConfig.BackupInterval.Interval,
|
||||
)
|
||||
|
||||
s.StartBackup(backupConfig.DatabaseID, remainedBackupTryCount == 1)
|
||||
database, err := s.databaseService.GetDatabaseByID(backupConfig.DatabaseID)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to get database by ID", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
s.StartBackup(database, remainedBackupTryCount == 1)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,7 +57,8 @@ func Test_RunPendingBackups_WhenLastBackupWasYesterday_CreatesNewBackup(t *testi
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
|
||||
@@ -126,7 +127,8 @@ func Test_RunPendingBackups_WhenLastBackupWasRecentlyCompleted_SkipsBackup(t *te
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
|
||||
@@ -194,7 +196,8 @@ func Test_RunPendingBackups_WhenLastBackupFailedAndRetriesDisabled_SkipsBackup(t
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
backupConfig.IsRetryIfFailed = false
|
||||
@@ -266,7 +269,8 @@ func Test_RunPendingBackups_WhenLastBackupFailedAndRetriesEnabled_CreatesNewBack
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
backupConfig.IsRetryIfFailed = true
|
||||
@@ -339,7 +343,8 @@ func Test_RunPendingBackups_WhenFailedBackupsExceedMaxRetries_SkipsBackup(t *tes
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
backupConfig.IsRetryIfFailed = true
|
||||
@@ -410,7 +415,8 @@ func Test_RunPendingBackups_WhenBackupsDisabled_SkipsBackup(t *testing.T) {
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = false
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
|
||||
@@ -479,7 +485,8 @@ func Test_CheckDeadNodesAndFailBackups_WhenNodeDies_FailsBackupAndCleansUpRegist
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
|
||||
@@ -492,7 +499,7 @@ func Test_CheckDeadNodesAndFailBackups_WhenNodeDies_FailsBackupAndCleansUpRegist
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Scheduler assigns backup to mock node
|
||||
GetBackupsScheduler().StartBackup(database.ID, false)
|
||||
GetBackupsScheduler().StartBackup(database, false)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
backups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
@@ -582,7 +589,8 @@ func Test_OnBackupCompleted_WhenTaskIsNotBackup_SkipsProcessing(t *testing.T) {
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
|
||||
@@ -595,7 +603,7 @@ func Test_OnBackupCompleted_WhenTaskIsNotBackup_SkipsProcessing(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Start a backup and assign it to the node
|
||||
GetBackupsScheduler().StartBackup(database.ID, false)
|
||||
GetBackupsScheduler().StartBackup(database, false)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
backups, err := backupRepository.FindByDatabaseID(database.ID)
|
||||
@@ -759,7 +767,8 @@ func Test_FailBackupsInProgress_WhenSchedulerStarts_CancelsBackupsAndUpdatesStat
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
|
||||
@@ -872,7 +881,8 @@ func Test_StartBackup_WhenBackupCompletes_DecrementsActiveTaskCount(t *testing.T
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
|
||||
@@ -892,7 +902,7 @@ func Test_StartBackup_WhenBackupCompletes_DecrementsActiveTaskCount(t *testing.T
|
||||
t.Logf("Initial active tasks: %d", initialActiveTasks)
|
||||
|
||||
// Start backup
|
||||
scheduler.StartBackup(database.ID, false)
|
||||
scheduler.StartBackup(database, false)
|
||||
|
||||
// Wait for backup to complete
|
||||
WaitForBackupCompletion(t, database.ID, 0, 10*time.Second)
|
||||
@@ -975,7 +985,8 @@ func Test_StartBackup_WhenBackupFails_DecrementsActiveTaskCount(t *testing.T) {
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
|
||||
@@ -995,7 +1006,7 @@ func Test_StartBackup_WhenBackupFails_DecrementsActiveTaskCount(t *testing.T) {
|
||||
t.Logf("Initial active tasks: %d", initialActiveTasks)
|
||||
|
||||
// Start backup
|
||||
scheduler.StartBackup(database.ID, false)
|
||||
scheduler.StartBackup(database, false)
|
||||
|
||||
// Wait for backup to fail
|
||||
WaitForBackupCompletion(t, database.ID, 0, 10*time.Second)
|
||||
@@ -1069,7 +1080,8 @@ func Test_StartBackup_WhenBackupAlreadyInProgress_SkipsNewBackup(t *testing.T) {
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
|
||||
@@ -1088,7 +1100,7 @@ func Test_StartBackup_WhenBackupAlreadyInProgress_SkipsNewBackup(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Try to start a new backup - should be skipped
|
||||
GetBackupsScheduler().StartBackup(database.ID, false)
|
||||
GetBackupsScheduler().StartBackup(database, false)
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
@@ -1140,7 +1152,8 @@ func Test_RunPendingBackups_WhenLastBackupFailedWithIsSkipRetry_SkipsBackupEvenW
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorePeriod = period.PeriodWeek
|
||||
backupConfig.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.StorageID = &storage.ID
|
||||
backupConfig.IsRetryIfFailed = true
|
||||
@@ -1242,7 +1255,8 @@ func Test_StartBackup_When2BackupsStartedForDifferentDatabases_BothUseCasesAreCa
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig1.IsBackupsEnabled = true
|
||||
backupConfig1.StorePeriod = period.PeriodWeek
|
||||
backupConfig1.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig1.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig1.Storage = storage
|
||||
backupConfig1.StorageID = &storage.ID
|
||||
|
||||
@@ -1259,7 +1273,8 @@ func Test_StartBackup_When2BackupsStartedForDifferentDatabases_BothUseCasesAreCa
|
||||
TimeOfDay: &timeOfDay,
|
||||
}
|
||||
backupConfig2.IsBackupsEnabled = true
|
||||
backupConfig2.StorePeriod = period.PeriodWeek
|
||||
backupConfig2.RetentionPolicyType = backups_config.RetentionPolicyTypeTimePeriod
|
||||
backupConfig2.RetentionTimePeriod = period.PeriodWeek
|
||||
backupConfig2.Storage = storage
|
||||
backupConfig2.StorageID = &storage.ID
|
||||
|
||||
@@ -1268,10 +1283,10 @@ func Test_StartBackup_When2BackupsStartedForDifferentDatabases_BothUseCasesAreCa
|
||||
|
||||
// Start 2 backups simultaneously
|
||||
t.Log("Starting backup for database1")
|
||||
scheduler.StartBackup(database1.ID, false)
|
||||
scheduler.StartBackup(database1, false)
|
||||
|
||||
t.Log("Starting backup for database2")
|
||||
scheduler.StartBackup(database2.ID, false)
|
||||
scheduler.StartBackup(database2, false)
|
||||
|
||||
// Wait up to 10 seconds for both backups to complete
|
||||
t.Log("Waiting for both backups to complete...")
|
||||
|
||||
@@ -1,17 +1,37 @@
|
||||
package common
|
||||
|
||||
import backups_config "databasus-backend/internal/features/backups/config"
|
||||
import (
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"errors"
|
||||
|
||||
type BackupType string
|
||||
|
||||
const (
|
||||
BackupTypeDefault BackupType = "DEFAULT" // For MySQL, MongoDB, PostgreSQL legacy (-Fc)
|
||||
BackupTypeDirectory BackupType = "DIRECTORY" // PostgreSQL directory type (-Fd)
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type BackupMetadata struct {
|
||||
EncryptionSalt *string
|
||||
EncryptionIV *string
|
||||
Encryption backups_config.BackupEncryption
|
||||
Type BackupType
|
||||
BackupID uuid.UUID `json:"backupId"`
|
||||
EncryptionSalt *string `json:"encryptionSalt"`
|
||||
EncryptionIV *string `json:"encryptionIV"`
|
||||
Encryption backups_config.BackupEncryption `json:"encryption"`
|
||||
}
|
||||
|
||||
func (m *BackupMetadata) Validate() error {
|
||||
if m.BackupID == uuid.Nil {
|
||||
return errors.New("backup ID is required")
|
||||
}
|
||||
|
||||
if m.Encryption == "" {
|
||||
return errors.New("encryption is required")
|
||||
}
|
||||
|
||||
if m.Encryption == backups_config.BackupEncryptionEncrypted {
|
||||
if m.EncryptionSalt == nil {
|
||||
return errors.New("encryption salt is required when encryption is enabled")
|
||||
}
|
||||
|
||||
if m.EncryptionIV == nil {
|
||||
return errors.New("encryption IV is required when encryption is enabled")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package backups
|
||||
package backups_controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backups_download "databasus-backend/internal/features/backups/backups/download"
|
||||
backups_dto "databasus-backend/internal/features/backups/backups/dto"
|
||||
backups_services "databasus-backend/internal/features/backups/backups/services"
|
||||
"databasus-backend/internal/features/databases"
|
||||
users_middleware "databasus-backend/internal/features/users/middleware"
|
||||
files_utils "databasus-backend/internal/util/files"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -16,7 +19,7 @@ import (
|
||||
)
|
||||
|
||||
type BackupController struct {
|
||||
backupService *BackupService
|
||||
backupService *backups_services.BackupService
|
||||
}
|
||||
|
||||
func (c *BackupController) RegisterRoutes(router *gin.RouterGroup) {
|
||||
@@ -41,7 +44,7 @@ func (c *BackupController) RegisterPublicRoutes(router *gin.RouterGroup) {
|
||||
// @Param database_id query string true "Database ID"
|
||||
// @Param limit query int false "Number of items per page" default(10)
|
||||
// @Param offset query int false "Offset for pagination" default(0)
|
||||
// @Success 200 {object} GetBackupsResponse
|
||||
// @Success 200 {object} backups_dto.GetBackupsResponse
|
||||
// @Failure 400
|
||||
// @Failure 401
|
||||
// @Failure 500
|
||||
@@ -53,7 +56,7 @@ func (c *BackupController) GetBackups(ctx *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
var request GetBackupsRequest
|
||||
var request backups_dto.GetBackupsRequest
|
||||
if err := ctx.ShouldBindQuery(&request); err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
@@ -80,7 +83,7 @@ func (c *BackupController) GetBackups(ctx *gin.Context) {
|
||||
// @Tags backups
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body MakeBackupRequest true "Backup creation data"
|
||||
// @Param request body backups_dto.MakeBackupRequest true "Backup creation data"
|
||||
// @Success 200 {object} map[string]string
|
||||
// @Failure 400
|
||||
// @Failure 401
|
||||
@@ -93,7 +96,7 @@ func (c *BackupController) MakeBackup(ctx *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
var request MakeBackupRequest
|
||||
var request backups_dto.MakeBackupRequest
|
||||
if err := ctx.ShouldBindJSON(&request); err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
@@ -304,16 +307,11 @@ func (c *BackupController) GetFile(ctx *gin.Context) {
|
||||
_, err = io.Copy(ctx.Writer, rateLimitedReader)
|
||||
if err != nil {
|
||||
fmt.Printf("Error streaming file: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
c.backupService.WriteAuditLogForDownload(downloadToken.UserID, backup, database)
|
||||
}
|
||||
|
||||
type MakeBackupRequest struct {
|
||||
DatabaseID uuid.UUID `json:"database_id" binding:"required"`
|
||||
}
|
||||
|
||||
func (c *BackupController) generateBackupFilename(
|
||||
backup *backups_core.Backup,
|
||||
database *databases.Database,
|
||||
@@ -322,7 +320,7 @@ func (c *BackupController) generateBackupFilename(
|
||||
timestamp := backup.CreatedAt.Format("2006-01-02_15-04-05")
|
||||
|
||||
// Sanitize database name for filename (replace spaces and special chars)
|
||||
safeName := sanitizeFilename(database.Name)
|
||||
safeName := files_utils.SanitizeFilename(database.Name)
|
||||
|
||||
// Determine extension based on database type
|
||||
extension := c.getBackupExtension(database.Type)
|
||||
@@ -346,33 +344,6 @@ func (c *BackupController) getBackupExtension(
|
||||
}
|
||||
}
|
||||
|
||||
func sanitizeFilename(name string) string {
|
||||
// Replace characters that are invalid in filenames
|
||||
replacer := map[rune]rune{
|
||||
' ': '_',
|
||||
'/': '-',
|
||||
'\\': '-',
|
||||
':': '-',
|
||||
'*': '-',
|
||||
'?': '-',
|
||||
'"': '-',
|
||||
'<': '-',
|
||||
'>': '-',
|
||||
'|': '-',
|
||||
}
|
||||
|
||||
result := make([]rune, 0, len(name))
|
||||
for _, char := range name {
|
||||
if replacement, exists := replacer[char]; exists {
|
||||
result = append(result, replacement)
|
||||
} else {
|
||||
result = append(result, char)
|
||||
}
|
||||
}
|
||||
|
||||
return string(result)
|
||||
}
|
||||
|
||||
func (c *BackupController) startDownloadHeartbeat(ctx context.Context, userID uuid.UUID) {
|
||||
ticker := time.NewTicker(backups_download.GetDownloadHeartbeatInterval())
|
||||
defer ticker.Stop()
|
||||
@@ -1,4 +1,4 @@
|
||||
package backups
|
||||
package backups_controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -18,13 +20,18 @@ import (
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
audit_logs "databasus-backend/internal/features/audit_logs"
|
||||
"databasus-backend/internal/features/backups/backups/backuping"
|
||||
backups_common "databasus-backend/internal/features/backups/backups/common"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backups_download "databasus-backend/internal/features/backups/backups/download"
|
||||
backups_dto "databasus-backend/internal/features/backups/backups/dto"
|
||||
backups_services "databasus-backend/internal/features/backups/backups/services"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/databases/databases/postgresql"
|
||||
"databasus-backend/internal/features/storages"
|
||||
local_storage "databasus-backend/internal/features/storages/models/local"
|
||||
task_cancellation "databasus-backend/internal/features/tasks/cancellation"
|
||||
users_dto "databasus-backend/internal/features/users/dto"
|
||||
users_enums "databasus-backend/internal/features/users/enums"
|
||||
users_services "databasus-backend/internal/features/users/services"
|
||||
@@ -32,6 +39,7 @@ import (
|
||||
workspaces_models "databasus-backend/internal/features/workspaces/models"
|
||||
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
|
||||
"databasus-backend/internal/util/encryption"
|
||||
files_utils "databasus-backend/internal/util/files"
|
||||
test_utils "databasus-backend/internal/util/testing"
|
||||
"databasus-backend/internal/util/tools"
|
||||
)
|
||||
@@ -114,7 +122,7 @@ func Test_GetBackups_PermissionsEnforced(t *testing.T) {
|
||||
)
|
||||
|
||||
if tt.expectSuccess {
|
||||
var response GetBackupsResponse
|
||||
var response backups_dto.GetBackupsResponse
|
||||
err := json.Unmarshal(testResp.Body, &response)
|
||||
assert.NoError(t, err)
|
||||
assert.GreaterOrEqual(t, len(response.Backups), 1)
|
||||
@@ -209,7 +217,7 @@ func Test_CreateBackup_PermissionsEnforced(t *testing.T) {
|
||||
testUserToken = nonMember.Token
|
||||
}
|
||||
|
||||
request := MakeBackupRequest{DatabaseID: database.ID}
|
||||
request := backups_dto.MakeBackupRequest{DatabaseID: database.ID}
|
||||
testResp := test_utils.MakePostRequest(
|
||||
t,
|
||||
router,
|
||||
@@ -240,7 +248,7 @@ func Test_CreateBackup_AuditLogWritten(t *testing.T) {
|
||||
database := createTestDatabase("Test Database", workspace.ID, owner.Token, router)
|
||||
enableBackupForDatabase(database.ID)
|
||||
|
||||
request := MakeBackupRequest{DatabaseID: database.ID}
|
||||
request := backups_dto.MakeBackupRequest{DatabaseID: database.ID}
|
||||
test_utils.MakePostRequest(
|
||||
t,
|
||||
router,
|
||||
@@ -368,7 +376,7 @@ func Test_DeleteBackup_PermissionsEnforced(t *testing.T) {
|
||||
ownerUser, err := userService.GetUserFromToken(owner.Token)
|
||||
assert.NoError(t, err)
|
||||
|
||||
response, err := GetBackupService().GetBackups(ownerUser, database.ID, 10, 0)
|
||||
response, err := backups_services.GetBackupService().GetBackups(ownerUser, database.ID, 10, 0)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 0, len(response.Backups))
|
||||
}
|
||||
@@ -956,7 +964,7 @@ func Test_SanitizeFilename(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
result := sanitizeFilename(tt.input)
|
||||
result := files_utils.SanitizeFilename(tt.input)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
@@ -994,7 +1002,7 @@ func Test_CancelBackup_InProgressBackup_SuccessfullyCancelled(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Register a cancellable context for the backup
|
||||
GetBackupService().taskCancelManager.RegisterTask(backup.ID, func() {})
|
||||
task_cancellation.GetTaskCancelManager().RegisterTask(backup.ID, func() {})
|
||||
|
||||
resp := test_utils.MakePostRequest(
|
||||
t,
|
||||
@@ -1086,7 +1094,7 @@ func Test_ConcurrentDownloadPrevention(t *testing.T) {
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
service := GetBackupService()
|
||||
service := backups_services.GetBackupService()
|
||||
if !service.IsDownloadInProgress(owner.UserID) {
|
||||
t.Log("Warning: First download completed before we could test concurrency")
|
||||
<-downloadComplete
|
||||
@@ -1187,7 +1195,7 @@ func Test_GenerateDownloadToken_BlockedWhenDownloadInProgress(t *testing.T) {
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
service := GetBackupService()
|
||||
service := backups_services.GetBackupService()
|
||||
if !service.IsDownloadInProgress(owner.UserID) {
|
||||
t.Log("Warning: First download completed before we could test token generation blocking")
|
||||
<-downloadComplete
|
||||
@@ -1244,6 +1252,86 @@ func Test_GenerateDownloadToken_BlockedWhenDownloadInProgress(t *testing.T) {
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}
|
||||
|
||||
func Test_MakeBackup_VerifyBackupAndMetadataFilesExistInStorage(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
|
||||
database, _, storage := createTestDatabaseWithBackups(workspace, owner, router)
|
||||
|
||||
backuperNode := backuping.CreateTestBackuperNode()
|
||||
backuperCancel := backuping.StartBackuperNodeForTest(t, backuperNode)
|
||||
defer backuping.StopBackuperNodeForTest(t, backuperCancel, backuperNode)
|
||||
|
||||
scheduler := backuping.CreateTestScheduler()
|
||||
schedulerCancel := backuping.StartSchedulerForTest(t, scheduler)
|
||||
defer schedulerCancel()
|
||||
|
||||
backupRepo := &backups_core.BackupRepository{}
|
||||
initialBackups, err := backupRepo.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
request := backups_dto.MakeBackupRequest{DatabaseID: database.ID}
|
||||
test_utils.MakePostRequest(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/backups",
|
||||
"Bearer "+owner.Token,
|
||||
request,
|
||||
http.StatusOK,
|
||||
)
|
||||
|
||||
backuping.WaitForBackupCompletion(t, database.ID, len(initialBackups), 30*time.Second)
|
||||
|
||||
backups, err := backupRepo.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Greater(t, len(backups), len(initialBackups))
|
||||
|
||||
backup := backups[0]
|
||||
assert.Equal(t, backups_core.BackupStatusCompleted, backup.Status)
|
||||
|
||||
storageService := storages.GetStorageService()
|
||||
backupStorage, err := storageService.GetStorageByID(backup.StorageID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
encryptor := encryption.GetFieldEncryptor()
|
||||
|
||||
backupFile, err := backupStorage.GetFile(encryptor, backup.FileName)
|
||||
assert.NoError(t, err)
|
||||
backupFile.Close()
|
||||
|
||||
metadataFile, err := backupStorage.GetFile(encryptor, backup.FileName+".metadata")
|
||||
assert.NoError(t, err)
|
||||
|
||||
metadataContent, err := io.ReadAll(metadataFile)
|
||||
assert.NoError(t, err)
|
||||
metadataFile.Close()
|
||||
|
||||
var storageMetadata backups_common.BackupMetadata
|
||||
err = json.Unmarshal(metadataContent, &storageMetadata)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, backup.ID, storageMetadata.BackupID)
|
||||
|
||||
if backup.EncryptionSalt != nil && storageMetadata.EncryptionSalt != nil {
|
||||
assert.Equal(t, *backup.EncryptionSalt, *storageMetadata.EncryptionSalt)
|
||||
}
|
||||
|
||||
if backup.EncryptionIV != nil && storageMetadata.EncryptionIV != nil {
|
||||
assert.Equal(t, *backup.EncryptionIV, *storageMetadata.EncryptionIV)
|
||||
}
|
||||
|
||||
assert.Equal(t, backup.Encryption, storageMetadata.Encryption)
|
||||
|
||||
err = backupRepo.DeleteByID(backup.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
databases.RemoveTestDatabase(database)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
storages.RemoveTestStorage(storage.ID)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}
|
||||
|
||||
func createTestRouter() *gin.Engine {
|
||||
return CreateTestRouter()
|
||||
}
|
||||
@@ -1366,11 +1454,24 @@ func createTestBackup(
|
||||
panic(err)
|
||||
}
|
||||
|
||||
storages, err := storages.GetStorageService().GetStorages(user, *database.WorkspaceID)
|
||||
if err != nil || len(storages) == 0 {
|
||||
loadedStorages, err := storages.GetStorageService().GetStorages(user, *database.WorkspaceID)
|
||||
if err != nil || len(loadedStorages) == 0 {
|
||||
panic("No storage found for workspace")
|
||||
}
|
||||
|
||||
// Filter out system storages
|
||||
var nonSystemStorages []*storages.Storage
|
||||
for _, storage := range loadedStorages {
|
||||
if !storage.IsSystem {
|
||||
nonSystemStorages = append(nonSystemStorages, storage)
|
||||
}
|
||||
}
|
||||
if len(nonSystemStorages) == 0 {
|
||||
panic("No non-system storage found for workspace")
|
||||
}
|
||||
|
||||
storages := nonSystemStorages
|
||||
|
||||
backup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
@@ -1394,7 +1495,7 @@ func createTestBackup(
|
||||
context.Background(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
logger,
|
||||
backup.ID,
|
||||
backup.ID.String(),
|
||||
reader,
|
||||
); err != nil {
|
||||
panic(fmt.Sprintf("Failed to create test backup file: %v", err))
|
||||
@@ -1404,7 +1505,7 @@ func createTestBackup(
|
||||
}
|
||||
|
||||
func createExpiredDownloadToken(backupID, userID uuid.UUID) string {
|
||||
tokenService := GetBackupService().downloadTokenService
|
||||
tokenService := backups_download.GetDownloadTokenService()
|
||||
token, err := tokenService.Generate(backupID, userID)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Failed to generate download token: %v", err))
|
||||
@@ -1707,3 +1808,84 @@ func Test_BandwidthThrottling_DynamicAdjustment(t *testing.T) {
|
||||
storages.RemoveTestStorage(storage.ID)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}
|
||||
|
||||
func Test_DeleteBackup_RemovesBackupAndMetadataFilesFromDisk(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
|
||||
database := createTestDatabase("Test Database", workspace.ID, owner.Token, router)
|
||||
storage := createTestStorage(workspace.ID)
|
||||
|
||||
configService := backups_config.GetBackupConfigService()
|
||||
backupConfig, err := configService.GetBackupConfigByDbId(database.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.StorageID = &storage.ID
|
||||
backupConfig.Storage = storage
|
||||
_, err = configService.SaveBackupConfig(backupConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
databases.RemoveTestDatabase(database)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
storages.RemoveTestStorage(storage.ID)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}()
|
||||
|
||||
backuperNode := backuping.CreateTestBackuperNode()
|
||||
backuperCancel := backuping.StartBackuperNodeForTest(t, backuperNode)
|
||||
defer backuping.StopBackuperNodeForTest(t, backuperCancel, backuperNode)
|
||||
|
||||
scheduler := backuping.CreateTestScheduler()
|
||||
schedulerCancel := backuping.StartSchedulerForTest(t, scheduler)
|
||||
defer schedulerCancel()
|
||||
|
||||
backupRepo := &backups_core.BackupRepository{}
|
||||
initialBackups, err := backupRepo.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
request := backups_dto.MakeBackupRequest{DatabaseID: database.ID}
|
||||
test_utils.MakePostRequest(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/backups",
|
||||
"Bearer "+owner.Token,
|
||||
request,
|
||||
http.StatusOK,
|
||||
)
|
||||
|
||||
backuping.WaitForBackupCompletion(t, database.ID, len(initialBackups), 30*time.Second)
|
||||
|
||||
backups, err := backupRepo.FindByDatabaseID(database.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Greater(t, len(backups), len(initialBackups))
|
||||
|
||||
backup := backups[0]
|
||||
assert.Equal(t, backups_core.BackupStatusCompleted, backup.Status)
|
||||
|
||||
dataFolder := config.GetEnv().DataFolder
|
||||
backupFilePath := filepath.Join(dataFolder, backup.FileName)
|
||||
metadataFilePath := filepath.Join(dataFolder, backup.FileName+".metadata")
|
||||
|
||||
_, err = os.Stat(backupFilePath)
|
||||
assert.NoError(t, err, "backup file should exist on disk before deletion")
|
||||
|
||||
_, err = os.Stat(metadataFilePath)
|
||||
assert.NoError(t, err, "metadata file should exist on disk before deletion")
|
||||
|
||||
test_utils.MakeDeleteRequest(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/backups/%s", backup.ID.String()),
|
||||
"Bearer "+owner.Token,
|
||||
http.StatusNoContent,
|
||||
)
|
||||
|
||||
_, err = os.Stat(backupFilePath)
|
||||
assert.True(t, os.IsNotExist(err), "backup file should be removed from disk after deletion")
|
||||
|
||||
_, err = os.Stat(metadataFilePath)
|
||||
assert.True(t, os.IsNotExist(err), "metadata file should be removed from disk after deletion")
|
||||
}
|
||||
23
backend/internal/features/backups/backups/controllers/di.go
Normal file
23
backend/internal/features/backups/backups/controllers/di.go
Normal file
@@ -0,0 +1,23 @@
|
||||
package backups_controllers
|
||||
|
||||
import (
|
||||
backups_services "databasus-backend/internal/features/backups/backups/services"
|
||||
"databasus-backend/internal/features/databases"
|
||||
)
|
||||
|
||||
var backupController = &BackupController{
|
||||
backups_services.GetBackupService(),
|
||||
}
|
||||
|
||||
func GetBackupController() *BackupController {
|
||||
return backupController
|
||||
}
|
||||
|
||||
var postgresWalBackupController = &PostgreWalBackupController{
|
||||
databases.GetDatabaseService(),
|
||||
backups_services.GetWalService(),
|
||||
}
|
||||
|
||||
func GetPostgresWalBackupController() *PostgreWalBackupController {
|
||||
return postgresWalBackupController
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
package backups_controllers
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backups_dto "databasus-backend/internal/features/backups/backups/dto"
|
||||
backups_services "databasus-backend/internal/features/backups/backups/services"
|
||||
"databasus-backend/internal/features/databases"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// PostgreWalBackupController handles WAL backup endpoints used by the databasus-cli agent.
|
||||
// Authentication is via a plain agent token in the Authorization header (no Bearer prefix).
|
||||
type PostgreWalBackupController struct {
|
||||
databaseService *databases.DatabaseService
|
||||
walService *backups_services.PostgreWalBackupService
|
||||
}
|
||||
|
||||
func (c *PostgreWalBackupController) RegisterRoutes(router *gin.RouterGroup) {
|
||||
walRoutes := router.Group("/backups/postgres/wal")
|
||||
|
||||
walRoutes.GET("/next-full-backup-time", c.GetNextFullBackupTime)
|
||||
walRoutes.POST("/error", c.ReportError)
|
||||
walRoutes.POST("/upload", c.Upload)
|
||||
walRoutes.GET("/restore/plan", c.GetRestorePlan)
|
||||
walRoutes.GET("/restore/download", c.DownloadBackupFile)
|
||||
}
|
||||
|
||||
// GetNextFullBackupTime
|
||||
// @Summary Get next full backup time
|
||||
// @Description Returns the next scheduled full basebackup time for the authenticated database
|
||||
// @Tags backups-wal
|
||||
// @Produce json
|
||||
// @Security AgentToken
|
||||
// @Success 200 {object} backups_dto.GetNextFullBackupTimeResponse
|
||||
// @Failure 401 {object} map[string]string
|
||||
// @Failure 500 {object} map[string]string
|
||||
// @Router /backups/postgres/wal/next-full-backup-time [get]
|
||||
func (c *PostgreWalBackupController) GetNextFullBackupTime(ctx *gin.Context) {
|
||||
database, err := c.getDatabase(ctx)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})
|
||||
return
|
||||
}
|
||||
|
||||
response, err := c.walService.GetNextFullBackupTime(database)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
ctx.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// ReportError
|
||||
// @Summary Report agent error
|
||||
// @Description Records a fatal error from the agent against the database record and marks it as errored
|
||||
// @Tags backups-wal
|
||||
// @Accept json
|
||||
// @Security AgentToken
|
||||
// @Param request body backups_dto.ReportErrorRequest true "Error details"
|
||||
// @Success 200
|
||||
// @Failure 400 {object} map[string]string
|
||||
// @Failure 401 {object} map[string]string
|
||||
// @Failure 500 {object} map[string]string
|
||||
// @Router /backups/postgres/wal/error [post]
|
||||
func (c *PostgreWalBackupController) ReportError(ctx *gin.Context) {
|
||||
database, err := c.getDatabase(ctx)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})
|
||||
return
|
||||
}
|
||||
|
||||
var request backups_dto.ReportErrorRequest
|
||||
if err := ctx.ShouldBindJSON(&request); err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if err := c.walService.ReportError(database, request.Error); err != nil {
|
||||
ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
ctx.Status(http.StatusOK)
|
||||
}
|
||||
|
||||
// Upload
|
||||
// @Summary Stream upload a basebackup or WAL segment
|
||||
// @Description Accepts a zstd-compressed binary stream and stores it in the database's configured storage.
|
||||
// The server generates the storage filename; agents do not control the destination path.
|
||||
// For WAL segment uploads the server validates the WAL chain and returns 409 if a gap is detected
|
||||
// or 400 if no full backup exists yet (agent should trigger a full basebackup in both cases).
|
||||
// @Tags backups-wal
|
||||
// @Accept application/octet-stream
|
||||
// @Produce json
|
||||
// @Security AgentToken
|
||||
// @Param X-Upload-Type header string true "Upload type" Enums(basebackup, wal)
|
||||
// @Param X-Wal-Segment-Name header string false "24-hex WAL segment identifier (required for wal uploads, e.g. 0000000100000001000000AB)"
|
||||
// @Param X-Wal-Segment-Size header int false "WAL segment size in bytes reported by the PostgreSQL instance (default: 16777216)"
|
||||
// @Param fullBackupWalStartSegment query string false "First WAL segment needed to make the basebackup consistent (required for basebackup uploads)"
|
||||
// @Param fullBackupWalStopSegment query string false "Last WAL segment included in the basebackup (required for basebackup uploads)"
|
||||
// @Success 204
|
||||
// @Failure 400 {object} backups_dto.UploadGapResponse "No full backup exists (error: no_full_backup)"
|
||||
// @Failure 401 {object} map[string]string
|
||||
// @Failure 409 {object} backups_dto.UploadGapResponse "WAL chain gap detected (error: gap_detected)"
|
||||
// @Failure 500 {object} map[string]string
|
||||
// @Router /backups/postgres/wal/upload [post]
|
||||
func (c *PostgreWalBackupController) Upload(ctx *gin.Context) {
|
||||
database, err := c.getDatabase(ctx)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})
|
||||
return
|
||||
}
|
||||
|
||||
uploadType := backups_core.PgWalUploadType(ctx.GetHeader("X-Upload-Type"))
|
||||
if uploadType != backups_core.PgWalUploadTypeBasebackup &&
|
||||
uploadType != backups_core.PgWalUploadTypeWal {
|
||||
ctx.JSON(
|
||||
http.StatusBadRequest,
|
||||
gin.H{"error": "X-Upload-Type must be 'basebackup' or 'wal'"},
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
walSegmentName := ""
|
||||
if uploadType == backups_core.PgWalUploadTypeWal {
|
||||
walSegmentName = ctx.GetHeader("X-Wal-Segment-Name")
|
||||
if walSegmentName == "" {
|
||||
ctx.JSON(
|
||||
http.StatusBadRequest,
|
||||
gin.H{"error": "X-Wal-Segment-Name is required for wal uploads"},
|
||||
)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if uploadType == backups_core.PgWalUploadTypeBasebackup {
|
||||
if ctx.Query("fullBackupWalStartSegment") == "" ||
|
||||
ctx.Query("fullBackupWalStopSegment") == "" {
|
||||
ctx.JSON(
|
||||
http.StatusBadRequest,
|
||||
gin.H{
|
||||
"error": "fullBackupWalStartSegment and fullBackupWalStopSegment are required for basebackup uploads",
|
||||
},
|
||||
)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
walSegmentSizeBytes := int64(0)
|
||||
if raw := ctx.GetHeader("X-Wal-Segment-Size"); raw != "" {
|
||||
parsed, parseErr := strconv.ParseInt(raw, 10, 64)
|
||||
if parseErr != nil || parsed <= 0 {
|
||||
ctx.JSON(
|
||||
http.StatusBadRequest,
|
||||
gin.H{"error": "X-Wal-Segment-Size must be a positive integer"},
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
walSegmentSizeBytes = parsed
|
||||
}
|
||||
|
||||
gapResp, uploadErr := c.walService.UploadWal(
|
||||
ctx.Request.Context(),
|
||||
database,
|
||||
uploadType,
|
||||
walSegmentName,
|
||||
ctx.Query("fullBackupWalStartSegment"),
|
||||
ctx.Query("fullBackupWalStopSegment"),
|
||||
walSegmentSizeBytes,
|
||||
ctx.Request.Body,
|
||||
)
|
||||
|
||||
if uploadErr != nil {
|
||||
ctx.JSON(http.StatusInternalServerError, gin.H{"error": uploadErr.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if gapResp != nil {
|
||||
if gapResp.Error == "no_full_backup" {
|
||||
ctx.JSON(http.StatusBadRequest, gapResp)
|
||||
return
|
||||
}
|
||||
|
||||
ctx.JSON(http.StatusConflict, gapResp)
|
||||
return
|
||||
}
|
||||
|
||||
ctx.Status(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// GetRestorePlan
|
||||
// @Summary Get restore plan
|
||||
// @Description Resolves the full backup and all required WAL segments needed for recovery. Validates the WAL chain is continuous.
|
||||
// @Tags backups-wal
|
||||
// @Produce json
|
||||
// @Security AgentToken
|
||||
// @Param backupId query string false "UUID of a specific full backup to restore from; defaults to the most recent"
|
||||
// @Success 200 {object} backups_dto.GetRestorePlanResponse
|
||||
// @Failure 400 {object} map[string]string "Broken WAL chain or no backups available"
|
||||
// @Failure 401 {object} map[string]string
|
||||
// @Failure 500 {object} map[string]string
|
||||
// @Router /backups/postgres/wal/restore/plan [get]
|
||||
func (c *PostgreWalBackupController) GetRestorePlan(ctx *gin.Context) {
|
||||
database, err := c.getDatabase(ctx)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})
|
||||
return
|
||||
}
|
||||
|
||||
var backupID *uuid.UUID
|
||||
if raw := ctx.Query("backupId"); raw != "" {
|
||||
parsed, parseErr := uuid.Parse(raw)
|
||||
if parseErr != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": "invalid backupId format"})
|
||||
return
|
||||
}
|
||||
|
||||
backupID = &parsed
|
||||
}
|
||||
|
||||
response, planErr, err := c.walService.GetRestorePlan(database, backupID)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if planErr != nil {
|
||||
ctx.JSON(http.StatusBadRequest, planErr)
|
||||
return
|
||||
}
|
||||
|
||||
ctx.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// DownloadBackupFile
|
||||
// @Summary Download a backup or WAL segment file for restore
|
||||
// @Description Retrieves the backup file by ID (validated against the authenticated database), decrypts it server-side if encrypted, and streams the zstd-compressed result to the agent
|
||||
// @Tags backups-wal
|
||||
// @Produce application/octet-stream
|
||||
// @Security AgentToken
|
||||
// @Param backupId query string true "Backup ID from the restore plan response"
|
||||
// @Success 200 {file} file
|
||||
// @Failure 400 {object} map[string]string
|
||||
// @Failure 401 {object} map[string]string
|
||||
// @Router /backups/postgres/wal/restore/download [get]
|
||||
func (c *PostgreWalBackupController) DownloadBackupFile(ctx *gin.Context) {
|
||||
database, err := c.getDatabase(ctx)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "invalid agent token"})
|
||||
return
|
||||
}
|
||||
|
||||
backupIDRaw := ctx.Query("backupId")
|
||||
if backupIDRaw == "" {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": "backupId is required"})
|
||||
return
|
||||
}
|
||||
|
||||
backupID, err := uuid.Parse(backupIDRaw)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": "invalid backupId format"})
|
||||
return
|
||||
}
|
||||
|
||||
reader, err := c.walService.DownloadBackupFile(database, backupID)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
ctx.Header("Content-Type", "application/octet-stream")
|
||||
ctx.Status(http.StatusOK)
|
||||
|
||||
_, _ = io.Copy(ctx.Writer, reader)
|
||||
}
|
||||
|
||||
func (c *PostgreWalBackupController) getDatabase(
|
||||
ctx *gin.Context,
|
||||
) (*databases.Database, error) {
|
||||
token := ctx.GetHeader("Authorization")
|
||||
return c.databaseService.GetDatabaseByAgentToken(token)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
package backups
|
||||
package backups_controllers
|
||||
|
||||
import (
|
||||
"testing"
|
||||
@@ -41,7 +41,7 @@ func WaitForBackupCompletion(
|
||||
deadline := time.Now().UTC().Add(timeout)
|
||||
|
||||
for time.Now().UTC().Before(deadline) {
|
||||
backups, err := backupRepository.FindByDatabaseID(databaseID)
|
||||
backups, err := backups_core.GetBackupRepository().FindByDatabaseID(databaseID)
|
||||
if err != nil {
|
||||
t.Logf("WaitForBackupCompletion: error finding backups: %v", err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
7
backend/internal/features/backups/backups/core/di.go
Normal file
7
backend/internal/features/backups/backups/core/di.go
Normal file
@@ -0,0 +1,7 @@
|
||||
package backups_core
|
||||
|
||||
var backupRepository = &BackupRepository{}
|
||||
|
||||
func GetBackupRepository() *BackupRepository {
|
||||
return backupRepository
|
||||
}
|
||||
@@ -8,3 +8,10 @@ const (
|
||||
BackupStatusFailed BackupStatus = "FAILED"
|
||||
BackupStatusCanceled BackupStatus = "CANCELED"
|
||||
)
|
||||
|
||||
type PgWalUploadType string
|
||||
|
||||
const (
|
||||
PgWalUploadTypeBasebackup PgWalUploadType = "basebackup"
|
||||
PgWalUploadTypeWal PgWalUploadType = "wal"
|
||||
)
|
||||
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/notifiers"
|
||||
"databasus-backend/internal/features/storages"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type NotificationSender interface {
|
||||
@@ -23,7 +21,7 @@ type NotificationSender interface {
|
||||
type CreateBackupUsecase interface {
|
||||
Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
database *databases.Database,
|
||||
storage *storages.Storage,
|
||||
|
||||
@@ -1,14 +1,25 @@
|
||||
package backups_core
|
||||
|
||||
import (
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
files_utils "databasus-backend/internal/util/files"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type PgWalBackupType string
|
||||
|
||||
const (
|
||||
PgWalBackupTypeFullBackup PgWalBackupType = "PG_FULL_BACKUP"
|
||||
PgWalBackupTypeWalSegment PgWalBackupType = "PG_WAL_SEGMENT"
|
||||
)
|
||||
|
||||
type Backup struct {
|
||||
ID uuid.UUID `json:"id" gorm:"column:id;type:uuid;primaryKey"`
|
||||
ID uuid.UUID `json:"id" gorm:"column:id;type:uuid;primaryKey"`
|
||||
FileName string `json:"fileName" gorm:"column:file_name;type:text;not null"`
|
||||
|
||||
DatabaseID uuid.UUID `json:"databaseId" gorm:"column:database_id;type:uuid;not null"`
|
||||
StorageID uuid.UUID `json:"storageId" gorm:"column:storage_id;type:uuid;not null"`
|
||||
@@ -25,5 +36,23 @@ type Backup struct {
|
||||
EncryptionIV *string `json:"-" gorm:"column:encryption_iv"`
|
||||
Encryption backups_config.BackupEncryption `json:"encryption" gorm:"column:encryption;type:text;not null;default:'NONE'"`
|
||||
|
||||
// Postgres WAL backup specific fields
|
||||
PgWalBackupType *PgWalBackupType `json:"pgWalBackupType" gorm:"column:pg_wal_backup_type;type:text"`
|
||||
PgFullBackupWalStartSegmentName *string `json:"pgFullBackupWalStartSegmentName" gorm:"column:pg_wal_start_segment;type:text"`
|
||||
PgFullBackupWalStopSegmentName *string `json:"pgFullBackupWalStopSegmentName" gorm:"column:pg_wal_stop_segment;type:text"`
|
||||
PgVersion *string `json:"pgVersion" gorm:"column:pg_version;type:text"`
|
||||
PgWalSegmentName *string `json:"pgWalSegmentName" gorm:"column:pg_wal_segment_name;type:text"`
|
||||
|
||||
CreatedAt time.Time `json:"createdAt" gorm:"column:created_at"`
|
||||
}
|
||||
|
||||
func (b *Backup) GenerateFilename(dbName string) {
|
||||
timestamp := time.Now().UTC()
|
||||
|
||||
b.FileName = fmt.Sprintf(
|
||||
"%s-%s-%s",
|
||||
files_utils.SanitizeFilename(dbName),
|
||||
timestamp.Format("20060102-150405"),
|
||||
b.ID.String(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -245,3 +245,134 @@ func (r *BackupRepository) FindOldestByDatabaseExcludingInProgress(
|
||||
|
||||
return backups, nil
|
||||
}
|
||||
|
||||
func (r *BackupRepository) FindCompletedFullWalBackupByID(
|
||||
databaseID uuid.UUID,
|
||||
backupID uuid.UUID,
|
||||
) (*Backup, error) {
|
||||
var backup Backup
|
||||
|
||||
err := storage.
|
||||
GetDb().
|
||||
Where(
|
||||
"database_id = ? AND id = ? AND pg_wal_backup_type = ? AND status = ?",
|
||||
databaseID,
|
||||
backupID,
|
||||
PgWalBackupTypeFullBackup,
|
||||
BackupStatusCompleted,
|
||||
).
|
||||
First(&backup).Error
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &backup, nil
|
||||
}
|
||||
|
||||
func (r *BackupRepository) FindCompletedWalSegmentsAfter(
|
||||
databaseID uuid.UUID,
|
||||
afterSegmentName string,
|
||||
) ([]*Backup, error) {
|
||||
var backups []*Backup
|
||||
|
||||
err := storage.
|
||||
GetDb().
|
||||
Where(
|
||||
"database_id = ? AND pg_wal_backup_type = ? AND pg_wal_segment_name >= ? AND status = ?",
|
||||
databaseID,
|
||||
PgWalBackupTypeWalSegment,
|
||||
afterSegmentName,
|
||||
BackupStatusCompleted,
|
||||
).
|
||||
Order("pg_wal_segment_name ASC").
|
||||
Find(&backups).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return backups, nil
|
||||
}
|
||||
|
||||
func (r *BackupRepository) FindLastCompletedFullWalBackupByDatabaseID(
|
||||
databaseID uuid.UUID,
|
||||
) (*Backup, error) {
|
||||
var backup Backup
|
||||
|
||||
err := storage.
|
||||
GetDb().
|
||||
Where(
|
||||
"database_id = ? AND pg_wal_backup_type = ? AND status = ?",
|
||||
databaseID,
|
||||
PgWalBackupTypeFullBackup,
|
||||
BackupStatusCompleted,
|
||||
).
|
||||
Order("created_at DESC").
|
||||
First(&backup).Error
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &backup, nil
|
||||
}
|
||||
|
||||
func (r *BackupRepository) FindWalSegmentByName(
|
||||
databaseID uuid.UUID,
|
||||
segmentName string,
|
||||
) (*Backup, error) {
|
||||
var backup Backup
|
||||
|
||||
err := storage.
|
||||
GetDb().
|
||||
Where(
|
||||
"database_id = ? AND pg_wal_backup_type = ? AND pg_wal_segment_name = ?",
|
||||
databaseID,
|
||||
PgWalBackupTypeWalSegment,
|
||||
segmentName,
|
||||
).
|
||||
First(&backup).Error
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &backup, nil
|
||||
}
|
||||
|
||||
func (r *BackupRepository) FindLastWalSegmentAfter(
|
||||
databaseID uuid.UUID,
|
||||
afterSegmentName string,
|
||||
) (*Backup, error) {
|
||||
var backup Backup
|
||||
|
||||
err := storage.
|
||||
GetDb().
|
||||
Where(
|
||||
"database_id = ? AND pg_wal_backup_type = ? AND pg_wal_segment_name > ? AND status = ?",
|
||||
databaseID,
|
||||
PgWalBackupTypeWalSegment,
|
||||
afterSegmentName,
|
||||
BackupStatusCompleted,
|
||||
).
|
||||
Order("pg_wal_segment_name DESC").
|
||||
First(&backup).Error
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &backup, nil
|
||||
}
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package backups
|
||||
|
||||
import (
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
"databasus-backend/internal/features/backups/backups/encryption"
|
||||
"io"
|
||||
)
|
||||
|
||||
type GetBackupsRequest struct {
|
||||
DatabaseID string `form:"database_id" binding:"required"`
|
||||
Limit int `form:"limit"`
|
||||
Offset int `form:"offset"`
|
||||
}
|
||||
|
||||
type GetBackupsResponse struct {
|
||||
Backups []*backups_core.Backup `json:"backups"`
|
||||
Total int64 `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
type DecryptionReaderCloser struct {
|
||||
*encryption.DecryptionReader
|
||||
BaseReader io.ReadCloser
|
||||
}
|
||||
|
||||
func (r *DecryptionReaderCloser) Close() error {
|
||||
return r.BaseReader.Close()
|
||||
}
|
||||
78
backend/internal/features/backups/backups/dto/dto.go
Normal file
78
backend/internal/features/backups/backups/dto/dto.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package backups_dto
|
||||
|
||||
import (
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
"databasus-backend/internal/features/backups/backups/encryption"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type GetBackupsRequest struct {
|
||||
DatabaseID string `form:"database_id" binding:"required"`
|
||||
Limit int `form:"limit"`
|
||||
Offset int `form:"offset"`
|
||||
}
|
||||
|
||||
type GetBackupsResponse struct {
|
||||
Backups []*backups_core.Backup `json:"backups"`
|
||||
Total int64 `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
type DecryptionReaderCloser struct {
|
||||
*encryption.DecryptionReader
|
||||
BaseReader io.ReadCloser
|
||||
}
|
||||
|
||||
func (r *DecryptionReaderCloser) Close() error {
|
||||
return r.BaseReader.Close()
|
||||
}
|
||||
|
||||
type MakeBackupRequest struct {
|
||||
DatabaseID uuid.UUID `json:"database_id" binding:"required"`
|
||||
}
|
||||
|
||||
type GetNextFullBackupTimeResponse struct {
|
||||
NextFullBackupTime *time.Time `json:"nextFullBackupTime"`
|
||||
}
|
||||
|
||||
type ReportErrorRequest struct {
|
||||
Error string `json:"error" binding:"required"`
|
||||
}
|
||||
|
||||
type UploadGapResponse struct {
|
||||
Error string `json:"error"`
|
||||
ExpectedSegmentName string `json:"expectedSegmentName"`
|
||||
ReceivedSegmentName string `json:"receivedSegmentName"`
|
||||
}
|
||||
|
||||
type RestorePlanFullBackup struct {
|
||||
BackupID uuid.UUID `json:"id"`
|
||||
FullBackupWalStartSegment string `json:"fullBackupWalStartSegment"`
|
||||
FullBackupWalStopSegment string `json:"fullBackupWalStopSegment"`
|
||||
PgVersion string `json:"pgVersion"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
}
|
||||
|
||||
type RestorePlanWalSegment struct {
|
||||
BackupID uuid.UUID `json:"backupId"`
|
||||
SegmentName string `json:"segmentName"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
}
|
||||
|
||||
type GetRestorePlanErrorResponse struct {
|
||||
Error string `json:"error"`
|
||||
Message string `json:"message"`
|
||||
LastContiguousSegment string `json:"lastContiguousSegment,omitempty"`
|
||||
}
|
||||
|
||||
type GetRestorePlanResponse struct {
|
||||
FullBackup RestorePlanFullBackup `json:"fullBackup"`
|
||||
WalSegments []RestorePlanWalSegment `json:"walSegments"`
|
||||
TotalSizeBytes int64 `json:"totalSizeBytes"`
|
||||
LatestAvailableSegment string `json:"latestAvailableSegment"`
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package encryption
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// EncryptionSetup holds the result of setting up encryption for a backup stream.
|
||||
type EncryptionSetup struct {
|
||||
Writer *EncryptionWriter
|
||||
SaltBase64 string
|
||||
NonceBase64 string
|
||||
}
|
||||
|
||||
// SetupEncryptionWriter generates salt/nonce, creates an EncryptionWriter, and
|
||||
// returns the base64-encoded salt and nonce for storage on the backup record.
|
||||
func SetupEncryptionWriter(
|
||||
baseWriter io.Writer,
|
||||
masterKey string,
|
||||
backupID uuid.UUID,
|
||||
) (*EncryptionSetup, error) {
|
||||
salt, err := GenerateSalt()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate salt: %w", err)
|
||||
}
|
||||
|
||||
nonce, err := GenerateNonce()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate nonce: %w", err)
|
||||
}
|
||||
|
||||
encWriter, err := NewEncryptionWriter(baseWriter, masterKey, backupID, salt, nonce)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create encryption writer: %w", err)
|
||||
}
|
||||
|
||||
return &EncryptionSetup{
|
||||
Writer: encWriter,
|
||||
SaltBase64: base64.StdEncoding.EncodeToString(salt),
|
||||
NonceBase64: base64.StdEncoding.EncodeToString(nonce),
|
||||
}, nil
|
||||
}
|
||||
@@ -1,9 +1,6 @@
|
||||
package backups
|
||||
package backups_services
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
audit_logs "databasus-backend/internal/features/audit_logs"
|
||||
"databasus-backend/internal/features/backups/backups/backuping"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
@@ -18,16 +15,16 @@ import (
|
||||
workspaces_services "databasus-backend/internal/features/workspaces/services"
|
||||
"databasus-backend/internal/util/encryption"
|
||||
"databasus-backend/internal/util/logger"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
var backupRepository = &backups_core.BackupRepository{}
|
||||
|
||||
var taskCancelManager = task_cancellation.GetTaskCancelManager()
|
||||
|
||||
var backupService = &BackupService{
|
||||
databases.GetDatabaseService(),
|
||||
storages.GetStorageService(),
|
||||
backupRepository,
|
||||
backups_core.GetBackupRepository(),
|
||||
notifiers.GetNotifierService(),
|
||||
notifiers.GetNotifierService(),
|
||||
backups_config.GetBackupConfigService(),
|
||||
@@ -44,16 +41,21 @@ var backupService = &BackupService{
|
||||
backuping.GetBackupCleaner(),
|
||||
}
|
||||
|
||||
var backupController = &BackupController{
|
||||
backupService: backupService,
|
||||
}
|
||||
|
||||
func GetBackupService() *BackupService {
|
||||
return backupService
|
||||
}
|
||||
|
||||
func GetBackupController() *BackupController {
|
||||
return backupController
|
||||
var walService = &PostgreWalBackupService{
|
||||
backups_config.GetBackupConfigService(),
|
||||
backups_core.GetBackupRepository(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
encryption_secrets.GetSecretKeyService(),
|
||||
logger.GetLogger(),
|
||||
backupService,
|
||||
}
|
||||
|
||||
func GetWalService() *PostgreWalBackupService {
|
||||
return walService
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -0,0 +1,613 @@
|
||||
package backups_services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backups_dto "databasus-backend/internal/features/backups/backups/dto"
|
||||
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/databases/databases/postgresql"
|
||||
encryption_secrets "databasus-backend/internal/features/encryption/secrets"
|
||||
util_encryption "databasus-backend/internal/util/encryption"
|
||||
util_wal "databasus-backend/internal/util/wal"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// PostgreWalBackupService handles WAL segment and basebackup uploads from the databasus-cli agent.
|
||||
type PostgreWalBackupService struct {
|
||||
backupConfigService *backups_config.BackupConfigService
|
||||
backupRepository *backups_core.BackupRepository
|
||||
fieldEncryptor util_encryption.FieldEncryptor
|
||||
secretKeyService *encryption_secrets.SecretKeyService
|
||||
logger *slog.Logger
|
||||
backupService *BackupService
|
||||
}
|
||||
|
||||
// UploadWal accepts a streaming WAL segment or basebackup upload from the agent.
|
||||
// For WAL segments it validates the WAL chain before accepting. Returns an UploadGapResponse
|
||||
// (409) when the chain is broken so the agent knows to trigger a full basebackup.
|
||||
func (s *PostgreWalBackupService) UploadWal(
|
||||
ctx context.Context,
|
||||
database *databases.Database,
|
||||
uploadType backups_core.PgWalUploadType,
|
||||
walSegmentName string,
|
||||
fullBackupWalStartSegment string,
|
||||
fullBackupWalStopSegment string,
|
||||
walSegmentSizeBytes int64,
|
||||
body io.Reader,
|
||||
) (*backups_dto.UploadGapResponse, error) {
|
||||
if err := s.validateWalBackupType(database); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if uploadType == backups_core.PgWalUploadTypeBasebackup {
|
||||
if fullBackupWalStartSegment == "" || fullBackupWalStopSegment == "" {
|
||||
return nil, fmt.Errorf(
|
||||
"fullBackupWalStartSegment and fullBackupWalStopSegment are required for basebackup uploads",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get backup config: %w", err)
|
||||
}
|
||||
|
||||
if backupConfig.Storage == nil {
|
||||
return nil, fmt.Errorf("no storage configured for database %s", database.ID)
|
||||
}
|
||||
|
||||
if uploadType == backups_core.PgWalUploadTypeWal {
|
||||
// Idempotency: check before chain validation so a successful re-upload is
|
||||
// not misidentified as a gap.
|
||||
existing, err := s.backupRepository.FindWalSegmentByName(database.ID, walSegmentName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to check for duplicate WAL segment: %w", err)
|
||||
}
|
||||
|
||||
if existing != nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
gapResp, err := s.validateWalChain(database.ID, walSegmentName, walSegmentSizeBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if gapResp != nil {
|
||||
return gapResp, nil
|
||||
}
|
||||
}
|
||||
|
||||
backup := s.createBackupRecord(
|
||||
database.ID,
|
||||
backupConfig.Storage.ID,
|
||||
uploadType,
|
||||
database.Name,
|
||||
walSegmentName,
|
||||
fullBackupWalStartSegment,
|
||||
fullBackupWalStopSegment,
|
||||
backupConfig.Encryption,
|
||||
)
|
||||
|
||||
if err := s.backupRepository.Save(backup); err != nil {
|
||||
return nil, fmt.Errorf("failed to create backup record: %w", err)
|
||||
}
|
||||
|
||||
sizeBytes, streamErr := s.streamToStorage(ctx, backup, backupConfig, body)
|
||||
if streamErr != nil {
|
||||
errMsg := streamErr.Error()
|
||||
s.markFailed(backup, errMsg)
|
||||
|
||||
return nil, fmt.Errorf("upload failed: %w", streamErr)
|
||||
}
|
||||
|
||||
s.markCompleted(backup, sizeBytes)
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *PostgreWalBackupService) GetRestorePlan(
|
||||
database *databases.Database,
|
||||
backupID *uuid.UUID,
|
||||
) (*backups_dto.GetRestorePlanResponse, *backups_dto.GetRestorePlanErrorResponse, error) {
|
||||
if err := s.validateWalBackupType(database); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
fullBackup, err := s.resolveFullBackup(database.ID, backupID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if fullBackup == nil {
|
||||
msg := "no full backups available for this database"
|
||||
if backupID != nil {
|
||||
msg = fmt.Sprintf("full backup %s not found or not completed", backupID)
|
||||
}
|
||||
|
||||
return nil, &backups_dto.GetRestorePlanErrorResponse{
|
||||
Error: "no_backups",
|
||||
Message: msg,
|
||||
}, nil
|
||||
}
|
||||
|
||||
startSegment := ""
|
||||
if fullBackup.PgFullBackupWalStartSegmentName != nil {
|
||||
startSegment = *fullBackup.PgFullBackupWalStartSegmentName
|
||||
}
|
||||
|
||||
walSegments, err := s.backupRepository.FindCompletedWalSegmentsAfter(database.ID, startSegment)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to query WAL segments: %w", err)
|
||||
}
|
||||
|
||||
chainErr := s.validateRestoreWalChain(fullBackup, walSegments)
|
||||
if chainErr != nil {
|
||||
return nil, chainErr, nil
|
||||
}
|
||||
|
||||
fullBackupSizeBytes := int64(fullBackup.BackupSizeMb * 1024 * 1024)
|
||||
|
||||
pgVersion := ""
|
||||
if fullBackup.PgVersion != nil {
|
||||
pgVersion = *fullBackup.PgVersion
|
||||
}
|
||||
|
||||
stopSegment := ""
|
||||
if fullBackup.PgFullBackupWalStopSegmentName != nil {
|
||||
stopSegment = *fullBackup.PgFullBackupWalStopSegmentName
|
||||
}
|
||||
|
||||
response := &backups_dto.GetRestorePlanResponse{
|
||||
FullBackup: backups_dto.RestorePlanFullBackup{
|
||||
BackupID: fullBackup.ID,
|
||||
FullBackupWalStartSegment: startSegment,
|
||||
FullBackupWalStopSegment: stopSegment,
|
||||
PgVersion: pgVersion,
|
||||
CreatedAt: fullBackup.CreatedAt,
|
||||
SizeBytes: fullBackupSizeBytes,
|
||||
},
|
||||
TotalSizeBytes: fullBackupSizeBytes,
|
||||
}
|
||||
|
||||
for _, seg := range walSegments {
|
||||
segName := ""
|
||||
if seg.PgWalSegmentName != nil {
|
||||
segName = *seg.PgWalSegmentName
|
||||
}
|
||||
|
||||
segSizeBytes := int64(seg.BackupSizeMb * 1024 * 1024)
|
||||
|
||||
response.WalSegments = append(response.WalSegments, backups_dto.RestorePlanWalSegment{
|
||||
BackupID: seg.ID,
|
||||
SegmentName: segName,
|
||||
SizeBytes: segSizeBytes,
|
||||
})
|
||||
|
||||
response.TotalSizeBytes += segSizeBytes
|
||||
response.LatestAvailableSegment = segName
|
||||
}
|
||||
|
||||
return response, nil, nil
|
||||
}
|
||||
|
||||
// DownloadBackupFile returns a reader for a backup file belonging to the given database.
|
||||
// Decryption is handled transparently if the backup is encrypted.
|
||||
func (s *PostgreWalBackupService) DownloadBackupFile(
|
||||
database *databases.Database,
|
||||
backupID uuid.UUID,
|
||||
) (io.ReadCloser, error) {
|
||||
if err := s.validateWalBackupType(database); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
backup, err := s.backupRepository.FindByID(backupID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup not found: %w", err)
|
||||
}
|
||||
|
||||
if backup.DatabaseID != database.ID {
|
||||
return nil, fmt.Errorf("backup does not belong to this database")
|
||||
}
|
||||
|
||||
if backup.Status != backups_core.BackupStatusCompleted {
|
||||
return nil, fmt.Errorf("backup is not completed")
|
||||
}
|
||||
|
||||
return s.backupService.GetBackupReader(backupID)
|
||||
}
|
||||
|
||||
func (s *PostgreWalBackupService) validateWalChain(
|
||||
databaseID uuid.UUID,
|
||||
incomingSegment string,
|
||||
walSegmentSizeBytes int64,
|
||||
) (*backups_dto.UploadGapResponse, error) {
|
||||
fullBackup, err := s.backupRepository.FindLastCompletedFullWalBackupByDatabaseID(databaseID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query full backup: %w", err)
|
||||
}
|
||||
|
||||
// No full backup exists yet: cannot accept WAL segments without a chain anchor.
|
||||
if fullBackup == nil || fullBackup.PgFullBackupWalStopSegmentName == nil {
|
||||
return &backups_dto.UploadGapResponse{
|
||||
Error: "no_full_backup",
|
||||
ExpectedSegmentName: "",
|
||||
ReceivedSegmentName: incomingSegment,
|
||||
}, nil
|
||||
}
|
||||
|
||||
stopSegment := *fullBackup.PgFullBackupWalStopSegmentName
|
||||
|
||||
lastWal, err := s.backupRepository.FindLastWalSegmentAfter(databaseID, stopSegment)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query last WAL segment: %w", err)
|
||||
}
|
||||
|
||||
walCalculator := util_wal.NewWalCalculator(walSegmentSizeBytes)
|
||||
|
||||
var chainTail string
|
||||
if lastWal != nil && lastWal.PgWalSegmentName != nil {
|
||||
chainTail = *lastWal.PgWalSegmentName
|
||||
} else {
|
||||
chainTail = stopSegment
|
||||
}
|
||||
|
||||
expectedNext, err := walCalculator.NextSegment(chainTail)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("WAL arithmetic failed for %q: %w", chainTail, err)
|
||||
}
|
||||
|
||||
if incomingSegment != expectedNext {
|
||||
return &backups_dto.UploadGapResponse{
|
||||
Error: "gap_detected",
|
||||
ExpectedSegmentName: expectedNext,
|
||||
ReceivedSegmentName: incomingSegment,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *PostgreWalBackupService) createBackupRecord(
|
||||
databaseID uuid.UUID,
|
||||
storageID uuid.UUID,
|
||||
uploadType backups_core.PgWalUploadType,
|
||||
dbName string,
|
||||
walSegmentName string,
|
||||
fullBackupWalStartSegment string,
|
||||
fullBackupWalStopSegment string,
|
||||
encryption backups_config.BackupEncryption,
|
||||
) *backups_core.Backup {
|
||||
now := time.Now().UTC()
|
||||
|
||||
backup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: databaseID,
|
||||
StorageID: storageID,
|
||||
Status: backups_core.BackupStatusInProgress,
|
||||
Encryption: encryption,
|
||||
CreatedAt: now,
|
||||
}
|
||||
|
||||
backup.GenerateFilename(dbName)
|
||||
|
||||
if uploadType == backups_core.PgWalUploadTypeBasebackup {
|
||||
walBackupType := backups_core.PgWalBackupTypeFullBackup
|
||||
backup.PgWalBackupType = &walBackupType
|
||||
|
||||
if fullBackupWalStartSegment != "" {
|
||||
backup.PgFullBackupWalStartSegmentName = &fullBackupWalStartSegment
|
||||
}
|
||||
|
||||
if fullBackupWalStopSegment != "" {
|
||||
backup.PgFullBackupWalStopSegmentName = &fullBackupWalStopSegment
|
||||
}
|
||||
} else {
|
||||
walBackupType := backups_core.PgWalBackupTypeWalSegment
|
||||
backup.PgWalBackupType = &walBackupType
|
||||
backup.PgWalSegmentName = &walSegmentName
|
||||
}
|
||||
|
||||
return backup
|
||||
}
|
||||
|
||||
func (s *PostgreWalBackupService) streamToStorage(
|
||||
ctx context.Context,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
body io.Reader,
|
||||
) (int64, error) {
|
||||
if backupConfig.Encryption == backups_config.BackupEncryptionEncrypted {
|
||||
return s.streamEncrypted(ctx, backup, backupConfig, body, backup.FileName)
|
||||
}
|
||||
|
||||
return s.streamDirect(ctx, backupConfig, body, backup.FileName)
|
||||
}
|
||||
|
||||
func (s *PostgreWalBackupService) streamDirect(
|
||||
ctx context.Context,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
body io.Reader,
|
||||
fileName string,
|
||||
) (int64, error) {
|
||||
cr := &countingReader{r: body}
|
||||
|
||||
if err := backupConfig.Storage.SaveFile(ctx, s.fieldEncryptor, s.logger, fileName, cr); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return cr.n, nil
|
||||
}
|
||||
|
||||
func (s *PostgreWalBackupService) streamEncrypted(
|
||||
ctx context.Context,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
body io.Reader,
|
||||
fileName string,
|
||||
) (int64, error) {
|
||||
masterKey, err := s.secretKeyService.GetSecretKey()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to get master encryption key: %w", err)
|
||||
}
|
||||
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
|
||||
encryptionSetup, err := backup_encryption.SetupEncryptionWriter(
|
||||
pipeWriter,
|
||||
masterKey,
|
||||
backup.ID,
|
||||
)
|
||||
if err != nil {
|
||||
_ = pipeWriter.Close()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
copyErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
_, copyErr := io.Copy(encryptionSetup.Writer, body)
|
||||
if copyErr != nil {
|
||||
_ = encryptionSetup.Writer.Close()
|
||||
_ = pipeWriter.CloseWithError(copyErr)
|
||||
copyErrCh <- copyErr
|
||||
return
|
||||
}
|
||||
|
||||
if closeErr := encryptionSetup.Writer.Close(); closeErr != nil {
|
||||
_ = pipeWriter.CloseWithError(closeErr)
|
||||
copyErrCh <- closeErr
|
||||
return
|
||||
}
|
||||
|
||||
copyErrCh <- pipeWriter.Close()
|
||||
}()
|
||||
|
||||
cr := &countingReader{r: pipeReader}
|
||||
saveErr := backupConfig.Storage.SaveFile(ctx, s.fieldEncryptor, s.logger, fileName, cr)
|
||||
copyErr := <-copyErrCh
|
||||
|
||||
if copyErr != nil {
|
||||
return 0, copyErr
|
||||
}
|
||||
|
||||
if saveErr != nil {
|
||||
return 0, saveErr
|
||||
}
|
||||
|
||||
backup.EncryptionSalt = &encryptionSetup.SaltBase64
|
||||
backup.EncryptionIV = &encryptionSetup.NonceBase64
|
||||
|
||||
return cr.n, nil
|
||||
}
|
||||
|
||||
func (s *PostgreWalBackupService) markCompleted(backup *backups_core.Backup, sizeBytes int64) {
|
||||
backup.Status = backups_core.BackupStatusCompleted
|
||||
backup.BackupSizeMb = float64(sizeBytes) / (1024 * 1024)
|
||||
|
||||
if err := s.backupRepository.Save(backup); err != nil {
|
||||
s.logger.Error(
|
||||
"failed to mark WAL backup as completed",
|
||||
"backupId",
|
||||
backup.ID,
|
||||
"error",
|
||||
err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostgreWalBackupService) markFailed(backup *backups_core.Backup, errMsg string) {
|
||||
backup.Status = backups_core.BackupStatusFailed
|
||||
backup.FailMessage = &errMsg
|
||||
|
||||
if err := s.backupRepository.Save(backup); err != nil {
|
||||
s.logger.Error("failed to mark WAL backup as failed", "backupId", backup.ID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PostgreWalBackupService) GetNextFullBackupTime(
|
||||
database *databases.Database,
|
||||
) (*backups_dto.GetNextFullBackupTimeResponse, error) {
|
||||
if err := s.validateWalBackupType(database); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get backup config: %w", err)
|
||||
}
|
||||
|
||||
if backupConfig.BackupInterval == nil {
|
||||
return nil, fmt.Errorf("no backup interval configured for database %s", database.ID)
|
||||
}
|
||||
|
||||
lastFullBackup, err := s.backupRepository.FindLastCompletedFullWalBackupByDatabaseID(
|
||||
database.ID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query last full backup: %w", err)
|
||||
}
|
||||
|
||||
var lastBackupTime *time.Time
|
||||
if lastFullBackup != nil {
|
||||
lastBackupTime = &lastFullBackup.CreatedAt
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
nextTime := backupConfig.BackupInterval.NextTriggerTime(now, lastBackupTime)
|
||||
|
||||
return &backups_dto.GetNextFullBackupTimeResponse{
|
||||
NextFullBackupTime: nextTime,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ReportError creates a FAILED backup record with the agent's error message.
|
||||
func (s *PostgreWalBackupService) ReportError(
|
||||
database *databases.Database,
|
||||
errorMsg string,
|
||||
) error {
|
||||
if err := s.validateWalBackupType(database); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get backup config: %w", err)
|
||||
}
|
||||
|
||||
if backupConfig.Storage == nil {
|
||||
return fmt.Errorf("no storage configured for database %s", database.ID)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
backup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: backupConfig.Storage.ID,
|
||||
Status: backups_core.BackupStatusFailed,
|
||||
FailMessage: &errorMsg,
|
||||
Encryption: backupConfig.Encryption,
|
||||
CreatedAt: now,
|
||||
}
|
||||
|
||||
backup.GenerateFilename(database.Name)
|
||||
|
||||
if err := s.backupRepository.Save(backup); err != nil {
|
||||
return fmt.Errorf("failed to save error backup record: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PostgreWalBackupService) resolveFullBackup(
|
||||
databaseID uuid.UUID,
|
||||
backupID *uuid.UUID,
|
||||
) (*backups_core.Backup, error) {
|
||||
if backupID != nil {
|
||||
return s.backupRepository.FindCompletedFullWalBackupByID(databaseID, *backupID)
|
||||
}
|
||||
|
||||
return s.backupRepository.FindLastCompletedFullWalBackupByDatabaseID(databaseID)
|
||||
}
|
||||
|
||||
func (s *PostgreWalBackupService) validateRestoreWalChain(
|
||||
fullBackup *backups_core.Backup,
|
||||
walSegments []*backups_core.Backup,
|
||||
) *backups_dto.GetRestorePlanErrorResponse {
|
||||
if len(walSegments) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
stopSegment := ""
|
||||
if fullBackup.PgFullBackupWalStopSegmentName != nil {
|
||||
stopSegment = *fullBackup.PgFullBackupWalStopSegmentName
|
||||
}
|
||||
|
||||
walCalculator := util_wal.NewWalCalculator(0)
|
||||
expectedNext, err := walCalculator.NextSegment(stopSegment)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, seg := range walSegments {
|
||||
segName := ""
|
||||
if seg.PgWalSegmentName != nil {
|
||||
segName = *seg.PgWalSegmentName
|
||||
}
|
||||
|
||||
cmp, cmpErr := walCalculator.Compare(segName, stopSegment)
|
||||
if cmpErr != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip segments that are <= stopSegment (they are part of the basebackup range)
|
||||
if cmp <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if segName != expectedNext {
|
||||
lastContiguous := stopSegment
|
||||
// Walk back to find the segment before the gap
|
||||
for _, prev := range walSegments {
|
||||
prevName := ""
|
||||
if prev.PgWalSegmentName != nil {
|
||||
prevName = *prev.PgWalSegmentName
|
||||
}
|
||||
|
||||
prevCmp, _ := walCalculator.Compare(prevName, stopSegment)
|
||||
if prevCmp <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if prevName == segName {
|
||||
break
|
||||
}
|
||||
|
||||
lastContiguous = prevName
|
||||
}
|
||||
|
||||
return &backups_dto.GetRestorePlanErrorResponse{
|
||||
Error: "wal_chain_broken",
|
||||
Message: fmt.Sprintf(
|
||||
"WAL chain has a gap after segment %s. Recovery is only possible up to this segment.",
|
||||
lastContiguous,
|
||||
),
|
||||
LastContiguousSegment: lastContiguous,
|
||||
}
|
||||
}
|
||||
|
||||
expectedNext, err = walCalculator.NextSegment(segName)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PostgreWalBackupService) validateWalBackupType(database *databases.Database) error {
|
||||
if database.Postgresql == nil ||
|
||||
database.Postgresql.BackupType != postgresql.PostgresBackupTypeWalV1 {
|
||||
return fmt.Errorf("database %s is not configured for WAL backups", database.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type countingReader struct {
|
||||
r io.Reader
|
||||
n int64
|
||||
}
|
||||
|
||||
func (cr *countingReader) Read(p []byte) (n int, err error) {
|
||||
n, err = cr.r.Read(p)
|
||||
cr.n += int64(n)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package backups
|
||||
package backups_services
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"databasus-backend/internal/features/backups/backups/backuping"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backups_download "databasus-backend/internal/features/backups/backups/download"
|
||||
backups_dto "databasus-backend/internal/features/backups/backups/dto"
|
||||
"databasus-backend/internal/features/backups/backups/encryption"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
@@ -21,6 +22,7 @@ import (
|
||||
users_models "databasus-backend/internal/features/users/models"
|
||||
workspaces_services "databasus-backend/internal/features/workspaces/services"
|
||||
util_encryption "databasus-backend/internal/util/encryption"
|
||||
files_utils "databasus-backend/internal/util/files"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -92,7 +94,7 @@ func (s *BackupService) MakeBackupWithAuth(
|
||||
return errors.New("insufficient permissions to create backup for this database")
|
||||
}
|
||||
|
||||
s.backupSchedulerService.StartBackup(databaseID, true)
|
||||
s.backupSchedulerService.StartBackup(database, true)
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Backup manually initiated for database: %s", database.Name),
|
||||
@@ -107,7 +109,7 @@ func (s *BackupService) GetBackups(
|
||||
user *users_models.User,
|
||||
databaseID uuid.UUID,
|
||||
limit, offset int,
|
||||
) (*GetBackupsResponse, error) {
|
||||
) (*backups_dto.GetBackupsResponse, error) {
|
||||
database, err := s.databaseService.GetDatabaseByID(databaseID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -142,7 +144,7 @@ func (s *BackupService) GetBackups(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &GetBackupsResponse{
|
||||
return &backups_dto.GetBackupsResponse{
|
||||
Backups: backups,
|
||||
Total: total,
|
||||
Limit: limit,
|
||||
@@ -181,11 +183,7 @@ func (s *BackupService) DeleteBackup(
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf(
|
||||
"Backup deleted for database: %s (ID: %s)",
|
||||
database.Name,
|
||||
backupID.String(),
|
||||
),
|
||||
fmt.Sprintf("Backup deleted for database: %s", database.Name),
|
||||
&user.ID,
|
||||
database.WorkspaceID,
|
||||
)
|
||||
@@ -232,11 +230,7 @@ func (s *BackupService) CancelBackup(
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf(
|
||||
"Backup cancelled for database: %s (ID: %s)",
|
||||
database.Name,
|
||||
backupID.String(),
|
||||
),
|
||||
fmt.Sprintf("Backup cancelled for database: %s", database.Name),
|
||||
&user.ID,
|
||||
database.WorkspaceID,
|
||||
)
|
||||
@@ -276,16 +270,12 @@ func (s *BackupService) GetBackupFile(
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf(
|
||||
"Backup file downloaded for database: %s (ID: %s)",
|
||||
database.Name,
|
||||
backupID.String(),
|
||||
),
|
||||
fmt.Sprintf("Backup file downloaded for database: %s", database.Name),
|
||||
&user.ID,
|
||||
database.WorkspaceID,
|
||||
)
|
||||
|
||||
reader, err := s.getBackupReader(backupID)
|
||||
reader, err := s.GetBackupReader(backupID)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
@@ -293,39 +283,9 @@ func (s *BackupService) GetBackupFile(
|
||||
return reader, backup, database, nil
|
||||
}
|
||||
|
||||
func (s *BackupService) deleteDbBackups(databaseID uuid.UUID) error {
|
||||
dbBackupsInProgress, err := s.backupRepository.FindByDatabaseIdAndStatus(
|
||||
databaseID,
|
||||
backups_core.BackupStatusInProgress,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(dbBackupsInProgress) > 0 {
|
||||
return errors.New("backup is in progress, storage cannot be removed")
|
||||
}
|
||||
|
||||
dbBackups, err := s.backupRepository.FindByDatabaseID(
|
||||
databaseID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, dbBackup := range dbBackups {
|
||||
err := s.backupCleaner.DeleteBackup(dbBackup)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBackupReader returns a reader for the backup file
|
||||
// If encrypted, wraps with DecryptionReader
|
||||
func (s *BackupService) getBackupReader(backupID uuid.UUID) (io.ReadCloser, error) {
|
||||
// GetBackupReader returns a reader for the backup file.
|
||||
// If encrypted, wraps with DecryptionReader.
|
||||
func (s *BackupService) GetBackupReader(backupID uuid.UUID) (io.ReadCloser, error) {
|
||||
backup, err := s.backupRepository.FindByID(backupID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find backup: %w", err)
|
||||
@@ -336,7 +296,7 @@ func (s *BackupService) getBackupReader(backupID uuid.UUID) (io.ReadCloser, erro
|
||||
return nil, fmt.Errorf("failed to get storage: %w", err)
|
||||
}
|
||||
|
||||
fileReader, err := storage.GetFile(s.fieldEncryptor, backup.ID)
|
||||
fileReader, err := storage.GetFile(s.fieldEncryptor, backup.FileName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get backup file: %w", err)
|
||||
}
|
||||
@@ -405,7 +365,7 @@ func (s *BackupService) getBackupReader(backupID uuid.UUID) (io.ReadCloser, erro
|
||||
|
||||
s.logger.Info("Returning encrypted backup with decryption", "backupId", backupID)
|
||||
|
||||
return &DecryptionReaderCloser{
|
||||
return &backups_dto.DecryptionReaderCloser{
|
||||
DecryptionReader: decryptionReader,
|
||||
BaseReader: fileReader,
|
||||
}, nil
|
||||
@@ -476,7 +436,7 @@ func (s *BackupService) GetBackupFileWithoutAuth(
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
reader, err := s.getBackupReader(backupID)
|
||||
reader, err := s.GetBackupReader(backupID)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
@@ -490,11 +450,7 @@ func (s *BackupService) WriteAuditLogForDownload(
|
||||
database *databases.Database,
|
||||
) {
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf(
|
||||
"Backup file downloaded for database: %s (ID: %s)",
|
||||
database.Name,
|
||||
backup.ID.String(),
|
||||
),
|
||||
fmt.Sprintf("Backup file downloaded for database: %s", database.Name),
|
||||
&userID,
|
||||
database.WorkspaceID,
|
||||
)
|
||||
@@ -516,12 +472,42 @@ func (s *BackupService) UnregisterDownload(userID uuid.UUID) {
|
||||
s.downloadTokenService.UnregisterDownload(userID)
|
||||
}
|
||||
|
||||
func (s *BackupService) deleteDbBackups(databaseID uuid.UUID) error {
|
||||
dbBackupsInProgress, err := s.backupRepository.FindByDatabaseIdAndStatus(
|
||||
databaseID,
|
||||
backups_core.BackupStatusInProgress,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(dbBackupsInProgress) > 0 {
|
||||
return errors.New("backup is in progress, storage cannot be removed")
|
||||
}
|
||||
|
||||
dbBackups, err := s.backupRepository.FindByDatabaseID(
|
||||
databaseID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, dbBackup := range dbBackups {
|
||||
err := s.backupCleaner.DeleteBackup(dbBackup)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BackupService) generateBackupFilename(
|
||||
backup *backups_core.Backup,
|
||||
database *databases.Database,
|
||||
) string {
|
||||
timestamp := backup.CreatedAt.Format("2006-01-02_15-04-05")
|
||||
safeName := sanitizeFilename(database.Name)
|
||||
safeName := files_utils.SanitizeFilename(database.Name)
|
||||
extension := s.getBackupExtension(database.Type)
|
||||
return fmt.Sprintf("%s_backup_%s%s", safeName, timestamp, extension)
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
|
||||
common "databasus-backend/internal/features/backups/backups/common"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
usecases_mariadb "databasus-backend/internal/features/backups/backups/usecases/mariadb"
|
||||
usecases_mongodb "databasus-backend/internal/features/backups/backups/usecases/mongodb"
|
||||
usecases_mysql "databasus-backend/internal/features/backups/backups/usecases/mysql"
|
||||
@@ -12,8 +13,6 @@ import (
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/storages"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type CreateBackupUsecase struct {
|
||||
@@ -25,7 +24,7 @@ type CreateBackupUsecase struct {
|
||||
|
||||
func (uc *CreateBackupUsecase) Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
database *databases.Database,
|
||||
storage *storages.Storage,
|
||||
@@ -35,7 +34,7 @@ func (uc *CreateBackupUsecase) Execute(
|
||||
case databases.DatabaseTypePostgres:
|
||||
return uc.CreatePostgresqlBackupUsecase.Execute(
|
||||
ctx,
|
||||
backupID,
|
||||
backup,
|
||||
backupConfig,
|
||||
database,
|
||||
storage,
|
||||
@@ -45,7 +44,7 @@ func (uc *CreateBackupUsecase) Execute(
|
||||
case databases.DatabaseTypeMysql:
|
||||
return uc.CreateMysqlBackupUsecase.Execute(
|
||||
ctx,
|
||||
backupID,
|
||||
backup,
|
||||
backupConfig,
|
||||
database,
|
||||
storage,
|
||||
@@ -55,7 +54,7 @@ func (uc *CreateBackupUsecase) Execute(
|
||||
case databases.DatabaseTypeMariadb:
|
||||
return uc.CreateMariadbBackupUsecase.Execute(
|
||||
ctx,
|
||||
backupID,
|
||||
backup,
|
||||
backupConfig,
|
||||
database,
|
||||
storage,
|
||||
@@ -65,7 +64,7 @@ func (uc *CreateBackupUsecase) Execute(
|
||||
case databases.DatabaseTypeMongodb:
|
||||
return uc.CreateMongodbBackupUsecase.Execute(
|
||||
ctx,
|
||||
backupID,
|
||||
backup,
|
||||
backupConfig,
|
||||
database,
|
||||
storage,
|
||||
|
||||
@@ -2,7 +2,6 @@ package usecases_mariadb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -19,6 +18,7 @@ import (
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
common "databasus-backend/internal/features/backups/backups/common"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
@@ -52,7 +52,7 @@ type writeResult struct {
|
||||
|
||||
func (uc *CreateMariadbBackupUsecase) Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
db *databases.Database,
|
||||
storage *storages.Storage,
|
||||
@@ -82,7 +82,7 @@ func (uc *CreateMariadbBackupUsecase) Execute(
|
||||
|
||||
return uc.streamToStorage(
|
||||
ctx,
|
||||
backupID,
|
||||
backup,
|
||||
backupConfig,
|
||||
tools.GetMariadbExecutable(
|
||||
tools.MariadbExecutableMariadbDump,
|
||||
@@ -108,18 +108,24 @@ func (uc *CreateMariadbBackupUsecase) buildMariadbDumpArgs(
|
||||
"--single-transaction",
|
||||
"--routines",
|
||||
"--quick",
|
||||
"--skip-extended-insert",
|
||||
"--verbose",
|
||||
}
|
||||
|
||||
if mdb.HasPrivilege("TRIGGER") {
|
||||
args = append(args, "--triggers")
|
||||
}
|
||||
if mdb.HasPrivilege("EVENT") {
|
||||
|
||||
if mdb.HasPrivilege("EVENT") && !mdb.IsExcludeEvents {
|
||||
args = append(args, "--events")
|
||||
}
|
||||
|
||||
args = append(args, "--compress")
|
||||
|
||||
if !config.GetEnv().IsCloud {
|
||||
args = append(args, "--max-allowed-packet=1G")
|
||||
}
|
||||
|
||||
if mdb.IsHttps {
|
||||
args = append(args, "--ssl")
|
||||
args = append(args, "--skip-ssl-verify-server-cert")
|
||||
@@ -134,7 +140,7 @@ func (uc *CreateMariadbBackupUsecase) buildMariadbDumpArgs(
|
||||
|
||||
func (uc *CreateMariadbBackupUsecase) streamToStorage(
|
||||
parentCtx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
mariadbBin string,
|
||||
args []string,
|
||||
@@ -185,7 +191,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
|
||||
storageReader, storageWriter := io.Pipe()
|
||||
|
||||
finalWriter, encryptionWriter, backupMetadata, err := uc.setupBackupEncryption(
|
||||
backupID,
|
||||
backup.ID,
|
||||
backupConfig,
|
||||
storageWriter,
|
||||
)
|
||||
@@ -202,7 +208,13 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
|
||||
|
||||
saveErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
saveErr := storage.SaveFile(ctx, uc.fieldEncryptor, uc.logger, backupID, storageReader)
|
||||
saveErr := storage.SaveFile(
|
||||
ctx,
|
||||
uc.fieldEncryptor,
|
||||
uc.logger,
|
||||
backup.FileName,
|
||||
storageReader,
|
||||
)
|
||||
saveErrCh <- saveErr
|
||||
}()
|
||||
|
||||
@@ -418,7 +430,9 @@ func (uc *CreateMariadbBackupUsecase) setupBackupEncryption(
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
storageWriter io.WriteCloser,
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, common.BackupMetadata, error) {
|
||||
metadata := common.BackupMetadata{}
|
||||
metadata := common.BackupMetadata{
|
||||
BackupID: backupID,
|
||||
}
|
||||
|
||||
if backupConfig.Encryption != backups_config.BackupEncryptionEncrypted {
|
||||
metadata.Encryption = backups_config.BackupEncryptionNone
|
||||
@@ -426,40 +440,22 @@ func (uc *CreateMariadbBackupUsecase) setupBackupEncryption(
|
||||
return storageWriter, nil, metadata, nil
|
||||
}
|
||||
|
||||
salt, err := backup_encryption.GenerateSalt()
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to generate salt: %w", err)
|
||||
}
|
||||
|
||||
nonce, err := backup_encryption.GenerateNonce()
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to generate nonce: %w", err)
|
||||
}
|
||||
|
||||
masterKey, err := uc.secretKeyService.GetSecretKey()
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to get master key: %w", err)
|
||||
}
|
||||
|
||||
encWriter, err := backup_encryption.NewEncryptionWriter(
|
||||
storageWriter,
|
||||
masterKey,
|
||||
backupID,
|
||||
salt,
|
||||
nonce,
|
||||
)
|
||||
encSetup, err := backup_encryption.SetupEncryptionWriter(storageWriter, masterKey, backupID)
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to create encrypting writer: %w", err)
|
||||
return nil, nil, metadata, err
|
||||
}
|
||||
|
||||
saltBase64 := base64.StdEncoding.EncodeToString(salt)
|
||||
nonceBase64 := base64.StdEncoding.EncodeToString(nonce)
|
||||
metadata.EncryptionSalt = &saltBase64
|
||||
metadata.EncryptionIV = &nonceBase64
|
||||
metadata.EncryptionSalt = &encSetup.SaltBase64
|
||||
metadata.EncryptionIV = &encSetup.NonceBase64
|
||||
metadata.Encryption = backups_config.BackupEncryptionEncrypted
|
||||
|
||||
uc.logger.Info("Encryption enabled for backup", "backupId", backupID)
|
||||
return encWriter, encWriter, metadata, nil
|
||||
return encSetup.Writer, encSetup.Writer, metadata, nil
|
||||
}
|
||||
|
||||
func (uc *CreateMariadbBackupUsecase) cleanupOnCancellation(
|
||||
|
||||
@@ -2,7 +2,6 @@ package usecases_mongodb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -16,6 +15,7 @@ import (
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
common "databasus-backend/internal/features/backups/backups/common"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
@@ -46,7 +46,7 @@ type writeResult struct {
|
||||
|
||||
func (uc *CreateMongodbBackupUsecase) Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
db *databases.Database,
|
||||
storage *storages.Storage,
|
||||
@@ -76,7 +76,7 @@ func (uc *CreateMongodbBackupUsecase) Execute(
|
||||
|
||||
return uc.streamToStorage(
|
||||
ctx,
|
||||
backupID,
|
||||
backup,
|
||||
backupConfig,
|
||||
tools.GetMongodbExecutable(
|
||||
tools.MongodbExecutableMongodump,
|
||||
@@ -114,7 +114,7 @@ func (uc *CreateMongodbBackupUsecase) buildMongodumpArgs(
|
||||
|
||||
func (uc *CreateMongodbBackupUsecase) streamToStorage(
|
||||
parentCtx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
mongodumpBin string,
|
||||
args []string,
|
||||
@@ -163,7 +163,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
|
||||
storageReader, storageWriter := io.Pipe()
|
||||
|
||||
finalWriter, encryptionWriter, backupMetadata, err := uc.setupBackupEncryption(
|
||||
backupID,
|
||||
backup.ID,
|
||||
backupConfig,
|
||||
storageWriter,
|
||||
)
|
||||
@@ -175,7 +175,13 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
|
||||
|
||||
saveErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
saveErr := storage.SaveFile(ctx, uc.fieldEncryptor, uc.logger, backupID, storageReader)
|
||||
saveErr := storage.SaveFile(
|
||||
ctx,
|
||||
uc.fieldEncryptor,
|
||||
uc.logger,
|
||||
backup.FileName,
|
||||
storageReader,
|
||||
)
|
||||
saveErrCh <- saveErr
|
||||
}()
|
||||
|
||||
@@ -262,6 +268,7 @@ func (uc *CreateMongodbBackupUsecase) setupBackupEncryption(
|
||||
storageWriter io.WriteCloser,
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, common.BackupMetadata, error) {
|
||||
backupMetadata := common.BackupMetadata{
|
||||
BackupID: backupID,
|
||||
Encryption: backups_config.BackupEncryptionNone,
|
||||
}
|
||||
|
||||
@@ -269,40 +276,21 @@ func (uc *CreateMongodbBackupUsecase) setupBackupEncryption(
|
||||
return storageWriter, nil, backupMetadata, nil
|
||||
}
|
||||
|
||||
salt, err := backup_encryption.GenerateSalt()
|
||||
if err != nil {
|
||||
return nil, nil, backupMetadata, fmt.Errorf("failed to generate salt: %w", err)
|
||||
}
|
||||
|
||||
nonce, err := backup_encryption.GenerateNonce()
|
||||
if err != nil {
|
||||
return nil, nil, backupMetadata, fmt.Errorf("failed to generate nonce: %w", err)
|
||||
}
|
||||
|
||||
masterKey, err := uc.secretKeyService.GetSecretKey()
|
||||
if err != nil {
|
||||
return nil, nil, backupMetadata, fmt.Errorf("failed to get master key: %w", err)
|
||||
}
|
||||
|
||||
encryptionWriter, err := backup_encryption.NewEncryptionWriter(
|
||||
storageWriter,
|
||||
masterKey,
|
||||
backupID,
|
||||
salt,
|
||||
nonce,
|
||||
)
|
||||
encSetup, err := backup_encryption.SetupEncryptionWriter(storageWriter, masterKey, backupID)
|
||||
if err != nil {
|
||||
return nil, nil, backupMetadata, fmt.Errorf("failed to create encryption writer: %w", err)
|
||||
return nil, nil, backupMetadata, err
|
||||
}
|
||||
|
||||
saltBase64 := base64.StdEncoding.EncodeToString(salt)
|
||||
nonceBase64 := base64.StdEncoding.EncodeToString(nonce)
|
||||
|
||||
backupMetadata.Encryption = backups_config.BackupEncryptionEncrypted
|
||||
backupMetadata.EncryptionSalt = &saltBase64
|
||||
backupMetadata.EncryptionIV = &nonceBase64
|
||||
backupMetadata.EncryptionSalt = &encSetup.SaltBase64
|
||||
backupMetadata.EncryptionIV = &encSetup.NonceBase64
|
||||
|
||||
return encryptionWriter, encryptionWriter, backupMetadata, nil
|
||||
return encSetup.Writer, encSetup.Writer, backupMetadata, nil
|
||||
}
|
||||
|
||||
func (uc *CreateMongodbBackupUsecase) copyWithShutdownCheck(
|
||||
|
||||
@@ -2,7 +2,6 @@ package usecases_mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -19,6 +18,7 @@ import (
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
common "databasus-backend/internal/features/backups/backups/common"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
@@ -52,7 +52,7 @@ type writeResult struct {
|
||||
|
||||
func (uc *CreateMysqlBackupUsecase) Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
db *databases.Database,
|
||||
storage *storages.Storage,
|
||||
@@ -82,7 +82,7 @@ func (uc *CreateMysqlBackupUsecase) Execute(
|
||||
|
||||
return uc.streamToStorage(
|
||||
ctx,
|
||||
backupID,
|
||||
backup,
|
||||
backupConfig,
|
||||
tools.GetMysqlExecutable(
|
||||
my.Version,
|
||||
@@ -107,6 +107,7 @@ func (uc *CreateMysqlBackupUsecase) buildMysqldumpArgs(my *mysqltypes.MysqlDatab
|
||||
"--routines",
|
||||
"--set-gtid-purged=OFF",
|
||||
"--quick",
|
||||
"--skip-extended-insert",
|
||||
"--verbose",
|
||||
}
|
||||
|
||||
@@ -117,7 +118,11 @@ func (uc *CreateMysqlBackupUsecase) buildMysqldumpArgs(my *mysqltypes.MysqlDatab
|
||||
args = append(args, "--events")
|
||||
}
|
||||
|
||||
args = append(args, uc.getNetworkCompressionArgs(my.Version)...)
|
||||
args = append(args, uc.getNetworkCompressionArgs(my)...)
|
||||
|
||||
if !config.GetEnv().IsCloud {
|
||||
args = append(args, "--max-allowed-packet=1G")
|
||||
}
|
||||
|
||||
if my.IsHttps {
|
||||
args = append(args, "--ssl-mode=REQUIRED")
|
||||
@@ -130,15 +135,21 @@ func (uc *CreateMysqlBackupUsecase) buildMysqldumpArgs(my *mysqltypes.MysqlDatab
|
||||
return args
|
||||
}
|
||||
|
||||
func (uc *CreateMysqlBackupUsecase) getNetworkCompressionArgs(version tools.MysqlVersion) []string {
|
||||
func (uc *CreateMysqlBackupUsecase) getNetworkCompressionArgs(
|
||||
my *mysqltypes.MysqlDatabase,
|
||||
) []string {
|
||||
const zstdCompressionLevel = 5
|
||||
|
||||
switch version {
|
||||
switch my.Version {
|
||||
case tools.MysqlVersion80, tools.MysqlVersion84, tools.MysqlVersion9:
|
||||
return []string{
|
||||
"--compression-algorithms=zstd",
|
||||
fmt.Sprintf("--zstd-compression-level=%d", zstdCompressionLevel),
|
||||
if my.IsZstdSupported {
|
||||
return []string{
|
||||
"--compression-algorithms=zstd",
|
||||
fmt.Sprintf("--zstd-compression-level=%d", zstdCompressionLevel),
|
||||
}
|
||||
}
|
||||
|
||||
return []string{"--compress"}
|
||||
case tools.MysqlVersion57:
|
||||
return []string{"--compress"}
|
||||
default:
|
||||
@@ -148,7 +159,7 @@ func (uc *CreateMysqlBackupUsecase) getNetworkCompressionArgs(version tools.Mysq
|
||||
|
||||
func (uc *CreateMysqlBackupUsecase) streamToStorage(
|
||||
parentCtx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
mysqlBin string,
|
||||
args []string,
|
||||
@@ -199,7 +210,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
|
||||
storageReader, storageWriter := io.Pipe()
|
||||
|
||||
finalWriter, encryptionWriter, backupMetadata, err := uc.setupBackupEncryption(
|
||||
backupID,
|
||||
backup.ID,
|
||||
backupConfig,
|
||||
storageWriter,
|
||||
)
|
||||
@@ -216,7 +227,13 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
|
||||
|
||||
saveErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
saveErr := storage.SaveFile(ctx, uc.fieldEncryptor, uc.logger, backupID, storageReader)
|
||||
saveErr := storage.SaveFile(
|
||||
ctx,
|
||||
uc.fieldEncryptor,
|
||||
uc.logger,
|
||||
backup.FileName,
|
||||
storageReader,
|
||||
)
|
||||
saveErrCh <- saveErr
|
||||
}()
|
||||
|
||||
@@ -430,7 +447,9 @@ func (uc *CreateMysqlBackupUsecase) setupBackupEncryption(
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
storageWriter io.WriteCloser,
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, common.BackupMetadata, error) {
|
||||
metadata := common.BackupMetadata{}
|
||||
metadata := common.BackupMetadata{
|
||||
BackupID: backupID,
|
||||
}
|
||||
|
||||
if backupConfig.Encryption != backups_config.BackupEncryptionEncrypted {
|
||||
metadata.Encryption = backups_config.BackupEncryptionNone
|
||||
@@ -438,40 +457,22 @@ func (uc *CreateMysqlBackupUsecase) setupBackupEncryption(
|
||||
return storageWriter, nil, metadata, nil
|
||||
}
|
||||
|
||||
salt, err := backup_encryption.GenerateSalt()
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to generate salt: %w", err)
|
||||
}
|
||||
|
||||
nonce, err := backup_encryption.GenerateNonce()
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to generate nonce: %w", err)
|
||||
}
|
||||
|
||||
masterKey, err := uc.secretKeyService.GetSecretKey()
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to get master key: %w", err)
|
||||
}
|
||||
|
||||
encWriter, err := backup_encryption.NewEncryptionWriter(
|
||||
storageWriter,
|
||||
masterKey,
|
||||
backupID,
|
||||
salt,
|
||||
nonce,
|
||||
)
|
||||
encSetup, err := backup_encryption.SetupEncryptionWriter(storageWriter, masterKey, backupID)
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to create encrypting writer: %w", err)
|
||||
return nil, nil, metadata, err
|
||||
}
|
||||
|
||||
saltBase64 := base64.StdEncoding.EncodeToString(salt)
|
||||
nonceBase64 := base64.StdEncoding.EncodeToString(nonce)
|
||||
metadata.EncryptionSalt = &saltBase64
|
||||
metadata.EncryptionIV = &nonceBase64
|
||||
metadata.EncryptionSalt = &encSetup.SaltBase64
|
||||
metadata.EncryptionIV = &encSetup.NonceBase64
|
||||
metadata.Encryption = backups_config.BackupEncryptionEncrypted
|
||||
|
||||
uc.logger.Info("Encryption enabled for backup", "backupId", backupID)
|
||||
return encWriter, encWriter, metadata, nil
|
||||
return encSetup.Writer, encSetup.Writer, metadata, nil
|
||||
}
|
||||
|
||||
func (uc *CreateMysqlBackupUsecase) cleanupOnCancellation(
|
||||
@@ -594,6 +595,15 @@ func (uc *CreateMysqlBackupUsecase) handleConnectionErrors(stderrStr string) err
|
||||
)
|
||||
}
|
||||
|
||||
if containsIgnoreCase(stderrStr, "compression algorithm") ||
|
||||
containsIgnoreCase(stderrStr, "2066") {
|
||||
return fmt.Errorf(
|
||||
"MySQL connection failed due to unsupported compression algorithm. "+
|
||||
"Try re-saving the database connection to re-detect compression support. stderr: %s",
|
||||
stderrStr,
|
||||
)
|
||||
}
|
||||
|
||||
if containsIgnoreCase(stderrStr, "unknown database") {
|
||||
return fmt.Errorf(
|
||||
"MySQL database does not exist. stderr: %s",
|
||||
|
||||
@@ -2,7 +2,6 @@ package usecases_postgresql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -16,6 +15,7 @@ import (
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
common "databasus-backend/internal/features/backups/backups/common"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
@@ -53,7 +53,7 @@ type writeResult struct {
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
db *databases.Database,
|
||||
storage *storages.Storage,
|
||||
@@ -88,7 +88,7 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(
|
||||
|
||||
return uc.streamToStorage(
|
||||
ctx,
|
||||
backupID,
|
||||
backup,
|
||||
backupConfig,
|
||||
tools.GetPostgresqlExecutable(
|
||||
pg.Version,
|
||||
@@ -107,7 +107,7 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(
|
||||
// streamToStorage streams pg_dump output directly to storage
|
||||
func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
|
||||
parentCtx context.Context,
|
||||
backupID uuid.UUID,
|
||||
backup *backups_core.Backup,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
pgBin string,
|
||||
args []string,
|
||||
@@ -166,7 +166,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
|
||||
storageReader, storageWriter := io.Pipe()
|
||||
|
||||
finalWriter, encryptionWriter, backupMetadata, err := uc.setupBackupEncryption(
|
||||
backupID,
|
||||
backup.ID,
|
||||
backupConfig,
|
||||
storageWriter,
|
||||
)
|
||||
@@ -181,7 +181,13 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
|
||||
// Start streaming into storage in its own goroutine
|
||||
saveErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
saveErr := storage.SaveFile(ctx, uc.fieldEncryptor, uc.logger, backupID, storageReader)
|
||||
saveErr := storage.SaveFile(
|
||||
ctx,
|
||||
uc.fieldEncryptor,
|
||||
uc.logger,
|
||||
backup.FileName,
|
||||
storageReader,
|
||||
)
|
||||
saveErrCh <- saveErr
|
||||
}()
|
||||
|
||||
@@ -475,7 +481,9 @@ func (uc *CreatePostgresqlBackupUsecase) setupBackupEncryption(
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
storageWriter io.WriteCloser,
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, common.BackupMetadata, error) {
|
||||
metadata := common.BackupMetadata{}
|
||||
metadata := common.BackupMetadata{
|
||||
BackupID: backupID,
|
||||
}
|
||||
|
||||
if backupConfig.Encryption != backups_config.BackupEncryptionEncrypted {
|
||||
metadata.Encryption = backups_config.BackupEncryptionNone
|
||||
@@ -483,40 +491,22 @@ func (uc *CreatePostgresqlBackupUsecase) setupBackupEncryption(
|
||||
return storageWriter, nil, metadata, nil
|
||||
}
|
||||
|
||||
salt, err := backup_encryption.GenerateSalt()
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to generate salt: %w", err)
|
||||
}
|
||||
|
||||
nonce, err := backup_encryption.GenerateNonce()
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to generate nonce: %w", err)
|
||||
}
|
||||
|
||||
masterKey, err := uc.secretKeyService.GetSecretKey()
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to get master key: %w", err)
|
||||
}
|
||||
|
||||
encWriter, err := backup_encryption.NewEncryptionWriter(
|
||||
storageWriter,
|
||||
masterKey,
|
||||
backupID,
|
||||
salt,
|
||||
nonce,
|
||||
)
|
||||
encSetup, err := backup_encryption.SetupEncryptionWriter(storageWriter, masterKey, backupID)
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to create encrypting writer: %w", err)
|
||||
return nil, nil, metadata, err
|
||||
}
|
||||
|
||||
saltBase64 := base64.StdEncoding.EncodeToString(salt)
|
||||
nonceBase64 := base64.StdEncoding.EncodeToString(nonce)
|
||||
metadata.EncryptionSalt = &saltBase64
|
||||
metadata.EncryptionIV = &nonceBase64
|
||||
metadata.EncryptionSalt = &encSetup.SaltBase64
|
||||
metadata.EncryptionIV = &encSetup.NonceBase64
|
||||
metadata.Encryption = backups_config.BackupEncryptionEncrypted
|
||||
|
||||
uc.logger.Info("Encryption enabled for backup", "backupId", backupID)
|
||||
return encWriter, encWriter, metadata, nil
|
||||
return encSetup.Writer, encSetup.Writer, metadata, nil
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) cleanupOnCancellation(
|
||||
|
||||
@@ -118,9 +118,10 @@ func Test_SaveBackupConfig_PermissionsEnforced(t *testing.T) {
|
||||
|
||||
timeOfDay := "04:00"
|
||||
request := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -146,7 +147,7 @@ func Test_SaveBackupConfig_PermissionsEnforced(t *testing.T) {
|
||||
if tt.expectSuccess {
|
||||
assert.Equal(t, database.ID, response.DatabaseID)
|
||||
assert.True(t, response.IsBackupsEnabled)
|
||||
assert.Equal(t, period.PeriodWeek, response.StorePeriod)
|
||||
assert.Equal(t, period.PeriodWeek, response.RetentionTimePeriod)
|
||||
} else {
|
||||
assert.Contains(t, string(testResp.Body), "insufficient permissions")
|
||||
}
|
||||
@@ -170,9 +171,10 @@ func Test_SaveBackupConfig_WhenUserIsNotWorkspaceMember_ReturnsForbidden(t *test
|
||||
|
||||
timeOfDay := "04:00"
|
||||
request := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -337,7 +339,7 @@ func Test_GetBackupConfigByDbID_ReturnsDefaultConfigForNewDatabase(t *testing.T)
|
||||
|
||||
assert.Equal(t, database.ID, response.DatabaseID)
|
||||
assert.False(t, response.IsBackupsEnabled)
|
||||
assert.Equal(t, plan.MaxStoragePeriod, response.StorePeriod)
|
||||
assert.Equal(t, plan.MaxStoragePeriod, response.RetentionTimePeriod)
|
||||
assert.Equal(t, plan.MaxBackupSizeMB, response.MaxBackupSizeMB)
|
||||
assert.Equal(t, plan.MaxBackupsTotalSizeMB, response.MaxBackupsTotalSizeMB)
|
||||
assert.True(t, response.IsRetryIfFailed)
|
||||
@@ -411,9 +413,10 @@ func Test_SaveBackupConfig_WhenPlanLimitsAreAdjusted_ValidationEnforced(t *testi
|
||||
// Test 1: Try to save backup config with exceeded backup size limit
|
||||
timeOfDay := "04:00"
|
||||
backupConfigExceededSize := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -440,9 +443,10 @@ func Test_SaveBackupConfig_WhenPlanLimitsAreAdjusted_ValidationEnforced(t *testi
|
||||
|
||||
// Test 2: Try to save backup config with exceeded total size limit
|
||||
backupConfigExceededTotal := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -469,9 +473,10 @@ func Test_SaveBackupConfig_WhenPlanLimitsAreAdjusted_ValidationEnforced(t *testi
|
||||
|
||||
// Test 3: Try to save backup config with exceeded storage period limit
|
||||
backupConfigExceededPeriod := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodYear, // Exceeds limit of Month
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodYear, // Exceeds limit of Month
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -498,9 +503,10 @@ func Test_SaveBackupConfig_WhenPlanLimitsAreAdjusted_ValidationEnforced(t *testi
|
||||
|
||||
// Test 4: Save backup config within all limits - should succeed
|
||||
backupConfigValid := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek, // Within Month limit
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek, // Within Month limit
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -529,7 +535,7 @@ func Test_SaveBackupConfig_WhenPlanLimitsAreAdjusted_ValidationEnforced(t *testi
|
||||
assert.Equal(t, database.ID, responseValid.DatabaseID)
|
||||
assert.Equal(t, int64(80), responseValid.MaxBackupSizeMB)
|
||||
assert.Equal(t, int64(800), responseValid.MaxBackupsTotalSizeMB)
|
||||
assert.Equal(t, period.PeriodWeek, responseValid.StorePeriod)
|
||||
assert.Equal(t, period.PeriodWeek, responseValid.RetentionTimePeriod)
|
||||
}
|
||||
|
||||
func Test_IsStorageUsing_PermissionsEnforced(t *testing.T) {
|
||||
@@ -618,9 +624,10 @@ func Test_SaveBackupConfig_WithEncryptionNone_ConfigSaved(t *testing.T) {
|
||||
|
||||
timeOfDay := "04:00"
|
||||
request := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -662,9 +669,10 @@ func Test_SaveBackupConfig_WithEncryptionEncrypted_ConfigSaved(t *testing.T) {
|
||||
|
||||
timeOfDay := "04:00"
|
||||
request := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -959,9 +967,10 @@ func Test_TransferDatabase_ToNewStorage_DatabaseTransferd(t *testing.T) {
|
||||
|
||||
timeOfDay := "04:00"
|
||||
backupConfigRequest := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -1045,9 +1054,10 @@ func Test_TransferDatabase_WithExistingStorage_DatabaseAndStorageTransferd(t *te
|
||||
|
||||
timeOfDay := "04:00"
|
||||
backupConfigRequest := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -1142,9 +1152,10 @@ func Test_TransferDatabase_StorageHasOtherDBs_CannotTransfer(t *testing.T) {
|
||||
|
||||
timeOfDay := "04:00"
|
||||
backupConfigRequest1 := BackupConfig{
|
||||
DatabaseID: database1.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database1.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -1168,9 +1179,10 @@ func Test_TransferDatabase_StorageHasOtherDBs_CannotTransfer(t *testing.T) {
|
||||
)
|
||||
|
||||
backupConfigRequest2 := BackupConfig{
|
||||
DatabaseID: database2.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database2.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -1244,9 +1256,10 @@ func Test_TransferDatabase_WithNotifiers_NotifiersTransferred(t *testing.T) {
|
||||
|
||||
timeOfDay := "04:00"
|
||||
backupConfigRequest := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -1364,9 +1377,10 @@ func Test_TransferDatabase_NotifierHasOtherDBs_NotifierSkipped(t *testing.T) {
|
||||
|
||||
timeOfDay := "04:00"
|
||||
backupConfigRequest := BackupConfig{
|
||||
DatabaseID: database1.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database1.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -1486,9 +1500,10 @@ func Test_TransferDatabase_WithMultipleNotifiers_OnlyExclusiveOnesTransferred(t
|
||||
|
||||
timeOfDay := "04:00"
|
||||
backupConfigRequest := BackupConfig{
|
||||
DatabaseID: database1.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database1.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -1585,9 +1600,10 @@ func Test_TransferDatabase_WithTargetNotifiers_NotifiersAssigned(t *testing.T) {
|
||||
|
||||
timeOfDay := "04:00"
|
||||
backupConfigRequest := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -1665,9 +1681,10 @@ func Test_TransferDatabase_TargetNotifierFromDifferentWorkspace_ReturnsBadReques
|
||||
|
||||
timeOfDay := "04:00"
|
||||
backupConfigRequest := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -1730,9 +1747,10 @@ func Test_TransferDatabase_TargetStorageFromDifferentWorkspace_ReturnsBadRequest
|
||||
|
||||
timeOfDay := "04:00"
|
||||
backupConfigRequest := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -1789,9 +1807,10 @@ func Test_SaveBackupConfig_WithSystemStorage_CanBeUsedByAnyDatabase(t *testing.T
|
||||
|
||||
timeOfDay := "04:00"
|
||||
backupConfigWithRegularStorage := BackupConfig{
|
||||
DatabaseID: databaseA.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: databaseA.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -1840,9 +1859,10 @@ func Test_SaveBackupConfig_WithSystemStorage_CanBeUsedByAnyDatabase(t *testing.T
|
||||
assert.True(t, savedSystemStorage.IsSystem)
|
||||
|
||||
backupConfigWithSystemStorage := BackupConfig{
|
||||
DatabaseID: databaseA.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: databaseA.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
|
||||
@@ -13,3 +13,11 @@ const (
|
||||
BackupEncryptionNone BackupEncryption = "NONE"
|
||||
BackupEncryptionEncrypted BackupEncryption = "ENCRYPTED"
|
||||
)
|
||||
|
||||
type RetentionPolicyType string
|
||||
|
||||
const (
|
||||
RetentionPolicyTypeTimePeriod RetentionPolicyType = "TIME_PERIOD"
|
||||
RetentionPolicyTypeCount RetentionPolicyType = "COUNT"
|
||||
RetentionPolicyTypeGFS RetentionPolicyType = "GFS"
|
||||
)
|
||||
|
||||
@@ -18,7 +18,15 @@ type BackupConfig struct {
|
||||
|
||||
IsBackupsEnabled bool `json:"isBackupsEnabled" gorm:"column:is_backups_enabled;type:boolean;not null"`
|
||||
|
||||
StorePeriod period.Period `json:"storePeriod" gorm:"column:store_period;type:text;not null"`
|
||||
RetentionPolicyType RetentionPolicyType `json:"retentionPolicyType" gorm:"column:retention_policy_type;type:text;not null;default:'TIME_PERIOD'"`
|
||||
RetentionTimePeriod period.TimePeriod `json:"retentionTimePeriod" gorm:"column:retention_time_period;type:text;not null;default:''"`
|
||||
|
||||
RetentionCount int `json:"retentionCount" gorm:"column:retention_count;type:int;not null;default:0"`
|
||||
RetentionGfsHours int `json:"retentionGfsHours" gorm:"column:retention_gfs_hours;type:int;not null;default:0"`
|
||||
RetentionGfsDays int `json:"retentionGfsDays" gorm:"column:retention_gfs_days;type:int;not null;default:0"`
|
||||
RetentionGfsWeeks int `json:"retentionGfsWeeks" gorm:"column:retention_gfs_weeks;type:int;not null;default:0"`
|
||||
RetentionGfsMonths int `json:"retentionGfsMonths" gorm:"column:retention_gfs_months;type:int;not null;default:0"`
|
||||
RetentionGfsYears int `json:"retentionGfsYears" gorm:"column:retention_gfs_years;type:int;not null;default:0"`
|
||||
|
||||
BackupIntervalID uuid.UUID `json:"backupIntervalId" gorm:"column:backup_interval_id;type:uuid;not null"`
|
||||
BackupInterval *intervals.Interval `json:"backupInterval,omitempty" gorm:"foreignKey:BackupIntervalID"`
|
||||
@@ -78,13 +86,12 @@ func (b *BackupConfig) AfterFind(tx *gorm.DB) error {
|
||||
}
|
||||
|
||||
func (b *BackupConfig) Validate(plan *plans.DatabasePlan) error {
|
||||
// Backup interval is required either as ID or as object
|
||||
if b.BackupIntervalID == uuid.Nil && b.BackupInterval == nil {
|
||||
return errors.New("backup interval is required")
|
||||
}
|
||||
|
||||
if b.StorePeriod == "" {
|
||||
return errors.New("store period is required")
|
||||
if err := b.validateRetentionPolicy(plan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if b.IsRetryIfFailed && b.MaxFailedTriesCount <= 0 {
|
||||
@@ -110,22 +117,12 @@ func (b *BackupConfig) Validate(plan *plans.DatabasePlan) error {
|
||||
return errors.New("max backups total size must be non-negative")
|
||||
}
|
||||
|
||||
// Validate against plan limits
|
||||
// Check storage period limit
|
||||
if plan.MaxStoragePeriod != period.PeriodForever {
|
||||
if b.StorePeriod.CompareTo(plan.MaxStoragePeriod) > 0 {
|
||||
return errors.New("storage period exceeds plan limit")
|
||||
}
|
||||
}
|
||||
|
||||
// Check max backup size limit (0 in plan means unlimited)
|
||||
if plan.MaxBackupSizeMB > 0 {
|
||||
if b.MaxBackupSizeMB == 0 || b.MaxBackupSizeMB > plan.MaxBackupSizeMB {
|
||||
return errors.New("max backup size exceeds plan limit")
|
||||
}
|
||||
}
|
||||
|
||||
// Check max total backups size limit (0 in plan means unlimited)
|
||||
if plan.MaxBackupsTotalSizeMB > 0 {
|
||||
if b.MaxBackupsTotalSizeMB == 0 ||
|
||||
b.MaxBackupsTotalSizeMB > plan.MaxBackupsTotalSizeMB {
|
||||
@@ -140,7 +137,14 @@ func (b *BackupConfig) Copy(newDatabaseID uuid.UUID) *BackupConfig {
|
||||
return &BackupConfig{
|
||||
DatabaseID: newDatabaseID,
|
||||
IsBackupsEnabled: b.IsBackupsEnabled,
|
||||
StorePeriod: b.StorePeriod,
|
||||
RetentionPolicyType: b.RetentionPolicyType,
|
||||
RetentionTimePeriod: b.RetentionTimePeriod,
|
||||
RetentionCount: b.RetentionCount,
|
||||
RetentionGfsHours: b.RetentionGfsHours,
|
||||
RetentionGfsDays: b.RetentionGfsDays,
|
||||
RetentionGfsWeeks: b.RetentionGfsWeeks,
|
||||
RetentionGfsMonths: b.RetentionGfsMonths,
|
||||
RetentionGfsYears: b.RetentionGfsYears,
|
||||
BackupIntervalID: uuid.Nil,
|
||||
BackupInterval: b.BackupInterval.Copy(),
|
||||
StorageID: b.StorageID,
|
||||
@@ -152,3 +156,34 @@ func (b *BackupConfig) Copy(newDatabaseID uuid.UUID) *BackupConfig {
|
||||
MaxBackupsTotalSizeMB: b.MaxBackupsTotalSizeMB,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BackupConfig) validateRetentionPolicy(plan *plans.DatabasePlan) error {
|
||||
switch b.RetentionPolicyType {
|
||||
case RetentionPolicyTypeTimePeriod, "":
|
||||
if b.RetentionTimePeriod == "" {
|
||||
return errors.New("retention time period is required")
|
||||
}
|
||||
|
||||
if plan.MaxStoragePeriod != period.PeriodForever {
|
||||
if b.RetentionTimePeriod.CompareTo(plan.MaxStoragePeriod) > 0 {
|
||||
return errors.New("storage period exceeds plan limit")
|
||||
}
|
||||
}
|
||||
|
||||
case RetentionPolicyTypeCount:
|
||||
if b.RetentionCount <= 0 {
|
||||
return errors.New("retention count must be greater than 0")
|
||||
}
|
||||
|
||||
case RetentionPolicyTypeGFS:
|
||||
if b.RetentionGfsHours <= 0 && b.RetentionGfsDays <= 0 && b.RetentionGfsWeeks <= 0 &&
|
||||
b.RetentionGfsMonths <= 0 && b.RetentionGfsYears <= 0 {
|
||||
return errors.New("at least one GFS retention field must be greater than 0")
|
||||
}
|
||||
|
||||
default:
|
||||
return errors.New("invalid retention policy type")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -11,9 +11,9 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func Test_Validate_WhenStoragePeriodIsWeekAndPlanAllowsMonth_ValidationPasses(t *testing.T) {
|
||||
func Test_Validate_WhenRetentionTimePeriodIsWeekAndPlanAllowsMonth_ValidationPasses(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.StorePeriod = period.PeriodWeek
|
||||
config.RetentionTimePeriod = period.PeriodWeek
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
plan.MaxStoragePeriod = period.PeriodMonth
|
||||
@@ -22,9 +22,9 @@ func Test_Validate_WhenStoragePeriodIsWeekAndPlanAllowsMonth_ValidationPasses(t
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func Test_Validate_WhenStoragePeriodIsYearAndPlanAllowsMonth_ValidationFails(t *testing.T) {
|
||||
func Test_Validate_WhenRetentionTimePeriodIsYearAndPlanAllowsMonth_ValidationFails(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.StorePeriod = period.PeriodYear
|
||||
config.RetentionTimePeriod = period.PeriodYear
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
plan.MaxStoragePeriod = period.PeriodMonth
|
||||
@@ -33,9 +33,11 @@ func Test_Validate_WhenStoragePeriodIsYearAndPlanAllowsMonth_ValidationFails(t *
|
||||
assert.EqualError(t, err, "storage period exceeds plan limit")
|
||||
}
|
||||
|
||||
func Test_Validate_WhenStoragePeriodIsForeverAndPlanAllowsForever_ValidationPasses(t *testing.T) {
|
||||
func Test_Validate_WhenRetentionTimePeriodIsForeverAndPlanAllowsForever_ValidationPasses(
|
||||
t *testing.T,
|
||||
) {
|
||||
config := createValidBackupConfig()
|
||||
config.StorePeriod = period.PeriodForever
|
||||
config.RetentionTimePeriod = period.PeriodForever
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
plan.MaxStoragePeriod = period.PeriodForever
|
||||
@@ -44,9 +46,9 @@ func Test_Validate_WhenStoragePeriodIsForeverAndPlanAllowsForever_ValidationPass
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func Test_Validate_WhenStoragePeriodIsForeverAndPlanAllowsYear_ValidationFails(t *testing.T) {
|
||||
func Test_Validate_WhenRetentionTimePeriodIsForeverAndPlanAllowsYear_ValidationFails(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.StorePeriod = period.PeriodForever
|
||||
config.RetentionTimePeriod = period.PeriodForever
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
plan.MaxStoragePeriod = period.PeriodYear
|
||||
@@ -55,9 +57,9 @@ func Test_Validate_WhenStoragePeriodIsForeverAndPlanAllowsYear_ValidationFails(t
|
||||
assert.EqualError(t, err, "storage period exceeds plan limit")
|
||||
}
|
||||
|
||||
func Test_Validate_WhenStoragePeriodEqualsExactPlanLimit_ValidationPasses(t *testing.T) {
|
||||
func Test_Validate_WhenRetentionTimePeriodEqualsExactPlanLimit_ValidationPasses(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.StorePeriod = period.PeriodMonth
|
||||
config.RetentionTimePeriod = period.PeriodMonth
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
plan.MaxStoragePeriod = period.PeriodMonth
|
||||
@@ -178,7 +180,7 @@ func Test_Validate_WhenTotalSizeEqualsExactPlanLimit_ValidationPasses(t *testing
|
||||
|
||||
func Test_Validate_WhenAllLimitsAreUnlimitedInPlan_AnyConfigurationPasses(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.StorePeriod = period.PeriodForever
|
||||
config.RetentionTimePeriod = period.PeriodForever
|
||||
config.MaxBackupSizeMB = 0
|
||||
config.MaxBackupsTotalSizeMB = 0
|
||||
|
||||
@@ -190,7 +192,7 @@ func Test_Validate_WhenAllLimitsAreUnlimitedInPlan_AnyConfigurationPasses(t *tes
|
||||
|
||||
func Test_Validate_WhenMultipleLimitsExceeded_ValidationFailsWithFirstError(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.StorePeriod = period.PeriodYear
|
||||
config.RetentionTimePeriod = period.PeriodYear
|
||||
config.MaxBackupSizeMB = 500
|
||||
config.MaxBackupsTotalSizeMB = 5000
|
||||
|
||||
@@ -249,14 +251,14 @@ func Test_Validate_WhenEncryptionIsInvalid_ValidationFailsRegardlessOfPlan(t *te
|
||||
assert.EqualError(t, err, "encryption must be NONE or ENCRYPTED")
|
||||
}
|
||||
|
||||
func Test_Validate_WhenStoragePeriodIsEmpty_ValidationFails(t *testing.T) {
|
||||
func Test_Validate_WhenRetentionTimePeriodIsEmpty_ValidationFails(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.StorePeriod = ""
|
||||
config.RetentionTimePeriod = ""
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
|
||||
err := config.Validate(plan)
|
||||
assert.EqualError(t, err, "store period is required")
|
||||
assert.EqualError(t, err, "retention time period is required")
|
||||
}
|
||||
|
||||
func Test_Validate_WhenMaxBackupSizeIsNegative_ValidationFails(t *testing.T) {
|
||||
@@ -282,8 +284,8 @@ func Test_Validate_WhenMaxTotalSizeIsNegative_ValidationFails(t *testing.T) {
|
||||
func Test_Validate_WhenPlanLimitsAreAtBoundary_ValidationWorks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
configPeriod period.Period
|
||||
planPeriod period.Period
|
||||
configPeriod period.TimePeriod
|
||||
planPeriod period.TimePeriod
|
||||
configSize int64
|
||||
planSize int64
|
||||
configTotal int64
|
||||
@@ -345,7 +347,7 @@ func Test_Validate_WhenPlanLimitsAreAtBoundary_ValidationWorks(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.StorePeriod = tt.configPeriod
|
||||
config.RetentionTimePeriod = tt.configPeriod
|
||||
config.MaxBackupSizeMB = tt.configSize
|
||||
config.MaxBackupsTotalSizeMB = tt.configTotal
|
||||
|
||||
@@ -364,12 +366,96 @@ func Test_Validate_WhenPlanLimitsAreAtBoundary_ValidationWorks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func Test_Validate_WhenPolicyTypeIsCount_RequiresPositiveCount(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.RetentionPolicyType = RetentionPolicyTypeCount
|
||||
config.RetentionCount = 0
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
|
||||
err := config.Validate(plan)
|
||||
assert.EqualError(t, err, "retention count must be greater than 0")
|
||||
}
|
||||
|
||||
func Test_Validate_WhenPolicyTypeIsCount_WithPositiveCount_ValidationPasses(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.RetentionPolicyType = RetentionPolicyTypeCount
|
||||
config.RetentionCount = 10
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
|
||||
err := config.Validate(plan)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func Test_Validate_WhenPolicyTypeIsGFS_RequiresAtLeastOneField(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.RetentionPolicyType = RetentionPolicyTypeGFS
|
||||
config.RetentionGfsDays = 0
|
||||
config.RetentionGfsWeeks = 0
|
||||
config.RetentionGfsMonths = 0
|
||||
config.RetentionGfsYears = 0
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
|
||||
err := config.Validate(plan)
|
||||
assert.EqualError(t, err, "at least one GFS retention field must be greater than 0")
|
||||
}
|
||||
|
||||
func Test_Validate_WhenPolicyTypeIsGFS_WithOnlyHours_ValidationPasses(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.RetentionPolicyType = RetentionPolicyTypeGFS
|
||||
config.RetentionGfsHours = 24
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
|
||||
err := config.Validate(plan)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func Test_Validate_WhenPolicyTypeIsGFS_WithOnlyDays_ValidationPasses(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.RetentionPolicyType = RetentionPolicyTypeGFS
|
||||
config.RetentionGfsDays = 7
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
|
||||
err := config.Validate(plan)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func Test_Validate_WhenPolicyTypeIsGFS_WithAllFields_ValidationPasses(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.RetentionPolicyType = RetentionPolicyTypeGFS
|
||||
config.RetentionGfsHours = 24
|
||||
config.RetentionGfsDays = 7
|
||||
config.RetentionGfsWeeks = 4
|
||||
config.RetentionGfsMonths = 12
|
||||
config.RetentionGfsYears = 3
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
|
||||
err := config.Validate(plan)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func Test_Validate_WhenPolicyTypeIsInvalid_ValidationFails(t *testing.T) {
|
||||
config := createValidBackupConfig()
|
||||
config.RetentionPolicyType = "INVALID"
|
||||
|
||||
plan := createUnlimitedPlan()
|
||||
|
||||
err := config.Validate(plan)
|
||||
assert.EqualError(t, err, "invalid retention policy type")
|
||||
}
|
||||
|
||||
func createValidBackupConfig() *BackupConfig {
|
||||
intervalID := uuid.New()
|
||||
return &BackupConfig{
|
||||
DatabaseID: uuid.New(),
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodMonth,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodMonth,
|
||||
BackupIntervalID: intervalID,
|
||||
BackupInterval: &intervals.Interval{ID: intervalID},
|
||||
SendNotificationsOn: []BackupNotificationType{},
|
||||
|
||||
@@ -227,7 +227,8 @@ func (s *BackupConfigService) initializeDefaultConfig(
|
||||
_, err = s.backupConfigRepository.Save(&BackupConfig{
|
||||
DatabaseID: databaseID,
|
||||
IsBackupsEnabled: false,
|
||||
StorePeriod: plan.MaxStoragePeriod,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: plan.MaxStoragePeriod,
|
||||
MaxBackupSizeMB: plan.MaxBackupSizeMB,
|
||||
MaxBackupsTotalSizeMB: plan.MaxBackupsTotalSizeMB,
|
||||
BackupInterval: &intervals.Interval{
|
||||
|
||||
@@ -35,9 +35,10 @@ func Test_AttachStorageFromSameWorkspace_SuccessfullyAttached(t *testing.T) {
|
||||
|
||||
timeOfDay := "04:00"
|
||||
request := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -87,9 +88,10 @@ func Test_AttachStorageFromDifferentWorkspace_ReturnsForbidden(t *testing.T) {
|
||||
|
||||
timeOfDay := "04:00"
|
||||
request := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -131,9 +133,10 @@ func Test_DeleteStorageWithAttachedDatabases_CannotDelete(t *testing.T) {
|
||||
|
||||
timeOfDay := "04:00"
|
||||
request := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
@@ -191,9 +194,10 @@ func Test_TransferStorageWithAttachedDatabase_CannotTransfer(t *testing.T) {
|
||||
|
||||
timeOfDay := "04:00"
|
||||
request := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
|
||||
@@ -15,9 +15,10 @@ func EnableBackupsForTestDatabase(
|
||||
timeOfDay := "16:00"
|
||||
|
||||
backupConfig := &BackupConfig{
|
||||
DatabaseID: databaseID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodDay,
|
||||
DatabaseID: databaseID,
|
||||
IsBackupsEnabled: true,
|
||||
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
|
||||
RetentionTimePeriod: period.PeriodDay,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
|
||||
@@ -29,6 +29,11 @@ func (c *DatabaseController) RegisterRoutes(router *gin.RouterGroup) {
|
||||
router.GET("/databases/notifier/:id/databases-count", c.CountDatabasesByNotifier)
|
||||
router.POST("/databases/is-readonly", c.IsUserReadOnly)
|
||||
router.POST("/databases/create-readonly-user", c.CreateReadOnlyUser)
|
||||
router.POST("/databases/:id/regenerate-token", c.RegenerateAgentToken)
|
||||
}
|
||||
|
||||
func (c *DatabaseController) RegisterPublicRoutes(router *gin.RouterGroup) {
|
||||
router.POST("/databases/verify-token", c.VerifyAgentToken)
|
||||
}
|
||||
|
||||
// CreateDatabase
|
||||
@@ -438,3 +443,61 @@ func (c *DatabaseController) CreateReadOnlyUser(ctx *gin.Context) {
|
||||
Password: password,
|
||||
})
|
||||
}
|
||||
|
||||
// RegenerateAgentToken
|
||||
// @Summary Regenerate agent token for a database
|
||||
// @Description Generate a new agent token for the database. The token is returned once and stored as a hash.
|
||||
// @Tags databases
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param id path string true "Database ID"
|
||||
// @Success 200 {object} map[string]string
|
||||
// @Failure 400 {object} map[string]string
|
||||
// @Failure 401 {object} map[string]string
|
||||
// @Router /databases/{id}/regenerate-token [post]
|
||||
func (c *DatabaseController) RegenerateAgentToken(ctx *gin.Context) {
|
||||
user, ok := users_middleware.GetUserFromContext(ctx)
|
||||
if !ok {
|
||||
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "User not authenticated"})
|
||||
return
|
||||
}
|
||||
|
||||
id, err := uuid.Parse(ctx.Param("id"))
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": "invalid database ID"})
|
||||
return
|
||||
}
|
||||
|
||||
token, err := c.databaseService.RegenerateAgentToken(user, id)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
ctx.JSON(http.StatusOK, gin.H{"token": token})
|
||||
}
|
||||
|
||||
// VerifyAgentToken
|
||||
// @Summary Verify agent token
|
||||
// @Description Verify that a given agent token is valid for any database
|
||||
// @Tags databases
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body VerifyAgentTokenRequest true "Token to verify"
|
||||
// @Success 200 {object} map[string]string
|
||||
// @Failure 401 {object} map[string]string
|
||||
// @Router /databases/verify-token [post]
|
||||
func (c *DatabaseController) VerifyAgentToken(ctx *gin.Context) {
|
||||
var request VerifyAgentTokenRequest
|
||||
if err := ctx.ShouldBindJSON(&request); err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
if err := c.databaseService.VerifyAgentToken(request.Token); err != nil {
|
||||
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "invalid token"})
|
||||
return
|
||||
}
|
||||
|
||||
ctx.JSON(http.StatusOK, gin.H{"message": "token is valid"})
|
||||
}
|
||||
|
||||
@@ -13,10 +13,13 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
"databasus-backend/internal/features/audit_logs"
|
||||
"databasus-backend/internal/features/databases/databases/mariadb"
|
||||
"databasus-backend/internal/features/databases/databases/mongodb"
|
||||
"databasus-backend/internal/features/databases/databases/postgresql"
|
||||
users_enums "databasus-backend/internal/features/users/enums"
|
||||
users_middleware "databasus-backend/internal/features/users/middleware"
|
||||
users_services "databasus-backend/internal/features/users/services"
|
||||
users_testing "databasus-backend/internal/features/users/testing"
|
||||
workspaces_controllers "databasus-backend/internal/features/workspaces/controllers"
|
||||
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
|
||||
@@ -144,6 +147,66 @@ func Test_CreateDatabase_WhenUserIsNotWorkspaceMember_ReturnsForbidden(t *testin
|
||||
assert.Contains(t, string(testResp.Body), "insufficient permissions")
|
||||
}
|
||||
|
||||
func Test_CreateDatabase_WalV1Type_NoConnectionFieldsRequired(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
defer workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
|
||||
request := Database{
|
||||
Name: "Test WAL Database",
|
||||
WorkspaceID: &workspace.ID,
|
||||
Type: DatabaseTypePostgres,
|
||||
Postgresql: &postgresql.PostgresqlDatabase{
|
||||
BackupType: postgresql.PostgresBackupTypeWalV1,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
var response Database
|
||||
test_utils.MakePostRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/databases/create",
|
||||
"Bearer "+owner.Token,
|
||||
request,
|
||||
http.StatusCreated,
|
||||
&response,
|
||||
)
|
||||
defer RemoveTestDatabase(&response)
|
||||
|
||||
assert.Equal(t, "Test WAL Database", response.Name)
|
||||
assert.NotEqual(t, uuid.Nil, response.ID)
|
||||
}
|
||||
|
||||
func Test_CreateDatabase_PgDumpType_ConnectionFieldsRequired(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
defer workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
|
||||
request := Database{
|
||||
Name: "Test PG_DUMP Database",
|
||||
WorkspaceID: &workspace.ID,
|
||||
Type: DatabaseTypePostgres,
|
||||
Postgresql: &postgresql.PostgresqlDatabase{
|
||||
BackupType: postgresql.PostgresBackupTypePgDump,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
testResp := test_utils.MakePostRequest(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/databases/create",
|
||||
"Bearer "+owner.Token,
|
||||
request,
|
||||
http.StatusBadRequest,
|
||||
)
|
||||
|
||||
assert.Contains(t, string(testResp.Body), "host is required")
|
||||
}
|
||||
|
||||
func Test_UpdateDatabase_PermissionsEnforced(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -256,6 +319,52 @@ func Test_UpdateDatabase_WhenUserIsNotWorkspaceMember_ReturnsForbidden(t *testin
|
||||
assert.Contains(t, string(testResp.Body), "insufficient permissions")
|
||||
}
|
||||
|
||||
func Test_UpdateDatabase_WhenDatabaseTypeChanged_ReturnsBadRequest(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
defer workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
|
||||
database := createTestDatabaseViaAPI("Test Database", workspace.ID, owner.Token, router)
|
||||
defer RemoveTestDatabase(database)
|
||||
|
||||
database.Type = DatabaseTypeMysql
|
||||
|
||||
testResp := test_utils.MakePostRequest(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/databases/update",
|
||||
"Bearer "+owner.Token,
|
||||
database,
|
||||
http.StatusBadRequest,
|
||||
)
|
||||
|
||||
assert.Contains(t, string(testResp.Body), "database type cannot be changed")
|
||||
}
|
||||
|
||||
func Test_UpdateDatabase_WhenBackupTypeChanged_ReturnsBadRequest(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
defer workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
|
||||
database := createTestDatabaseViaAPI("Test Database", workspace.ID, owner.Token, router)
|
||||
defer RemoveTestDatabase(database)
|
||||
|
||||
database.Postgresql.BackupType = postgresql.PostgresBackupTypeWalV1
|
||||
|
||||
testResp := test_utils.MakePostRequest(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/databases/update",
|
||||
"Bearer "+owner.Token,
|
||||
database,
|
||||
http.StatusBadRequest,
|
||||
)
|
||||
|
||||
assert.Contains(t, string(testResp.Body), "backup type cannot be changed")
|
||||
}
|
||||
|
||||
func Test_DeleteDatabase_PermissionsEnforced(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -1050,6 +1159,87 @@ func Test_TestConnection_PermissionsEnforced(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func Test_RegenerateAgentToken_ReturnsToken(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
defer workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
|
||||
database := createTestDatabaseViaAPI("Test Database", workspace.ID, owner.Token, router)
|
||||
defer RemoveTestDatabase(database)
|
||||
|
||||
var response map[string]string
|
||||
test_utils.MakePostRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/databases/"+database.ID.String()+"/regenerate-token",
|
||||
"Bearer "+owner.Token,
|
||||
nil,
|
||||
http.StatusOK,
|
||||
&response,
|
||||
)
|
||||
|
||||
assert.NotEmpty(t, response["token"])
|
||||
assert.Len(t, response["token"], 32)
|
||||
|
||||
var updatedDatabase Database
|
||||
test_utils.MakeGetRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/databases/"+database.ID.String(),
|
||||
"Bearer "+owner.Token,
|
||||
http.StatusOK,
|
||||
&updatedDatabase,
|
||||
)
|
||||
assert.True(t, updatedDatabase.IsAgentTokenGenerated)
|
||||
}
|
||||
|
||||
func Test_VerifyAgentToken_WithValidToken_Succeeds(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
defer workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
|
||||
database := createTestDatabaseViaAPI("Test Database", workspace.ID, owner.Token, router)
|
||||
defer RemoveTestDatabase(database)
|
||||
|
||||
var regenerateResponse map[string]string
|
||||
test_utils.MakePostRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/databases/"+database.ID.String()+"/regenerate-token",
|
||||
"Bearer "+owner.Token,
|
||||
nil,
|
||||
http.StatusOK,
|
||||
®enerateResponse,
|
||||
)
|
||||
|
||||
token := regenerateResponse["token"]
|
||||
assert.NotEmpty(t, token)
|
||||
|
||||
w := workspaces_testing.MakeAPIRequest(
|
||||
router,
|
||||
"POST",
|
||||
"/api/v1/databases/verify-token",
|
||||
"",
|
||||
VerifyAgentTokenRequest{Token: token},
|
||||
)
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
}
|
||||
|
||||
func Test_VerifyAgentToken_WithInvalidToken_Returns401(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
|
||||
w := workspaces_testing.MakeAPIRequest(
|
||||
router,
|
||||
"POST",
|
||||
"/api/v1/databases/verify-token",
|
||||
"",
|
||||
VerifyAgentTokenRequest{Token: "invalidtoken00000000000000000000"},
|
||||
)
|
||||
assert.Equal(t, http.StatusUnauthorized, w.Code)
|
||||
}
|
||||
|
||||
func createTestDatabaseViaAPI(
|
||||
name string,
|
||||
workspaceID uuid.UUID,
|
||||
@@ -1101,11 +1291,20 @@ func createTestDatabaseViaAPI(
|
||||
}
|
||||
|
||||
func createTestRouter() *gin.Engine {
|
||||
router := workspaces_testing.CreateTestRouter(
|
||||
workspaces_controllers.GetWorkspaceController(),
|
||||
workspaces_controllers.GetMembershipController(),
|
||||
GetDatabaseController(),
|
||||
)
|
||||
gin.SetMode(gin.TestMode)
|
||||
router := gin.New()
|
||||
|
||||
v1 := router.Group("/api/v1")
|
||||
protected := v1.Group("").Use(users_middleware.AuthMiddleware(users_services.GetUserService()))
|
||||
|
||||
workspaces_controllers.GetWorkspaceController().RegisterRoutes(protected.(*gin.RouterGroup))
|
||||
workspaces_controllers.GetMembershipController().RegisterRoutes(protected.(*gin.RouterGroup))
|
||||
GetDatabaseController().RegisterRoutes(protected.(*gin.RouterGroup))
|
||||
|
||||
GetDatabaseController().RegisterPublicRoutes(v1)
|
||||
|
||||
audit_logs.SetupDependencies()
|
||||
|
||||
return router
|
||||
}
|
||||
|
||||
@@ -1118,13 +1317,14 @@ func getTestPostgresConfig() *postgresql.PostgresqlDatabase {
|
||||
|
||||
testDbName := "testdb"
|
||||
return &postgresql.PostgresqlDatabase{
|
||||
Version: tools.PostgresqlVersion16,
|
||||
Host: config.GetEnv().TestLocalhost,
|
||||
Port: port,
|
||||
Username: "testuser",
|
||||
Password: "testpassword",
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
BackupType: postgresql.PostgresBackupTypePgDump,
|
||||
Version: tools.PostgresqlVersion16,
|
||||
Host: config.GetEnv().TestLocalhost,
|
||||
Port: port,
|
||||
Username: "testuser",
|
||||
Password: "testpassword",
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1164,12 +1364,13 @@ func getTestMongodbConfig() *mongodb.MongodbDatabase {
|
||||
return &mongodb.MongodbDatabase{
|
||||
Version: tools.MongodbVersion7,
|
||||
Host: config.GetEnv().TestLocalhost,
|
||||
Port: port,
|
||||
Port: &port,
|
||||
Username: "root",
|
||||
Password: "rootpassword",
|
||||
Database: "testdb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
IsSrv: false,
|
||||
CpuCount: 1,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,13 +25,14 @@ type MariadbDatabase struct {
|
||||
|
||||
Version tools.MariadbVersion `json:"version" gorm:"type:text;not null"`
|
||||
|
||||
Host string `json:"host" gorm:"type:text;not null"`
|
||||
Port int `json:"port" gorm:"type:int;not null"`
|
||||
Username string `json:"username" gorm:"type:text;not null"`
|
||||
Password string `json:"password" gorm:"type:text;not null"`
|
||||
Database *string `json:"database" gorm:"type:text"`
|
||||
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
|
||||
Privileges string `json:"privileges" gorm:"column:privileges;type:text;not null;default:''"`
|
||||
Host string `json:"host" gorm:"type:text;not null"`
|
||||
Port int `json:"port" gorm:"type:int;not null"`
|
||||
Username string `json:"username" gorm:"type:text;not null"`
|
||||
Password string `json:"password" gorm:"type:text;not null"`
|
||||
Database *string `json:"database" gorm:"type:text"`
|
||||
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
|
||||
IsExcludeEvents bool `json:"isExcludeEvents" gorm:"type:boolean;default:false"`
|
||||
Privileges string `json:"privileges" gorm:"column:privileges;type:text;not null;default:''"`
|
||||
}
|
||||
|
||||
func (m *MariadbDatabase) TableName() string {
|
||||
@@ -124,6 +125,7 @@ func (m *MariadbDatabase) Update(incoming *MariadbDatabase) {
|
||||
m.Username = incoming.Username
|
||||
m.Database = incoming.Database
|
||||
m.IsHttps = incoming.IsHttps
|
||||
m.IsExcludeEvents = incoming.IsExcludeEvents
|
||||
m.Privileges = incoming.Privileges
|
||||
|
||||
if incoming.Password != "" {
|
||||
|
||||
@@ -25,14 +25,16 @@ type MongodbDatabase struct {
|
||||
|
||||
Version tools.MongodbVersion `json:"version" gorm:"type:text;not null"`
|
||||
|
||||
Host string `json:"host" gorm:"type:text;not null"`
|
||||
Port int `json:"port" gorm:"type:int;not null"`
|
||||
Username string `json:"username" gorm:"type:text;not null"`
|
||||
Password string `json:"password" gorm:"type:text;not null"`
|
||||
Database string `json:"database" gorm:"type:text;not null"`
|
||||
AuthDatabase string `json:"authDatabase" gorm:"type:text;not null;default:'admin'"`
|
||||
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
|
||||
CpuCount int `json:"cpuCount" gorm:"column:cpu_count;type:int;not null;default:1"`
|
||||
Host string `json:"host" gorm:"type:text;not null"`
|
||||
Port *int `json:"port" gorm:"type:int"`
|
||||
Username string `json:"username" gorm:"type:text;not null"`
|
||||
Password string `json:"password" gorm:"type:text;not null"`
|
||||
Database string `json:"database" gorm:"type:text;not null"`
|
||||
AuthDatabase string `json:"authDatabase" gorm:"type:text;not null;default:'admin'"`
|
||||
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
|
||||
IsSrv bool `json:"isSrv" gorm:"column:is_srv;type:boolean;not null;default:false"`
|
||||
IsDirectConnection bool `json:"isDirectConnection" gorm:"column:is_direct_connection;type:boolean;not null;default:false"`
|
||||
CpuCount int `json:"cpuCount" gorm:"column:cpu_count;type:int;not null;default:1"`
|
||||
}
|
||||
|
||||
func (m *MongodbDatabase) TableName() string {
|
||||
@@ -43,9 +45,13 @@ func (m *MongodbDatabase) Validate() error {
|
||||
if m.Host == "" {
|
||||
return errors.New("host is required")
|
||||
}
|
||||
if m.Port == 0 {
|
||||
return errors.New("port is required")
|
||||
|
||||
if !m.IsSrv {
|
||||
if m.Port == nil || *m.Port == 0 {
|
||||
return errors.New("port is required for standard connections")
|
||||
}
|
||||
}
|
||||
|
||||
if m.Username == "" {
|
||||
return errors.New("username is required")
|
||||
}
|
||||
@@ -58,6 +64,7 @@ func (m *MongodbDatabase) Validate() error {
|
||||
if m.CpuCount <= 0 {
|
||||
return errors.New("cpu count must be greater than 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -125,6 +132,8 @@ func (m *MongodbDatabase) Update(incoming *MongodbDatabase) {
|
||||
m.Database = incoming.Database
|
||||
m.AuthDatabase = incoming.AuthDatabase
|
||||
m.IsHttps = incoming.IsHttps
|
||||
m.IsSrv = incoming.IsSrv
|
||||
m.IsDirectConnection = incoming.IsDirectConnection
|
||||
m.CpuCount = incoming.CpuCount
|
||||
|
||||
if incoming.Password != "" {
|
||||
@@ -450,9 +459,29 @@ func (m *MongodbDatabase) buildConnectionURI(password string) string {
|
||||
authDB = "admin"
|
||||
}
|
||||
|
||||
tlsParams := ""
|
||||
extraParams := ""
|
||||
if m.IsHttps {
|
||||
tlsParams = "&tls=true&tlsInsecure=true"
|
||||
extraParams += "&tls=true&tlsInsecure=true"
|
||||
}
|
||||
if m.IsDirectConnection {
|
||||
extraParams += "&directConnection=true"
|
||||
}
|
||||
|
||||
if m.IsSrv {
|
||||
return fmt.Sprintf(
|
||||
"mongodb+srv://%s:%s@%s/%s?authSource=%s&connectTimeoutMS=15000%s",
|
||||
url.QueryEscape(m.Username),
|
||||
url.QueryEscape(password),
|
||||
m.Host,
|
||||
m.Database,
|
||||
authDB,
|
||||
extraParams,
|
||||
)
|
||||
}
|
||||
|
||||
port := 27017
|
||||
if m.Port != nil {
|
||||
port = *m.Port
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
@@ -460,10 +489,10 @@ func (m *MongodbDatabase) buildConnectionURI(password string) string {
|
||||
url.QueryEscape(m.Username),
|
||||
url.QueryEscape(password),
|
||||
m.Host,
|
||||
m.Port,
|
||||
port,
|
||||
m.Database,
|
||||
authDB,
|
||||
tlsParams,
|
||||
extraParams,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -474,9 +503,28 @@ func (m *MongodbDatabase) BuildMongodumpURI(password string) string {
|
||||
authDB = "admin"
|
||||
}
|
||||
|
||||
tlsParams := ""
|
||||
extraParams := ""
|
||||
if m.IsHttps {
|
||||
tlsParams = "&tls=true&tlsInsecure=true"
|
||||
extraParams += "&tls=true&tlsInsecure=true"
|
||||
}
|
||||
if m.IsDirectConnection {
|
||||
extraParams += "&directConnection=true"
|
||||
}
|
||||
|
||||
if m.IsSrv {
|
||||
return fmt.Sprintf(
|
||||
"mongodb+srv://%s:%s@%s/?authSource=%s&connectTimeoutMS=15000%s",
|
||||
url.QueryEscape(m.Username),
|
||||
url.QueryEscape(password),
|
||||
m.Host,
|
||||
authDB,
|
||||
extraParams,
|
||||
)
|
||||
}
|
||||
|
||||
port := 27017
|
||||
if m.Port != nil {
|
||||
port = *m.Port
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
@@ -484,9 +532,9 @@ func (m *MongodbDatabase) BuildMongodumpURI(password string) string {
|
||||
url.QueryEscape(m.Username),
|
||||
url.QueryEscape(password),
|
||||
m.Host,
|
||||
m.Port,
|
||||
port,
|
||||
authDB,
|
||||
tlsParams,
|
||||
extraParams,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -64,15 +64,17 @@ func Test_TestConnection_InsufficientPermissions_ReturnsError(t *testing.T) {
|
||||
|
||||
defer dropUserSafe(container.Client, limitedUsername, container.AuthDatabase)
|
||||
|
||||
port := container.Port
|
||||
mongodbModel := &MongodbDatabase{
|
||||
Version: tc.version,
|
||||
Host: container.Host,
|
||||
Port: container.Port,
|
||||
Port: &port,
|
||||
Username: limitedUsername,
|
||||
Password: limitedPassword,
|
||||
Database: container.Database,
|
||||
AuthDatabase: container.AuthDatabase,
|
||||
IsHttps: false,
|
||||
IsSrv: false,
|
||||
CpuCount: 1,
|
||||
}
|
||||
|
||||
@@ -133,15 +135,17 @@ func Test_TestConnection_SufficientPermissions_Success(t *testing.T) {
|
||||
|
||||
defer dropUserSafe(container.Client, backupUsername, container.AuthDatabase)
|
||||
|
||||
port := container.Port
|
||||
mongodbModel := &MongodbDatabase{
|
||||
Version: tc.version,
|
||||
Host: container.Host,
|
||||
Port: container.Port,
|
||||
Port: &port,
|
||||
Username: backupUsername,
|
||||
Password: backupPassword,
|
||||
Database: container.Database,
|
||||
AuthDatabase: container.AuthDatabase,
|
||||
IsHttps: false,
|
||||
IsSrv: false,
|
||||
CpuCount: 1,
|
||||
}
|
||||
|
||||
@@ -442,15 +446,17 @@ func connectToMongodbContainer(
|
||||
}
|
||||
|
||||
func createMongodbModel(container *MongodbContainer) *MongodbDatabase {
|
||||
port := container.Port
|
||||
return &MongodbDatabase{
|
||||
Version: container.Version,
|
||||
Host: container.Host,
|
||||
Port: container.Port,
|
||||
Port: &port,
|
||||
Username: container.Username,
|
||||
Password: container.Password,
|
||||
Database: container.Database,
|
||||
AuthDatabase: container.AuthDatabase,
|
||||
IsHttps: false,
|
||||
IsSrv: false,
|
||||
CpuCount: 1,
|
||||
}
|
||||
}
|
||||
@@ -489,3 +495,240 @@ func assertWriteDenied(t *testing.T, err error) {
|
||||
strings.Contains(errStr, "permission denied"),
|
||||
"Expected authorization error, got: %v", err)
|
||||
}
|
||||
|
||||
func Test_BuildConnectionURI_WithSrvFormat_ReturnsCorrectUri(t *testing.T) {
|
||||
port := 27017
|
||||
model := &MongodbDatabase{
|
||||
Host: "cluster0.example.mongodb.net",
|
||||
Port: &port,
|
||||
Username: "testuser",
|
||||
Password: "testpass123",
|
||||
Database: "mydb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
IsSrv: true,
|
||||
}
|
||||
|
||||
uri := model.buildConnectionURI("testpass123")
|
||||
|
||||
assert.Contains(t, uri, "mongodb+srv://")
|
||||
assert.Contains(t, uri, "testuser")
|
||||
assert.Contains(t, uri, "testpass123")
|
||||
assert.Contains(t, uri, "cluster0.example.mongodb.net")
|
||||
assert.Contains(t, uri, "/mydb")
|
||||
assert.Contains(t, uri, "authSource=admin")
|
||||
assert.Contains(t, uri, "connectTimeoutMS=15000")
|
||||
assert.NotContains(t, uri, ":27017")
|
||||
}
|
||||
|
||||
func Test_BuildConnectionURI_WithStandardFormat_ReturnsCorrectUri(t *testing.T) {
|
||||
port := 27017
|
||||
model := &MongodbDatabase{
|
||||
Host: "localhost",
|
||||
Port: &port,
|
||||
Username: "testuser",
|
||||
Password: "testpass123",
|
||||
Database: "mydb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
IsSrv: false,
|
||||
}
|
||||
|
||||
uri := model.buildConnectionURI("testpass123")
|
||||
|
||||
assert.Contains(t, uri, "mongodb://")
|
||||
assert.Contains(t, uri, "testuser")
|
||||
assert.Contains(t, uri, "testpass123")
|
||||
assert.Contains(t, uri, "localhost:27017")
|
||||
assert.Contains(t, uri, "/mydb")
|
||||
assert.Contains(t, uri, "authSource=admin")
|
||||
assert.Contains(t, uri, "connectTimeoutMS=15000")
|
||||
assert.NotContains(t, uri, "mongodb+srv://")
|
||||
}
|
||||
|
||||
func Test_BuildConnectionURI_WithNullPort_UsesDefault(t *testing.T) {
|
||||
model := &MongodbDatabase{
|
||||
Host: "localhost",
|
||||
Port: nil,
|
||||
Username: "testuser",
|
||||
Password: "testpass123",
|
||||
Database: "mydb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
IsSrv: false,
|
||||
}
|
||||
|
||||
uri := model.buildConnectionURI("testpass123")
|
||||
|
||||
assert.Contains(t, uri, "localhost:27017")
|
||||
}
|
||||
|
||||
func Test_BuildMongodumpURI_WithSrvFormat_ReturnsCorrectUri(t *testing.T) {
|
||||
port := 27017
|
||||
model := &MongodbDatabase{
|
||||
Host: "cluster0.example.mongodb.net",
|
||||
Port: &port,
|
||||
Username: "testuser",
|
||||
Password: "testpass123",
|
||||
Database: "mydb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
IsSrv: true,
|
||||
}
|
||||
|
||||
uri := model.BuildMongodumpURI("testpass123")
|
||||
|
||||
assert.Contains(t, uri, "mongodb+srv://")
|
||||
assert.Contains(t, uri, "testuser")
|
||||
assert.Contains(t, uri, "testpass123")
|
||||
assert.Contains(t, uri, "cluster0.example.mongodb.net")
|
||||
assert.Contains(t, uri, "/?authSource=admin")
|
||||
assert.Contains(t, uri, "connectTimeoutMS=15000")
|
||||
assert.NotContains(t, uri, ":27017")
|
||||
assert.NotContains(t, uri, "/mydb")
|
||||
}
|
||||
|
||||
func Test_BuildMongodumpURI_WithStandardFormat_ReturnsCorrectUri(t *testing.T) {
|
||||
port := 27017
|
||||
model := &MongodbDatabase{
|
||||
Host: "localhost",
|
||||
Port: &port,
|
||||
Username: "testuser",
|
||||
Password: "testpass123",
|
||||
Database: "mydb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
IsSrv: false,
|
||||
}
|
||||
|
||||
uri := model.BuildMongodumpURI("testpass123")
|
||||
|
||||
assert.Contains(t, uri, "mongodb://")
|
||||
assert.Contains(t, uri, "testuser")
|
||||
assert.Contains(t, uri, "testpass123")
|
||||
assert.Contains(t, uri, "localhost:27017")
|
||||
assert.Contains(t, uri, "/?authSource=admin")
|
||||
assert.Contains(t, uri, "connectTimeoutMS=15000")
|
||||
assert.NotContains(t, uri, "mongodb+srv://")
|
||||
assert.NotContains(t, uri, "/mydb")
|
||||
}
|
||||
|
||||
func Test_Validate_SrvConnection_AllowsNullPort(t *testing.T) {
|
||||
model := &MongodbDatabase{
|
||||
Host: "cluster0.example.mongodb.net",
|
||||
Port: nil,
|
||||
Username: "testuser",
|
||||
Password: "testpass123",
|
||||
Database: "mydb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
IsSrv: true,
|
||||
CpuCount: 1,
|
||||
}
|
||||
|
||||
err := model.Validate()
|
||||
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func Test_BuildConnectionURI_WithDirectConnection_ReturnsCorrectUri(t *testing.T) {
|
||||
port := 27017
|
||||
model := &MongodbDatabase{
|
||||
Host: "mongo.example.local",
|
||||
Port: &port,
|
||||
Username: "testuser",
|
||||
Password: "testpass123",
|
||||
Database: "mydb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
IsSrv: false,
|
||||
IsDirectConnection: true,
|
||||
}
|
||||
|
||||
uri := model.buildConnectionURI("testpass123")
|
||||
|
||||
assert.Contains(t, uri, "mongodb://")
|
||||
assert.Contains(t, uri, "directConnection=true")
|
||||
assert.Contains(t, uri, "mongo.example.local:27017")
|
||||
assert.Contains(t, uri, "authSource=admin")
|
||||
}
|
||||
|
||||
func Test_BuildConnectionURI_WithoutDirectConnection_OmitsParam(t *testing.T) {
|
||||
port := 27017
|
||||
model := &MongodbDatabase{
|
||||
Host: "localhost",
|
||||
Port: &port,
|
||||
Username: "testuser",
|
||||
Password: "testpass123",
|
||||
Database: "mydb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
IsSrv: false,
|
||||
IsDirectConnection: false,
|
||||
}
|
||||
|
||||
uri := model.buildConnectionURI("testpass123")
|
||||
|
||||
assert.NotContains(t, uri, "directConnection")
|
||||
}
|
||||
|
||||
func Test_BuildMongodumpURI_WithDirectConnection_ReturnsCorrectUri(t *testing.T) {
|
||||
port := 27017
|
||||
model := &MongodbDatabase{
|
||||
Host: "mongo.example.local",
|
||||
Port: &port,
|
||||
Username: "testuser",
|
||||
Password: "testpass123",
|
||||
Database: "mydb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
IsSrv: false,
|
||||
IsDirectConnection: true,
|
||||
}
|
||||
|
||||
uri := model.BuildMongodumpURI("testpass123")
|
||||
|
||||
assert.Contains(t, uri, "mongodb://")
|
||||
assert.Contains(t, uri, "directConnection=true")
|
||||
assert.NotContains(t, uri, "/mydb")
|
||||
}
|
||||
|
||||
func Test_BuildConnectionURI_WithDirectConnectionAndTls_ReturnsBothParams(t *testing.T) {
|
||||
port := 27017
|
||||
model := &MongodbDatabase{
|
||||
Host: "mongo.example.local",
|
||||
Port: &port,
|
||||
Username: "testuser",
|
||||
Password: "testpass123",
|
||||
Database: "mydb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: true,
|
||||
IsSrv: false,
|
||||
IsDirectConnection: true,
|
||||
}
|
||||
|
||||
uri := model.buildConnectionURI("testpass123")
|
||||
|
||||
assert.Contains(t, uri, "directConnection=true")
|
||||
assert.Contains(t, uri, "tls=true")
|
||||
assert.Contains(t, uri, "tlsInsecure=true")
|
||||
}
|
||||
|
||||
func Test_Validate_StandardConnection_RequiresPort(t *testing.T) {
|
||||
model := &MongodbDatabase{
|
||||
Host: "localhost",
|
||||
Port: nil,
|
||||
Username: "testuser",
|
||||
Password: "testpass123",
|
||||
Database: "mydb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
IsSrv: false,
|
||||
CpuCount: 1,
|
||||
}
|
||||
|
||||
err := model.Validate()
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "port is required for standard connections")
|
||||
}
|
||||
|
||||
@@ -25,13 +25,14 @@ type MysqlDatabase struct {
|
||||
|
||||
Version tools.MysqlVersion `json:"version" gorm:"type:text;not null"`
|
||||
|
||||
Host string `json:"host" gorm:"type:text;not null"`
|
||||
Port int `json:"port" gorm:"type:int;not null"`
|
||||
Username string `json:"username" gorm:"type:text;not null"`
|
||||
Password string `json:"password" gorm:"type:text;not null"`
|
||||
Database *string `json:"database" gorm:"type:text"`
|
||||
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
|
||||
Privileges string `json:"privileges" gorm:"column:privileges;type:text;not null;default:''"`
|
||||
Host string `json:"host" gorm:"type:text;not null"`
|
||||
Port int `json:"port" gorm:"type:int;not null"`
|
||||
Username string `json:"username" gorm:"type:text;not null"`
|
||||
Password string `json:"password" gorm:"type:text;not null"`
|
||||
Database *string `json:"database" gorm:"type:text"`
|
||||
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
|
||||
Privileges string `json:"privileges" gorm:"column:privileges;type:text;not null;default:''"`
|
||||
IsZstdSupported bool `json:"isZstdSupported" gorm:"column:is_zstd_supported;type:boolean;not null;default:true"`
|
||||
}
|
||||
|
||||
func (m *MysqlDatabase) TableName() string {
|
||||
@@ -102,6 +103,7 @@ func (m *MysqlDatabase) TestConnection(
|
||||
return err
|
||||
}
|
||||
m.Privileges = privileges
|
||||
m.IsZstdSupported = detectZstdSupport(ctx, db)
|
||||
|
||||
if err := checkBackupPermissions(m.Privileges); err != nil {
|
||||
return err
|
||||
@@ -125,6 +127,7 @@ func (m *MysqlDatabase) Update(incoming *MysqlDatabase) {
|
||||
m.Database = incoming.Database
|
||||
m.IsHttps = incoming.IsHttps
|
||||
m.Privileges = incoming.Privileges
|
||||
m.IsZstdSupported = incoming.IsZstdSupported
|
||||
|
||||
if incoming.Password != "" {
|
||||
m.Password = incoming.Password
|
||||
@@ -185,6 +188,7 @@ func (m *MysqlDatabase) PopulateDbData(
|
||||
return err
|
||||
}
|
||||
m.Privileges = privileges
|
||||
m.IsZstdSupported = detectZstdSupport(ctx, db)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -223,6 +227,7 @@ func (m *MysqlDatabase) PopulateVersion(
|
||||
return err
|
||||
}
|
||||
m.Version = detectedVersion
|
||||
m.IsZstdSupported = detectZstdSupport(ctx, db)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -575,6 +580,22 @@ func checkBackupPermissions(privileges string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// detectZstdSupport checks if the MySQL server supports zstd network compression.
|
||||
// The protocol_compression_algorithms variable was introduced in MySQL 8.0.18.
|
||||
// Managed MySQL providers (e.g. PlanetScale) may not support zstd even on 8.0+.
|
||||
func detectZstdSupport(ctx context.Context, db *sql.DB) bool {
|
||||
var varName, value string
|
||||
|
||||
err := db.QueryRowContext(ctx,
|
||||
"SHOW VARIABLES LIKE 'protocol_compression_algorithms'",
|
||||
).Scan(&varName, &value)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return strings.Contains(strings.ToLower(value), "zstd")
|
||||
}
|
||||
|
||||
func decryptPasswordIfNeeded(
|
||||
password string,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
|
||||
@@ -177,6 +177,38 @@ func Test_TestConnection_SufficientPermissions_Success(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func Test_TestConnection_DetectsZstdSupport(t *testing.T) {
|
||||
env := config.GetEnv()
|
||||
cases := []struct {
|
||||
name string
|
||||
version tools.MysqlVersion
|
||||
port string
|
||||
isExpectZstd bool
|
||||
}{
|
||||
{"MySQL 5.7", tools.MysqlVersion57, env.TestMysql57Port, false},
|
||||
{"MySQL 8.0", tools.MysqlVersion80, env.TestMysql80Port, true},
|
||||
{"MySQL 8.4", tools.MysqlVersion84, env.TestMysql84Port, true},
|
||||
{"MySQL 9", tools.MysqlVersion9, env.TestMysql90Port, true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
container := connectToMysqlContainer(t, tc.port, tc.version)
|
||||
defer container.DB.Close()
|
||||
|
||||
mysqlModel := createMysqlModel(container)
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
|
||||
|
||||
err := mysqlModel.TestConnection(logger, nil, uuid.New())
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tc.isExpectZstd, mysqlModel.IsZstdSupported,
|
||||
"IsZstdSupported mismatch for %s", tc.name)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_IsUserReadOnly_AdminUser_ReturnsFalse(t *testing.T) {
|
||||
env := config.GetEnv()
|
||||
cases := []struct {
|
||||
|
||||
@@ -2,6 +2,7 @@ package postgresql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"databasus-backend/internal/config"
|
||||
"databasus-backend/internal/util/encryption"
|
||||
"databasus-backend/internal/util/tools"
|
||||
"errors"
|
||||
@@ -17,6 +18,13 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type PostgresBackupType string
|
||||
|
||||
const (
|
||||
PostgresBackupTypePgDump PostgresBackupType = "PG_DUMP"
|
||||
PostgresBackupTypeWalV1 PostgresBackupType = "WAL_V1"
|
||||
)
|
||||
|
||||
type PostgresqlDatabase struct {
|
||||
ID uuid.UUID `json:"id" gorm:"primaryKey;type:uuid;default:gen_random_uuid()"`
|
||||
|
||||
@@ -24,11 +32,13 @@ type PostgresqlDatabase struct {
|
||||
|
||||
Version tools.PostgresqlVersion `json:"version" gorm:"type:text;not null"`
|
||||
|
||||
// connection data
|
||||
Host string `json:"host" gorm:"type:text;not null"`
|
||||
Port int `json:"port" gorm:"type:int;not null"`
|
||||
Username string `json:"username" gorm:"type:text;not null"`
|
||||
Password string `json:"password" gorm:"type:text;not null"`
|
||||
BackupType PostgresBackupType `json:"backupType" gorm:"column:backup_type;type:text;not null;default:'PG_DUMP'"`
|
||||
|
||||
// connection data — required for PG_DUMP, optional for WAL_V1
|
||||
Host string `json:"host" gorm:"type:text"`
|
||||
Port int `json:"port" gorm:"type:int"`
|
||||
Username string `json:"username" gorm:"type:text"`
|
||||
Password string `json:"password" gorm:"type:text"`
|
||||
Database *string `json:"database" gorm:"type:text"`
|
||||
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
|
||||
|
||||
@@ -66,20 +76,30 @@ func (p *PostgresqlDatabase) AfterFind(_ *gorm.DB) error {
|
||||
}
|
||||
|
||||
func (p *PostgresqlDatabase) Validate() error {
|
||||
if p.Host == "" {
|
||||
return errors.New("host is required")
|
||||
if p.BackupType == "" {
|
||||
p.BackupType = PostgresBackupTypePgDump
|
||||
}
|
||||
|
||||
if p.Port == 0 {
|
||||
return errors.New("port is required")
|
||||
if p.BackupType == PostgresBackupTypePgDump && config.GetEnv().IsCloud {
|
||||
return errors.New("PG_DUMP backup type is not supported in cloud mode")
|
||||
}
|
||||
|
||||
if p.Username == "" {
|
||||
return errors.New("username is required")
|
||||
}
|
||||
if p.BackupType == PostgresBackupTypePgDump {
|
||||
if p.Host == "" {
|
||||
return errors.New("host is required")
|
||||
}
|
||||
|
||||
if p.Password == "" {
|
||||
return errors.New("password is required")
|
||||
if p.Port == 0 {
|
||||
return errors.New("port is required")
|
||||
}
|
||||
|
||||
if p.Username == "" {
|
||||
return errors.New("username is required")
|
||||
}
|
||||
|
||||
if p.Password == "" {
|
||||
return errors.New("password is required")
|
||||
}
|
||||
}
|
||||
|
||||
if p.CpuCount <= 0 {
|
||||
@@ -90,7 +110,7 @@ func (p *PostgresqlDatabase) Validate() error {
|
||||
// Databasus runs an internal PostgreSQL instance that should not be backed up through the UI
|
||||
// because it would expose internal metadata to non-system administrators.
|
||||
// To properly backup Databasus, see: https://databasus.com/faq#backup-databasus
|
||||
if p.Database != nil && *p.Database != "" {
|
||||
if p.BackupType == PostgresBackupTypePgDump && p.Database != nil && *p.Database != "" {
|
||||
localhostHosts := []string{
|
||||
"localhost",
|
||||
"127.0.0.1",
|
||||
@@ -130,6 +150,10 @@ func (p *PostgresqlDatabase) TestConnection(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
databaseID uuid.UUID,
|
||||
) error {
|
||||
if p.BackupType == PostgresBackupTypeWalV1 {
|
||||
return errors.New("test connection is not supported for WAL backup type")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
@@ -144,7 +168,21 @@ func (p *PostgresqlDatabase) HideSensitiveData() {
|
||||
p.Password = ""
|
||||
}
|
||||
|
||||
func (p *PostgresqlDatabase) ValidateUpdate(old *PostgresqlDatabase) error {
|
||||
// BackupType cannot be changed after creation — the full backup structure
|
||||
// (WAL hierarchy, storage files, cleanup logic) is built around
|
||||
// the type chosen at creation time. Automatically migrating this state is
|
||||
// error-prone; it is safer for the user to create a new database and
|
||||
// remove the old one.
|
||||
if old.BackupType != p.BackupType {
|
||||
return errors.New("backup type cannot be changed; create a new database instead")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *PostgresqlDatabase) Update(incoming *PostgresqlDatabase) {
|
||||
p.BackupType = incoming.BackupType
|
||||
p.Version = incoming.Version
|
||||
p.Host = incoming.Host
|
||||
p.Port = incoming.Port
|
||||
@@ -181,6 +219,10 @@ func (p *PostgresqlDatabase) PopulateDbData(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
databaseID uuid.UUID,
|
||||
) error {
|
||||
if p.BackupType == PostgresBackupTypeWalV1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return p.PopulateVersion(logger, encryptor, databaseID)
|
||||
}
|
||||
|
||||
@@ -243,6 +285,10 @@ func (p *PostgresqlDatabase) IsUserReadOnly(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
databaseID uuid.UUID,
|
||||
) (bool, []string, error) {
|
||||
if p.BackupType == PostgresBackupTypeWalV1 {
|
||||
return false, nil, errors.New("read-only check is not supported for WAL backup type")
|
||||
}
|
||||
|
||||
password, err := decryptPasswordIfNeeded(p.Password, encryptor, databaseID)
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("failed to decrypt password: %w", err)
|
||||
@@ -415,6 +461,10 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
databaseID uuid.UUID,
|
||||
) (string, string, error) {
|
||||
if p.BackupType == PostgresBackupTypeWalV1 {
|
||||
return "", "", errors.New("read-only user creation is not supported for WAL backup type")
|
||||
}
|
||||
|
||||
password, err := decryptPasswordIfNeeded(p.Password, encryptor, databaseID)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to decrypt password: %w", err)
|
||||
@@ -564,12 +614,23 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
|
||||
logger.Warn("Failed to revoke TEMP privilege", "error", err, "username", baseUsername)
|
||||
}
|
||||
|
||||
// Step 4: Discover all user-created schemas
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT schema_name
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name NOT IN ('pg_catalog', 'information_schema')
|
||||
`)
|
||||
// Step 4: Discover schemas to grant privileges on
|
||||
// If IncludeSchemas is specified, only use those schemas; otherwise use all non-system schemas
|
||||
var rows pgx.Rows
|
||||
if len(p.IncludeSchemas) > 0 {
|
||||
rows, err = tx.Query(ctx, `
|
||||
SELECT schema_name
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name NOT IN ('pg_catalog', 'information_schema')
|
||||
AND schema_name = ANY($1::text[])
|
||||
`, p.IncludeSchemas)
|
||||
} else {
|
||||
rows, err = tx.Query(ctx, `
|
||||
SELECT schema_name
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name NOT IN ('pg_catalog', 'information_schema')
|
||||
`)
|
||||
}
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to get schemas: %w", err)
|
||||
}
|
||||
@@ -619,50 +680,197 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
|
||||
}
|
||||
|
||||
// Step 6: Grant SELECT on ALL existing tables and sequences
|
||||
grantSelectSQL := fmt.Sprintf(`
|
||||
DO $$
|
||||
DECLARE
|
||||
schema_rec RECORD;
|
||||
BEGIN
|
||||
FOR schema_rec IN
|
||||
SELECT schema_name
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name NOT IN ('pg_catalog', 'information_schema')
|
||||
LOOP
|
||||
EXECUTE format('GRANT SELECT ON ALL TABLES IN SCHEMA %%I TO "%s"', schema_rec.schema_name);
|
||||
EXECUTE format('GRANT SELECT ON ALL SEQUENCES IN SCHEMA %%I TO "%s"', schema_rec.schema_name);
|
||||
END LOOP;
|
||||
END $$;
|
||||
`, baseUsername, baseUsername)
|
||||
// Use the already-filtered schemas list from Step 4
|
||||
for _, schema := range schemas {
|
||||
_, err = tx.Exec(
|
||||
ctx,
|
||||
fmt.Sprintf(
|
||||
`GRANT SELECT ON ALL TABLES IN SCHEMA "%s" TO "%s"`,
|
||||
schema,
|
||||
baseUsername,
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf(
|
||||
"failed to grant select on tables in schema %s: %w",
|
||||
schema,
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
_, err = tx.Exec(ctx, grantSelectSQL)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to grant select on tables: %w", err)
|
||||
_, err = tx.Exec(
|
||||
ctx,
|
||||
fmt.Sprintf(
|
||||
`GRANT SELECT ON ALL SEQUENCES IN SCHEMA "%s" TO "%s"`,
|
||||
schema,
|
||||
baseUsername,
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf(
|
||||
"failed to grant select on sequences in schema %s: %w",
|
||||
schema,
|
||||
err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 7: Set default privileges for FUTURE tables and sequences
|
||||
defaultPrivilegesSQL := fmt.Sprintf(`
|
||||
DO $$
|
||||
DECLARE
|
||||
schema_rec RECORD;
|
||||
BEGIN
|
||||
FOR schema_rec IN
|
||||
SELECT schema_name
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name NOT IN ('pg_catalog', 'information_schema')
|
||||
LOOP
|
||||
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %%I GRANT SELECT ON TABLES TO "%s"', schema_rec.schema_name);
|
||||
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %%I GRANT SELECT ON SEQUENCES TO "%s"', schema_rec.schema_name);
|
||||
END LOOP;
|
||||
END $$;
|
||||
`, baseUsername, baseUsername)
|
||||
// First, set default privileges for objects created by the current user
|
||||
// Use the already-filtered schemas list from Step 4
|
||||
for _, schema := range schemas {
|
||||
_, err = tx.Exec(
|
||||
ctx,
|
||||
fmt.Sprintf(
|
||||
`ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT SELECT ON TABLES TO "%s"`,
|
||||
schema,
|
||||
baseUsername,
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf(
|
||||
"failed to set default privileges for tables in schema %s: %w",
|
||||
schema,
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
_, err = tx.Exec(ctx, defaultPrivilegesSQL)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to set default privileges: %w", err)
|
||||
_, err = tx.Exec(
|
||||
ctx,
|
||||
fmt.Sprintf(
|
||||
`ALTER DEFAULT PRIVILEGES IN SCHEMA "%s" GRANT SELECT ON SEQUENCES TO "%s"`,
|
||||
schema,
|
||||
baseUsername,
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf(
|
||||
"failed to set default privileges for sequences in schema %s: %w",
|
||||
schema,
|
||||
err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 8: Verify user creation before committing
|
||||
// Step 8: Discover all roles that own objects in each schema
|
||||
// This is needed because ALTER DEFAULT PRIVILEGES only applies to objects created by the current role.
|
||||
// To handle tables created by OTHER users (like the GitHub issue with partitioned tables),
|
||||
// we need to set "ALTER DEFAULT PRIVILEGES FOR ROLE <owner>" for each object owner.
|
||||
// Filter by IncludeSchemas if specified.
|
||||
type SchemaOwner struct {
|
||||
SchemaName string
|
||||
RoleName string
|
||||
}
|
||||
|
||||
var ownerRows pgx.Rows
|
||||
if len(p.IncludeSchemas) > 0 {
|
||||
ownerRows, err = tx.Query(ctx, `
|
||||
SELECT DISTINCT n.nspname as schema_name, pg_get_userbyid(c.relowner) as role_name
|
||||
FROM pg_class c
|
||||
JOIN pg_namespace n ON c.relnamespace = n.oid
|
||||
WHERE n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
|
||||
AND n.nspname = ANY($1::text[])
|
||||
AND c.relkind IN ('r', 'p', 'v', 'm', 'f')
|
||||
AND pg_get_userbyid(c.relowner) != current_user
|
||||
ORDER BY n.nspname, role_name
|
||||
`, p.IncludeSchemas)
|
||||
} else {
|
||||
ownerRows, err = tx.Query(ctx, `
|
||||
SELECT DISTINCT n.nspname as schema_name, pg_get_userbyid(c.relowner) as role_name
|
||||
FROM pg_class c
|
||||
JOIN pg_namespace n ON c.relnamespace = n.oid
|
||||
WHERE n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
|
||||
AND c.relkind IN ('r', 'p', 'v', 'm', 'f')
|
||||
AND pg_get_userbyid(c.relowner) != current_user
|
||||
ORDER BY n.nspname, role_name
|
||||
`)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// Log warning but continue - this is a best-effort enhancement
|
||||
logger.Warn("Failed to query object owners for default privileges", "error", err)
|
||||
} else {
|
||||
var schemaOwners []SchemaOwner
|
||||
for ownerRows.Next() {
|
||||
var so SchemaOwner
|
||||
if err := ownerRows.Scan(&so.SchemaName, &so.RoleName); err != nil {
|
||||
ownerRows.Close()
|
||||
logger.Warn("Failed to scan schema owner", "error", err)
|
||||
break
|
||||
}
|
||||
schemaOwners = append(schemaOwners, so)
|
||||
}
|
||||
ownerRows.Close()
|
||||
|
||||
if err := ownerRows.Err(); err != nil {
|
||||
logger.Warn("Error iterating schema owners", "error", err)
|
||||
}
|
||||
|
||||
// Step 9: Set default privileges FOR ROLE for each object owner
|
||||
// Note: This may fail for some roles due to permission issues (e.g., roles owned by other superusers)
|
||||
// We log warnings but continue - user creation should succeed even if some roles can't be configured
|
||||
for _, so := range schemaOwners {
|
||||
// Try to set default privileges for tables
|
||||
_, err = tx.Exec(
|
||||
ctx,
|
||||
fmt.Sprintf(
|
||||
`ALTER DEFAULT PRIVILEGES FOR ROLE "%s" IN SCHEMA "%s" GRANT SELECT ON TABLES TO "%s"`,
|
||||
so.RoleName,
|
||||
so.SchemaName,
|
||||
baseUsername,
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Warn(
|
||||
"Failed to set default privileges for role (tables)",
|
||||
"error",
|
||||
err,
|
||||
"role",
|
||||
so.RoleName,
|
||||
"schema",
|
||||
so.SchemaName,
|
||||
"readonly_user",
|
||||
baseUsername,
|
||||
)
|
||||
}
|
||||
|
||||
// Try to set default privileges for sequences
|
||||
_, err = tx.Exec(
|
||||
ctx,
|
||||
fmt.Sprintf(
|
||||
`ALTER DEFAULT PRIVILEGES FOR ROLE "%s" IN SCHEMA "%s" GRANT SELECT ON SEQUENCES TO "%s"`,
|
||||
so.RoleName,
|
||||
so.SchemaName,
|
||||
baseUsername,
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Warn(
|
||||
"Failed to set default privileges for role (sequences)",
|
||||
"error",
|
||||
err,
|
||||
"role",
|
||||
so.RoleName,
|
||||
"schema",
|
||||
so.SchemaName,
|
||||
"readonly_user",
|
||||
baseUsername,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if len(schemaOwners) > 0 {
|
||||
logger.Info(
|
||||
"Set default privileges for existing object owners",
|
||||
"readonly_user",
|
||||
baseUsername,
|
||||
"owner_count",
|
||||
len(schemaOwners),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 10: Verify user creation before committing
|
||||
var verifyUsername string
|
||||
err = tx.QueryRow(ctx, fmt.Sprintf(`SELECT rolname FROM pg_roles WHERE rolname = '%s'`, baseUsername)).
|
||||
Scan(&verifyUsername)
|
||||
|
||||
@@ -1319,6 +1319,346 @@ type PostgresContainer struct {
|
||||
DB *sqlx.DB
|
||||
}
|
||||
|
||||
func Test_CreateReadOnlyUser_TablesCreatedByDifferentUser_ReadOnlyUserCanRead(t *testing.T) {
|
||||
env := config.GetEnv()
|
||||
container := connectToPostgresContainer(t, env.TestPostgres16Port)
|
||||
defer container.DB.Close()
|
||||
|
||||
// Step 1: Create a second database user who will create tables
|
||||
userCreatorUsername := fmt.Sprintf("user_creator_%s", uuid.New().String()[:8])
|
||||
userCreatorPassword := "creator_password_123"
|
||||
|
||||
_, err := container.DB.Exec(fmt.Sprintf(
|
||||
`CREATE USER "%s" WITH PASSWORD '%s' LOGIN`,
|
||||
userCreatorUsername,
|
||||
userCreatorPassword,
|
||||
))
|
||||
assert.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
_, _ = container.DB.Exec(fmt.Sprintf(`DROP OWNED BY "%s" CASCADE`, userCreatorUsername))
|
||||
_, _ = container.DB.Exec(fmt.Sprintf(`DROP USER IF EXISTS "%s"`, userCreatorUsername))
|
||||
}()
|
||||
|
||||
// Step 2: Grant the user_creator privileges to connect and create tables
|
||||
_, err = container.DB.Exec(fmt.Sprintf(
|
||||
`GRANT CONNECT ON DATABASE "%s" TO "%s"`,
|
||||
container.Database,
|
||||
userCreatorUsername,
|
||||
))
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = container.DB.Exec(fmt.Sprintf(
|
||||
`GRANT USAGE ON SCHEMA public TO "%s"`,
|
||||
userCreatorUsername,
|
||||
))
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = container.DB.Exec(fmt.Sprintf(
|
||||
`GRANT CREATE ON SCHEMA public TO "%s"`,
|
||||
userCreatorUsername,
|
||||
))
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Step 2b: Create an initial table by user_creator so they become an object owner
|
||||
// This is important because our fix discovers existing object owners
|
||||
userCreatorDSN := fmt.Sprintf(
|
||||
"host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||||
container.Host,
|
||||
container.Port,
|
||||
userCreatorUsername,
|
||||
userCreatorPassword,
|
||||
container.Database,
|
||||
)
|
||||
userCreatorConn, err := sqlx.Connect("postgres", userCreatorDSN)
|
||||
assert.NoError(t, err)
|
||||
defer userCreatorConn.Close()
|
||||
|
||||
initialTableName := fmt.Sprintf(
|
||||
"public.initial_table_%s",
|
||||
strings.ReplaceAll(uuid.New().String()[:8], "-", ""),
|
||||
)
|
||||
_, err = userCreatorConn.Exec(fmt.Sprintf(`
|
||||
CREATE TABLE %s (
|
||||
id SERIAL PRIMARY KEY,
|
||||
data TEXT NOT NULL
|
||||
);
|
||||
INSERT INTO %s (data) VALUES ('initial_data');
|
||||
`, initialTableName, initialTableName))
|
||||
assert.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
_, _ = container.DB.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s CASCADE`, initialTableName))
|
||||
}()
|
||||
|
||||
// Step 3: NOW create read-only user via Databasus (as admin)
|
||||
// At this point, user_creator already owns objects, so ALTER DEFAULT PRIVILEGES FOR ROLE should apply
|
||||
pgModel := createPostgresModel(container)
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
|
||||
ctx := context.Background()
|
||||
|
||||
readonlyUsername, readonlyPassword, err := pgModel.CreateReadOnlyUser(
|
||||
ctx,
|
||||
logger,
|
||||
nil,
|
||||
uuid.New(),
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, readonlyUsername)
|
||||
assert.NotEmpty(t, readonlyPassword)
|
||||
|
||||
defer func() {
|
||||
_, _ = container.DB.Exec(fmt.Sprintf(`DROP OWNED BY "%s" CASCADE`, readonlyUsername))
|
||||
_, _ = container.DB.Exec(fmt.Sprintf(`DROP USER IF EXISTS "%s"`, readonlyUsername))
|
||||
}()
|
||||
|
||||
// Step 4: user_creator creates a NEW table AFTER the read-only user was created
|
||||
// This table should automatically grant SELECT to the read-only user via ALTER DEFAULT PRIVILEGES FOR ROLE
|
||||
tableName := fmt.Sprintf(
|
||||
"public.future_table_%s",
|
||||
strings.ReplaceAll(uuid.New().String()[:8], "-", ""),
|
||||
)
|
||||
_, err = userCreatorConn.Exec(fmt.Sprintf(`
|
||||
CREATE TABLE %s (
|
||||
id SERIAL PRIMARY KEY,
|
||||
data TEXT NOT NULL
|
||||
);
|
||||
INSERT INTO %s (data) VALUES ('test_data_1'), ('test_data_2');
|
||||
`, tableName, tableName))
|
||||
assert.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
_, _ = container.DB.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s CASCADE`, tableName))
|
||||
}()
|
||||
|
||||
// Step 5: Connect as read-only user and verify it can SELECT from the new table
|
||||
readonlyDSN := fmt.Sprintf(
|
||||
"host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||||
container.Host,
|
||||
container.Port,
|
||||
readonlyUsername,
|
||||
readonlyPassword,
|
||||
container.Database,
|
||||
)
|
||||
readonlyConn, err := sqlx.Connect("postgres", readonlyDSN)
|
||||
assert.NoError(t, err)
|
||||
defer readonlyConn.Close()
|
||||
|
||||
var count int
|
||||
err = readonlyConn.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(
|
||||
t,
|
||||
2,
|
||||
count,
|
||||
"Read-only user should be able to SELECT from table created by different user",
|
||||
)
|
||||
|
||||
// Step 6: Verify read-only user cannot write to the table
|
||||
_, err = readonlyConn.Exec(
|
||||
fmt.Sprintf("INSERT INTO %s (data) VALUES ('should-fail')", tableName),
|
||||
)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "permission denied")
|
||||
|
||||
// Step 7: Verify pg_dump operations (LOCK TABLE) work
|
||||
// pg_dump needs to lock tables in ACCESS SHARE MODE for consistent backup
|
||||
tx, err := readonlyConn.Begin()
|
||||
assert.NoError(t, err)
|
||||
defer tx.Rollback()
|
||||
|
||||
_, err = tx.Exec(fmt.Sprintf("LOCK TABLE %s IN ACCESS SHARE MODE", tableName))
|
||||
assert.NoError(t, err, "Read-only user should be able to LOCK TABLE (needed for pg_dump)")
|
||||
|
||||
err = tx.Commit()
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func Test_CreateReadOnlyUser_WithIncludeSchemas_OnlyGrantsAccessToSpecifiedSchemas(t *testing.T) {
|
||||
env := config.GetEnv()
|
||||
container := connectToPostgresContainer(t, env.TestPostgres16Port)
|
||||
defer container.DB.Close()
|
||||
|
||||
// Step 1: Create multiple schemas and tables
|
||||
_, err := container.DB.Exec(`
|
||||
DROP SCHEMA IF EXISTS included_schema CASCADE;
|
||||
DROP SCHEMA IF EXISTS excluded_schema CASCADE;
|
||||
CREATE SCHEMA included_schema;
|
||||
CREATE SCHEMA excluded_schema;
|
||||
|
||||
CREATE TABLE public.public_table (id INT, data TEXT);
|
||||
INSERT INTO public.public_table VALUES (1, 'public_data');
|
||||
|
||||
CREATE TABLE included_schema.included_table (id INT, data TEXT);
|
||||
INSERT INTO included_schema.included_table VALUES (2, 'included_data');
|
||||
|
||||
CREATE TABLE excluded_schema.excluded_table (id INT, data TEXT);
|
||||
INSERT INTO excluded_schema.excluded_table VALUES (3, 'excluded_data');
|
||||
`)
|
||||
assert.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
_, _ = container.DB.Exec(`DROP SCHEMA IF EXISTS included_schema CASCADE`)
|
||||
_, _ = container.DB.Exec(`DROP SCHEMA IF EXISTS excluded_schema CASCADE`)
|
||||
}()
|
||||
|
||||
// Step 2: Create a second user who owns tables in both included and excluded schemas
|
||||
userCreatorUsername := fmt.Sprintf("user_creator_%s", uuid.New().String()[:8])
|
||||
userCreatorPassword := "creator_password_123"
|
||||
|
||||
_, err = container.DB.Exec(fmt.Sprintf(
|
||||
`CREATE USER "%s" WITH PASSWORD '%s' LOGIN`,
|
||||
userCreatorUsername,
|
||||
userCreatorPassword,
|
||||
))
|
||||
assert.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
_, _ = container.DB.Exec(fmt.Sprintf(`DROP OWNED BY "%s" CASCADE`, userCreatorUsername))
|
||||
_, _ = container.DB.Exec(fmt.Sprintf(`DROP USER IF EXISTS "%s"`, userCreatorUsername))
|
||||
}()
|
||||
|
||||
// Grant privileges to user_creator
|
||||
_, err = container.DB.Exec(fmt.Sprintf(
|
||||
`GRANT CONNECT ON DATABASE "%s" TO "%s"`,
|
||||
container.Database,
|
||||
userCreatorUsername,
|
||||
))
|
||||
assert.NoError(t, err)
|
||||
|
||||
for _, schema := range []string{"public", "included_schema", "excluded_schema"} {
|
||||
_, err = container.DB.Exec(fmt.Sprintf(
|
||||
`GRANT USAGE, CREATE ON SCHEMA %s TO "%s"`,
|
||||
schema,
|
||||
userCreatorUsername,
|
||||
))
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// User_creator creates tables in included and excluded schemas
|
||||
userCreatorDSN := fmt.Sprintf(
|
||||
"host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||||
container.Host,
|
||||
container.Port,
|
||||
userCreatorUsername,
|
||||
userCreatorPassword,
|
||||
container.Database,
|
||||
)
|
||||
userCreatorConn, err := sqlx.Connect("postgres", userCreatorDSN)
|
||||
assert.NoError(t, err)
|
||||
defer userCreatorConn.Close()
|
||||
|
||||
_, err = userCreatorConn.Exec(`
|
||||
CREATE TABLE included_schema.user_table (id INT, data TEXT);
|
||||
INSERT INTO included_schema.user_table VALUES (4, 'user_included_data');
|
||||
|
||||
CREATE TABLE excluded_schema.user_excluded_table (id INT, data TEXT);
|
||||
INSERT INTO excluded_schema.user_excluded_table VALUES (5, 'user_excluded_data');
|
||||
`)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Step 3: Create read-only user with IncludeSchemas = ["public", "included_schema"]
|
||||
pgModel := createPostgresModel(container)
|
||||
pgModel.IncludeSchemas = []string{"public", "included_schema"}
|
||||
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
|
||||
ctx := context.Background()
|
||||
|
||||
readonlyUsername, readonlyPassword, err := pgModel.CreateReadOnlyUser(
|
||||
ctx,
|
||||
logger,
|
||||
nil,
|
||||
uuid.New(),
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, readonlyUsername)
|
||||
assert.NotEmpty(t, readonlyPassword)
|
||||
|
||||
defer func() {
|
||||
_, _ = container.DB.Exec(fmt.Sprintf(`DROP OWNED BY "%s" CASCADE`, readonlyUsername))
|
||||
_, _ = container.DB.Exec(fmt.Sprintf(`DROP USER IF EXISTS "%s"`, readonlyUsername))
|
||||
}()
|
||||
|
||||
// Step 4: Connect as read-only user
|
||||
readonlyDSN := fmt.Sprintf(
|
||||
"host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||||
container.Host,
|
||||
container.Port,
|
||||
readonlyUsername,
|
||||
readonlyPassword,
|
||||
container.Database,
|
||||
)
|
||||
readonlyConn, err := sqlx.Connect("postgres", readonlyDSN)
|
||||
assert.NoError(t, err)
|
||||
defer readonlyConn.Close()
|
||||
|
||||
// Step 5: Verify read-only user CAN access included schemas
|
||||
var publicData string
|
||||
err = readonlyConn.Get(&publicData, "SELECT data FROM public.public_table LIMIT 1")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "public_data", publicData)
|
||||
|
||||
var includedData string
|
||||
err = readonlyConn.Get(&includedData, "SELECT data FROM included_schema.included_table LIMIT 1")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "included_data", includedData)
|
||||
|
||||
var userIncludedData string
|
||||
err = readonlyConn.Get(&userIncludedData, "SELECT data FROM included_schema.user_table LIMIT 1")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "user_included_data", userIncludedData)
|
||||
|
||||
// Step 6: Verify read-only user CANNOT access excluded schema
|
||||
var excludedData string
|
||||
err = readonlyConn.Get(&excludedData, "SELECT data FROM excluded_schema.excluded_table LIMIT 1")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "permission denied")
|
||||
|
||||
err = readonlyConn.Get(
|
||||
&excludedData,
|
||||
"SELECT data FROM excluded_schema.user_excluded_table LIMIT 1",
|
||||
)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "permission denied")
|
||||
|
||||
// Step 7: Verify future tables in included schemas are accessible
|
||||
_, err = userCreatorConn.Exec(`
|
||||
CREATE TABLE included_schema.future_table (id INT, data TEXT);
|
||||
INSERT INTO included_schema.future_table VALUES (6, 'future_data');
|
||||
`)
|
||||
assert.NoError(t, err)
|
||||
|
||||
var futureData string
|
||||
err = readonlyConn.Get(&futureData, "SELECT data FROM included_schema.future_table LIMIT 1")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(
|
||||
t,
|
||||
"future_data",
|
||||
futureData,
|
||||
"Read-only user should access future tables in included schemas via ALTER DEFAULT PRIVILEGES FOR ROLE",
|
||||
)
|
||||
|
||||
// Step 8: Verify future tables in excluded schema are NOT accessible
|
||||
_, err = userCreatorConn.Exec(`
|
||||
CREATE TABLE excluded_schema.future_excluded_table (id INT, data TEXT);
|
||||
INSERT INTO excluded_schema.future_excluded_table VALUES (7, 'future_excluded_data');
|
||||
`)
|
||||
assert.NoError(t, err)
|
||||
|
||||
var futureExcludedData string
|
||||
err = readonlyConn.Get(
|
||||
&futureExcludedData,
|
||||
"SELECT data FROM excluded_schema.future_excluded_table LIMIT 1",
|
||||
)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(
|
||||
t,
|
||||
err.Error(),
|
||||
"permission denied",
|
||||
"Read-only user should NOT access tables in excluded schemas",
|
||||
)
|
||||
}
|
||||
|
||||
func connectToPostgresContainer(t *testing.T, port string) *PostgresContainer {
|
||||
dbName := "testdb"
|
||||
password := "testpassword"
|
||||
|
||||
@@ -9,3 +9,7 @@ type IsReadOnlyResponse struct {
|
||||
IsReadOnly bool `json:"isReadOnly"`
|
||||
Privileges []string `json:"privileges"`
|
||||
}
|
||||
|
||||
type VerifyAgentTokenRequest struct {
|
||||
Token string `json:"token" binding:"required"`
|
||||
}
|
||||
|
||||
@@ -37,6 +37,9 @@ type Database struct {
|
||||
LastBackupErrorMessage *string `json:"lastBackupErrorMessage,omitempty" gorm:"column:last_backup_error_message;type:text"`
|
||||
|
||||
HealthStatus *HealthStatus `json:"healthStatus" gorm:"column:health_status;type:text;not null"`
|
||||
|
||||
AgentToken *string `json:"-" gorm:"column:agent_token;type:text"`
|
||||
IsAgentTokenGenerated bool `json:"isAgentTokenGenerated" gorm:"column:is_agent_token_generated;not null;default:false"`
|
||||
}
|
||||
|
||||
func (d *Database) Validate() error {
|
||||
@@ -71,8 +74,19 @@ func (d *Database) Validate() error {
|
||||
}
|
||||
|
||||
func (d *Database) ValidateUpdate(old, new Database) error {
|
||||
// Database type cannot be changed after creation — the entire backup
|
||||
// structure (storage files, schedulers, WAL hierarchy, etc.) is tied to
|
||||
// the type at creation time. Recreating that state automatically is
|
||||
// error-prone; it is safer for the user to create a new database and
|
||||
// remove the old one.
|
||||
if old.Type != new.Type {
|
||||
return errors.New("database type is not allowed to change")
|
||||
return errors.New("database type cannot be changed; create a new database instead")
|
||||
}
|
||||
|
||||
if old.Type == DatabaseTypePostgres && old.Postgresql != nil && new.Postgresql != nil {
|
||||
if err := new.Postgresql.ValidateUpdate(old.Postgresql); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -244,6 +244,18 @@ func (r *DatabaseRepository) GetAllDatabases() ([]*Database, error) {
|
||||
return databases, nil
|
||||
}
|
||||
|
||||
func (r *DatabaseRepository) FindByAgentTokenHash(hash string) (*Database, error) {
|
||||
var database Database
|
||||
|
||||
if err := storage.GetDb().
|
||||
Where("agent_token = ?", hash).
|
||||
First(&database).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &database, nil
|
||||
}
|
||||
|
||||
func (r *DatabaseRepository) GetDatabasesIDsByNotifierID(
|
||||
notifierID uuid.UUID,
|
||||
) ([]uuid.UUID, error) {
|
||||
|
||||
@@ -2,9 +2,11 @@ package databases
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
@@ -87,21 +89,8 @@ func (s *DatabaseService) CreateDatabase(
|
||||
return nil, fmt.Errorf("failed to auto-detect database data: %w", err)
|
||||
}
|
||||
|
||||
if config.GetEnv().IsCloud {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
isReadOnly, permissions, err := database.IsUserReadOnly(ctx, s.logger, s.fieldEncryptor)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to verify user permissions: %w", err)
|
||||
}
|
||||
|
||||
if !isReadOnly {
|
||||
return nil, fmt.Errorf(
|
||||
"in cloud mode, only read-only database users are allowed (user has permissions: %v)",
|
||||
permissions,
|
||||
)
|
||||
}
|
||||
if err := s.verifyReadOnlyUserIfNeeded(database); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := database.EncryptSensitiveFields(s.fieldEncryptor); err != nil {
|
||||
@@ -171,27 +160,12 @@ func (s *DatabaseService) UpdateDatabase(
|
||||
return fmt.Errorf("failed to auto-detect database data: %w", err)
|
||||
}
|
||||
|
||||
if config.GetEnv().IsCloud {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
isReadOnly, permissions, err := existingDatabase.IsUserReadOnly(
|
||||
ctx,
|
||||
s.logger,
|
||||
s.fieldEncryptor,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to verify user permissions: %w", err)
|
||||
}
|
||||
|
||||
if !isReadOnly {
|
||||
return fmt.Errorf(
|
||||
"in cloud mode, only read-only database users are allowed (user has permissions: %v)",
|
||||
permissions,
|
||||
)
|
||||
}
|
||||
if err := s.verifyReadOnlyUserIfNeeded(existingDatabase); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
oldName := existingDatabase.Name
|
||||
|
||||
if err := existingDatabase.EncryptSensitiveFields(s.fieldEncryptor); err != nil {
|
||||
return fmt.Errorf("failed to encrypt sensitive fields: %w", err)
|
||||
}
|
||||
@@ -201,11 +175,23 @@ func (s *DatabaseService) UpdateDatabase(
|
||||
return err
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Database updated: %s", existingDatabase.Name),
|
||||
&user.ID,
|
||||
existingDatabase.WorkspaceID,
|
||||
)
|
||||
if oldName != existingDatabase.Name {
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf(
|
||||
"Database updated and renamed from '%s' to '%s'",
|
||||
oldName,
|
||||
existingDatabase.Name,
|
||||
),
|
||||
&user.ID,
|
||||
existingDatabase.WorkspaceID,
|
||||
)
|
||||
} else {
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Database updated: %s", existingDatabase.Name),
|
||||
&user.ID,
|
||||
existingDatabase.WorkspaceID,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -471,6 +457,7 @@ func (s *DatabaseService) CopyDatabase(
|
||||
newDatabase.Postgresql = &postgresql.PostgresqlDatabase{
|
||||
ID: uuid.Nil,
|
||||
DatabaseID: nil,
|
||||
BackupType: existingDatabase.Postgresql.BackupType,
|
||||
Version: existingDatabase.Postgresql.Version,
|
||||
Host: existingDatabase.Postgresql.Host,
|
||||
Port: existingDatabase.Postgresql.Port,
|
||||
@@ -571,9 +558,19 @@ func (s *DatabaseService) TransferDatabaseToWorkspace(
|
||||
return err
|
||||
}
|
||||
|
||||
sourceWorkspace, err := s.workspaceService.GetWorkspaceByID(*sourceWorkspaceID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get source workspace: %w", err)
|
||||
}
|
||||
|
||||
targetWorkspace, err := s.workspaceService.GetWorkspaceByID(targetWorkspaceID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get target workspace: %w", err)
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Database transferred: %s from workspace %s to workspace %s",
|
||||
database.Name, sourceWorkspaceID, targetWorkspaceID),
|
||||
fmt.Sprintf("Database transferred: %s from workspace '%s' to workspace '%s'",
|
||||
database.Name, sourceWorkspace.Name, targetWorkspace.Name),
|
||||
nil,
|
||||
&targetWorkspaceID,
|
||||
)
|
||||
@@ -614,6 +611,71 @@ func (s *DatabaseService) SetHealthStatus(
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DatabaseService) RegenerateAgentToken(
|
||||
user *users_models.User,
|
||||
databaseID uuid.UUID,
|
||||
) (string, error) {
|
||||
database, err := s.dbRepository.FindByID(databaseID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if database.WorkspaceID == nil {
|
||||
return "", errors.New("cannot regenerate token for database without workspace")
|
||||
}
|
||||
|
||||
canManage, err := s.workspaceService.CanUserManageDBs(*database.WorkspaceID, user)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !canManage {
|
||||
return "", errors.New(
|
||||
"insufficient permissions to regenerate agent token for this database",
|
||||
)
|
||||
}
|
||||
|
||||
plainToken := strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
tokenHash := hashAgentToken(plainToken)
|
||||
|
||||
database.AgentToken = &tokenHash
|
||||
database.IsAgentTokenGenerated = true
|
||||
|
||||
_, err = s.dbRepository.Save(database)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Agent token regenerated for database: %s", database.Name),
|
||||
&user.ID,
|
||||
database.WorkspaceID,
|
||||
)
|
||||
|
||||
return plainToken, nil
|
||||
}
|
||||
|
||||
func (s *DatabaseService) VerifyAgentToken(token string) error {
|
||||
hash := hashAgentToken(token)
|
||||
|
||||
_, err := s.dbRepository.FindByAgentTokenHash(hash)
|
||||
if err != nil {
|
||||
return errors.New("invalid token")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DatabaseService) GetDatabaseByAgentToken(token string) (*Database, error) {
|
||||
hash := hashAgentToken(token)
|
||||
|
||||
partial, err := s.dbRepository.FindByAgentTokenHash(hash)
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid agent token")
|
||||
}
|
||||
|
||||
return s.dbRepository.FindByID(partial.ID)
|
||||
}
|
||||
|
||||
func (s *DatabaseService) OnBeforeWorkspaceDeletion(workspaceID uuid.UUID) error {
|
||||
databases, err := s.dbRepository.FindByWorkspaceID(workspaceID)
|
||||
if err != nil {
|
||||
@@ -785,3 +847,36 @@ func (s *DatabaseService) CreateReadOnlyUser(
|
||||
|
||||
return username, password, nil
|
||||
}
|
||||
|
||||
func (s *DatabaseService) verifyReadOnlyUserIfNeeded(database *Database) error {
|
||||
if !config.GetEnv().IsCloud {
|
||||
return nil
|
||||
}
|
||||
|
||||
if database.Postgresql != nil &&
|
||||
database.Postgresql.BackupType == postgresql.PostgresBackupTypeWalV1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
isReadOnly, permissions, err := database.IsUserReadOnly(ctx, s.logger, s.fieldEncryptor)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to verify user permissions: %w", err)
|
||||
}
|
||||
|
||||
if !isReadOnly {
|
||||
return fmt.Errorf(
|
||||
"in cloud mode, only read-only database users are allowed (user has permissions: %v)",
|
||||
permissions,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func hashAgentToken(token string) string {
|
||||
hash := sha256.Sum256([]byte(token))
|
||||
return fmt.Sprintf("%x", hash)
|
||||
}
|
||||
|
||||
@@ -71,12 +71,13 @@ func GetTestMongodbConfig() *mongodb.MongodbDatabase {
|
||||
return &mongodb.MongodbDatabase{
|
||||
Version: tools.MongodbVersion7,
|
||||
Host: config.GetEnv().TestLocalhost,
|
||||
Port: port,
|
||||
Port: &port,
|
||||
Username: "root",
|
||||
Password: "rootpassword",
|
||||
Database: "testdb",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
IsSrv: false,
|
||||
CpuCount: 1,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ var emailSMTPSender = &EmailSMTPSender{
|
||||
env.SMTPPort,
|
||||
env.SMTPUser,
|
||||
env.SMTPPassword,
|
||||
env.SMTPFrom,
|
||||
env.SMTPHost != "" && env.SMTPPort != 0,
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ type EmailSMTPSender struct {
|
||||
smtpPort int
|
||||
smtpUser string
|
||||
smtpPassword string
|
||||
smtpFrom string
|
||||
isConfigured bool
|
||||
}
|
||||
|
||||
@@ -33,7 +34,10 @@ func (s *EmailSMTPSender) SendEmail(to, subject, body string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
from := s.smtpUser
|
||||
from := s.smtpFrom
|
||||
if from == "" {
|
||||
from = s.smtpUser
|
||||
}
|
||||
if from == "" {
|
||||
from = "noreply@" + s.smtpHost
|
||||
}
|
||||
|
||||
@@ -79,6 +79,38 @@ func (i *Interval) ShouldTriggerBackup(now time.Time, lastBackupTime *time.Time)
|
||||
}
|
||||
}
|
||||
|
||||
// NextTriggerTime computes the next time a backup should trigger based on the interval and last backup time.
|
||||
// Returns nil when a backup is due immediately (no previous backup exists).
|
||||
func (i *Interval) NextTriggerTime(now time.Time, lastBackupTime *time.Time) *time.Time {
|
||||
if lastBackupTime == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch i.Interval {
|
||||
case IntervalHourly:
|
||||
next := lastBackupTime.Add(time.Hour)
|
||||
return &next
|
||||
|
||||
case IntervalDaily:
|
||||
next := i.nextDailyTrigger(now)
|
||||
return &next
|
||||
|
||||
case IntervalWeekly:
|
||||
next := i.nextWeeklyTrigger(now)
|
||||
return &next
|
||||
|
||||
case IntervalMonthly:
|
||||
next := i.nextMonthlyTrigger(now)
|
||||
return &next
|
||||
|
||||
case IntervalCron:
|
||||
return i.nextCronTrigger(*lastBackupTime)
|
||||
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Interval) Copy() *Interval {
|
||||
return &Interval{
|
||||
ID: uuid.Nil,
|
||||
@@ -240,6 +272,99 @@ func (i *Interval) shouldTriggerCron(now, lastBackup time.Time) bool {
|
||||
return now.After(nextAfterLastBackup) || now.Equal(nextAfterLastBackup)
|
||||
}
|
||||
|
||||
func (i *Interval) nextDailyTrigger(now time.Time) time.Time {
|
||||
t, err := time.Parse("15:04", *i.TimeOfDay)
|
||||
if err != nil {
|
||||
return now
|
||||
}
|
||||
|
||||
todaySlot := time.Date(
|
||||
now.Year(), now.Month(), now.Day(),
|
||||
t.Hour(), t.Minute(), 0, 0, now.Location(),
|
||||
)
|
||||
|
||||
if now.Before(todaySlot) {
|
||||
return todaySlot
|
||||
}
|
||||
|
||||
return todaySlot.AddDate(0, 0, 1)
|
||||
}
|
||||
|
||||
func (i *Interval) nextWeeklyTrigger(now time.Time) time.Time {
|
||||
targetWd := time.Weekday(0)
|
||||
if i.Weekday != nil {
|
||||
targetWd = time.Weekday(*i.Weekday)
|
||||
}
|
||||
|
||||
startOfWeek := getStartOfWeek(now)
|
||||
|
||||
var daysFromMonday int
|
||||
if targetWd == time.Sunday {
|
||||
daysFromMonday = 6
|
||||
} else {
|
||||
daysFromMonday = int(targetWd) - 1
|
||||
}
|
||||
|
||||
targetThisWeek := startOfWeek.AddDate(0, 0, daysFromMonday)
|
||||
|
||||
if i.TimeOfDay != nil {
|
||||
t, err := time.Parse("15:04", *i.TimeOfDay)
|
||||
if err == nil {
|
||||
targetThisWeek = time.Date(
|
||||
targetThisWeek.Year(), targetThisWeek.Month(), targetThisWeek.Day(),
|
||||
t.Hour(), t.Minute(), 0, 0, targetThisWeek.Location(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if now.Before(targetThisWeek) {
|
||||
return targetThisWeek
|
||||
}
|
||||
|
||||
return targetThisWeek.AddDate(0, 0, 7)
|
||||
}
|
||||
|
||||
func (i *Interval) nextMonthlyTrigger(now time.Time) time.Time {
|
||||
day := 1
|
||||
if i.DayOfMonth != nil {
|
||||
day = *i.DayOfMonth
|
||||
}
|
||||
|
||||
targetThisMonth := time.Date(now.Year(), now.Month(), day, 0, 0, 0, 0, now.Location())
|
||||
|
||||
if i.TimeOfDay != nil {
|
||||
t, err := time.Parse("15:04", *i.TimeOfDay)
|
||||
if err == nil {
|
||||
targetThisMonth = time.Date(
|
||||
targetThisMonth.Year(), targetThisMonth.Month(), targetThisMonth.Day(),
|
||||
t.Hour(), t.Minute(), 0, 0, targetThisMonth.Location(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if now.Before(targetThisMonth) {
|
||||
return targetThisMonth
|
||||
}
|
||||
|
||||
return targetThisMonth.AddDate(0, 1, 0)
|
||||
}
|
||||
|
||||
func (i *Interval) nextCronTrigger(lastBackup time.Time) *time.Time {
|
||||
if i.CronExpression == nil || *i.CronExpression == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
|
||||
schedule, err := parser.Parse(*i.CronExpression)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
next := schedule.Next(lastBackup)
|
||||
|
||||
return &next
|
||||
}
|
||||
|
||||
func (i *Interval) validateCronExpression(expr string) error {
|
||||
parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
|
||||
_, err := parser.Parse(expr)
|
||||
|
||||
@@ -721,3 +721,265 @@ func TestInterval_Validate(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestInterval_NextTriggerTime_NilLastBackup(t *testing.T) {
|
||||
now := time.Date(2024, 1, 15, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
t.Run("Hourly with nil lastBackup returns nil", func(t *testing.T) {
|
||||
interval := &Interval{ID: uuid.New(), Interval: IntervalHourly}
|
||||
result := interval.NextTriggerTime(now, nil)
|
||||
assert.Nil(t, result)
|
||||
})
|
||||
|
||||
t.Run("Daily with nil lastBackup returns nil", func(t *testing.T) {
|
||||
timeOfDay := "09:00"
|
||||
interval := &Interval{ID: uuid.New(), Interval: IntervalDaily, TimeOfDay: &timeOfDay}
|
||||
result := interval.NextTriggerTime(now, nil)
|
||||
assert.Nil(t, result)
|
||||
})
|
||||
|
||||
t.Run("Weekly with nil lastBackup returns nil", func(t *testing.T) {
|
||||
timeOfDay := "15:00"
|
||||
weekday := 3
|
||||
interval := &Interval{
|
||||
ID: uuid.New(),
|
||||
Interval: IntervalWeekly,
|
||||
TimeOfDay: &timeOfDay,
|
||||
Weekday: &weekday,
|
||||
}
|
||||
result := interval.NextTriggerTime(now, nil)
|
||||
assert.Nil(t, result)
|
||||
})
|
||||
|
||||
t.Run("Monthly with nil lastBackup returns nil", func(t *testing.T) {
|
||||
timeOfDay := "08:00"
|
||||
dayOfMonth := 10
|
||||
interval := &Interval{
|
||||
ID: uuid.New(),
|
||||
Interval: IntervalMonthly,
|
||||
TimeOfDay: &timeOfDay,
|
||||
DayOfMonth: &dayOfMonth,
|
||||
}
|
||||
result := interval.NextTriggerTime(now, nil)
|
||||
assert.Nil(t, result)
|
||||
})
|
||||
|
||||
t.Run("Cron with nil lastBackup returns nil", func(t *testing.T) {
|
||||
cronExpr := "0 2 * * *"
|
||||
interval := &Interval{ID: uuid.New(), Interval: IntervalCron, CronExpression: &cronExpr}
|
||||
result := interval.NextTriggerTime(now, nil)
|
||||
assert.Nil(t, result)
|
||||
})
|
||||
}
|
||||
|
||||
func TestInterval_NextTriggerTime_Hourly(t *testing.T) {
|
||||
interval := &Interval{ID: uuid.New(), Interval: IntervalHourly}
|
||||
now := time.Date(2024, 1, 15, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
t.Run("Returns lastBackup + 1 hour", func(t *testing.T) {
|
||||
lastBackup := time.Date(2024, 1, 15, 11, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, time.Date(2024, 1, 15, 12, 0, 0, 0, time.UTC), *result)
|
||||
})
|
||||
|
||||
t.Run("Returns future time when last backup was recent", func(t *testing.T) {
|
||||
lastBackup := time.Date(2024, 1, 15, 11, 30, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, time.Date(2024, 1, 15, 12, 30, 0, 0, time.UTC), *result)
|
||||
})
|
||||
}
|
||||
|
||||
func TestInterval_NextTriggerTime_Daily(t *testing.T) {
|
||||
timeOfDay := "09:00"
|
||||
interval := &Interval{ID: uuid.New(), Interval: IntervalDaily, TimeOfDay: &timeOfDay}
|
||||
|
||||
t.Run("Before today's slot: returns today's slot", func(t *testing.T) {
|
||||
now := time.Date(2024, 1, 15, 8, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 14, 9, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, time.Date(2024, 1, 15, 9, 0, 0, 0, time.UTC), *result)
|
||||
})
|
||||
|
||||
t.Run("After today's slot: returns tomorrow's slot", func(t *testing.T) {
|
||||
now := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 15, 9, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, time.Date(2024, 1, 16, 9, 0, 0, 0, time.UTC), *result)
|
||||
})
|
||||
|
||||
t.Run("Exactly at today's slot: returns tomorrow's slot", func(t *testing.T) {
|
||||
now := time.Date(2024, 1, 15, 9, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 14, 9, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, time.Date(2024, 1, 16, 9, 0, 0, 0, time.UTC), *result)
|
||||
})
|
||||
}
|
||||
|
||||
func TestInterval_NextTriggerTime_Weekly(t *testing.T) {
|
||||
timeOfDay := "15:00"
|
||||
weekday := 3 // Wednesday
|
||||
interval := &Interval{
|
||||
ID: uuid.New(),
|
||||
Interval: IntervalWeekly,
|
||||
TimeOfDay: &timeOfDay,
|
||||
Weekday: &weekday,
|
||||
}
|
||||
|
||||
t.Run("Before this week's target: returns this week's target", func(t *testing.T) {
|
||||
// Tuesday Jan 16, 2024
|
||||
now := time.Date(2024, 1, 16, 10, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 10, 15, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
// Wednesday Jan 17 at 15:00
|
||||
assert.Equal(t, time.Date(2024, 1, 17, 15, 0, 0, 0, time.UTC), *result)
|
||||
})
|
||||
|
||||
t.Run("After this week's target: returns next week's target", func(t *testing.T) {
|
||||
// Thursday Jan 18, 2024
|
||||
now := time.Date(2024, 1, 18, 10, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 17, 15, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
// Next Wednesday Jan 24 at 15:00
|
||||
assert.Equal(t, time.Date(2024, 1, 24, 15, 0, 0, 0, time.UTC), *result)
|
||||
})
|
||||
|
||||
t.Run("Friday interval: returns correct target", func(t *testing.T) {
|
||||
fridayTimeOfDay := "00:00"
|
||||
fridayWeekday := 5 // Friday
|
||||
fridayInterval := &Interval{
|
||||
ID: uuid.New(),
|
||||
Interval: IntervalWeekly,
|
||||
TimeOfDay: &fridayTimeOfDay,
|
||||
Weekday: &fridayWeekday,
|
||||
}
|
||||
|
||||
// Wednesday Jan 17, 2024
|
||||
now := time.Date(2024, 1, 17, 10, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 12, 0, 0, 0, 0, time.UTC)
|
||||
result := fridayInterval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
// Friday Jan 19 at 00:00
|
||||
assert.Equal(t, time.Date(2024, 1, 19, 0, 0, 0, 0, time.UTC), *result)
|
||||
})
|
||||
}
|
||||
|
||||
func TestInterval_NextTriggerTime_Monthly(t *testing.T) {
|
||||
timeOfDay := "08:00"
|
||||
dayOfMonth := 10
|
||||
interval := &Interval{
|
||||
ID: uuid.New(),
|
||||
Interval: IntervalMonthly,
|
||||
TimeOfDay: &timeOfDay,
|
||||
DayOfMonth: &dayOfMonth,
|
||||
}
|
||||
|
||||
t.Run("Before this month's target: returns this month's target", func(t *testing.T) {
|
||||
now := time.Date(2024, 1, 5, 10, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2023, 12, 10, 8, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, time.Date(2024, 1, 10, 8, 0, 0, 0, time.UTC), *result)
|
||||
})
|
||||
|
||||
t.Run("After this month's target: returns next month's target", func(t *testing.T) {
|
||||
now := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 10, 8, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, time.Date(2024, 2, 10, 8, 0, 0, 0, time.UTC), *result)
|
||||
})
|
||||
|
||||
t.Run("Exactly at this month's target: returns next month's target", func(t *testing.T) {
|
||||
now := time.Date(2024, 1, 10, 8, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2023, 12, 10, 8, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, time.Date(2024, 2, 10, 8, 0, 0, 0, time.UTC), *result)
|
||||
})
|
||||
}
|
||||
|
||||
func TestInterval_NextTriggerTime_Cron(t *testing.T) {
|
||||
t.Run("Daily cron: returns next trigger after lastBackup", func(t *testing.T) {
|
||||
cronExpr := "0 2 * * *" // Daily at 2:00 AM
|
||||
interval := &Interval{ID: uuid.New(), Interval: IntervalCron, CronExpression: &cronExpr}
|
||||
|
||||
now := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 14, 2, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, time.Date(2024, 1, 15, 2, 0, 0, 0, time.UTC), *result)
|
||||
})
|
||||
|
||||
t.Run("Complex cron: 1st and 15th at 4:30", func(t *testing.T) {
|
||||
cronExpr := "30 4 1,15 * *"
|
||||
interval := &Interval{ID: uuid.New(), Interval: IntervalCron, CronExpression: &cronExpr}
|
||||
|
||||
now := time.Date(2024, 1, 10, 10, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 1, 4, 30, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, time.Date(2024, 1, 15, 4, 30, 0, 0, time.UTC), *result)
|
||||
})
|
||||
|
||||
t.Run("Invalid cron expression returns nil", func(t *testing.T) {
|
||||
invalidCron := "invalid cron"
|
||||
interval := &Interval{ID: uuid.New(), Interval: IntervalCron, CronExpression: &invalidCron}
|
||||
|
||||
now := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 14, 10, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.Nil(t, result)
|
||||
})
|
||||
|
||||
t.Run("Empty cron expression returns nil", func(t *testing.T) {
|
||||
emptyCron := ""
|
||||
interval := &Interval{ID: uuid.New(), Interval: IntervalCron, CronExpression: &emptyCron}
|
||||
|
||||
now := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 14, 10, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.Nil(t, result)
|
||||
})
|
||||
|
||||
t.Run("Nil cron expression returns nil", func(t *testing.T) {
|
||||
interval := &Interval{ID: uuid.New(), Interval: IntervalCron, CronExpression: nil}
|
||||
|
||||
now := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 14, 10, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.Nil(t, result)
|
||||
})
|
||||
}
|
||||
|
||||
func TestInterval_NextTriggerTime_UnknownInterval(t *testing.T) {
|
||||
interval := &Interval{ID: uuid.New(), Interval: IntervalType("UNKNOWN")}
|
||||
|
||||
now := time.Date(2024, 1, 15, 12, 0, 0, 0, time.UTC)
|
||||
lastBackup := time.Date(2024, 1, 14, 12, 0, 0, 0, time.UTC)
|
||||
result := interval.NextTriggerTime(now, &lastBackup)
|
||||
|
||||
assert.Nil(t, result)
|
||||
}
|
||||
|
||||
@@ -9,27 +9,28 @@ import (
|
||||
"mime"
|
||||
"net"
|
||||
"net/smtp"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
ImplicitTLSPort = 465
|
||||
DefaultTimeout = 5 * time.Second
|
||||
DefaultHelloName = "localhost"
|
||||
MIMETypeHTML = "text/html"
|
||||
MIMECharsetUTF8 = "UTF-8"
|
||||
ImplicitTLSPort = 465
|
||||
DefaultTimeout = 5 * time.Second
|
||||
MIMETypeHTML = "text/html"
|
||||
MIMECharsetUTF8 = "UTF-8"
|
||||
)
|
||||
|
||||
type EmailNotifier struct {
|
||||
NotifierID uuid.UUID `json:"notifierId" gorm:"primaryKey;type:uuid;column:notifier_id"`
|
||||
TargetEmail string `json:"targetEmail" gorm:"not null;type:varchar(255);column:target_email"`
|
||||
SMTPHost string `json:"smtpHost" gorm:"not null;type:varchar(255);column:smtp_host"`
|
||||
SMTPPort int `json:"smtpPort" gorm:"not null;column:smtp_port"`
|
||||
SMTPUser string `json:"smtpUser" gorm:"type:varchar(255);column:smtp_user"`
|
||||
SMTPPassword string `json:"smtpPassword" gorm:"type:varchar(255);column:smtp_password"`
|
||||
From string `json:"from" gorm:"type:varchar(255);column:from_email"`
|
||||
NotifierID uuid.UUID `json:"notifierId" gorm:"primaryKey;type:uuid;column:notifier_id"`
|
||||
TargetEmail string `json:"targetEmail" gorm:"not null;type:varchar(255);column:target_email"`
|
||||
SMTPHost string `json:"smtpHost" gorm:"not null;type:varchar(255);column:smtp_host"`
|
||||
SMTPPort int `json:"smtpPort" gorm:"not null;column:smtp_port"`
|
||||
SMTPUser string `json:"smtpUser" gorm:"type:varchar(255);column:smtp_user"`
|
||||
SMTPPassword string `json:"smtpPassword" gorm:"type:varchar(255);column:smtp_password"`
|
||||
From string `json:"from" gorm:"type:varchar(255);column:from_email"`
|
||||
IsInsecureSkipVerify bool `json:"isInsecureSkipVerify" gorm:"default:false;column:is_insecure_skip_verify"`
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) TableName() string {
|
||||
@@ -99,6 +100,7 @@ func (e *EmailNotifier) Update(incoming *EmailNotifier) {
|
||||
e.SMTPPort = incoming.SMTPPort
|
||||
e.SMTPUser = incoming.SMTPUser
|
||||
e.From = incoming.From
|
||||
e.IsInsecureSkipVerify = incoming.IsInsecureSkipVerify
|
||||
|
||||
if incoming.SMTPPassword != "" {
|
||||
e.SMTPPassword = incoming.SMTPPassword
|
||||
@@ -116,6 +118,16 @@ func (e *EmailNotifier) EncryptSensitiveData(encryptor encryption.FieldEncryptor
|
||||
return nil
|
||||
}
|
||||
|
||||
func getHelloName() string {
|
||||
hostname, err := os.Hostname()
|
||||
|
||||
if err != nil || hostname == "" {
|
||||
return "localhost"
|
||||
}
|
||||
|
||||
return hostname
|
||||
}
|
||||
|
||||
// encodeRFC2047 encodes a string using RFC 2047 MIME encoding for email headers
|
||||
// This ensures compatibility with SMTP servers that don't support SMTPUTF8
|
||||
func encodeRFC2047(s string) string {
|
||||
@@ -131,6 +143,7 @@ func (e *EmailNotifier) buildEmailContent(heading, message, from string) []byte
|
||||
encodedSubject := encodeRFC2047(heading)
|
||||
subject := fmt.Sprintf("Subject: %s\r\n", encodedSubject)
|
||||
dateHeader := fmt.Sprintf("Date: %s\r\n", time.Now().UTC().Format(time.RFC1123Z))
|
||||
messageID := fmt.Sprintf("Message-ID: <%s@%s>\r\n", uuid.New().String(), e.SMTPHost)
|
||||
|
||||
mimeHeaders := fmt.Sprintf(
|
||||
"MIME-version: 1.0;\nContent-Type: %s; charset=\"%s\";\n\n",
|
||||
@@ -144,7 +157,7 @@ func (e *EmailNotifier) buildEmailContent(heading, message, from string) []byte
|
||||
|
||||
toHeader := fmt.Sprintf("To: %s\r\n", e.TargetEmail)
|
||||
|
||||
return []byte(fromHeader + toHeader + subject + dateHeader + mimeHeaders + message)
|
||||
return []byte(fromHeader + toHeader + subject + dateHeader + messageID + mimeHeaders + message)
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) sendImplicitTLS(
|
||||
@@ -187,7 +200,10 @@ func (e *EmailNotifier) sendStartTLS(
|
||||
|
||||
func (e *EmailNotifier) createImplicitTLSClient() (*smtp.Client, func(), error) {
|
||||
addr := net.JoinHostPort(e.SMTPHost, fmt.Sprintf("%d", e.SMTPPort))
|
||||
tlsConfig := &tls.Config{ServerName: e.SMTPHost}
|
||||
tlsConfig := &tls.Config{
|
||||
ServerName: e.SMTPHost,
|
||||
InsecureSkipVerify: e.IsInsecureSkipVerify,
|
||||
}
|
||||
dialer := &net.Dialer{Timeout: DefaultTimeout}
|
||||
|
||||
conn, err := tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
|
||||
@@ -219,14 +235,17 @@ func (e *EmailNotifier) createStartTLSClient() (*smtp.Client, func(), error) {
|
||||
return nil, nil, fmt.Errorf("failed to create SMTP client: %w", err)
|
||||
}
|
||||
|
||||
if err := client.Hello(DefaultHelloName); err != nil {
|
||||
if err := client.Hello(getHelloName()); err != nil {
|
||||
_ = client.Quit()
|
||||
_ = conn.Close()
|
||||
return nil, nil, fmt.Errorf("SMTP hello failed: %w", err)
|
||||
}
|
||||
|
||||
if ok, _ := client.Extension("STARTTLS"); ok {
|
||||
if err := client.StartTLS(&tls.Config{ServerName: e.SMTPHost}); err != nil {
|
||||
if err := client.StartTLS(&tls.Config{
|
||||
ServerName: e.SMTPHost,
|
||||
InsecureSkipVerify: e.IsInsecureSkipVerify,
|
||||
}); err != nil {
|
||||
_ = client.Quit()
|
||||
_ = conn.Close()
|
||||
return nil, nil, fmt.Errorf("STARTTLS failed: %w", err)
|
||||
|
||||
@@ -58,6 +58,8 @@ func (s *NotifierService) SaveNotifier(
|
||||
return err
|
||||
}
|
||||
|
||||
oldName := existingNotifier.Name
|
||||
|
||||
if err := existingNotifier.Validate(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -67,11 +69,23 @@ func (s *NotifierService) SaveNotifier(
|
||||
return err
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Notifier updated: %s", existingNotifier.Name),
|
||||
&user.ID,
|
||||
&workspaceID,
|
||||
)
|
||||
if oldName != existingNotifier.Name {
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf(
|
||||
"Notifier updated and renamed from '%s' to '%s'",
|
||||
oldName,
|
||||
existingNotifier.Name,
|
||||
),
|
||||
&user.ID,
|
||||
&workspaceID,
|
||||
)
|
||||
} else {
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Notifier updated: %s", existingNotifier.Name),
|
||||
&user.ID,
|
||||
&workspaceID,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
notifier.WorkspaceID = workspaceID
|
||||
|
||||
@@ -343,9 +357,19 @@ func (s *NotifierService) TransferNotifierToWorkspace(
|
||||
return err
|
||||
}
|
||||
|
||||
sourceWorkspace, err := s.workspaceService.GetWorkspaceByID(sourceWorkspaceID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get source workspace: %w", err)
|
||||
}
|
||||
|
||||
targetWorkspace, err := s.workspaceService.GetWorkspaceByID(targetWorkspaceID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get target workspace: %w", err)
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Notifier transferred: %s from workspace %s to workspace %s",
|
||||
existingNotifier.Name, sourceWorkspaceID, targetWorkspaceID),
|
||||
fmt.Sprintf("Notifier transferred: %s from workspace '%s' to workspace '%s'",
|
||||
existingNotifier.Name, sourceWorkspace.Name, targetWorkspace.Name),
|
||||
&user.ID,
|
||||
&targetWorkspaceID,
|
||||
)
|
||||
|
||||
@@ -9,9 +9,9 @@ import (
|
||||
type DatabasePlan struct {
|
||||
DatabaseID uuid.UUID `json:"databaseId" gorm:"column:database_id;type:uuid;primaryKey;not null"`
|
||||
|
||||
MaxBackupSizeMB int64 `json:"maxBackupSizeMb" gorm:"column:max_backup_size_mb;type:int;not null"`
|
||||
MaxBackupsTotalSizeMB int64 `json:"maxBackupsTotalSizeMb" gorm:"column:max_backups_total_size_mb;type:int;not null"`
|
||||
MaxStoragePeriod period.Period `json:"maxStoragePeriod" gorm:"column:max_storage_period;type:text;not null"`
|
||||
MaxBackupSizeMB int64 `json:"maxBackupSizeMb" gorm:"column:max_backup_size_mb;type:int;not null"`
|
||||
MaxBackupsTotalSizeMB int64 `json:"maxBackupsTotalSizeMb" gorm:"column:max_backups_total_size_mb;type:int;not null"`
|
||||
MaxStoragePeriod period.TimePeriod `json:"maxStoragePeriod" gorm:"column:max_storage_period;type:text;not null"`
|
||||
}
|
||||
|
||||
func (p *DatabasePlan) TableName() string {
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
|
||||
env_config "databasus-backend/internal/config"
|
||||
audit_logs "databasus-backend/internal/features/audit_logs"
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
backups_controllers "databasus-backend/internal/features/backups/backups/controllers"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
@@ -32,7 +32,6 @@ import (
|
||||
tasks_cancellation "databasus-backend/internal/features/tasks/cancellation"
|
||||
users_dto "databasus-backend/internal/features/users/dto"
|
||||
users_enums "databasus-backend/internal/features/users/enums"
|
||||
users_services "databasus-backend/internal/features/users/services"
|
||||
users_testing "databasus-backend/internal/features/users/testing"
|
||||
workspaces_models "databasus-backend/internal/features/workspaces/models"
|
||||
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
|
||||
@@ -262,7 +261,7 @@ func Test_RestoreBackup_AuditLogWritten(t *testing.T) {
|
||||
|
||||
found := false
|
||||
for _, log := range auditLogs.AuditLogs {
|
||||
if strings.Contains(log.Message, "Database restored from backup") &&
|
||||
if strings.Contains(log.Message, "Database restored for database") &&
|
||||
strings.Contains(log.Message, database.Name) {
|
||||
found = true
|
||||
break
|
||||
@@ -358,7 +357,7 @@ func Test_RestoreBackup_DiskSpaceValidation(t *testing.T) {
|
||||
_, err = configService.SaveBackupConfig(config)
|
||||
assert.NoError(t, err)
|
||||
|
||||
backup = createTestBackup(mysqlDB, owner)
|
||||
backup = createTestBackup(mysqlDB, storage)
|
||||
|
||||
request = restores_core.RestoreBackupRequest{
|
||||
MysqlDatabase: &mysql.MysqlDatabase{
|
||||
@@ -441,7 +440,7 @@ func Test_CancelRestore_InProgressRestore_SuccessfullyCancelled(t *testing.T) {
|
||||
}()
|
||||
|
||||
backups_config.EnableBackupsForTestDatabase(database.ID, storage)
|
||||
backup := backups.CreateTestBackup(database.ID, storage.ID)
|
||||
backup := backups_controllers.CreateTestBackup(database.ID, storage.ID)
|
||||
|
||||
mockUsecase := &restoring.MockBlockingRestoreUsecase{
|
||||
StartedChan: make(chan bool, 1),
|
||||
@@ -610,7 +609,7 @@ func createTestDatabaseWithBackupForRestore(
|
||||
panic(err)
|
||||
}
|
||||
|
||||
backup := createTestBackup(database, owner)
|
||||
backup := createTestBackup(database, storage)
|
||||
|
||||
return database, backup
|
||||
}
|
||||
@@ -727,24 +726,14 @@ func createTestStorage(workspaceID uuid.UUID) *storages.Storage {
|
||||
|
||||
func createTestBackup(
|
||||
database *databases.Database,
|
||||
owner *users_dto.SignInResponseDTO,
|
||||
storage *storages.Storage,
|
||||
) *backups_core.Backup {
|
||||
fieldEncryptor := util_encryption.GetFieldEncryptor()
|
||||
userService := users_services.GetUserService()
|
||||
user, err := userService.GetUserFromToken(owner.Token)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
storages, err := storages.GetStorageService().GetStorages(user, *database.WorkspaceID)
|
||||
if err != nil || len(storages) == 0 {
|
||||
panic("No storage found for workspace")
|
||||
}
|
||||
|
||||
backup := &backups_core.Backup{
|
||||
ID: uuid.New(),
|
||||
DatabaseID: database.ID,
|
||||
StorageID: storages[0].ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups_core.BackupStatusCompleted,
|
||||
BackupSizeMb: 10.5,
|
||||
BackupDurationMs: 1000,
|
||||
@@ -759,11 +748,11 @@ func createTestBackup(
|
||||
dummyContent := []byte("dummy backup content for testing")
|
||||
reader := strings.NewReader(string(dummyContent))
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
if err := storages[0].SaveFile(
|
||||
if err := storage.SaveFile(
|
||||
context.Background(),
|
||||
fieldEncryptor,
|
||||
logger,
|
||||
backup.ID,
|
||||
backup.ID.String(),
|
||||
reader,
|
||||
); err != nil {
|
||||
panic(fmt.Sprintf("Failed to create test backup file: %v", err))
|
||||
|
||||
@@ -5,8 +5,8 @@ import (
|
||||
"sync/atomic"
|
||||
|
||||
audit_logs "databasus-backend/internal/features/audit_logs"
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
"databasus-backend/internal/features/backups/backups/backuping"
|
||||
backups_services "databasus-backend/internal/features/backups/backups/services"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/disk"
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
|
||||
var restoreRepository = &restores_core.RestoreRepository{}
|
||||
var restoreService = &RestoreService{
|
||||
backups.GetBackupService(),
|
||||
backups_services.GetBackupService(),
|
||||
restoreRepository,
|
||||
storages.GetStorageService(),
|
||||
backups_config.GetBackupConfigService(),
|
||||
@@ -51,7 +51,7 @@ func SetupDependencies() {
|
||||
wasAlreadySetup := isSetup.Load()
|
||||
|
||||
setupOnce.Do(func() {
|
||||
backups.GetBackupService().AddBackupRemoveListener(restoreService)
|
||||
backups_services.GetBackupService().AddBackupRemoveListener(restoreService)
|
||||
backuping.GetBackupCleaner().AddBackupRemoveListener(restoreService)
|
||||
|
||||
isSetup.Store(true)
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
backups_services "databasus-backend/internal/features/backups/backups/services"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
restores_core "databasus-backend/internal/features/restores/core"
|
||||
@@ -39,37 +39,37 @@ var restoreDatabaseCache = cache_utils.NewCacheUtil[RestoreDatabaseCache](
|
||||
var restoreCancelManager = tasks_cancellation.GetTaskCancelManager()
|
||||
|
||||
var restorerNode = &RestorerNode{
|
||||
nodeID: uuid.New(),
|
||||
databaseService: databases.GetDatabaseService(),
|
||||
backupService: backups.GetBackupService(),
|
||||
fieldEncryptor: encryption.GetFieldEncryptor(),
|
||||
restoreRepository: restoreRepository,
|
||||
backupConfigService: backups_config.GetBackupConfigService(),
|
||||
storageService: storages.GetStorageService(),
|
||||
restoreNodesRegistry: restoreNodesRegistry,
|
||||
logger: logger.GetLogger(),
|
||||
restoreBackupUsecase: usecases.GetRestoreBackupUsecase(),
|
||||
cacheUtil: restoreDatabaseCache,
|
||||
restoreCancelManager: restoreCancelManager,
|
||||
lastHeartbeat: time.Time{},
|
||||
runOnce: sync.Once{},
|
||||
hasRun: atomic.Bool{},
|
||||
uuid.New(),
|
||||
databases.GetDatabaseService(),
|
||||
backups_services.GetBackupService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
restoreRepository,
|
||||
backups_config.GetBackupConfigService(),
|
||||
storages.GetStorageService(),
|
||||
restoreNodesRegistry,
|
||||
logger.GetLogger(),
|
||||
usecases.GetRestoreBackupUsecase(),
|
||||
restoreDatabaseCache,
|
||||
restoreCancelManager,
|
||||
time.Time{},
|
||||
sync.Once{},
|
||||
atomic.Bool{},
|
||||
}
|
||||
|
||||
var restoresScheduler = &RestoresScheduler{
|
||||
restoreRepository: restoreRepository,
|
||||
backupService: backups.GetBackupService(),
|
||||
storageService: storages.GetStorageService(),
|
||||
backupConfigService: backups_config.GetBackupConfigService(),
|
||||
restoreNodesRegistry: restoreNodesRegistry,
|
||||
lastCheckTime: time.Now().UTC(),
|
||||
logger: logger.GetLogger(),
|
||||
restoreToNodeRelations: make(map[uuid.UUID]RestoreToNodeRelation),
|
||||
restorerNode: restorerNode,
|
||||
cacheUtil: restoreDatabaseCache,
|
||||
completionSubscriptionID: uuid.Nil,
|
||||
runOnce: sync.Once{},
|
||||
hasRun: atomic.Bool{},
|
||||
restoreRepository,
|
||||
backups_services.GetBackupService(),
|
||||
storages.GetStorageService(),
|
||||
backups_config.GetBackupConfigService(),
|
||||
restoreNodesRegistry,
|
||||
time.Now().UTC(),
|
||||
logger.GetLogger(),
|
||||
make(map[uuid.UUID]RestoreToNodeRelation),
|
||||
restorerNode,
|
||||
restoreDatabaseCache,
|
||||
uuid.Nil,
|
||||
sync.Once{},
|
||||
atomic.Bool{},
|
||||
}
|
||||
|
||||
func GetRestoresScheduler() *RestoresScheduler {
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
backups_services "databasus-backend/internal/features/backups/backups/services"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
restores_core "databasus-backend/internal/features/restores/core"
|
||||
@@ -32,7 +32,7 @@ type RestorerNode struct {
|
||||
nodeID uuid.UUID
|
||||
|
||||
databaseService *databases.DatabaseService
|
||||
backupService *backups.BackupService
|
||||
backupService *backups_services.BackupService
|
||||
fieldEncryptor util_encryption.FieldEncryptor
|
||||
restoreRepository *restores_core.RestoreRepository
|
||||
backupConfigService *backups_config.BackupConfigService
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
backups_controllers "databasus-backend/internal/features/backups/backups/controllers"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
@@ -58,7 +58,7 @@ func Test_MakeRestore_WhenCacheMissed_RestoreFails(t *testing.T) {
|
||||
cache_utils.ClearAllCache()
|
||||
}()
|
||||
|
||||
backup := backups.CreateTestBackup(database.ID, storage.ID)
|
||||
backup := backups_controllers.CreateTestBackup(database.ID, storage.ID)
|
||||
|
||||
// Create restore but DON'T cache DB credentials
|
||||
// Also don't set embedded DB fields to avoid schema issues
|
||||
@@ -126,7 +126,7 @@ func Test_MakeRestore_WhenTaskStarts_CacheDeletedImmediately(t *testing.T) {
|
||||
cache_utils.ClearAllCache()
|
||||
}()
|
||||
|
||||
backup := backups.CreateTestBackup(database.ID, storage.ID)
|
||||
backup := backups_controllers.CreateTestBackup(database.ID, storage.ID)
|
||||
|
||||
// Create restore with cached DB credentials
|
||||
// Don't set embedded DB fields in the restore model itself
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
backups_services "databasus-backend/internal/features/backups/backups/services"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
restores_core "databasus-backend/internal/features/restores/core"
|
||||
"databasus-backend/internal/features/storages"
|
||||
@@ -26,7 +26,7 @@ const (
|
||||
|
||||
type RestoresScheduler struct {
|
||||
restoreRepository *restores_core.RestoreRepository
|
||||
backupService *backups.BackupService
|
||||
backupService *backups_services.BackupService
|
||||
storageService *storages.StorageService
|
||||
backupConfigService *backups_config.BackupConfigService
|
||||
restoreNodesRegistry *RestoreNodesRegistry
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"time"
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
backups_controllers "databasus-backend/internal/features/backups/backups/controllers"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
@@ -68,7 +68,7 @@ func Test_CheckDeadNodesAndFailRestores_NodeDies_FailsRestoreAndCleansUpRegistry
|
||||
backups_config.EnableBackupsForTestDatabase(database.ID, storage)
|
||||
|
||||
// Create a test backup
|
||||
backup := backups.CreateTestBackup(database.ID, storage.ID)
|
||||
backup := backups_controllers.CreateTestBackup(database.ID, storage.ID)
|
||||
|
||||
var err error
|
||||
// Register mock node without subscribing to restores (simulates node crash after registration)
|
||||
@@ -171,7 +171,7 @@ func Test_OnRestoreCompleted_TaskIsNotRestore_SkipsProcessing(t *testing.T) {
|
||||
backups_config.EnableBackupsForTestDatabase(database.ID, storage)
|
||||
|
||||
// Create a test backup
|
||||
backup := backups.CreateTestBackup(database.ID, storage.ID)
|
||||
backup := backups_controllers.CreateTestBackup(database.ID, storage.ID)
|
||||
|
||||
// Register mock node
|
||||
mockNodeID = uuid.New()
|
||||
@@ -357,7 +357,7 @@ func Test_FailRestoresInProgress_SchedulerStarts_UpdatesStatus(t *testing.T) {
|
||||
backups_config.EnableBackupsForTestDatabase(database.ID, storage)
|
||||
|
||||
// Create a test backup
|
||||
backup := backups.CreateTestBackup(database.ID, storage.ID)
|
||||
backup := backups_controllers.CreateTestBackup(database.ID, storage.ID)
|
||||
|
||||
// Create two in-progress restores that should be failed on scheduler restart
|
||||
restore1 := &restores_core.Restore{
|
||||
@@ -465,7 +465,7 @@ func Test_StartRestore_RestoreCompletes_DecrementsActiveTaskCount(t *testing.T)
|
||||
backups_config.EnableBackupsForTestDatabase(database.ID, storage)
|
||||
|
||||
// Create a test backup
|
||||
backup := backups.CreateTestBackup(database.ID, storage.ID)
|
||||
backup := backups_controllers.CreateTestBackup(database.ID, storage.ID)
|
||||
|
||||
// Get initial active task count
|
||||
stats, err := restoreNodesRegistry.GetRestoreNodesStats()
|
||||
@@ -566,7 +566,7 @@ func Test_StartRestore_RestoreFails_DecrementsActiveTaskCount(t *testing.T) {
|
||||
backups_config.EnableBackupsForTestDatabase(database.ID, storage)
|
||||
|
||||
// Create a test backup
|
||||
backup := backups.CreateTestBackup(database.ID, storage.ID)
|
||||
backup := backups_controllers.CreateTestBackup(database.ID, storage.ID)
|
||||
|
||||
// Get initial active task count
|
||||
stats, err := restoreNodesRegistry.GetRestoreNodesStats()
|
||||
@@ -664,7 +664,7 @@ func Test_StartRestore_CredentialsStoredEncryptedInCache(t *testing.T) {
|
||||
backups_config.EnableBackupsForTestDatabase(database.ID, storage)
|
||||
|
||||
// Create a test backup
|
||||
backup := backups.CreateTestBackup(database.ID, storage.ID)
|
||||
backup := backups_controllers.CreateTestBackup(database.ID, storage.ID)
|
||||
|
||||
// Register mock node so scheduler can assign restore to it
|
||||
mockNodeID = uuid.New()
|
||||
@@ -779,7 +779,7 @@ func Test_StartRestore_CredentialsRemovedAfterRestoreStarts(t *testing.T) {
|
||||
backups_config.EnableBackupsForTestDatabase(database.ID, storage)
|
||||
|
||||
// Create a test backup
|
||||
backup := backups.CreateTestBackup(database.ID, storage.ID)
|
||||
backup := backups_controllers.CreateTestBackup(database.ID, storage.ID)
|
||||
|
||||
// Create restore with credentials
|
||||
plaintextPassword := "test_password_456"
|
||||
|
||||
@@ -12,8 +12,8 @@ import (
|
||||
"github.com/google/uuid"
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backups_services "databasus-backend/internal/features/backups/backups/services"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/databases/databases/postgresql"
|
||||
@@ -40,48 +40,48 @@ func CreateTestRouter() *gin.Engine {
|
||||
|
||||
func CreateTestRestorerNode() *RestorerNode {
|
||||
return &RestorerNode{
|
||||
nodeID: uuid.New(),
|
||||
databaseService: databases.GetDatabaseService(),
|
||||
backupService: backups.GetBackupService(),
|
||||
fieldEncryptor: encryption.GetFieldEncryptor(),
|
||||
restoreRepository: restoreRepository,
|
||||
backupConfigService: backups_config.GetBackupConfigService(),
|
||||
storageService: storages.GetStorageService(),
|
||||
restoreNodesRegistry: restoreNodesRegistry,
|
||||
logger: logger.GetLogger(),
|
||||
restoreBackupUsecase: usecases.GetRestoreBackupUsecase(),
|
||||
cacheUtil: restoreDatabaseCache,
|
||||
restoreCancelManager: tasks_cancellation.GetTaskCancelManager(),
|
||||
lastHeartbeat: time.Time{},
|
||||
runOnce: sync.Once{},
|
||||
hasRun: atomic.Bool{},
|
||||
uuid.New(),
|
||||
databases.GetDatabaseService(),
|
||||
backups_services.GetBackupService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
restoreRepository,
|
||||
backups_config.GetBackupConfigService(),
|
||||
storages.GetStorageService(),
|
||||
restoreNodesRegistry,
|
||||
logger.GetLogger(),
|
||||
usecases.GetRestoreBackupUsecase(),
|
||||
restoreDatabaseCache,
|
||||
tasks_cancellation.GetTaskCancelManager(),
|
||||
time.Time{},
|
||||
sync.Once{},
|
||||
atomic.Bool{},
|
||||
}
|
||||
}
|
||||
|
||||
func CreateTestRestorerNodeWithUsecase(usecase restores_core.RestoreBackupUsecase) *RestorerNode {
|
||||
return &RestorerNode{
|
||||
nodeID: uuid.New(),
|
||||
databaseService: databases.GetDatabaseService(),
|
||||
backupService: backups.GetBackupService(),
|
||||
fieldEncryptor: encryption.GetFieldEncryptor(),
|
||||
restoreRepository: restoreRepository,
|
||||
backupConfigService: backups_config.GetBackupConfigService(),
|
||||
storageService: storages.GetStorageService(),
|
||||
restoreNodesRegistry: restoreNodesRegistry,
|
||||
logger: logger.GetLogger(),
|
||||
restoreBackupUsecase: usecase,
|
||||
cacheUtil: restoreDatabaseCache,
|
||||
restoreCancelManager: tasks_cancellation.GetTaskCancelManager(),
|
||||
lastHeartbeat: time.Time{},
|
||||
runOnce: sync.Once{},
|
||||
hasRun: atomic.Bool{},
|
||||
uuid.New(),
|
||||
databases.GetDatabaseService(),
|
||||
backups_services.GetBackupService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
restoreRepository,
|
||||
backups_config.GetBackupConfigService(),
|
||||
storages.GetStorageService(),
|
||||
restoreNodesRegistry,
|
||||
logger.GetLogger(),
|
||||
usecase,
|
||||
restoreDatabaseCache,
|
||||
tasks_cancellation.GetTaskCancelManager(),
|
||||
time.Time{},
|
||||
sync.Once{},
|
||||
atomic.Bool{},
|
||||
}
|
||||
}
|
||||
|
||||
func CreateTestRestoresScheduler() *RestoresScheduler {
|
||||
return &RestoresScheduler{
|
||||
restoreRepository,
|
||||
backups.GetBackupService(),
|
||||
backups_services.GetBackupService(),
|
||||
storages.GetStorageService(),
|
||||
backups_config.GetBackupConfigService(),
|
||||
restoreNodesRegistry,
|
||||
|
||||
@@ -3,8 +3,8 @@ package restores
|
||||
import (
|
||||
"databasus-backend/internal/config"
|
||||
audit_logs "databasus-backend/internal/features/audit_logs"
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
backups_core "databasus-backend/internal/features/backups/backups/core"
|
||||
backups_services "databasus-backend/internal/features/backups/backups/services"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/disk"
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
)
|
||||
|
||||
type RestoreService struct {
|
||||
backupService *backups.BackupService
|
||||
backupService *backups_services.BackupService
|
||||
restoreRepository *restores_core.RestoreRepository
|
||||
storageService *storages.StorageService
|
||||
backupConfigService *backups_config.BackupConfigService
|
||||
@@ -190,11 +190,7 @@ func (s *RestoreService) RestoreBackupWithAuth(
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf(
|
||||
"Database restored from backup %s for database: %s",
|
||||
backupID.String(),
|
||||
database.Name,
|
||||
),
|
||||
fmt.Sprintf("Database restored for database: %s", database.Name),
|
||||
&user.ID,
|
||||
database.WorkspaceID,
|
||||
)
|
||||
@@ -412,11 +408,7 @@ func (s *RestoreService) CancelRestore(
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf(
|
||||
"Restore cancelled for database: %s (ID: %s)",
|
||||
database.Name,
|
||||
restoreID.String(),
|
||||
),
|
||||
fmt.Sprintf("Restore cancelled for database: %s", database.Name),
|
||||
&user.ID,
|
||||
database.WorkspaceID,
|
||||
)
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
backups_controllers "databasus-backend/internal/features/backups/backups/controllers"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/restores/restoring"
|
||||
@@ -22,12 +22,12 @@ func CreateTestRouter() *gin.Engine {
|
||||
workspaces_controllers.GetMembershipController(),
|
||||
databases.GetDatabaseController(),
|
||||
backups_config.GetBackupConfigController(),
|
||||
backups.GetBackupController(),
|
||||
backups_controllers.GetBackupController(),
|
||||
GetRestoreController(),
|
||||
)
|
||||
|
||||
v1 := router.Group("/api/v1")
|
||||
backups.GetBackupController().RegisterPublicRoutes(v1)
|
||||
backups_controllers.GetBackupController().RegisterPublicRoutes(v1)
|
||||
|
||||
return router
|
||||
}
|
||||
|
||||
@@ -70,6 +70,10 @@ func (uc *RestoreMariadbBackupUsecase) Execute(
|
||||
"--verbose",
|
||||
}
|
||||
|
||||
if !config.GetEnv().IsCloud {
|
||||
args = append(args, "--max-allowed-packet=1G")
|
||||
}
|
||||
|
||||
if mdb.IsHttps {
|
||||
args = append(args, "--ssl")
|
||||
args = append(args, "--skip-ssl-verify-server-cert")
|
||||
@@ -106,7 +110,7 @@ func (uc *RestoreMariadbBackupUsecase) restoreFromStorage(
|
||||
storage *storages.Storage,
|
||||
mdbConfig *mariadbtypes.MariadbDatabase,
|
||||
) error {
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 60*time.Minute)
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 23*time.Hour)
|
||||
defer cancel()
|
||||
|
||||
go func() {
|
||||
@@ -141,7 +145,7 @@ func (uc *RestoreMariadbBackupUsecase) restoreFromStorage(
|
||||
defer func() { _ = os.RemoveAll(filepath.Dir(myCnfFile)) }()
|
||||
|
||||
// Stream backup directly from storage
|
||||
rawReader, err := storage.GetFile(fieldEncryptor, backup.ID)
|
||||
rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get backup file from storage: %w", err)
|
||||
}
|
||||
|
||||
@@ -154,7 +154,7 @@ func (uc *RestoreMongodbBackupUsecase) restoreFromStorage(
|
||||
|
||||
// Stream backup directly from storage
|
||||
fieldEncryptor := util_encryption.GetFieldEncryptor()
|
||||
rawReader, err := storage.GetFile(fieldEncryptor, backup.ID)
|
||||
rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get backup file from storage: %w", err)
|
||||
}
|
||||
|
||||
@@ -70,6 +70,10 @@ func (uc *RestoreMysqlBackupUsecase) Execute(
|
||||
"--verbose",
|
||||
}
|
||||
|
||||
if !config.GetEnv().IsCloud {
|
||||
args = append(args, "--max-allowed-packet=1G")
|
||||
}
|
||||
|
||||
if my.IsHttps {
|
||||
args = append(args, "--ssl-mode=REQUIRED")
|
||||
}
|
||||
@@ -105,7 +109,7 @@ func (uc *RestoreMysqlBackupUsecase) restoreFromStorage(
|
||||
storage *storages.Storage,
|
||||
myConfig *mysqltypes.MysqlDatabase,
|
||||
) error {
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 60*time.Minute)
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 23*time.Hour)
|
||||
defer cancel()
|
||||
|
||||
go func() {
|
||||
@@ -140,7 +144,7 @@ func (uc *RestoreMysqlBackupUsecase) restoreFromStorage(
|
||||
defer func() { _ = os.RemoveAll(filepath.Dir(myCnfFile)) }()
|
||||
|
||||
// Stream backup directly from storage
|
||||
rawReader, err := storage.GetFile(fieldEncryptor, backup.ID)
|
||||
rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get backup file from storage: %w", err)
|
||||
}
|
||||
|
||||
@@ -152,7 +152,7 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
|
||||
"--no-acl",
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 60*time.Minute)
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 23*time.Hour)
|
||||
defer cancel()
|
||||
|
||||
// Monitor for shutdown and parent cancellation
|
||||
@@ -209,7 +209,7 @@ func (uc *RestorePostgresqlBackupUsecase) restoreViaStdin(
|
||||
}
|
||||
|
||||
// Get backup stream from storage
|
||||
rawReader, err := storage.GetFile(fieldEncryptor, backup.ID)
|
||||
rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get backup file from storage: %w", err)
|
||||
}
|
||||
@@ -429,7 +429,7 @@ func (uc *RestorePostgresqlBackupUsecase) restoreFromStorage(
|
||||
isExcludeExtensions,
|
||||
)
|
||||
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 60*time.Minute)
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 23*time.Hour)
|
||||
defer cancel()
|
||||
|
||||
// Monitor for shutdown and parent cancellation
|
||||
@@ -540,12 +540,14 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(
|
||||
"encrypted",
|
||||
backup.Encryption == backups_config.BackupEncryptionEncrypted,
|
||||
)
|
||||
|
||||
fieldEncryptor := util_encryption.GetFieldEncryptor()
|
||||
rawReader, err := storage.GetFile(fieldEncryptor, backup.ID)
|
||||
rawReader, err := storage.GetFile(fieldEncryptor, backup.FileName)
|
||||
if err != nil {
|
||||
cleanupFunc()
|
||||
return "", nil, fmt.Errorf("failed to get backup file from storage: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := rawReader.Close(); err != nil {
|
||||
uc.logger.Error("Failed to close backup reader", "error", err)
|
||||
|
||||
@@ -14,13 +14,13 @@ type StorageFileSaver interface {
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
file io.Reader,
|
||||
) error
|
||||
|
||||
GetFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) (io.ReadCloser, error)
|
||||
GetFile(encryptor encryption.FieldEncryptor, fileName string) (io.ReadCloser, error)
|
||||
|
||||
DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error
|
||||
DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error
|
||||
|
||||
Validate(encryptor encryption.FieldEncryptor) error
|
||||
|
||||
|
||||
@@ -41,10 +41,10 @@ func (s *Storage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
file io.Reader,
|
||||
) error {
|
||||
err := s.getSpecificStorage().SaveFile(ctx, encryptor, logger, fileID, file)
|
||||
err := s.getSpecificStorage().SaveFile(ctx, encryptor, logger, fileName, file)
|
||||
if err != nil {
|
||||
lastSaveError := err.Error()
|
||||
s.LastSaveError = &lastSaveError
|
||||
@@ -58,13 +58,13 @@ func (s *Storage) SaveFile(
|
||||
|
||||
func (s *Storage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
) (io.ReadCloser, error) {
|
||||
return s.getSpecificStorage().GetFile(encryptor, fileID)
|
||||
return s.getSpecificStorage().GetFile(encryptor, fileName)
|
||||
}
|
||||
|
||||
func (s *Storage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
return s.getSpecificStorage().DeleteFile(encryptor, fileID)
|
||||
func (s *Storage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {
|
||||
return s.getSpecificStorage().DeleteFile(encryptor, fileName)
|
||||
}
|
||||
|
||||
func (s *Storage) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
|
||||
@@ -229,12 +229,12 @@ acl = private`, s3Container.accessKey, s3Container.secretKey, s3Container.endpoi
|
||||
context.Background(),
|
||||
encryptor,
|
||||
logger.GetLogger(),
|
||||
fileID,
|
||||
fileID.String(),
|
||||
bytes.NewReader(fileData),
|
||||
)
|
||||
require.NoError(t, err, "SaveFile should succeed")
|
||||
|
||||
file, err := tc.storage.GetFile(encryptor, fileID)
|
||||
file, err := tc.storage.GetFile(encryptor, fileID.String())
|
||||
assert.NoError(t, err, "GetFile should succeed")
|
||||
defer file.Close()
|
||||
|
||||
@@ -252,15 +252,15 @@ acl = private`, s3Container.accessKey, s3Container.secretKey, s3Container.endpoi
|
||||
context.Background(),
|
||||
encryptor,
|
||||
logger.GetLogger(),
|
||||
fileID,
|
||||
fileID.String(),
|
||||
bytes.NewReader(fileData),
|
||||
)
|
||||
require.NoError(t, err, "SaveFile should succeed")
|
||||
|
||||
err = tc.storage.DeleteFile(encryptor, fileID)
|
||||
err = tc.storage.DeleteFile(encryptor, fileID.String())
|
||||
assert.NoError(t, err, "DeleteFile should succeed")
|
||||
|
||||
file, err := tc.storage.GetFile(encryptor, fileID)
|
||||
file, err := tc.storage.GetFile(encryptor, fileID.String())
|
||||
assert.Error(t, err, "GetFile should fail for non-existent file")
|
||||
if file != nil {
|
||||
file.Close()
|
||||
@@ -270,7 +270,7 @@ acl = private`, s3Container.accessKey, s3Container.secretKey, s3Container.endpoi
|
||||
t.Run("Test_TestDeleteNonExistentFile_DoesNotError", func(t *testing.T) {
|
||||
// Try to delete a non-existent file
|
||||
nonExistentID := uuid.New()
|
||||
err := tc.storage.DeleteFile(encryptor, nonExistentID)
|
||||
err := tc.storage.DeleteFile(encryptor, nonExistentID.String())
|
||||
assert.NoError(t, err, "DeleteFile should not error for non-existent file")
|
||||
})
|
||||
})
|
||||
|
||||
@@ -68,7 +68,7 @@ func (s *AzureBlobStorage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
file io.Reader,
|
||||
) error {
|
||||
select {
|
||||
@@ -82,7 +82,7 @@ func (s *AzureBlobStorage) SaveFile(
|
||||
return err
|
||||
}
|
||||
|
||||
blobName := s.buildBlobName(fileID.String())
|
||||
blobName := s.buildBlobName(fileName)
|
||||
blockBlobClient := client.ServiceClient().
|
||||
NewContainerClient(s.ContainerName).
|
||||
NewBlockBlobClient(blobName)
|
||||
@@ -157,14 +157,14 @@ func (s *AzureBlobStorage) SaveFile(
|
||||
|
||||
func (s *AzureBlobStorage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
) (io.ReadCloser, error) {
|
||||
client, err := s.getClient(encryptor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
blobName := s.buildBlobName(fileID.String())
|
||||
blobName := s.buildBlobName(fileName)
|
||||
|
||||
response, err := client.DownloadStream(
|
||||
context.TODO(),
|
||||
@@ -179,13 +179,13 @@ func (s *AzureBlobStorage) GetFile(
|
||||
return response.Body, nil
|
||||
}
|
||||
|
||||
func (s *AzureBlobStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
func (s *AzureBlobStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {
|
||||
client, err := s.getClient(encryptor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
blobName := s.buildBlobName(fileID.String())
|
||||
blobName := s.buildBlobName(fileName)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), azureDeleteTimeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -41,7 +41,7 @@ func (f *FTPStorage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
file io.Reader,
|
||||
) error {
|
||||
select {
|
||||
@@ -50,19 +50,19 @@ func (f *FTPStorage) SaveFile(
|
||||
default:
|
||||
}
|
||||
|
||||
logger.Info("Starting to save file to FTP storage", "fileId", fileID.String(), "host", f.Host)
|
||||
logger.Info("Starting to save file to FTP storage", "fileName", fileName, "host", f.Host)
|
||||
|
||||
conn, err := f.connect(encryptor, ftpConnectTimeout)
|
||||
if err != nil {
|
||||
logger.Error("Failed to connect to FTP", "fileId", fileID.String(), "error", err)
|
||||
logger.Error("Failed to connect to FTP", "fileName", fileName, "error", err)
|
||||
return fmt.Errorf("failed to connect to FTP: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if quitErr := conn.Quit(); quitErr != nil {
|
||||
logger.Error(
|
||||
"Failed to close FTP connection",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"error",
|
||||
quitErr,
|
||||
)
|
||||
@@ -73,8 +73,8 @@ func (f *FTPStorage) SaveFile(
|
||||
if err := f.ensureDirectory(conn, f.Path); err != nil {
|
||||
logger.Error(
|
||||
"Failed to ensure directory",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"path",
|
||||
f.Path,
|
||||
"error",
|
||||
@@ -84,8 +84,8 @@ func (f *FTPStorage) SaveFile(
|
||||
}
|
||||
}
|
||||
|
||||
filePath := f.getFilePath(fileID.String())
|
||||
logger.Debug("Uploading file to FTP", "fileId", fileID.String(), "filePath", filePath)
|
||||
filePath := f.getFilePath(fileName)
|
||||
logger.Debug("Uploading file to FTP", "fileName", fileName, "filePath", filePath)
|
||||
|
||||
ctxReader := &contextReader{ctx: ctx, reader: file}
|
||||
|
||||
@@ -93,18 +93,18 @@ func (f *FTPStorage) SaveFile(
|
||||
if err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.Info("FTP upload cancelled", "fileId", fileID.String())
|
||||
logger.Info("FTP upload cancelled", "fileName", fileName)
|
||||
return ctx.Err()
|
||||
default:
|
||||
logger.Error("Failed to upload file to FTP", "fileId", fileID.String(), "error", err)
|
||||
logger.Error("Failed to upload file to FTP", "fileName", fileName, "error", err)
|
||||
return fmt.Errorf("failed to upload file to FTP: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info(
|
||||
"Successfully saved file to FTP storage",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"filePath",
|
||||
filePath,
|
||||
)
|
||||
@@ -113,14 +113,14 @@ func (f *FTPStorage) SaveFile(
|
||||
|
||||
func (f *FTPStorage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
) (io.ReadCloser, error) {
|
||||
conn, err := f.connect(encryptor, ftpConnectTimeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to FTP: %w", err)
|
||||
}
|
||||
|
||||
filePath := f.getFilePath(fileID.String())
|
||||
filePath := f.getFilePath(fileName)
|
||||
|
||||
resp, err := conn.Retr(filePath)
|
||||
if err != nil {
|
||||
@@ -134,7 +134,7 @@ func (f *FTPStorage) GetFile(
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *FTPStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
func (f *FTPStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), ftpDeleteTimeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -146,7 +146,7 @@ func (f *FTPStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid
|
||||
_ = conn.Quit()
|
||||
}()
|
||||
|
||||
filePath := f.getFilePath(fileID.String())
|
||||
filePath := f.getFilePath(fileName)
|
||||
|
||||
_, err = conn.FileSize(filePath)
|
||||
if err != nil {
|
||||
|
||||
@@ -50,21 +50,19 @@ func (s *GoogleDriveStorage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
file io.Reader,
|
||||
) error {
|
||||
return s.withRetryOnAuth(ctx, encryptor, func(driveService *drive.Service) error {
|
||||
filename := fileID.String()
|
||||
|
||||
folderID, err := s.ensureBackupsFolderExists(ctx, driveService)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create/find backups folder: %w", err)
|
||||
}
|
||||
|
||||
_ = s.deleteByName(ctx, driveService, filename, folderID)
|
||||
_ = s.deleteByName(ctx, driveService, fileName, folderID)
|
||||
|
||||
fileMeta := &drive.File{
|
||||
Name: filename,
|
||||
Name: fileName,
|
||||
Parents: []string{folderID},
|
||||
}
|
||||
|
||||
@@ -91,7 +89,7 @@ func (s *GoogleDriveStorage) SaveFile(
|
||||
logger.Info(
|
||||
"file uploaded to Google Drive",
|
||||
"name",
|
||||
filename,
|
||||
fileName,
|
||||
"folder",
|
||||
"databasus_backups",
|
||||
)
|
||||
@@ -152,7 +150,7 @@ func (r *backpressureReader) Read(p []byte) (n int, err error) {
|
||||
|
||||
func (s *GoogleDriveStorage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
) (io.ReadCloser, error) {
|
||||
var result io.ReadCloser
|
||||
err := s.withRetryOnAuth(
|
||||
@@ -164,7 +162,7 @@ func (s *GoogleDriveStorage) GetFile(
|
||||
return fmt.Errorf("failed to find backups folder: %w", err)
|
||||
}
|
||||
|
||||
fileIDGoogle, err := s.lookupFileID(driveService, fileID.String(), folderID)
|
||||
fileIDGoogle, err := s.lookupFileID(driveService, fileName, folderID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -184,7 +182,7 @@ func (s *GoogleDriveStorage) GetFile(
|
||||
|
||||
func (s *GoogleDriveStorage) DeleteFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), gdDeleteTimeout)
|
||||
defer cancel()
|
||||
@@ -195,7 +193,7 @@ func (s *GoogleDriveStorage) DeleteFile(
|
||||
return fmt.Errorf("failed to find backups folder: %w", err)
|
||||
}
|
||||
|
||||
return s.deleteByName(ctx, driveService, fileID.String(), folderID)
|
||||
return s.deleteByName(ctx, driveService, fileName, folderID)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ func (l *LocalStorage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
file io.Reader,
|
||||
) error {
|
||||
select {
|
||||
@@ -45,24 +45,25 @@ func (l *LocalStorage) SaveFile(
|
||||
default:
|
||||
}
|
||||
|
||||
logger.Info("Starting to save file to local storage", "fileId", fileID.String())
|
||||
logger.Info("Starting to save file to local storage", "fileName", fileName)
|
||||
|
||||
tempFilePath := filepath.Join(config.GetEnv().TempFolder, fileName)
|
||||
|
||||
err := files_utils.EnsureDirectories([]string{
|
||||
config.GetEnv().TempFolder,
|
||||
filepath.Dir(tempFilePath),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to ensure directories: %w", err)
|
||||
}
|
||||
|
||||
tempFilePath := filepath.Join(config.GetEnv().TempFolder, fileID.String())
|
||||
logger.Debug("Creating temp file", "fileId", fileID.String(), "tempPath", tempFilePath)
|
||||
logger.Debug("Creating temp file", "fileName", fileName, "tempPath", tempFilePath)
|
||||
|
||||
tempFile, err := os.Create(tempFilePath)
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
"Failed to create temp file",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"tempPath",
|
||||
tempFilePath,
|
||||
"error",
|
||||
@@ -74,39 +75,43 @@ func (l *LocalStorage) SaveFile(
|
||||
_ = tempFile.Close()
|
||||
}()
|
||||
|
||||
logger.Debug("Copying file data to temp file", "fileId", fileID.String())
|
||||
logger.Debug("Copying file data to temp file", "fileName", fileName)
|
||||
_, err = copyWithContext(ctx, tempFile, file)
|
||||
if err != nil {
|
||||
logger.Error("Failed to write to temp file", "fileId", fileID.String(), "error", err)
|
||||
logger.Error("Failed to write to temp file", "fileName", fileName, "error", err)
|
||||
return fmt.Errorf("failed to write to temp file: %w", err)
|
||||
}
|
||||
|
||||
if err = tempFile.Sync(); err != nil {
|
||||
logger.Error("Failed to sync temp file", "fileId", fileID.String(), "error", err)
|
||||
logger.Error("Failed to sync temp file", "fileName", fileName, "error", err)
|
||||
return fmt.Errorf("failed to sync temp file: %w", err)
|
||||
}
|
||||
|
||||
// Close the temp file explicitly before moving it (required on Windows)
|
||||
if err = tempFile.Close(); err != nil {
|
||||
logger.Error("Failed to close temp file", "fileId", fileID.String(), "error", err)
|
||||
logger.Error("Failed to close temp file", "fileName", fileName, "error", err)
|
||||
return fmt.Errorf("failed to close temp file: %w", err)
|
||||
}
|
||||
|
||||
finalPath := filepath.Join(config.GetEnv().DataFolder, fileID.String())
|
||||
finalPath := filepath.Join(config.GetEnv().DataFolder, fileName)
|
||||
logger.Debug(
|
||||
"Moving file from temp to final location",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"finalPath",
|
||||
finalPath,
|
||||
)
|
||||
|
||||
if err = files_utils.EnsureDirectories([]string{filepath.Dir(finalPath)}); err != nil {
|
||||
return fmt.Errorf("failed to ensure final directory: %w", err)
|
||||
}
|
||||
|
||||
// Move the file from temp to backups directory
|
||||
if err = os.Rename(tempFilePath, finalPath); err != nil {
|
||||
logger.Error(
|
||||
"Failed to move file from temp to backups",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"tempPath",
|
||||
tempFilePath,
|
||||
"finalPath",
|
||||
@@ -119,8 +124,8 @@ func (l *LocalStorage) SaveFile(
|
||||
|
||||
logger.Info(
|
||||
"Successfully saved file to local storage",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"finalPath",
|
||||
finalPath,
|
||||
)
|
||||
@@ -130,12 +135,12 @@ func (l *LocalStorage) SaveFile(
|
||||
|
||||
func (l *LocalStorage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
) (io.ReadCloser, error) {
|
||||
filePath := filepath.Join(config.GetEnv().DataFolder, fileID.String())
|
||||
filePath := filepath.Join(config.GetEnv().DataFolder, fileName)
|
||||
|
||||
if _, err := os.Stat(filePath); os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("file not found: %s", fileID.String())
|
||||
return nil, fmt.Errorf("file not found: %s", fileName)
|
||||
}
|
||||
|
||||
file, err := os.Open(filePath)
|
||||
@@ -146,8 +151,8 @@ func (l *LocalStorage) GetFile(
|
||||
return file, nil
|
||||
}
|
||||
|
||||
func (l *LocalStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
filePath := filepath.Join(config.GetEnv().DataFolder, fileID.String())
|
||||
func (l *LocalStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {
|
||||
filePath := filepath.Join(config.GetEnv().DataFolder, fileName)
|
||||
|
||||
if _, err := os.Stat(filePath); os.IsNotExist(err) {
|
||||
return nil
|
||||
|
||||
@@ -46,7 +46,7 @@ func (n *NASStorage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
file io.Reader,
|
||||
) error {
|
||||
select {
|
||||
@@ -55,19 +55,19 @@ func (n *NASStorage) SaveFile(
|
||||
default:
|
||||
}
|
||||
|
||||
logger.Info("Starting to save file to NAS storage", "fileId", fileID.String(), "host", n.Host)
|
||||
logger.Info("Starting to save file to NAS storage", "fileName", fileName, "host", n.Host)
|
||||
|
||||
session, err := n.createSessionWithContext(ctx, encryptor)
|
||||
if err != nil {
|
||||
logger.Error("Failed to create NAS session", "fileId", fileID.String(), "error", err)
|
||||
logger.Error("Failed to create NAS session", "fileName", fileName, "error", err)
|
||||
return fmt.Errorf("failed to create NAS session: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if logoffErr := session.Logoff(); logoffErr != nil {
|
||||
logger.Error(
|
||||
"Failed to logoff NAS session",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"error",
|
||||
logoffErr,
|
||||
)
|
||||
@@ -78,8 +78,8 @@ func (n *NASStorage) SaveFile(
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
"Failed to mount NAS share",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"share",
|
||||
n.Share,
|
||||
"error",
|
||||
@@ -91,8 +91,8 @@ func (n *NASStorage) SaveFile(
|
||||
if umountErr := fs.Umount(); umountErr != nil {
|
||||
logger.Error(
|
||||
"Failed to unmount NAS share",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"error",
|
||||
umountErr,
|
||||
)
|
||||
@@ -104,8 +104,8 @@ func (n *NASStorage) SaveFile(
|
||||
if err := n.ensureDirectory(fs, n.Path); err != nil {
|
||||
logger.Error(
|
||||
"Failed to ensure directory",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"path",
|
||||
n.Path,
|
||||
"error",
|
||||
@@ -115,15 +115,15 @@ func (n *NASStorage) SaveFile(
|
||||
}
|
||||
}
|
||||
|
||||
filePath := n.getFilePath(fileID.String())
|
||||
logger.Debug("Creating file on NAS", "fileId", fileID.String(), "filePath", filePath)
|
||||
filePath := n.getFilePath(fileName)
|
||||
logger.Debug("Creating file on NAS", "fileName", fileName, "filePath", filePath)
|
||||
|
||||
nasFile, err := fs.Create(filePath)
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
"Failed to create file on NAS",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"filePath",
|
||||
filePath,
|
||||
"error",
|
||||
@@ -133,21 +133,21 @@ func (n *NASStorage) SaveFile(
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := nasFile.Close(); closeErr != nil {
|
||||
logger.Error("Failed to close NAS file", "fileId", fileID.String(), "error", closeErr)
|
||||
logger.Error("Failed to close NAS file", "fileName", fileName, "error", closeErr)
|
||||
}
|
||||
}()
|
||||
|
||||
logger.Debug("Copying file data to NAS", "fileId", fileID.String())
|
||||
logger.Debug("Copying file data to NAS", "fileName", fileName)
|
||||
_, err = copyWithContext(ctx, nasFile, file)
|
||||
if err != nil {
|
||||
logger.Error("Failed to write file to NAS", "fileId", fileID.String(), "error", err)
|
||||
logger.Error("Failed to write file to NAS", "fileName", fileName, "error", err)
|
||||
return fmt.Errorf("failed to write file to NAS: %w", err)
|
||||
}
|
||||
|
||||
logger.Info(
|
||||
"Successfully saved file to NAS storage",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"filePath",
|
||||
filePath,
|
||||
)
|
||||
@@ -156,7 +156,7 @@ func (n *NASStorage) SaveFile(
|
||||
|
||||
func (n *NASStorage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
) (io.ReadCloser, error) {
|
||||
session, err := n.createSession(encryptor)
|
||||
if err != nil {
|
||||
@@ -169,14 +169,14 @@ func (n *NASStorage) GetFile(
|
||||
return nil, fmt.Errorf("failed to mount share '%s': %w", n.Share, err)
|
||||
}
|
||||
|
||||
filePath := n.getFilePath(fileID.String())
|
||||
filePath := n.getFilePath(fileName)
|
||||
|
||||
// Check if file exists
|
||||
_, err = fs.Stat(filePath)
|
||||
if err != nil {
|
||||
_ = fs.Umount()
|
||||
_ = session.Logoff()
|
||||
return nil, fmt.Errorf("file not found: %s", fileID.String())
|
||||
return nil, fmt.Errorf("file not found: %s", fileName)
|
||||
}
|
||||
|
||||
nasFile, err := fs.Open(filePath)
|
||||
@@ -194,7 +194,7 @@ func (n *NASStorage) GetFile(
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (n *NASStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
func (n *NASStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), nasDeleteTimeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -214,7 +214,7 @@ func (n *NASStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid
|
||||
_ = fs.Umount()
|
||||
}()
|
||||
|
||||
filePath := n.getFilePath(fileID.String())
|
||||
filePath := n.getFilePath(fileName)
|
||||
|
||||
_, err = fs.Stat(filePath)
|
||||
if err != nil {
|
||||
|
||||
@@ -41,7 +41,7 @@ func (r *RcloneStorage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
file io.Reader,
|
||||
) error {
|
||||
select {
|
||||
@@ -50,28 +50,28 @@ func (r *RcloneStorage) SaveFile(
|
||||
default:
|
||||
}
|
||||
|
||||
logger.Info("Starting to save file to rclone storage", "fileId", fileID.String())
|
||||
logger.Info("Starting to save file to rclone storage", "fileName", fileName)
|
||||
|
||||
remoteFs, err := r.getFs(ctx, encryptor)
|
||||
if err != nil {
|
||||
logger.Error("Failed to create rclone filesystem", "fileId", fileID.String(), "error", err)
|
||||
logger.Error("Failed to create rclone filesystem", "fileName", fileName, "error", err)
|
||||
return fmt.Errorf("failed to create rclone filesystem: %w", err)
|
||||
}
|
||||
|
||||
filePath := r.getFilePath(fileID.String())
|
||||
logger.Debug("Uploading file via rclone", "fileId", fileID.String(), "filePath", filePath)
|
||||
filePath := r.getFilePath(fileName)
|
||||
logger.Debug("Uploading file via rclone", "fileName", fileName, "filePath", filePath)
|
||||
|
||||
_, err = operations.Rcat(ctx, remoteFs, filePath, io.NopCloser(file), time.Now().UTC(), nil)
|
||||
if err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.Info("Rclone upload cancelled", "fileId", fileID.String())
|
||||
logger.Info("Rclone upload cancelled", "fileName", fileName)
|
||||
return ctx.Err()
|
||||
default:
|
||||
logger.Error(
|
||||
"Failed to upload file via rclone",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"error",
|
||||
err,
|
||||
)
|
||||
@@ -81,8 +81,8 @@ func (r *RcloneStorage) SaveFile(
|
||||
|
||||
logger.Info(
|
||||
"Successfully saved file to rclone storage",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"filePath",
|
||||
filePath,
|
||||
)
|
||||
@@ -91,7 +91,7 @@ func (r *RcloneStorage) SaveFile(
|
||||
|
||||
func (r *RcloneStorage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
) (io.ReadCloser, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -100,7 +100,7 @@ func (r *RcloneStorage) GetFile(
|
||||
return nil, fmt.Errorf("failed to create rclone filesystem: %w", err)
|
||||
}
|
||||
|
||||
filePath := r.getFilePath(fileID.String())
|
||||
filePath := r.getFilePath(fileName)
|
||||
|
||||
obj, err := remoteFs.NewObject(ctx, filePath)
|
||||
if err != nil {
|
||||
@@ -115,7 +115,7 @@ func (r *RcloneStorage) GetFile(
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func (r *RcloneStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
func (r *RcloneStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), rcloneDeleteTimeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -124,7 +124,7 @@ func (r *RcloneStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID u
|
||||
return fmt.Errorf("failed to create rclone filesystem: %w", err)
|
||||
}
|
||||
|
||||
filePath := r.getFilePath(fileID.String())
|
||||
filePath := r.getFilePath(fileName)
|
||||
|
||||
obj, err := remoteFs.NewObject(ctx, filePath)
|
||||
if err != nil {
|
||||
|
||||
@@ -55,7 +55,7 @@ func (s *S3Storage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
file io.Reader,
|
||||
) error {
|
||||
select {
|
||||
@@ -69,7 +69,7 @@ func (s *S3Storage) SaveFile(
|
||||
return err
|
||||
}
|
||||
|
||||
objectKey := s.buildObjectKey(fileID.String())
|
||||
objectKey := s.buildObjectKey(fileName)
|
||||
|
||||
uploadID, err := coreClient.NewMultipartUpload(
|
||||
ctx,
|
||||
@@ -184,14 +184,14 @@ func (s *S3Storage) SaveFile(
|
||||
|
||||
func (s *S3Storage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
) (io.ReadCloser, error) {
|
||||
client, err := s.getClient(encryptor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objectKey := s.buildObjectKey(fileID.String())
|
||||
objectKey := s.buildObjectKey(fileName)
|
||||
|
||||
object, err := client.GetObject(
|
||||
context.TODO(),
|
||||
@@ -221,13 +221,13 @@ func (s *S3Storage) GetFile(
|
||||
return object, nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
func (s *S3Storage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {
|
||||
client, err := s.getClient(encryptor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
objectKey := s.buildObjectKey(fileID.String())
|
||||
objectKey := s.buildObjectKey(fileName)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), s3DeleteTimeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -41,7 +41,7 @@ func (s *SFTPStorage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
file io.Reader,
|
||||
) error {
|
||||
select {
|
||||
@@ -50,19 +50,19 @@ func (s *SFTPStorage) SaveFile(
|
||||
default:
|
||||
}
|
||||
|
||||
logger.Info("Starting to save file to SFTP storage", "fileId", fileID.String(), "host", s.Host)
|
||||
logger.Info("Starting to save file to SFTP storage", "fileName", fileName, "host", s.Host)
|
||||
|
||||
client, sshConn, err := s.connect(encryptor, sftpConnectTimeout)
|
||||
if err != nil {
|
||||
logger.Error("Failed to connect to SFTP", "fileId", fileID.String(), "error", err)
|
||||
logger.Error("Failed to connect to SFTP", "fileName", fileName, "error", err)
|
||||
return fmt.Errorf("failed to connect to SFTP: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := client.Close(); closeErr != nil {
|
||||
logger.Error(
|
||||
"Failed to close SFTP client",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"error",
|
||||
closeErr,
|
||||
)
|
||||
@@ -70,8 +70,8 @@ func (s *SFTPStorage) SaveFile(
|
||||
if closeErr := sshConn.Close(); closeErr != nil {
|
||||
logger.Error(
|
||||
"Failed to close SSH connection",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"error",
|
||||
closeErr,
|
||||
)
|
||||
@@ -82,8 +82,8 @@ func (s *SFTPStorage) SaveFile(
|
||||
if err := s.ensureDirectory(client, s.Path); err != nil {
|
||||
logger.Error(
|
||||
"Failed to ensure directory",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"path",
|
||||
s.Path,
|
||||
"error",
|
||||
@@ -93,12 +93,12 @@ func (s *SFTPStorage) SaveFile(
|
||||
}
|
||||
}
|
||||
|
||||
filePath := s.getFilePath(fileID.String())
|
||||
logger.Debug("Uploading file to SFTP", "fileId", fileID.String(), "filePath", filePath)
|
||||
filePath := s.getFilePath(fileName)
|
||||
logger.Debug("Uploading file to SFTP", "fileName", fileName, "filePath", filePath)
|
||||
|
||||
remoteFile, err := client.Create(filePath)
|
||||
if err != nil {
|
||||
logger.Error("Failed to create remote file", "fileId", fileID.String(), "error", err)
|
||||
logger.Error("Failed to create remote file", "fileName", fileName, "error", err)
|
||||
return fmt.Errorf("failed to create remote file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
@@ -111,18 +111,18 @@ func (s *SFTPStorage) SaveFile(
|
||||
if err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
logger.Info("SFTP upload cancelled", "fileId", fileID.String())
|
||||
logger.Info("SFTP upload cancelled", "fileName", fileName)
|
||||
return ctx.Err()
|
||||
default:
|
||||
logger.Error("Failed to upload file to SFTP", "fileId", fileID.String(), "error", err)
|
||||
logger.Error("Failed to upload file to SFTP", "fileName", fileName, "error", err)
|
||||
return fmt.Errorf("failed to upload file to SFTP: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info(
|
||||
"Successfully saved file to SFTP storage",
|
||||
"fileId",
|
||||
fileID.String(),
|
||||
"fileName",
|
||||
fileName,
|
||||
"filePath",
|
||||
filePath,
|
||||
)
|
||||
@@ -131,14 +131,14 @@ func (s *SFTPStorage) SaveFile(
|
||||
|
||||
func (s *SFTPStorage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
fileName string,
|
||||
) (io.ReadCloser, error) {
|
||||
client, sshConn, err := s.connect(encryptor, sftpConnectTimeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to SFTP: %w", err)
|
||||
}
|
||||
|
||||
filePath := s.getFilePath(fileID.String())
|
||||
filePath := s.getFilePath(fileName)
|
||||
|
||||
remoteFile, err := client.Open(filePath)
|
||||
if err != nil {
|
||||
@@ -154,7 +154,7 @@ func (s *SFTPStorage) GetFile(
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *SFTPStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
func (s *SFTPStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileName string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), sftpDeleteTimeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -167,7 +167,7 @@ func (s *SFTPStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uui
|
||||
_ = sshConn.Close()
|
||||
}()
|
||||
|
||||
filePath := s.getFilePath(fileID.String())
|
||||
filePath := s.getFilePath(fileName)
|
||||
|
||||
_, err = client.Stat(filePath)
|
||||
if err != nil {
|
||||
|
||||
@@ -92,6 +92,8 @@ func (s *StorageService) SaveStorage(
|
||||
|
||||
existingStorage.Update(storage)
|
||||
|
||||
oldName := existingStorage.Name
|
||||
|
||||
if err := existingStorage.EncryptSensitiveData(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -105,11 +107,19 @@ func (s *StorageService) SaveStorage(
|
||||
return err
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Storage updated: %s", existingStorage.Name),
|
||||
&user.ID,
|
||||
&workspaceID,
|
||||
)
|
||||
if oldName != existingStorage.Name {
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Storage renamed from '%s' to '%s'", oldName, existingStorage.Name),
|
||||
&user.ID,
|
||||
&workspaceID,
|
||||
)
|
||||
} else {
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Storage updated: %s", existingStorage.Name),
|
||||
&user.ID,
|
||||
&workspaceID,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
storage.WorkspaceID = workspaceID
|
||||
|
||||
@@ -368,9 +378,26 @@ func (s *StorageService) TransferStorageToWorkspace(
|
||||
return err
|
||||
}
|
||||
|
||||
sourceWorkspace, err := s.workspaceService.GetWorkspaceByID(sourceWorkspaceID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get source workspace: %w", err)
|
||||
}
|
||||
|
||||
targetWorkspace, err := s.workspaceService.GetWorkspaceByID(targetWorkspaceID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get target workspace: %w", err)
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Storage transferred: %s from workspace %s to workspace %s",
|
||||
existingStorage.Name, sourceWorkspaceID, targetWorkspaceID),
|
||||
fmt.Sprintf("Storage transferred out: %s to workspace '%s'",
|
||||
existingStorage.Name, targetWorkspace.Name),
|
||||
&user.ID,
|
||||
&sourceWorkspaceID,
|
||||
)
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf("Storage transferred in: %s from workspace '%s'",
|
||||
existingStorage.Name, sourceWorkspace.Name),
|
||||
&user.ID,
|
||||
&targetWorkspaceID,
|
||||
)
|
||||
|
||||
30
backend/internal/features/system/version/controller.go
Normal file
30
backend/internal/features/system/version/controller.go
Normal file
@@ -0,0 +1,30 @@
|
||||
package system_version
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type VersionController struct{}
|
||||
|
||||
func (c *VersionController) RegisterRoutes(router *gin.RouterGroup) {
|
||||
router.GET("/system/version", c.GetVersion)
|
||||
}
|
||||
|
||||
// GetVersion
|
||||
// @Summary Get application version
|
||||
// @Description Returns the current application version
|
||||
// @Tags system/version
|
||||
// @Produce json
|
||||
// @Success 200 {object} VersionResponse
|
||||
// @Router /system/version [get]
|
||||
func (c *VersionController) GetVersion(ctx *gin.Context) {
|
||||
version := os.Getenv("APP_VERSION")
|
||||
if version == "" {
|
||||
version = "dev"
|
||||
}
|
||||
|
||||
ctx.JSON(http.StatusOK, VersionResponse{Version: version})
|
||||
}
|
||||
7
backend/internal/features/system/version/di.go
Normal file
7
backend/internal/features/system/version/di.go
Normal file
@@ -0,0 +1,7 @@
|
||||
package system_version
|
||||
|
||||
var versionController = &VersionController{}
|
||||
|
||||
func GetVersionController() *VersionController {
|
||||
return versionController
|
||||
}
|
||||
5
backend/internal/features/system/version/dto.go
Normal file
5
backend/internal/features/system/version/dto.go
Normal file
@@ -0,0 +1,5 @@
|
||||
package system_version
|
||||
|
||||
type VersionResponse struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
@@ -8,8 +8,8 @@ import (
|
||||
"time"
|
||||
|
||||
"databasus-backend/internal/features/audit_logs"
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
"databasus-backend/internal/features/backups/backups/backuping"
|
||||
backups_services "databasus-backend/internal/features/backups/backups/services"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
healthcheck_config "databasus-backend/internal/features/healthcheck/config"
|
||||
@@ -26,8 +26,8 @@ func Test_SetupDependencies_CalledTwice_LogsWarning(t *testing.T) {
|
||||
audit_logs.SetupDependencies()
|
||||
audit_logs.SetupDependencies()
|
||||
|
||||
backups.SetupDependencies()
|
||||
backups.SetupDependencies()
|
||||
backups_services.SetupDependencies()
|
||||
backups_services.SetupDependencies()
|
||||
|
||||
backups_config.SetupDependencies()
|
||||
backups_config.SetupDependencies()
|
||||
|
||||
@@ -147,6 +147,26 @@ func Test_BackupAndRestoreMariadb_WithReadOnlyUser_RestoreIsSuccessful(t *testin
|
||||
}
|
||||
}
|
||||
|
||||
func Test_BackupAndRestoreMariadb_WithExcludeEvents_EventsNotRestored(t *testing.T) {
|
||||
env := config.GetEnv()
|
||||
cases := []struct {
|
||||
name string
|
||||
version tools.MariadbVersion
|
||||
port string
|
||||
}{
|
||||
{"MariaDB 10.5", tools.MariadbVersion105, env.TestMariadb105Port},
|
||||
{"MariaDB 10.11", tools.MariadbVersion1011, env.TestMariadb1011Port},
|
||||
{"MariaDB 11.4", tools.MariadbVersion114, env.TestMariadb114Port},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
testMariadbBackupRestoreWithExcludeEventsForVersion(t, tc.version, tc.port)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testMariadbBackupRestoreForVersion(
|
||||
t *testing.T,
|
||||
mariadbVersion tools.MariadbVersion,
|
||||
@@ -702,3 +722,145 @@ func updateMariadbDatabaseCredentialsViaAPI(
|
||||
|
||||
return &updatedDatabase
|
||||
}
|
||||
|
||||
func testMariadbBackupRestoreWithExcludeEventsForVersion(
|
||||
t *testing.T,
|
||||
mariadbVersion tools.MariadbVersion,
|
||||
port string,
|
||||
) {
|
||||
container, err := connectToMariadbContainer(mariadbVersion, port)
|
||||
if err != nil {
|
||||
t.Skipf("Skipping MariaDB %s test: %v", mariadbVersion, err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if container.DB != nil {
|
||||
container.DB.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
setupMariadbTestData(t, container.DB)
|
||||
|
||||
_, err = container.DB.Exec(`
|
||||
CREATE EVENT IF NOT EXISTS test_event
|
||||
ON SCHEDULE EVERY 1 DAY
|
||||
DO BEGIN
|
||||
INSERT INTO test_data (name, value) VALUES ('event_test', 999);
|
||||
END
|
||||
`)
|
||||
if err != nil {
|
||||
t.Skipf(
|
||||
"Skipping test: MariaDB version doesn't support events or event scheduler disabled: %v",
|
||||
err,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
router := createTestRouter()
|
||||
user := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace(
|
||||
"MariaDB Exclude Events Test Workspace",
|
||||
user,
|
||||
router,
|
||||
)
|
||||
|
||||
storage := storages.CreateTestStorage(workspace.ID)
|
||||
|
||||
database := createMariadbDatabaseViaAPI(
|
||||
t, router, "MariaDB Exclude Events Test Database", workspace.ID,
|
||||
container.Host, container.Port,
|
||||
container.Username, container.Password, container.Database,
|
||||
container.Version,
|
||||
user.Token,
|
||||
)
|
||||
|
||||
database.Mariadb.IsExcludeEvents = true
|
||||
w := workspaces_testing.MakeAPIRequest(
|
||||
router,
|
||||
"POST",
|
||||
"/api/v1/databases/update",
|
||||
"Bearer "+user.Token,
|
||||
database,
|
||||
)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf(
|
||||
"Failed to update database with IsExcludeEvents. Status: %d, Body: %s",
|
||||
w.Code,
|
||||
w.Body.String(),
|
||||
)
|
||||
}
|
||||
|
||||
enableBackupsViaAPI(
|
||||
t, router, database.ID, storage.ID,
|
||||
backups_config.BackupEncryptionNone, user.Token,
|
||||
)
|
||||
|
||||
createBackupViaAPI(t, router, database.ID, user.Token)
|
||||
|
||||
backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute)
|
||||
assert.Equal(t, backups_core.BackupStatusCompleted, backup.Status)
|
||||
|
||||
newDBName := "restoreddb_mariadb_no_events"
|
||||
_, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName))
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName))
|
||||
assert.NoError(t, err)
|
||||
|
||||
newDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true",
|
||||
container.Username, container.Password, container.Host, container.Port, newDBName)
|
||||
newDB, err := sqlx.Connect("mysql", newDSN)
|
||||
assert.NoError(t, err)
|
||||
defer newDB.Close()
|
||||
|
||||
createMariadbRestoreViaAPI(
|
||||
t, router, backup.ID,
|
||||
container.Host, container.Port,
|
||||
container.Username, container.Password, newDBName,
|
||||
container.Version,
|
||||
user.Token,
|
||||
)
|
||||
|
||||
restore := waitForMariadbRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute)
|
||||
assert.Equal(t, restores_core.RestoreStatusCompleted, restore.Status)
|
||||
|
||||
var tableExists int
|
||||
err = newDB.Get(
|
||||
&tableExists,
|
||||
"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = ? AND table_name = 'test_data'",
|
||||
newDBName,
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, tableExists, "Table 'test_data' should exist in restored database")
|
||||
|
||||
verifyMariadbDataIntegrity(t, container.DB, newDB)
|
||||
|
||||
var eventCount int
|
||||
err = newDB.Get(
|
||||
&eventCount,
|
||||
"SELECT COUNT(*) FROM information_schema.events WHERE event_schema = ? AND event_name = 'test_event'",
|
||||
newDBName,
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(
|
||||
t,
|
||||
0,
|
||||
eventCount,
|
||||
"Event 'test_event' should NOT exist in restored database when IsExcludeEvents is true",
|
||||
)
|
||||
|
||||
err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))
|
||||
if err != nil {
|
||||
t.Logf("Warning: Failed to delete backup file: %v", err)
|
||||
}
|
||||
|
||||
test_utils.MakeDeleteRequest(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/databases/"+database.ID.String(),
|
||||
"Bearer "+user.Token,
|
||||
http.StatusNoContent,
|
||||
)
|
||||
storages.RemoveTestStorage(storage.ID)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user