Compare commits

...

23 Commits

Author SHA1 Message Date
Rostislav Dugin
1b011bdcd4 Merge pull request #432 from databasus/develop
Develop
2026-03-13 18:51:50 +03:00
Rostislav Dugin
7e209ff537 REFACTOR (linters): Apply linters fixes 2026-03-13 18:50:57 +03:00
Rostislav Dugin
f712e3a437 FEATURE (linters): Introduce more strict linters 2026-03-13 18:03:38 +03:00
Rostislav Dugin
bcd7d8e1aa REFACTOR (formatters): Apply formatters auto fixes 2026-03-13 17:53:00 +03:00
Rostislav Dugin
880a7488e9 FEATURE (formatters): Add gofumpt and gci formatters 2026-03-13 17:50:07 +03:00
Rostislav Dugin
ca4d483f2c REFACTOR (golines): Apply golines fixes 2026-03-13 17:47:46 +03:00
Rostislav Dugin
1b511410a6 FEATURE (formatters): Fix config of golines 2026-03-13 17:47:29 +03:00
Rostislav Dugin
c8edff8046 FEATURE (golangci-lint): Upgrade golangci-lint to 2.11.3 in CI \ CD 2026-03-13 17:41:14 +03:00
Rostislav Dugin
f60e3d956b FEAUTRE (go): Upgrade Go version to 1.26.1 2026-03-13 17:37:39 +03:00
Rostislav Dugin
f2cb9022f2 FEATURE (agent): Setup agent directory, pre-commit and CI\CD workflow 2026-03-13 17:23:00 +03:00
Rostislav Dugin
4b3f36eea2 Merge pull request #429 from databasus/develop
FIX (readme): Add info about Anthropic and Open AI support via OSS pr…
2026-03-12 09:38:01 +03:00
Rostislav Dugin
460063e7a5 FIX (readme): Add info about Anthropic and Open AI support via OSS programs 2026-03-12 09:34:42 +03:00
Rostislav Dugin
a0f02b253e Merge pull request #427 from databasus/develop
FIX (retention): Fix GFS retention while hourly backups prevent daily…
2026-03-11 15:36:27 +03:00
Rostislav Dugin
812f11bc2f FIX (retention): Fix GFS retention while hourly backups prevent daily from cleanup 2026-03-11 15:35:53 +03:00
Rostislav Dugin
e796e3ddf0 Merge pull request #426 from databasus/develop
FIX (mysql): Detect supported compression levels
2026-03-11 12:53:35 +03:00
Rostislav Dugin
c96d3db337 FIX (mysql): Detect supported compression levels 2026-03-11 12:52:41 +03:00
Rostislav Dugin
ed6c3a2034 Merge pull request #425 from databasus/develop
Develop
2026-03-11 12:31:19 +03:00
Rostislav Dugin
05115047c3 FEATURE (version): Reload frontend if faced version mismatch with backend 2026-03-11 12:28:07 +03:00
Rostislav Dugin
446b96c6c0 FEATURE (arch): Add architecture to Databasus version in the bottom left of UI 2026-03-11 11:39:53 +03:00
Rostislav Dugin
36a0448da1 Merge pull request #420 from databasus/develop
FEATURE (email): Add skipping TLS for email notifier
2026-03-08 22:53:45 +03:00
Rostislav Dugin
8e392cfeab FEATURE (email): Add skipping TLS for email notifier 2026-03-08 22:48:28 +03:00
Rostislav Dugin
6683db1e52 Merge pull request #419 from databasus/develop
FIX (issues): Add DB version to issues template
2026-03-08 22:22:52 +03:00
Rostislav Dugin
703b883936 FIX (issues): Add DB version to issues template 2026-03-08 22:22:26 +03:00
219 changed files with 3581 additions and 1499 deletions

View File

@@ -4,7 +4,9 @@ about: Report a bug or unexpected behavior in Databasus
labels: bug
---
## Databasus version
## Databasus version (screenshot)
It is displayed in the bottom left corner of the Databasus UI. Please attach screenshot, not just version text
<!-- e.g. 1.4.2 -->
@@ -12,6 +14,10 @@ labels: bug
<!-- e.g. Ubuntu 22.04 x64, macOS 14 ARM, Windows 11 x64 -->
## Database type and version (optional, for DB-related bugs)
<!-- e.g. PostgreSQL 16 in Docker, MySQL 8.0 installed on server, MariaDB 11.4 in AWS Cloud -->
## Describe the bug (please write manually, do not ask AI to summarize)
**What happened:**

View File

@@ -11,7 +11,7 @@ jobs:
lint-backend:
runs-on: self-hosted
container:
image: golang:1.24.9
image: golang:1.26.1
volumes:
- /runner-cache/go-pkg:/go/pkg/mod
- /runner-cache/go-build:/root/.cache/go-build
@@ -32,7 +32,7 @@ jobs:
- name: Install golangci-lint
run: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.7.2
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
- name: Install swag for swagger generation
@@ -86,6 +86,39 @@ jobs:
cd frontend
npm run build
lint-agent:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.26.1"
cache-dependency-path: agent/go.sum
- name: Download Go modules
run: |
cd agent
go mod download
- name: Install golangci-lint
run: |
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
- name: Run golangci-lint
run: |
cd agent
golangci-lint run
- name: Verify go mod tidy
run: |
cd agent
go mod tidy
git diff --exit-code go.mod go.sum || (echo "go mod tidy made changes, please run 'go mod tidy' and commit the changes" && exit 1)
test-frontend:
runs-on: ubuntu-latest
needs: [lint-frontend]
@@ -108,11 +141,34 @@ jobs:
cd frontend
npm run test
test-agent:
runs-on: ubuntu-latest
needs: [lint-agent]
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.26.1"
cache-dependency-path: agent/go.sum
- name: Download Go modules
run: |
cd agent
go mod download
- name: Run Go tests
run: |
cd agent
go test -count=1 -failfast ./internal/...
test-backend:
runs-on: self-hosted
needs: [lint-backend]
container:
image: golang:1.24.9
image: golang:1.26.1
options: --privileged -v /var/run/docker.sock:/var/run/docker.sock --add-host=host.docker.internal:host-gateway
volumes:
- /runner-cache/go-pkg:/go/pkg/mod
@@ -441,7 +497,7 @@ jobs:
runs-on: self-hosted
container:
image: node:20
needs: [test-backend, test-frontend]
needs: [test-backend, test-frontend, test-agent]
if: ${{ github.ref == 'refs/heads/main' && !contains(github.event.head_commit.message, '[skip-release]') }}
outputs:
should_release: ${{ steps.version_bump.outputs.should_release }}
@@ -534,7 +590,7 @@ jobs:
build-only:
runs-on: self-hosted
needs: [test-backend, test-frontend]
needs: [test-backend, test-frontend, test-agent]
if: ${{ github.ref == 'refs/heads/main' && contains(github.event.head_commit.message, '[skip-release]') }}
steps:
- name: Clean workspace

View File

@@ -41,3 +41,20 @@ repos:
language: system
files: ^backend/.*\.go$
pass_filenames: false
# Agent checks
- repo: local
hooks:
- id: agent-format-and-lint
name: Agent Format & Lint (golangci-lint)
entry: bash -c "cd agent && golangci-lint fmt ./internal/... ./cmd/... && golangci-lint run ./internal/... ./cmd/..."
language: system
files: ^agent/.*\.go$
pass_filenames: false
- id: agent-go-mod-tidy
name: Agent Go Mod Tidy
entry: bash -c "cd agent && go mod tidy"
language: system
files: ^agent/.*\.go$
pass_filenames: false

View File

@@ -22,7 +22,7 @@ RUN npm run build
# ========= BUILD BACKEND =========
# Backend build stage
FROM --platform=$BUILDPLATFORM golang:1.24.9 AS backend-build
FROM --platform=$BUILDPLATFORM golang:1.26.1 AS backend-build
# Make TARGET args available early so tools built here match the final image arch
ARG TARGETOS
@@ -66,13 +66,52 @@ RUN CGO_ENABLED=0 \
go build -o /app/main ./cmd/main.go
# ========= BUILD AGENT =========
# Builds the databasus-agent CLI binary for BOTH x86_64 and ARM64.
# Both architectures are always built because:
# - Databasus server runs on one arch (e.g. amd64)
# - The agent runs on remote PostgreSQL servers that may be on a
# different arch (e.g. arm64)
# - The backend serves the correct binary based on the agent's
# ?arch= query parameter
#
# We cross-compile from the build platform (no QEMU needed) because the
# agent is pure Go with zero C dependencies.
# CGO_ENABLED=0 produces fully static binaries — no glibc/musl dependency,
# so the agent runs on any Linux distro (Alpine, Debian, Ubuntu, RHEL, etc.).
# APP_VERSION is baked into the binary via -ldflags so the agent can
# compare its version against the server and auto-update when needed.
FROM --platform=$BUILDPLATFORM golang:1.26.1 AS agent-build
ARG APP_VERSION=dev
WORKDIR /agent
COPY agent/go.mod ./
RUN go mod download
COPY agent/ ./
# Build for x86_64 (amd64) — static binary, no glibc dependency
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
go build -ldflags "-X main.Version=${APP_VERSION}" \
-o /agent-binaries/databasus-agent-linux-amd64 ./cmd/main.go
# Build for ARM64 (arm64) — static binary, no glibc dependency
RUN CGO_ENABLED=0 GOOS=linux GOARCH=arm64 \
go build -ldflags "-X main.Version=${APP_VERSION}" \
-o /agent-binaries/databasus-agent-linux-arm64 ./cmd/main.go
# ========= RUNTIME =========
FROM debian:bookworm-slim
# Add version metadata to runtime image
ARG APP_VERSION=dev
ARG TARGETARCH
LABEL org.opencontainers.image.version=$APP_VERSION
ENV APP_VERSION=$APP_VERSION
ENV CONTAINER_ARCH=$TARGETARCH
# Set production mode for Docker containers
ENV ENV_MODE=production
@@ -218,6 +257,10 @@ COPY backend/migrations ./migrations
# Copy UI files
COPY --from=backend-build /app/ui/build ./ui/build
# Copy agent binaries (both architectures) — served by the backend
# at GET /api/v1/system/agent?arch=amd64|arm64
COPY --from=agent-build /agent-binaries ./agent-binaries
# Copy .env file (with fallback to .env.production.example)
COPY backend/.env* /app/
RUN if [ ! -f /app/.env ]; then \
@@ -269,7 +312,8 @@ window.__RUNTIME_CONFIG__ = {
GITHUB_CLIENT_ID: '\${GITHUB_CLIENT_ID:-}',
GOOGLE_CLIENT_ID: '\${GOOGLE_CLIENT_ID:-}',
IS_EMAIL_CONFIGURED: '\$IS_EMAIL_CONFIGURED',
CLOUDFLARE_TURNSTILE_SITE_KEY: '\${CLOUDFLARE_TURNSTILE_SITE_KEY:-}'
CLOUDFLARE_TURNSTILE_SITE_KEY: '\${CLOUDFLARE_TURNSTILE_SITE_KEY:-}',
CONTAINER_ARCH: '\${CONTAINER_ARCH:-unknown}'
};
JSEOF
@@ -394,6 +438,8 @@ fi
# Create database and set password for postgres user
echo "Setting up database and user..."
gosu postgres \$PG_BIN/psql -p 5437 -h localhost -d postgres << 'SQL'
# We use stub password, because internal DB is not exposed outside container
ALTER USER postgres WITH PASSWORD 'Q1234567';
SELECT 'CREATE DATABASE databasus OWNER postgres'
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'databasus')

View File

@@ -261,12 +261,17 @@ Also you can join our large community of developers, DBAs and DevOps engineers o
There have been questions about AI usage in project development in issues and discussions. As the project focuses on security, reliability and production usage, it's important to explain how AI is used in the development process.
First of all, we are proud to say that Databasus has been accepted into both [Claude for Open Source](https://claude.com/contact-sales/claude-for-oss) by Anthropic and [Codex for Open Source](https://developers.openai.com/codex/community/codex-for-oss/) by OpenAI in March 2026. For us it is one more signal that the project was recognized as important open-source software and was as critical infrastructure worth supporting independently by two of the world's leading AI companies. Read more at [databasus.com/faq](https://databasus.com/faq#oss-programs).
Despite of this, we have the following rules how AI is used in the development process:
AI is used as a helper for:
- verification of code quality and searching for vulnerabilities
- cleaning up and improving documentation, comments and code
- assistance during development
- double-checking PRs and commits after human review
- additional security analysis of PRs via Codex Security
AI is not used for:

1
agent/.env.example Normal file
View File

@@ -0,0 +1 @@
ENV_MODE=development

23
agent/.gitignore vendored Normal file
View File

@@ -0,0 +1,23 @@
main
.env
docker-compose.yml
pgdata
pgdata_test/
mysqldata/
mariadbdata/
main.exe
swagger/
swagger/*
swagger/docs.go
swagger/swagger.json
swagger/swagger.yaml
postgresus-backend.exe
databasus-backend.exe
ui/build/*
pgdata-for-restore/
temp/
cmd.exe
temp/
valkey-data/
victoria-logs-data/
databasus.json

41
agent/.golangci.yml Normal file
View File

@@ -0,0 +1,41 @@
version: "2"
run:
timeout: 5m
tests: false
concurrency: 4
linters:
default: standard
enable:
- funcorder
- bodyclose
- errorlint
- gocritic
- unconvert
- misspell
- errname
- noctx
- modernize
settings:
errcheck:
check-type-assertions: true
formatters:
enable:
- gofumpt
- golines
- gci
settings:
golines:
max-len: 120
gofumpt:
module-path: databasus-agent
extra-rules: true
gci:
sections:
- standard
- default
- localmodule

12
agent/Makefile Normal file
View File

@@ -0,0 +1,12 @@
# Usage: make run ARGS="start --pg-host localhost"
run:
go run cmd/main.go $(ARGS)
build:
CGO_ENABLED=0 go build -ldflags "-X main.Version=$(VERSION)" -o databasus-agent ./cmd/main.go
test:
go test -count=1 -failfast ./internal/...
lint:
golangci-lint fmt ./cmd/... ./internal/... && golangci-lint run ./cmd/... ./internal/...

174
agent/cmd/main.go Normal file
View File

@@ -0,0 +1,174 @@
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"strings"
"databasus-agent/internal/config"
"databasus-agent/internal/features/start"
"databasus-agent/internal/features/upgrade"
"databasus-agent/internal/logger"
)
var Version = "dev"
func main() {
if len(os.Args) < 2 {
printUsage()
os.Exit(1)
}
switch os.Args[1] {
case "start":
runStart(os.Args[2:])
case "stop":
runStop()
case "status":
runStatus()
case "restore":
runRestore(os.Args[2:])
case "version":
fmt.Println(Version)
default:
fmt.Fprintf(os.Stderr, "unknown command: %s\n", os.Args[1])
printUsage()
os.Exit(1)
}
}
func runStart(args []string) {
fs := flag.NewFlagSet("start", flag.ExitOnError)
isDebug := fs.Bool("debug", false, "Enable debug logging")
isSkipUpdate := fs.Bool("skip-update", false, "Skip auto-update check")
cfg := &config.Config{}
cfg.LoadFromJSONAndArgs(fs, args)
if err := cfg.SaveToJSON(); err != nil {
fmt.Fprintf(os.Stderr, "Failed to save config: %v\n", err)
}
logger.Init(*isDebug)
log := logger.GetLogger()
isDev := checkIsDevelopment()
runUpdateCheck(cfg.DatabasusHost, *isSkipUpdate, isDev, log)
if err := start.Run(cfg, log); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
func runStop() {
logger.Init(false)
logger.GetLogger().Info("stop: stub — not yet implemented")
}
func runStatus() {
logger.Init(false)
logger.GetLogger().Info("status: stub — not yet implemented")
}
func runRestore(args []string) {
fs := flag.NewFlagSet("restore", flag.ExitOnError)
targetDir := fs.String("target-dir", "", "Target pgdata directory")
backupID := fs.String("backup-id", "", "Full backup UUID (optional)")
targetTime := fs.String("target-time", "", "PITR target time in RFC3339 (optional)")
isYes := fs.Bool("yes", false, "Skip confirmation prompt")
isDebug := fs.Bool("debug", false, "Enable debug logging")
isSkipUpdate := fs.Bool("skip-update", false, "Skip auto-update check")
cfg := &config.Config{}
cfg.LoadFromJSONAndArgs(fs, args)
if err := cfg.SaveToJSON(); err != nil {
fmt.Fprintf(os.Stderr, "Failed to save config: %v\n", err)
}
logger.Init(*isDebug)
log := logger.GetLogger()
isDev := checkIsDevelopment()
runUpdateCheck(cfg.DatabasusHost, *isSkipUpdate, isDev, log)
log.Info("restore: stub — not yet implemented",
"targetDir", *targetDir,
"backupId", *backupID,
"targetTime", *targetTime,
"yes", *isYes,
)
}
func printUsage() {
fmt.Fprintln(os.Stderr, "Usage: databasus-agent <command> [flags]")
fmt.Fprintln(os.Stderr, "")
fmt.Fprintln(os.Stderr, "Commands:")
fmt.Fprintln(os.Stderr, " start Start the agent (WAL archiving + basebackups)")
fmt.Fprintln(os.Stderr, " stop Stop a running agent")
fmt.Fprintln(os.Stderr, " status Show agent status")
fmt.Fprintln(os.Stderr, " restore Restore a database from backup")
fmt.Fprintln(os.Stderr, " version Print agent version")
}
func runUpdateCheck(host string, isSkipUpdate, isDev bool, log interface {
Info(string, ...any)
Warn(string, ...any)
Error(string, ...any)
},
) {
if isSkipUpdate {
return
}
if host == "" {
return
}
if err := upgrade.CheckAndUpdate(host, Version, isDev, log); err != nil {
log.Error("Auto-update failed", "error", err)
os.Exit(1)
}
}
func checkIsDevelopment() bool {
dir, err := os.Getwd()
if err != nil {
return false
}
for range 3 {
if data, err := os.ReadFile(filepath.Join(dir, ".env")); err == nil {
return parseEnvMode(data)
}
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
return false
}
dir = filepath.Dir(dir)
}
return false
}
func parseEnvMode(data []byte) bool {
for line := range strings.SplitSeq(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
parts := strings.SplitN(line, "=", 2)
if len(parts) == 2 && strings.TrimSpace(parts[0]) == "ENV_MODE" {
return strings.TrimSpace(parts[1]) == "development"
}
}
return false
}

11
agent/go.mod Normal file
View File

@@ -0,0 +1,11 @@
module databasus-agent
go 1.26.1
require github.com/stretchr/testify v1.11.1
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

10
agent/go.sum Normal file
View File

@@ -0,0 +1,10 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -0,0 +1,136 @@
package config
import (
"encoding/json"
"flag"
"os"
"databasus-agent/internal/logger"
)
var log = logger.GetLogger()
const configFileName = "databasus.json"
type Config struct {
DatabasusHost string `json:"databasusHost"`
DbID string `json:"dbId"`
Token string `json:"token"`
flags parsedFlags
}
// LoadFromJSONAndArgs reads databasus.json into the struct
// and overrides JSON values with any explicitly provided CLI flags.
func (c *Config) LoadFromJSONAndArgs(fs *flag.FlagSet, args []string) {
c.loadFromJSON()
c.initSources()
c.flags.host = fs.String(
"databasus-host",
"",
"Databasus server URL (e.g. http://your-server:4005)",
)
c.flags.dbID = fs.String("db-id", "", "Database ID")
c.flags.token = fs.String("token", "", "Agent token")
if err := fs.Parse(args); err != nil {
os.Exit(1)
}
c.applyFlags()
log.Info("========= Loading config ============")
c.logConfigSources()
log.Info("========= Config has been loaded ====")
}
// SaveToJSON writes the current struct to databasus.json.
func (c *Config) SaveToJSON() error {
data, err := json.MarshalIndent(c, "", " ")
if err != nil {
return err
}
return os.WriteFile(configFileName, data, 0o644)
}
func (c *Config) loadFromJSON() {
data, err := os.ReadFile(configFileName)
if err != nil {
if os.IsNotExist(err) {
log.Info("No databasus.json found, will create on save")
return
}
log.Warn("Failed to read databasus.json", "error", err)
return
}
if err := json.Unmarshal(data, c); err != nil {
log.Warn("Failed to parse databasus.json", "error", err)
return
}
log.Info("Configuration loaded from " + configFileName)
}
func (c *Config) initSources() {
c.flags.sources = map[string]string{
"databasus-host": "not configured",
"db-id": "not configured",
"token": "not configured",
}
if c.DatabasusHost != "" {
c.flags.sources["databasus-host"] = configFileName
}
if c.DbID != "" {
c.flags.sources["db-id"] = configFileName
}
if c.Token != "" {
c.flags.sources["token"] = configFileName
}
}
func (c *Config) applyFlags() {
if c.flags.host != nil && *c.flags.host != "" {
c.DatabasusHost = *c.flags.host
c.flags.sources["databasus-host"] = "command line args"
}
if c.flags.dbID != nil && *c.flags.dbID != "" {
c.DbID = *c.flags.dbID
c.flags.sources["db-id"] = "command line args"
}
if c.flags.token != nil && *c.flags.token != "" {
c.Token = *c.flags.token
c.flags.sources["token"] = "command line args"
}
}
func (c *Config) logConfigSources() {
log.Info(
"databasus-host",
"value",
c.DatabasusHost,
"source",
c.flags.sources["databasus-host"],
)
log.Info("db-id", "value", c.DbID, "source", c.flags.sources["db-id"])
log.Info("token", "value", maskSensitive(c.Token), "source", c.flags.sources["token"])
}
func maskSensitive(value string) string {
if value == "" {
return "(not set)"
}
visibleLen := max(len(value)/4, 1)
return value[:visibleLen] + "***"
}

View File

@@ -0,0 +1,162 @@
package config
import (
"encoding/json"
"flag"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_LoadFromJSONAndArgs_ValuesLoadedFromJSON(t *testing.T) {
dir := setupTempDir(t)
writeConfigJSON(t, dir, Config{
DatabasusHost: "http://json-host:4005",
DbID: "json-db-id",
Token: "json-token",
})
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{})
assert.Equal(t, "http://json-host:4005", cfg.DatabasusHost)
assert.Equal(t, "json-db-id", cfg.DbID)
assert.Equal(t, "json-token", cfg.Token)
}
func Test_LoadFromJSONAndArgs_ValuesLoadedFromArgs_WhenNoJSON(t *testing.T) {
setupTempDir(t)
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{
"--databasus-host", "http://arg-host:4005",
"--db-id", "arg-db-id",
"--token", "arg-token",
})
assert.Equal(t, "http://arg-host:4005", cfg.DatabasusHost)
assert.Equal(t, "arg-db-id", cfg.DbID)
assert.Equal(t, "arg-token", cfg.Token)
}
func Test_LoadFromJSONAndArgs_ArgsOverrideJSON(t *testing.T) {
dir := setupTempDir(t)
writeConfigJSON(t, dir, Config{
DatabasusHost: "http://json-host:4005",
DbID: "json-db-id",
Token: "json-token",
})
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{
"--databasus-host", "http://arg-host:9999",
"--db-id", "arg-db-id-override",
"--token", "arg-token-override",
})
assert.Equal(t, "http://arg-host:9999", cfg.DatabasusHost)
assert.Equal(t, "arg-db-id-override", cfg.DbID)
assert.Equal(t, "arg-token-override", cfg.Token)
}
func Test_LoadFromJSONAndArgs_PartialArgsOverrideJSON(t *testing.T) {
dir := setupTempDir(t)
writeConfigJSON(t, dir, Config{
DatabasusHost: "http://json-host:4005",
DbID: "json-db-id",
Token: "json-token",
})
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{
"--databasus-host", "http://arg-host-only:4005",
})
assert.Equal(t, "http://arg-host-only:4005", cfg.DatabasusHost)
assert.Equal(t, "json-db-id", cfg.DbID)
assert.Equal(t, "json-token", cfg.Token)
}
func Test_SaveToJSON_ConfigSavedCorrectly(t *testing.T) {
setupTempDir(t)
cfg := &Config{
DatabasusHost: "http://save-host:4005",
DbID: "save-db-id",
Token: "save-token",
}
err := cfg.SaveToJSON()
require.NoError(t, err)
saved := readConfigJSON(t)
assert.Equal(t, "http://save-host:4005", saved.DatabasusHost)
assert.Equal(t, "save-db-id", saved.DbID)
assert.Equal(t, "save-token", saved.Token)
}
func Test_SaveToJSON_AfterArgsOverrideJSON_SavedFileContainsMergedValues(t *testing.T) {
dir := setupTempDir(t)
writeConfigJSON(t, dir, Config{
DatabasusHost: "http://json-host:4005",
DbID: "json-db-id",
Token: "json-token",
})
cfg := &Config{}
fs := flag.NewFlagSet("test", flag.ContinueOnError)
cfg.LoadFromJSONAndArgs(fs, []string{
"--databasus-host", "http://override-host:9999",
})
err := cfg.SaveToJSON()
require.NoError(t, err)
saved := readConfigJSON(t)
assert.Equal(t, "http://override-host:9999", saved.DatabasusHost)
assert.Equal(t, "json-db-id", saved.DbID)
assert.Equal(t, "json-token", saved.Token)
}
func setupTempDir(t *testing.T) string {
t.Helper()
origDir, err := os.Getwd()
require.NoError(t, err)
dir := t.TempDir()
require.NoError(t, os.Chdir(dir))
t.Cleanup(func() { os.Chdir(origDir) })
return dir
}
func writeConfigJSON(t *testing.T, dir string, cfg Config) {
t.Helper()
data, err := json.MarshalIndent(cfg, "", " ")
require.NoError(t, err)
require.NoError(t, os.WriteFile(dir+"/"+configFileName, data, 0o644))
}
func readConfigJSON(t *testing.T) Config {
t.Helper()
data, err := os.ReadFile(configFileName)
require.NoError(t, err)
var cfg Config
require.NoError(t, json.Unmarshal(data, &cfg))
return cfg
}

View File

@@ -0,0 +1,9 @@
package config
type parsedFlags struct {
host *string
dbID *string
token *string
sources map[string]string
}

View File

@@ -0,0 +1,37 @@
package start
import (
"errors"
"log/slog"
"databasus-agent/internal/config"
)
func Run(cfg *config.Config, log *slog.Logger) error {
if err := validateConfig(cfg); err != nil {
return err
}
log.Info("start: stub — not yet implemented",
"dbId", cfg.DbID,
"hasToken", cfg.Token != "",
)
return nil
}
func validateConfig(cfg *config.Config) error {
if cfg.DatabasusHost == "" {
return errors.New("argument databasus-host is required")
}
if cfg.DbID == "" {
return errors.New("argument db-id is required")
}
if cfg.Token == "" {
return errors.New("argument token is required")
}
return nil
}

View File

@@ -0,0 +1,159 @@
package upgrade
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"runtime"
"strings"
"syscall"
"time"
)
type Logger interface {
Info(msg string, args ...any)
Warn(msg string, args ...any)
Error(msg string, args ...any)
}
type versionResponse struct {
Version string `json:"version"`
}
func CheckAndUpdate(databasusHost, currentVersion string, isDev bool, log Logger) error {
if isDev {
log.Info("Skipping update check (development mode)")
return nil
}
serverVersion, err := fetchServerVersion(databasusHost, log)
if err != nil {
return nil
}
if serverVersion == currentVersion {
log.Info("Agent version is up to date", "version", currentVersion)
return nil
}
log.Info("Updating agent...", "current", currentVersion, "target", serverVersion)
selfPath, err := os.Executable()
if err != nil {
return fmt.Errorf("failed to determine executable path: %w", err)
}
tempPath := selfPath + ".update"
defer func() {
_ = os.Remove(tempPath)
}()
if err := downloadBinary(databasusHost, tempPath); err != nil {
return fmt.Errorf("failed to download update: %w", err)
}
if err := os.Chmod(tempPath, 0o755); err != nil {
return fmt.Errorf("failed to set permissions on update: %w", err)
}
if err := verifyBinary(tempPath, serverVersion); err != nil {
return fmt.Errorf("update verification failed: %w", err)
}
if err := os.Rename(tempPath, selfPath); err != nil {
return fmt.Errorf("failed to replace binary (try --skip-update if this persists): %w", err)
}
log.Info("Update complete, re-executing...")
return syscall.Exec(selfPath, os.Args, os.Environ())
}
func fetchServerVersion(host string, log Logger) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, host+"/api/v1/system/version", nil)
if err != nil {
return "", err
}
resp, err := client.Do(req)
if err != nil {
log.Warn("Could not reach server for update check, continuing", "error", err)
return "", err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
log.Warn(
"Server returned non-OK status for version check, continuing",
"status",
resp.StatusCode,
)
return "", fmt.Errorf("status %d", resp.StatusCode)
}
var ver versionResponse
if err := json.NewDecoder(resp.Body).Decode(&ver); err != nil {
log.Warn("Failed to parse server version response, continuing", "error", err)
return "", err
}
return ver.Version, nil
}
func downloadBinary(host, destPath string) error {
url := fmt.Sprintf("%s/api/v1/system/agent?arch=%s", host, runtime.GOARCH)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("server returned %d for agent download", resp.StatusCode)
}
f, err := os.Create(destPath)
if err != nil {
return err
}
defer func() { _ = f.Close() }()
_, err = io.Copy(f, resp.Body)
return err
}
func verifyBinary(binaryPath, expectedVersion string) error {
cmd := exec.CommandContext(context.Background(), binaryPath, "version")
output, err := cmd.Output()
if err != nil {
return fmt.Errorf("binary failed to execute: %w", err)
}
got := strings.TrimSpace(string(output))
if got != expectedVersion {
return fmt.Errorf("version mismatch: expected %q, got %q", expectedVersion, got)
}
return nil
}

View File

@@ -0,0 +1,47 @@
package logger
import (
"log/slog"
"os"
"sync"
"time"
)
var (
loggerInstance *slog.Logger
once sync.Once
)
func Init(isDebug bool) {
level := slog.LevelInfo
if isDebug {
level = slog.LevelDebug
}
once.Do(func() {
loggerInstance = slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
Level: level,
ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
if a.Key == slog.TimeKey {
a.Value = slog.StringValue(time.Now().Format("2006/01/02 15:04:05"))
}
if a.Key == slog.LevelKey {
return slog.Attr{}
}
return a
},
}))
loggerInstance.Info("Text structured logger initialized")
})
}
// GetLogger returns a singleton slog.Logger that logs to the console
func GetLogger() *slog.Logger {
if loggerInstance == nil {
Init(false)
}
return loggerInstance
}

View File

@@ -7,6 +7,16 @@ run:
linters:
default: standard
enable:
- funcorder
- bodyclose
- errorlint
- gocritic
- unconvert
- misspell
- errname
- noctx
- modernize
settings:
errcheck:
@@ -14,6 +24,18 @@ linters:
formatters:
enable:
- gofmt
- gofumpt
- golines
- goimports
- gci
settings:
golines:
max-len: 120
gofumpt:
module-path: databasus-backend
extra-rules: true
gci:
sections:
- standard
- default
- localmodule

View File

@@ -12,6 +12,12 @@ import (
"syscall"
"time"
"github.com/gin-contrib/cors"
"github.com/gin-contrib/gzip"
"github.com/gin-gonic/gin"
swaggerFiles "github.com/swaggo/files"
ginSwagger "github.com/swaggo/gin-swagger"
"databasus-backend/internal/config"
"databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/backups/backups/backuping"
@@ -28,7 +34,9 @@ import (
"databasus-backend/internal/features/restores"
"databasus-backend/internal/features/restores/restoring"
"databasus-backend/internal/features/storages"
system_agent "databasus-backend/internal/features/system/agent"
system_healthcheck "databasus-backend/internal/features/system/healthcheck"
system_version "databasus-backend/internal/features/system/version"
task_cancellation "databasus-backend/internal/features/tasks/cancellation"
users_controllers "databasus-backend/internal/features/users/controllers"
users_middleware "databasus-backend/internal/features/users/middleware"
@@ -39,12 +47,6 @@ import (
files_utils "databasus-backend/internal/util/files"
"databasus-backend/internal/util/logger"
_ "databasus-backend/swagger" // swagger docs
"github.com/gin-contrib/cors"
"github.com/gin-contrib/gzip"
"github.com/gin-gonic/gin"
swaggerFiles "github.com/swaggo/files"
ginSwagger "github.com/swaggo/gin-swagger"
)
// @title Databasus Backend API
@@ -81,7 +83,6 @@ func main() {
config.GetEnv().TempFolder,
config.GetEnv().DataFolder,
})
if err != nil {
log.Error("Failed to ensure directories", "error", err)
os.Exit(1)
@@ -148,7 +149,7 @@ func handlePasswordReset(log *slog.Logger) {
resetPassword(*email, *newPassword, log)
}
func resetPassword(email string, newPassword string, log *slog.Logger) {
func resetPassword(email, newPassword string, log *slog.Logger) {
log.Info("Resetting password...")
userService := users_services.GetUserService()
@@ -210,6 +211,8 @@ func setUpRoutes(r *gin.Engine) {
userController := users_controllers.GetUserController()
userController.RegisterRoutes(v1)
system_healthcheck.GetHealthcheckController().RegisterRoutes(v1)
system_version.GetVersionController().RegisterRoutes(v1)
system_agent.GetAgentController().RegisterRoutes(v1)
backups_controllers.GetBackupController().RegisterPublicRoutes(v1)
backups_controllers.GetPostgresWalBackupController().RegisterRoutes(v1)
databases.GetDatabaseController().RegisterPublicRoutes(v1)
@@ -350,7 +353,9 @@ func generateSwaggerDocs(log *slog.Logger) {
return
}
cmd := exec.Command("swag", "init", "-d", currentDir, "-g", "cmd/main.go", "-o", "swagger")
cmd := exec.CommandContext(
context.Background(), "swag", "init", "-d", currentDir, "-g", "cmd/main.go", "-o", "swagger",
)
output, err := cmd.CombinedOutput()
if err != nil {
@@ -364,7 +369,7 @@ func generateSwaggerDocs(log *slog.Logger) {
func runMigrations(log *slog.Logger) {
log.Info("Running database migrations...")
cmd := exec.Command("goose", "-dir", "./migrations", "up")
cmd := exec.CommandContext(context.Background(), "goose", "-dir", "./migrations", "up")
cmd.Env = append(
os.Environ(),
"GOOSE_DRIVER=postgres",

View File

@@ -1,6 +1,6 @@
module databasus-backend
go 1.24.9
go 1.26.1
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0

View File

@@ -1,9 +1,6 @@
package config
import (
env_utils "databasus-backend/internal/util/env"
"databasus-backend/internal/util/logger"
"databasus-backend/internal/util/tools"
"os"
"path/filepath"
"strings"
@@ -11,6 +8,10 @@ import (
"github.com/ilyakaznacheev/cleanenv"
"github.com/joho/godotenv"
env_utils "databasus-backend/internal/util/env"
"databasus-backend/internal/util/logger"
"databasus-backend/internal/util/tools"
)
var log = logger.GetLogger()
@@ -29,7 +30,7 @@ type EnvVariables struct {
MongodbInstallDir string `env:"MONGODB_INSTALL_DIR"`
// Internal database
DatabaseDsn string `env:"DATABASE_DSN" required:"true"`
DatabaseDsn string `env:"DATABASE_DSN" required:"true"`
// Internal Valkey
ValkeyHost string `env:"VALKEY_HOST" required:"true"`
ValkeyPort string `env:"VALKEY_PORT" required:"true"`

View File

@@ -1,17 +1,17 @@
package audit_logs
import (
"databasus-backend/internal/storage"
"fmt"
"testing"
"time"
user_enums "databasus-backend/internal/features/users/enums"
users_testing "databasus-backend/internal/features/users/testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"gorm.io/gorm"
user_enums "databasus-backend/internal/features/users/enums"
users_testing "databasus-backend/internal/features/users/testing"
"databasus-backend/internal/storage"
)
func Test_CleanOldAuditLogs_DeletesLogsOlderThanOneYear(t *testing.T) {

View File

@@ -4,10 +4,10 @@ import (
"errors"
"net/http"
user_models "databasus-backend/internal/features/users/models"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
user_models "databasus-backend/internal/features/users/models"
)
type AuditLogController struct {

View File

@@ -6,15 +6,15 @@ import (
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
user_enums "databasus-backend/internal/features/users/enums"
users_middleware "databasus-backend/internal/features/users/middleware"
users_services "databasus-backend/internal/features/users/services"
users_testing "databasus-backend/internal/features/users/testing"
test_utils "databasus-backend/internal/util/testing"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
func Test_GetGlobalAuditLogs_WithDifferentUserRoles_EnforcesPermissionsCorrectly(t *testing.T) {

View File

@@ -8,14 +8,18 @@ import (
"databasus-backend/internal/util/logger"
)
var auditLogRepository = &AuditLogRepository{}
var auditLogService = &AuditLogService{
auditLogRepository,
logger.GetLogger(),
}
var (
auditLogRepository = &AuditLogRepository{}
auditLogService = &AuditLogService{
auditLogRepository,
logger.GetLogger(),
}
)
var auditLogController = &AuditLogController{
auditLogService,
}
var auditLogBackgroundService = &AuditLogBackgroundService{
auditLogService: auditLogService,
logger: logger.GetLogger(),

View File

@@ -1,10 +1,11 @@
package audit_logs
import (
"databasus-backend/internal/storage"
"time"
"github.com/google/uuid"
"databasus-backend/internal/storage"
)
type AuditLogRepository struct{}
@@ -21,7 +22,7 @@ func (r *AuditLogRepository) GetGlobal(
limit, offset int,
beforeDate *time.Time,
) ([]*AuditLogDTO, error) {
var auditLogs = make([]*AuditLogDTO, 0)
auditLogs := make([]*AuditLogDTO, 0)
sql := `
SELECT
@@ -37,7 +38,7 @@ func (r *AuditLogRepository) GetGlobal(
LEFT JOIN users u ON al.user_id = u.id
LEFT JOIN workspaces w ON al.workspace_id = w.id`
args := []interface{}{}
args := []any{}
if beforeDate != nil {
sql += " WHERE al.created_at < ?"
@@ -57,7 +58,7 @@ func (r *AuditLogRepository) GetByUser(
limit, offset int,
beforeDate *time.Time,
) ([]*AuditLogDTO, error) {
var auditLogs = make([]*AuditLogDTO, 0)
auditLogs := make([]*AuditLogDTO, 0)
sql := `
SELECT
@@ -74,7 +75,7 @@ func (r *AuditLogRepository) GetByUser(
LEFT JOIN workspaces w ON al.workspace_id = w.id
WHERE al.user_id = ?`
args := []interface{}{userID}
args := []any{userID}
if beforeDate != nil {
sql += " AND al.created_at < ?"
@@ -94,7 +95,7 @@ func (r *AuditLogRepository) GetByWorkspace(
limit, offset int,
beforeDate *time.Time,
) ([]*AuditLogDTO, error) {
var auditLogs = make([]*AuditLogDTO, 0)
auditLogs := make([]*AuditLogDTO, 0)
sql := `
SELECT
@@ -111,7 +112,7 @@ func (r *AuditLogRepository) GetByWorkspace(
LEFT JOIN workspaces w ON al.workspace_id = w.id
WHERE al.workspace_id = ?`
args := []interface{}{workspaceID}
args := []any{workspaceID}
if beforeDate != nil {
sql += " AND al.created_at < ?"

View File

@@ -4,10 +4,10 @@ import (
"log/slog"
"time"
"github.com/google/uuid"
user_enums "databasus-backend/internal/features/users/enums"
user_models "databasus-backend/internal/features/users/models"
"github.com/google/uuid"
)
type AuditLogService struct {

View File

@@ -4,11 +4,11 @@ import (
"testing"
"time"
user_enums "databasus-backend/internal/features/users/enums"
users_testing "databasus-backend/internal/features/users/testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
user_enums "databasus-backend/internal/features/users/enums"
users_testing "databasus-backend/internal/features/users/testing"
)
func Test_AuditLogs_WorkspaceSpecificLogs(t *testing.T) {

View File

@@ -5,6 +5,9 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
@@ -14,9 +17,6 @@ import (
users_testing "databasus-backend/internal/features/users/testing"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
cache_utils "databasus-backend/internal/util/cache"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
func Test_BackupExecuted_NotificationSent(t *testing.T) {

View File

@@ -80,8 +80,7 @@ func (c *BackupCleaner) DeleteBackup(backup *backups_core.Backup) error {
return err
}
err = storage.DeleteFile(c.fieldEncryptor, backup.FileName)
if err != nil {
if err := storage.DeleteFile(c.fieldEncryptor, backup.FileName); err != nil {
// we do not return error here, because sometimes clean up performed
// before unavailable storage removal or change - therefore we should
// proceed even in case of error. It's possible that some S3 or
@@ -408,6 +407,10 @@ func buildGFSKeepSet(
) map[uuid.UUID]bool {
keep := make(map[uuid.UUID]bool)
if len(backups) == 0 {
return keep
}
hoursSeen := make(map[string]bool)
daysSeen := make(map[string]bool)
weeksSeen := make(map[string]bool)
@@ -416,6 +419,54 @@ func buildGFSKeepSet(
hoursKept, daysKept, weeksKept, monthsKept, yearsKept := 0, 0, 0, 0, 0
// Compute per-level time-window cutoffs so higher-frequency slots
// cannot absorb backups that belong to lower-frequency levels.
ref := backups[0].CreatedAt
rawHourlyCutoff := ref.Add(-time.Duration(hours) * time.Hour)
rawDailyCutoff := ref.Add(-time.Duration(days) * 24 * time.Hour)
rawWeeklyCutoff := ref.Add(-time.Duration(weeks) * 7 * 24 * time.Hour)
rawMonthlyCutoff := ref.AddDate(0, -months, 0)
rawYearlyCutoff := ref.AddDate(-years, 0, 0)
// Hierarchical capping: each level's window cannot extend further back
// than the nearest active lower-frequency level's window.
yearlyCutoff := rawYearlyCutoff
monthlyCutoff := rawMonthlyCutoff
if years > 0 {
monthlyCutoff = laterOf(monthlyCutoff, yearlyCutoff)
}
weeklyCutoff := rawWeeklyCutoff
if months > 0 {
weeklyCutoff = laterOf(weeklyCutoff, monthlyCutoff)
} else if years > 0 {
weeklyCutoff = laterOf(weeklyCutoff, yearlyCutoff)
}
dailyCutoff := rawDailyCutoff
switch {
case weeks > 0:
dailyCutoff = laterOf(dailyCutoff, weeklyCutoff)
case months > 0:
dailyCutoff = laterOf(dailyCutoff, monthlyCutoff)
case years > 0:
dailyCutoff = laterOf(dailyCutoff, yearlyCutoff)
}
hourlyCutoff := rawHourlyCutoff
switch {
case days > 0:
hourlyCutoff = laterOf(hourlyCutoff, dailyCutoff)
case weeks > 0:
hourlyCutoff = laterOf(hourlyCutoff, weeklyCutoff)
case months > 0:
hourlyCutoff = laterOf(hourlyCutoff, monthlyCutoff)
case years > 0:
hourlyCutoff = laterOf(hourlyCutoff, yearlyCutoff)
}
for _, backup := range backups {
t := backup.CreatedAt
@@ -426,31 +477,31 @@ func buildGFSKeepSet(
monthKey := t.Format("2006-01")
yearKey := t.Format("2006")
if hours > 0 && hoursKept < hours && !hoursSeen[hourKey] {
if hours > 0 && hoursKept < hours && !hoursSeen[hourKey] && t.After(hourlyCutoff) {
keep[backup.ID] = true
hoursSeen[hourKey] = true
hoursKept++
}
if days > 0 && daysKept < days && !daysSeen[dayKey] {
if days > 0 && daysKept < days && !daysSeen[dayKey] && t.After(dailyCutoff) {
keep[backup.ID] = true
daysSeen[dayKey] = true
daysKept++
}
if weeks > 0 && weeksKept < weeks && !weeksSeen[weekKey] {
if weeks > 0 && weeksKept < weeks && !weeksSeen[weekKey] && t.After(weeklyCutoff) {
keep[backup.ID] = true
weeksSeen[weekKey] = true
weeksKept++
}
if months > 0 && monthsKept < months && !monthsSeen[monthKey] {
if months > 0 && monthsKept < months && !monthsSeen[monthKey] && t.After(monthlyCutoff) {
keep[backup.ID] = true
monthsSeen[monthKey] = true
monthsKept++
}
if years > 0 && yearsKept < years && !yearsSeen[yearKey] {
if years > 0 && yearsKept < years && !yearsSeen[yearKey] && t.After(yearlyCutoff) {
keep[backup.ID] = true
yearsSeen[yearKey] = true
yearsKept++
@@ -459,3 +510,11 @@ func buildGFSKeepSet(
return keep
}
func laterOf(a, b time.Time) time.Time {
if a.After(b) {
return a
}
return b
}

File diff suppressed because it is too large Load Diff

View File

@@ -4,6 +4,9 @@ import (
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
@@ -15,9 +18,6 @@ import (
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
"databasus-backend/internal/storage"
"databasus-backend/internal/util/period"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
func Test_CleanOldBackups_DeletesBackupsOlderThanRetentionTimePeriod(t *testing.T) {
@@ -697,160 +697,6 @@ func Test_CleanByCount_DoesNotDeleteInProgressBackups(t *testing.T) {
assert.True(t, inProgressFound, "In-progress backup should not be deleted by count policy")
}
func Test_CleanByGFS_KeepsCorrectBackupsPerSlot(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
storage := storages.CreateTestStorage(workspace.ID)
notifier := notifiers.CreateTestNotifier(workspace.ID)
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
defer func() {
backups, _ := backupRepository.FindByDatabaseID(database.ID)
for _, backup := range backups {
backupRepository.DeleteByID(backup.ID)
}
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
interval := createTestInterval()
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: backups_config.RetentionPolicyTypeGFS,
RetentionGfsDays: 3,
RetentionGfsWeeks: 0,
RetentionGfsMonths: 0,
RetentionGfsYears: 0,
StorageID: &storage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
now := time.Now().UTC()
// Create 5 backups on 5 different days; only the 3 newest days should be kept
var backupIDs []uuid.UUID
for i := 0; i < 5; i++ {
backup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-time.Duration(4-i) * 24 * time.Hour).Truncate(24 * time.Hour),
}
err = backupRepository.Save(backup)
assert.NoError(t, err)
backupIDs = append(backupIDs, backup.ID)
}
cleaner := GetBackupCleaner()
err = cleaner.cleanByRetentionPolicy()
assert.NoError(t, err)
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
assert.Equal(t, 3, len(remainingBackups))
remainingIDs := make(map[uuid.UUID]bool)
for _, backup := range remainingBackups {
remainingIDs[backup.ID] = true
}
assert.False(t, remainingIDs[backupIDs[0]], "Oldest daily backup should be deleted")
assert.False(t, remainingIDs[backupIDs[1]], "2nd oldest daily backup should be deleted")
assert.True(t, remainingIDs[backupIDs[2]], "3rd backup should remain")
assert.True(t, remainingIDs[backupIDs[3]], "4th backup should remain")
assert.True(t, remainingIDs[backupIDs[4]], "Newest backup should remain")
}
func Test_CleanByGFS_WithWeeklyAndMonthlySlots_KeepsWiderSpread(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
storage := storages.CreateTestStorage(workspace.ID)
notifier := notifiers.CreateTestNotifier(workspace.ID)
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
defer func() {
backups, _ := backupRepository.FindByDatabaseID(database.ID)
for _, backup := range backups {
backupRepository.DeleteByID(backup.ID)
}
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
interval := createTestInterval()
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: backups_config.RetentionPolicyTypeGFS,
RetentionGfsDays: 2,
RetentionGfsWeeks: 2,
RetentionGfsMonths: 1,
RetentionGfsYears: 0,
StorageID: &storage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
now := time.Now().UTC()
// Create one backup per week for 6 weeks (each on Monday of that week)
// GFS should keep: 2 daily (most recent 2 unique days) + 2 weekly + 1 monthly = up to 5 unique
var createdIDs []uuid.UUID
for i := 0; i < 6; i++ {
weekOffset := time.Duration(5-i) * 7 * 24 * time.Hour
backup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-weekOffset).Truncate(24 * time.Hour),
}
err = backupRepository.Save(backup)
assert.NoError(t, err)
createdIDs = append(createdIDs, backup.ID)
}
cleaner := GetBackupCleaner()
err = cleaner.cleanByRetentionPolicy()
assert.NoError(t, err)
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
// We should have at most 5 backups kept (2 daily + 2 weekly + 1 monthly, but with overlap possible)
// The exact count depends on how many unique periods are covered
assert.LessOrEqual(t, len(remainingBackups), 5)
assert.GreaterOrEqual(t, len(remainingBackups), 1)
// The two most recent backups should always be retained (daily slots)
remainingIDs := make(map[uuid.UUID]bool)
for _, backup := range remainingBackups {
remainingIDs[backup.ID] = true
}
assert.True(t, remainingIDs[createdIDs[4]], "Second newest backup should be retained (daily)")
assert.True(t, remainingIDs[createdIDs[5]], "Newest backup should be retained (daily)")
}
// Test_DeleteBackup_WhenStorageDeleteFails_BackupStillRemovedFromDatabase verifies resilience
// when storage becomes unavailable. Even if storage.DeleteFile fails (e.g., storage is offline,
// credentials changed, or storage was deleted), the backup record should still be removed from
@@ -897,292 +743,6 @@ func Test_DeleteBackup_WhenStorageDeleteFails_BackupStillRemovedFromDatabase(t *
assert.Nil(t, deletedBackup)
}
func Test_CleanByGFS_WithHourlySlots_KeepsCorrectBackups(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
testStorage := storages.CreateTestStorage(workspace.ID)
notifier := notifiers.CreateTestNotifier(workspace.ID)
database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier)
defer func() {
backups, _ := backupRepository.FindByDatabaseID(database.ID)
for _, backup := range backups {
backupRepository.DeleteByID(backup.ID)
}
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(testStorage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
interval := createTestInterval()
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: backups_config.RetentionPolicyTypeGFS,
RetentionGfsHours: 3,
StorageID: &testStorage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
now := time.Now().UTC()
// Create 5 backups spaced 1 hour apart; only the 3 newest hours should be kept
var backupIDs []uuid.UUID
for i := 0; i < 5; i++ {
backup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: testStorage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-time.Duration(4-i) * time.Hour).Truncate(time.Hour),
}
err = backupRepository.Save(backup)
assert.NoError(t, err)
backupIDs = append(backupIDs, backup.ID)
}
cleaner := GetBackupCleaner()
err = cleaner.cleanByRetentionPolicy()
assert.NoError(t, err)
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
assert.Equal(t, 3, len(remainingBackups))
remainingIDs := make(map[uuid.UUID]bool)
for _, backup := range remainingBackups {
remainingIDs[backup.ID] = true
}
assert.False(t, remainingIDs[backupIDs[0]], "Oldest hourly backup should be deleted")
assert.False(t, remainingIDs[backupIDs[1]], "2nd oldest hourly backup should be deleted")
assert.True(t, remainingIDs[backupIDs[2]], "3rd backup should remain")
assert.True(t, remainingIDs[backupIDs[3]], "4th backup should remain")
assert.True(t, remainingIDs[backupIDs[4]], "Newest backup should remain")
}
func Test_BuildGFSKeepSet(t *testing.T) {
// Fixed reference time: a Wednesday mid-month to avoid boundary edge cases in the default tests.
// Use time.Date for determinism across test runs.
ref := time.Date(2025, 6, 18, 12, 0, 0, 0, time.UTC) // Wednesday, 2025-06-18
day := 24 * time.Hour
week := 7 * day
newBackup := func(createdAt time.Time) *backups_core.Backup {
return &backups_core.Backup{ID: uuid.New(), CreatedAt: createdAt}
}
// backupsEveryDay returns n backups, newest-first, each 1 day apart.
backupsEveryDay := func(n int) []*backups_core.Backup {
bs := make([]*backups_core.Backup, n)
for i := 0; i < n; i++ {
bs[i] = newBackup(ref.Add(-time.Duration(i) * day))
}
return bs
}
// backupsEveryWeek returns n backups, newest-first, each 7 days apart.
backupsEveryWeek := func(n int) []*backups_core.Backup {
bs := make([]*backups_core.Backup, n)
for i := 0; i < n; i++ {
bs[i] = newBackup(ref.Add(-time.Duration(i) * week))
}
return bs
}
hour := time.Hour
// backupsEveryHour returns n backups, newest-first, each 1 hour apart.
backupsEveryHour := func(n int) []*backups_core.Backup {
bs := make([]*backups_core.Backup, n)
for i := 0; i < n; i++ {
bs[i] = newBackup(ref.Add(-time.Duration(i) * hour))
}
return bs
}
tests := []struct {
name string
backups []*backups_core.Backup
hours int
days int
weeks int
months int
years int
keptIndices []int // which indices in backups should be kept
deletedRange *[2]int // optional: all indices in [from, to) must be deleted
}{
{
name: "OnlyHourlySlots_KeepsNewest3Of5",
backups: backupsEveryHour(5),
hours: 3,
keptIndices: []int{0, 1, 2},
},
{
name: "SameHourDedup_OnlyNewestKeptForHourlySlot",
backups: []*backups_core.Backup{
newBackup(ref.Truncate(hour).Add(45 * time.Minute)),
newBackup(ref.Truncate(hour).Add(10 * time.Minute)),
},
hours: 1,
keptIndices: []int{0},
},
{
name: "OnlyDailySlots_KeepsNewest3Of5",
backups: backupsEveryDay(5),
days: 3,
keptIndices: []int{0, 1, 2},
},
{
name: "OnlyDailySlots_FewerBackupsThanSlots_KeepsAll",
backups: backupsEveryDay(2),
days: 5,
keptIndices: []int{0, 1},
},
{
name: "OnlyWeeklySlots_KeepsNewest2Weeks",
backups: backupsEveryWeek(4),
weeks: 2,
keptIndices: []int{0, 1},
},
{
name: "OnlyMonthlySlots_KeepsNewest2Months",
backups: []*backups_core.Backup{
newBackup(time.Date(2025, 6, 1, 12, 0, 0, 0, time.UTC)),
newBackup(time.Date(2025, 5, 1, 12, 0, 0, 0, time.UTC)),
newBackup(time.Date(2025, 4, 1, 12, 0, 0, 0, time.UTC)),
},
months: 2,
keptIndices: []int{0, 1},
},
{
name: "OnlyYearlySlots_KeepsNewest2Years",
backups: []*backups_core.Backup{
newBackup(time.Date(2025, 6, 1, 12, 0, 0, 0, time.UTC)),
newBackup(time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC)),
newBackup(time.Date(2023, 6, 1, 12, 0, 0, 0, time.UTC)),
},
years: 2,
keptIndices: []int{0, 1},
},
{
name: "SameDayDedup_OnlyNewestKeptForDailySlot",
backups: []*backups_core.Backup{
// Two backups on the same day; newest-first order
newBackup(ref.Truncate(day).Add(10 * time.Hour)),
newBackup(ref.Truncate(day).Add(2 * time.Hour)),
},
days: 1,
keptIndices: []int{0},
},
{
name: "SameWeekDedup_OnlyNewestKeptForWeeklySlot",
backups: []*backups_core.Backup{
// ref is Wednesday; add Thursday of same week
newBackup(ref.Add(1 * day)), // Thursday same week
newBackup(ref), // Wednesday same week
},
weeks: 1,
keptIndices: []int{0},
},
{
name: "AdditiveSlots_NewestFillsDailyAndWeeklyAndMonthly",
// Newest backup fills daily + weekly + monthly simultaneously
backups: []*backups_core.Backup{
newBackup(time.Date(2025, 6, 18, 12, 0, 0, 0, time.UTC)), // newest
newBackup(time.Date(2025, 6, 11, 12, 0, 0, 0, time.UTC)), // 1 week ago
newBackup(time.Date(2025, 5, 18, 12, 0, 0, 0, time.UTC)), // 1 month ago
newBackup(time.Date(2025, 4, 18, 12, 0, 0, 0, time.UTC)), // 2 months ago
},
days: 1,
weeks: 2,
months: 2,
keptIndices: []int{0, 1, 2},
},
{
name: "YearBoundary_CorrectlySplitsAcrossYears",
backups: []*backups_core.Backup{
newBackup(time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC)),
newBackup(time.Date(2024, 12, 31, 12, 0, 0, 0, time.UTC)),
newBackup(time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC)),
newBackup(time.Date(2023, 6, 1, 12, 0, 0, 0, time.UTC)),
},
years: 2,
keptIndices: []int{0, 1}, // 2025 and 2024 kept; 2024-06 and 2023 deleted
},
{
name: "ISOWeekBoundary_Jan1UsesCorrectISOWeek",
// 2025-01-01 is ISO week 1 of 2025; 2024-12-28 is ISO week 52 of 2024
backups: []*backups_core.Backup{
newBackup(time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC)), // ISO week 2025-W01
newBackup(time.Date(2024, 12, 28, 12, 0, 0, 0, time.UTC)), // ISO week 2024-W52
},
weeks: 2,
keptIndices: []int{0, 1}, // different ISO weeks → both kept
},
{
name: "EmptyBackups_ReturnsEmptyKeepSet",
backups: []*backups_core.Backup{},
hours: 3,
days: 3,
weeks: 2,
months: 1,
years: 1,
keptIndices: []int{},
},
{
name: "AllZeroSlots_KeepsNothing",
backups: backupsEveryDay(5),
hours: 0,
days: 0,
weeks: 0,
months: 0,
years: 0,
keptIndices: []int{},
},
{
name: "AllSlotsActive_FullCombination",
backups: backupsEveryWeek(12),
days: 2,
weeks: 3,
months: 2,
years: 1,
// 2 daily (indices 0,1) + 3rd weekly slot (index 2) + 2nd monthly slot (index 3 or later).
// Additive slots: newest fills daily+weekly+monthly+yearly; each subsequent week fills another weekly,
// and a backup ~4 weeks later fills the 2nd monthly slot.
keptIndices: []int{0, 1, 2, 3},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
keepSet := buildGFSKeepSet(tc.backups, tc.hours, tc.days, tc.weeks, tc.months, tc.years)
keptIndexSet := make(map[int]bool, len(tc.keptIndices))
for _, idx := range tc.keptIndices {
keptIndexSet[idx] = true
}
for i, backup := range tc.backups {
if keptIndexSet[i] {
assert.True(t, keepSet[backup.ID], "backup at index %d should be kept", i)
} else {
assert.False(t, keepSet[backup.ID], "backup at index %d should be deleted", i)
}
}
})
}
}
func Test_CleanByTimePeriod_SkipsRecentBackup_EvenIfOlderThanRetention(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
@@ -1354,114 +914,6 @@ func Test_CleanByCount_SkipsRecentBackup_EvenIfOverLimit(t *testing.T) {
assert.True(t, remainingIDs[newestBackup.ID], "Newest backup should be preserved")
}
func Test_CleanByGFS_SkipsRecentBackup_WhenNotInKeepSet(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
storage := storages.CreateTestStorage(workspace.ID)
notifier := notifiers.CreateTestNotifier(workspace.ID)
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
defer func() {
backups, _ := backupRepository.FindByDatabaseID(database.ID)
for _, backup := range backups {
backupRepository.DeleteByID(backup.ID)
}
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
interval := createTestInterval()
// Keep only 1 daily slot. We create 2 old backups plus two recent backups on today.
// Backups are ordered newest-first, so the 15-min-old backup fills the single daily slot.
// The 30-min-old backup is the same day → not in the GFS keep-set, but it is still recent
// (within grace period) and must be preserved.
backupConfig := &backups_config.BackupConfig{
DatabaseID: database.ID,
IsBackupsEnabled: true,
RetentionPolicyType: backups_config.RetentionPolicyTypeGFS,
RetentionGfsDays: 1,
StorageID: &storage.ID,
BackupIntervalID: interval.ID,
BackupInterval: interval,
}
_, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
assert.NoError(t, err)
now := time.Now().UTC()
oldBackup1 := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-3 * 24 * time.Hour).Truncate(24 * time.Hour),
}
oldBackup2 := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-2 * 24 * time.Hour).Truncate(24 * time.Hour),
}
// Newest backup today — will fill the single GFS daily slot.
newestTodayBackup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-15 * time.Minute),
}
// Slightly older backup, also today — NOT in GFS keep-set (duplicate day),
// but within the 60-minute grace period so it must survive.
recentNotInKeepSet := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: storage.ID,
Status: backups_core.BackupStatusCompleted,
BackupSizeMb: 10,
CreatedAt: now.Add(-30 * time.Minute),
}
for _, b := range []*backups_core.Backup{oldBackup1, oldBackup2, newestTodayBackup, recentNotInKeepSet} {
err = backupRepository.Save(b)
assert.NoError(t, err)
}
cleaner := GetBackupCleaner()
err = cleaner.cleanByRetentionPolicy()
assert.NoError(t, err)
remainingBackups, err := backupRepository.FindByDatabaseID(database.ID)
assert.NoError(t, err)
remainingIDs := make(map[uuid.UUID]bool)
for _, backup := range remainingBackups {
remainingIDs[backup.ID] = true
}
assert.False(t, remainingIDs[oldBackup1.ID], "Old backup 1 should be deleted by GFS")
assert.False(t, remainingIDs[oldBackup2.ID], "Old backup 2 should be deleted by GFS")
assert.True(
t,
remainingIDs[newestTodayBackup.ID],
"Newest backup fills GFS daily slot and must remain",
)
assert.True(
t,
remainingIDs[recentNotInKeepSet.ID],
"Recent backup not in keep-set must be preserved by grace period",
)
}
func Test_CleanExceededBackups_SkipsRecentBackup_WhenOverTotalSizeLimit(t *testing.T) {
router := CreateTestRouter()
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)

View File

@@ -6,15 +6,15 @@ import (
"sync/atomic"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/mock"
common "databasus-backend/internal/features/backups/backups/common"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/notifiers"
"databasus-backend/internal/features/storages"
"github.com/google/uuid"
"github.com/stretchr/testify/mock"
)
type MockNotificationSender struct {

View File

@@ -10,10 +10,10 @@ import (
"sync/atomic"
"time"
cache_utils "databasus-backend/internal/util/cache"
"github.com/google/uuid"
"github.com/valkey-io/valkey-go"
cache_utils "databasus-backend/internal/util/cache"
)
const (
@@ -415,7 +415,7 @@ func (r *BackupNodesRegistry) UnsubscribeNodeForBackupsAssignments() error {
return nil
}
func (r *BackupNodesRegistry) PublishBackupCompletion(nodeID uuid.UUID, backupID uuid.UUID) error {
func (r *BackupNodesRegistry) PublishBackupCompletion(nodeID, backupID uuid.UUID) error {
ctx := context.Background()
message := BackupCompletionMessage{
@@ -437,7 +437,7 @@ func (r *BackupNodesRegistry) PublishBackupCompletion(nodeID uuid.UUID, backupID
}
func (r *BackupNodesRegistry) SubscribeForBackupsCompletions(
handler func(nodeID uuid.UUID, backupID uuid.UUID),
handler func(nodeID, backupID uuid.UUID),
) error {
ctx := context.Background()

View File

@@ -9,11 +9,11 @@ import (
"testing"
"time"
cache_utils "databasus-backend/internal/util/cache"
"databasus-backend/internal/util/logger"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
cache_utils "databasus-backend/internal/util/cache"
"databasus-backend/internal/util/logger"
)
func Test_HearthbeatNodeInRegistry_RegistersNodeWithTTL(t *testing.T) {
@@ -903,7 +903,7 @@ func Test_SubscribeForBackupsCompletions_ReceivesCompletedBackups(t *testing.T)
receivedBackupID := make(chan uuid.UUID, 1)
receivedNodeID := make(chan uuid.UUID, 1)
handler := func(nodeID uuid.UUID, backupID uuid.UUID) {
handler := func(nodeID, backupID uuid.UUID) {
receivedNodeID <- nodeID
receivedBackupID <- backupID
}
@@ -940,7 +940,7 @@ func Test_SubscribeForBackupsCompletions_ParsesJsonCorrectly(t *testing.T) {
defer registry.UnsubscribeForBackupsCompletions()
receivedBackups := make(chan uuid.UUID, 2)
handler := func(nodeID uuid.UUID, backupID uuid.UUID) {
handler := func(nodeID, backupID uuid.UUID) {
receivedBackups <- backupID
}
@@ -969,7 +969,7 @@ func Test_SubscribeForBackupsCompletions_HandlesInvalidJson(t *testing.T) {
defer registry.UnsubscribeForBackupsCompletions()
receivedBackupID := make(chan uuid.UUID, 1)
handler := func(nodeID uuid.UUID, backupID uuid.UUID) {
handler := func(nodeID, backupID uuid.UUID) {
receivedBackupID <- backupID
}
@@ -997,7 +997,7 @@ func Test_UnsubscribeForBackupsCompletions_StopsReceivingMessages(t *testing.T)
backupID2 := uuid.New()
receivedBackupID := make(chan uuid.UUID, 2)
handler := func(nodeID uuid.UUID, backupID uuid.UUID) {
handler := func(nodeID, backupID uuid.UUID) {
receivedBackupID <- backupID
}
@@ -1032,7 +1032,7 @@ func Test_SubscribeForBackupsCompletions_WhenAlreadySubscribed_ReturnsError(t *t
registry := createTestRegistry()
defer registry.UnsubscribeForBackupsCompletions()
handler := func(nodeID uuid.UUID, backupID uuid.UUID) {}
handler := func(nodeID, backupID uuid.UUID) {}
err := registry.SubscribeForBackupsCompletions(handler)
assert.NoError(t, err)
@@ -1064,9 +1064,9 @@ func Test_MultipleSubscribers_EachReceivesCompletionMessages(t *testing.T) {
receivedBackups2 := make(chan uuid.UUID, 3)
receivedBackups3 := make(chan uuid.UUID, 3)
handler1 := func(nodeID uuid.UUID, backupID uuid.UUID) { receivedBackups1 <- backupID }
handler2 := func(nodeID uuid.UUID, backupID uuid.UUID) { receivedBackups2 <- backupID }
handler3 := func(nodeID uuid.UUID, backupID uuid.UUID) { receivedBackups3 <- backupID }
handler1 := func(nodeID, backupID uuid.UUID) { receivedBackups1 <- backupID }
handler2 := func(nodeID, backupID uuid.UUID) { receivedBackups2 <- backupID }
handler3 := func(nodeID, backupID uuid.UUID) { receivedBackups3 <- backupID }
err := registry1.SubscribeForBackupsCompletions(handler1)
assert.NoError(t, err)

View File

@@ -441,7 +441,7 @@ func (s *BackupsScheduler) calculateLeastBusyNode() (*uuid.UUID, error) {
return &bestNode.ID, nil
}
func (s *BackupsScheduler) onBackupCompleted(nodeID uuid.UUID, backupID uuid.UUID) {
func (s *BackupsScheduler) onBackupCompleted(nodeID, backupID uuid.UUID) {
// Verify this task is actually a backup (registry contains multiple task types)
_, err := s.backupRepository.FindByID(backupID)
if err != nil {

View File

@@ -1,6 +1,12 @@
package backuping
import (
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
@@ -12,11 +18,6 @@ import (
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
cache_utils "databasus-backend/internal/util/cache"
"databasus-backend/internal/util/period"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
func Test_RunPendingBackups_WhenLastBackupWasYesterday_CreatesNewBackup(t *testing.T) {

View File

@@ -8,6 +8,9 @@ import (
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
backups_core "databasus-backend/internal/features/backups/backups/core"
"databasus-backend/internal/features/backups/backups/usecases"
backups_config "databasus-backend/internal/features/backups/config"
@@ -19,9 +22,6 @@ import (
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/logger"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
)
func CreateTestRouter() *gin.Engine {

View File

@@ -1,10 +1,11 @@
package common
import (
backups_config "databasus-backend/internal/features/backups/config"
"errors"
"github.com/google/uuid"
backups_config "databasus-backend/internal/features/backups/config"
)
type BackupMetadata struct {

View File

@@ -7,6 +7,10 @@ type CountingWriter struct {
BytesWritten int64
}
func NewCountingWriter(writer io.Writer) *CountingWriter {
return &CountingWriter{Writer: writer}
}
func (cw *CountingWriter) Write(p []byte) (n int, err error) {
n, err = cw.Writer.Write(p)
cw.BytesWritten += int64(n)
@@ -16,7 +20,3 @@ func (cw *CountingWriter) Write(p []byte) (n int, err error) {
func (cw *CountingWriter) GetBytesWritten() int64 {
return cw.BytesWritten
}
func NewCountingWriter(writer io.Writer) *CountingWriter {
return &CountingWriter{Writer: writer}
}

View File

@@ -2,13 +2,7 @@ package backups_controllers
import (
"context"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_download "databasus-backend/internal/features/backups/backups/download"
backups_dto "databasus-backend/internal/features/backups/backups/dto"
backups_services "databasus-backend/internal/features/backups/backups/services"
"databasus-backend/internal/features/databases"
users_middleware "databasus-backend/internal/features/users/middleware"
files_utils "databasus-backend/internal/util/files"
"errors"
"fmt"
"io"
"net/http"
@@ -16,6 +10,14 @@ import (
"github.com/gin-gonic/gin"
"github.com/google/uuid"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_download "databasus-backend/internal/features/backups/backups/download"
backups_dto "databasus-backend/internal/features/backups/backups/dto"
backups_services "databasus-backend/internal/features/backups/backups/services"
"databasus-backend/internal/features/databases"
users_middleware "databasus-backend/internal/features/users/middleware"
files_utils "databasus-backend/internal/util/files"
)
type BackupController struct {
@@ -197,7 +199,7 @@ func (c *BackupController) GenerateDownloadToken(ctx *gin.Context) {
response, err := c.backupService.GenerateDownloadToken(user, id)
if err != nil {
if err == backups_download.ErrDownloadAlreadyInProgress {
if errors.Is(err, backups_download.ErrDownloadAlreadyInProgress) {
ctx.JSON(
http.StatusConflict,
gin.H{
@@ -248,7 +250,7 @@ func (c *BackupController) GetFile(ctx *gin.Context) {
downloadToken, rateLimiter, err := c.backupService.ValidateDownloadToken(token)
if err != nil {
if err == backups_download.ErrDownloadAlreadyInProgress {
if errors.Is(err, backups_download.ErrDownloadAlreadyInProgress) {
ctx.JSON(
http.StatusConflict,
gin.H{

View File

@@ -5,13 +5,13 @@ import (
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_dto "databasus-backend/internal/features/backups/backups/dto"
backups_services "databasus-backend/internal/features/backups/backups/services"
"databasus-backend/internal/features/databases"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
)
// PostgreWalBackupController handles WAL backup endpoints used by the databasus-cli agent.

View File

@@ -10,6 +10,11 @@ import (
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_dto "databasus-backend/internal/features/backups/backups/dto"
backups_config "databasus-backend/internal/features/backups/config"
@@ -23,11 +28,6 @@ import (
workspaces_controllers "databasus-backend/internal/features/workspaces/controllers"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
test_utils "databasus-backend/internal/util/testing"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_WalUpload_InProgressStatusSetBeforeStream(t *testing.T) {

View File

@@ -4,14 +4,14 @@ import (
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
workspaces_controllers "databasus-backend/internal/features/workspaces/controllers"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
)
func CreateTestRouter() *gin.Engine {

View File

@@ -4,10 +4,10 @@ import (
"fmt"
"time"
"github.com/google/uuid"
backups_config "databasus-backend/internal/features/backups/config"
files_utils "databasus-backend/internal/util/files"
"github.com/google/uuid"
)
type PgWalBackupType string

View File

@@ -1,13 +1,13 @@
package backups_core
import (
"databasus-backend/internal/storage"
"errors"
"time"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/storage"
)
type BackupRepository struct{}
@@ -88,7 +88,7 @@ func (r *BackupRepository) FindLastByDatabaseID(databaseID uuid.UUID) (*Backup,
Where("database_id = ?", databaseID).
Order("created_at DESC").
First(&backup).Error; err != nil {
if err == gorm.ErrRecordNotFound {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}

View File

@@ -13,9 +13,11 @@ var downloadTokenRepository = &DownloadTokenRepository{}
var downloadTracker = NewDownloadTracker(cache_utils.GetValkeyClient())
var bandwidthManager *BandwidthManager
var downloadTokenService *DownloadTokenService
var downloadTokenBackgroundService *DownloadTokenBackgroundService
var (
bandwidthManager *BandwidthManager
downloadTokenService *DownloadTokenService
downloadTokenBackgroundService *DownloadTokenBackgroundService
)
func init() {
env := config.GetEnv()

View File

@@ -66,9 +66,7 @@ func (rl *RateLimiter) Wait(bytes int64) {
tokensNeeded := float64(bytes) - rl.availableTokens
waitTime := time.Duration(tokensNeeded/float64(rl.bytesPerSecond)*1000) * time.Millisecond
if waitTime < time.Millisecond {
waitTime = time.Millisecond
}
waitTime = max(waitTime, time.Millisecond)
rl.mu.Unlock()
time.Sleep(waitTime)

View File

@@ -2,12 +2,14 @@ package backups_download
import (
"crypto/rand"
"databasus-backend/internal/storage"
"encoding/base64"
"errors"
"time"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/storage"
)
type DownloadTokenRepository struct{}
@@ -28,9 +30,8 @@ func (r *DownloadTokenRepository) FindByToken(token string) (*DownloadToken, err
err := storage.GetDb().
Where("token = ?", token).
First(&downloadToken).Error
if err != nil {
if err == gorm.ErrRecordNotFound {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return nil, err

View File

@@ -1,12 +1,13 @@
package backups_download
import (
cache_utils "databasus-backend/internal/util/cache"
"errors"
"time"
"github.com/google/uuid"
"github.com/valkey-io/valkey-go"
cache_utils "databasus-backend/internal/util/cache"
)
const (
@@ -16,9 +17,7 @@ const (
downloadHeartbeatDelay = 3 * time.Second
)
var (
ErrDownloadAlreadyInProgress = errors.New("download already in progress for this user")
)
var ErrDownloadAlreadyInProgress = errors.New("download already in progress for this user")
type DownloadTracker struct {
cache *cache_utils.CacheUtil[string]

View File

@@ -1,12 +1,13 @@
package backups_dto
import (
backups_core "databasus-backend/internal/features/backups/backups/core"
"databasus-backend/internal/features/backups/backups/encryption"
"io"
"time"
"github.com/google/uuid"
backups_core "databasus-backend/internal/features/backups/backups/core"
"databasus-backend/internal/features/backups/backups/encryption"
)
type GetBackupsRequest struct {

View File

@@ -4,6 +4,7 @@ import (
"crypto/aes"
"crypto/cipher"
"encoding/binary"
"errors"
"fmt"
"io"
@@ -69,7 +70,7 @@ func NewDecryptionReader(
func (r *DecryptionReader) Read(p []byte) (n int, err error) {
for len(r.buffer) < len(p) && !r.eof {
if err := r.readAndDecryptChunk(); err != nil {
if err == io.EOF {
if errors.Is(err, io.EOF) {
r.eof = true
break
}

View File

@@ -1,6 +1,9 @@
package backups_services
import (
"sync"
"sync/atomic"
audit_logs "databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/backups/backups/backuping"
backups_core "databasus-backend/internal/features/backups/backups/core"
@@ -15,8 +18,6 @@ import (
workspaces_services "databasus-backend/internal/features/workspaces/services"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/logger"
"sync"
"sync/atomic"
)
var taskCancelManager = task_cancellation.GetTaskCancelManager()

View File

@@ -7,6 +7,8 @@ import (
"log/slog"
"time"
"github.com/google/uuid"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_dto "databasus-backend/internal/features/backups/backups/dto"
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
@@ -16,8 +18,6 @@ import (
encryption_secrets "databasus-backend/internal/features/encryption/secrets"
util_encryption "databasus-backend/internal/util/encryption"
util_wal "databasus-backend/internal/util/wal"
"github.com/google/uuid"
)
// PostgreWalBackupService handles WAL segment and basebackup uploads from the databasus-cli agent.
@@ -225,6 +225,80 @@ func (s *PostgreWalBackupService) DownloadBackupFile(
return s.backupService.GetBackupReader(backupID)
}
func (s *PostgreWalBackupService) GetNextFullBackupTime(
database *databases.Database,
) (*backups_dto.GetNextFullBackupTimeResponse, error) {
if err := s.validateWalBackupType(database); err != nil {
return nil, err
}
backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)
if err != nil {
return nil, fmt.Errorf("failed to get backup config: %w", err)
}
if backupConfig.BackupInterval == nil {
return nil, fmt.Errorf("no backup interval configured for database %s", database.ID)
}
lastFullBackup, err := s.backupRepository.FindLastCompletedFullWalBackupByDatabaseID(
database.ID,
)
if err != nil {
return nil, fmt.Errorf("failed to query last full backup: %w", err)
}
var lastBackupTime *time.Time
if lastFullBackup != nil {
lastBackupTime = &lastFullBackup.CreatedAt
}
now := time.Now().UTC()
nextTime := backupConfig.BackupInterval.NextTriggerTime(now, lastBackupTime)
return &backups_dto.GetNextFullBackupTimeResponse{
NextFullBackupTime: nextTime,
}, nil
}
// ReportError creates a FAILED backup record with the agent's error message.
func (s *PostgreWalBackupService) ReportError(
database *databases.Database,
errorMsg string,
) error {
if err := s.validateWalBackupType(database); err != nil {
return err
}
backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)
if err != nil {
return fmt.Errorf("failed to get backup config: %w", err)
}
if backupConfig.Storage == nil {
return fmt.Errorf("no storage configured for database %s", database.ID)
}
now := time.Now().UTC()
backup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: backupConfig.Storage.ID,
Status: backups_core.BackupStatusFailed,
FailMessage: &errorMsg,
Encryption: backupConfig.Encryption,
CreatedAt: now,
}
backup.GenerateFilename(database.Name)
if err := s.backupRepository.Save(backup); err != nil {
return fmt.Errorf("failed to save error backup record: %w", err)
}
return nil
}
func (s *PostgreWalBackupService) validateWalChain(
databaseID uuid.UUID,
incomingSegment string,
@@ -432,80 +506,6 @@ func (s *PostgreWalBackupService) markFailed(backup *backups_core.Backup, errMsg
}
}
func (s *PostgreWalBackupService) GetNextFullBackupTime(
database *databases.Database,
) (*backups_dto.GetNextFullBackupTimeResponse, error) {
if err := s.validateWalBackupType(database); err != nil {
return nil, err
}
backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)
if err != nil {
return nil, fmt.Errorf("failed to get backup config: %w", err)
}
if backupConfig.BackupInterval == nil {
return nil, fmt.Errorf("no backup interval configured for database %s", database.ID)
}
lastFullBackup, err := s.backupRepository.FindLastCompletedFullWalBackupByDatabaseID(
database.ID,
)
if err != nil {
return nil, fmt.Errorf("failed to query last full backup: %w", err)
}
var lastBackupTime *time.Time
if lastFullBackup != nil {
lastBackupTime = &lastFullBackup.CreatedAt
}
now := time.Now().UTC()
nextTime := backupConfig.BackupInterval.NextTriggerTime(now, lastBackupTime)
return &backups_dto.GetNextFullBackupTimeResponse{
NextFullBackupTime: nextTime,
}, nil
}
// ReportError creates a FAILED backup record with the agent's error message.
func (s *PostgreWalBackupService) ReportError(
database *databases.Database,
errorMsg string,
) error {
if err := s.validateWalBackupType(database); err != nil {
return err
}
backupConfig, err := s.backupConfigService.GetBackupConfigByDbId(database.ID)
if err != nil {
return fmt.Errorf("failed to get backup config: %w", err)
}
if backupConfig.Storage == nil {
return fmt.Errorf("no storage configured for database %s", database.ID)
}
now := time.Now().UTC()
backup := &backups_core.Backup{
ID: uuid.New(),
DatabaseID: database.ID,
StorageID: backupConfig.Storage.ID,
Status: backups_core.BackupStatusFailed,
FailMessage: &errorMsg,
Encryption: backupConfig.Encryption,
CreatedAt: now,
}
backup.GenerateFilename(database.Name)
if err := s.backupRepository.Save(backup); err != nil {
return fmt.Errorf("failed to save error backup record: %w", err)
}
return nil
}
func (s *PostgreWalBackupService) resolveFullBackup(
databaseID uuid.UUID,
backupID *uuid.UUID,
@@ -609,5 +609,5 @@ func (cr *countingReader) Read(p []byte) (n int, err error) {
n, err = cr.r.Read(p)
cr.n += int64(n)
return
return n, err
}

View File

@@ -7,6 +7,8 @@ import (
"io"
"log/slog"
"github.com/google/uuid"
audit_logs "databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/backups/backups/backuping"
backups_core "databasus-backend/internal/features/backups/backups/core"
@@ -23,8 +25,6 @@ import (
workspaces_services "databasus-backend/internal/features/workspaces/services"
util_encryption "databasus-backend/internal/util/encryption"
files_utils "databasus-backend/internal/util/files"
"github.com/google/uuid"
)
type BackupService struct {

View File

@@ -279,10 +279,10 @@ func (uc *CreateMariadbBackupUsecase) createTempMyCnfFile(
password string,
) (string, error) {
tempFolder := config.GetEnv().TempFolder
if err := os.MkdirAll(tempFolder, 0700); err != nil {
if err := os.MkdirAll(tempFolder, 0o700); err != nil {
return "", fmt.Errorf("failed to ensure temp folder exists: %w", err)
}
if err := os.Chmod(tempFolder, 0700); err != nil {
if err := os.Chmod(tempFolder, 0o700); err != nil {
return "", fmt.Errorf("failed to set temp folder permissions: %w", err)
}
@@ -291,7 +291,7 @@ func (uc *CreateMariadbBackupUsecase) createTempMyCnfFile(
return "", fmt.Errorf("failed to create temp directory: %w", err)
}
if err := os.Chmod(tempDir, 0700); err != nil {
if err := os.Chmod(tempDir, 0o700); err != nil {
_ = os.RemoveAll(tempDir)
return "", fmt.Errorf("failed to set temp directory permissions: %w", err)
}
@@ -311,7 +311,7 @@ port=%d
content += "ssl=false\n"
}
err = os.WriteFile(myCnfFile, []byte(content), 0600)
err = os.WriteFile(myCnfFile, []byte(content), 0o600)
if err != nil {
_ = os.RemoveAll(tempDir)
return "", fmt.Errorf("failed to write .my.cnf: %w", err)
@@ -548,8 +548,8 @@ func (uc *CreateMariadbBackupUsecase) buildMariadbDumpErrorMessage(
stderrStr,
)
exitErr, ok := waitErr.(*exec.ExitError)
if !ok {
var exitErr *exec.ExitError
if !errors.As(waitErr, &exitErr) {
return errors.New(errorMsg)
}

View File

@@ -118,7 +118,7 @@ func (uc *CreateMysqlBackupUsecase) buildMysqldumpArgs(my *mysqltypes.MysqlDatab
args = append(args, "--events")
}
args = append(args, uc.getNetworkCompressionArgs(my.Version)...)
args = append(args, uc.getNetworkCompressionArgs(my)...)
if !config.GetEnv().IsCloud {
args = append(args, "--max-allowed-packet=1G")
@@ -135,15 +135,21 @@ func (uc *CreateMysqlBackupUsecase) buildMysqldumpArgs(my *mysqltypes.MysqlDatab
return args
}
func (uc *CreateMysqlBackupUsecase) getNetworkCompressionArgs(version tools.MysqlVersion) []string {
func (uc *CreateMysqlBackupUsecase) getNetworkCompressionArgs(
my *mysqltypes.MysqlDatabase,
) []string {
const zstdCompressionLevel = 5
switch version {
switch my.Version {
case tools.MysqlVersion80, tools.MysqlVersion84, tools.MysqlVersion9:
return []string{
"--compression-algorithms=zstd",
fmt.Sprintf("--zstd-compression-level=%d", zstdCompressionLevel),
if my.IsZstdSupported {
return []string{
"--compression-algorithms=zstd",
fmt.Sprintf("--zstd-compression-level=%d", zstdCompressionLevel),
}
}
return []string{"--compress"}
case tools.MysqlVersion57:
return []string{"--compress"}
default:
@@ -292,10 +298,10 @@ func (uc *CreateMysqlBackupUsecase) createTempMyCnfFile(
password string,
) (string, error) {
tempFolder := config.GetEnv().TempFolder
if err := os.MkdirAll(tempFolder, 0700); err != nil {
if err := os.MkdirAll(tempFolder, 0o700); err != nil {
return "", fmt.Errorf("failed to ensure temp folder exists: %w", err)
}
if err := os.Chmod(tempFolder, 0700); err != nil {
if err := os.Chmod(tempFolder, 0o700); err != nil {
return "", fmt.Errorf("failed to set temp folder permissions: %w", err)
}
@@ -304,7 +310,7 @@ func (uc *CreateMysqlBackupUsecase) createTempMyCnfFile(
return "", fmt.Errorf("failed to create temp directory: %w", err)
}
if err := os.Chmod(tempDir, 0700); err != nil {
if err := os.Chmod(tempDir, 0o700); err != nil {
_ = os.RemoveAll(tempDir)
return "", fmt.Errorf("failed to set temp directory permissions: %w", err)
}
@@ -322,7 +328,7 @@ port=%d
content += "ssl-mode=REQUIRED\n"
}
err = os.WriteFile(myCnfFile, []byte(content), 0600)
err = os.WriteFile(myCnfFile, []byte(content), 0o600)
if err != nil {
_ = os.RemoveAll(tempDir)
return "", fmt.Errorf("failed to write .my.cnf: %w", err)
@@ -559,8 +565,8 @@ func (uc *CreateMysqlBackupUsecase) buildMysqldumpErrorMessage(
stderrStr,
)
exitErr, ok := waitErr.(*exec.ExitError)
if !ok {
var exitErr *exec.ExitError
if !errors.As(waitErr, &exitErr) {
return errors.New(errorMsg)
}
@@ -589,6 +595,15 @@ func (uc *CreateMysqlBackupUsecase) handleConnectionErrors(stderrStr string) err
)
}
if containsIgnoreCase(stderrStr, "compression algorithm") ||
containsIgnoreCase(stderrStr, "2066") {
return fmt.Errorf(
"MySQL connection failed due to unsupported compression algorithm. "+
"Try re-saving the database connection to re-detect compression support. stderr: %s",
stderrStr,
)
}
if containsIgnoreCase(stderrStr, "unknown database") {
return fmt.Errorf(
"MySQL database does not exist. stderr: %s",

View File

@@ -13,6 +13,8 @@ import (
"strings"
"time"
"github.com/google/uuid"
"databasus-backend/internal/config"
common "databasus-backend/internal/features/backups/backups/common"
backups_core "databasus-backend/internal/features/backups/backups/core"
@@ -24,8 +26,6 @@ import (
"databasus-backend/internal/features/storages"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
"github.com/google/uuid"
)
const (
@@ -595,8 +595,8 @@ func (uc *CreatePostgresqlBackupUsecase) buildPgDumpErrorMessage(
stderrStr := string(stderrOutput)
errorMsg := fmt.Sprintf("%s failed: %v stderr: %s", filepath.Base(pgBin), waitErr, stderrStr)
exitErr, ok := waitErr.(*exec.ExitError)
if !ok {
var exitErr *exec.ExitError
if !errors.As(waitErr, &exitErr) {
return errors.New(errorMsg)
}
@@ -748,10 +748,10 @@ func (uc *CreatePostgresqlBackupUsecase) createTempPgpassFile(
)
tempFolder := config.GetEnv().TempFolder
if err := os.MkdirAll(tempFolder, 0700); err != nil {
if err := os.MkdirAll(tempFolder, 0o700); err != nil {
return "", fmt.Errorf("failed to ensure temp folder exists: %w", err)
}
if err := os.Chmod(tempFolder, 0700); err != nil {
if err := os.Chmod(tempFolder, 0o700); err != nil {
return "", fmt.Errorf("failed to set temp folder permissions: %w", err)
}
@@ -760,13 +760,13 @@ func (uc *CreatePostgresqlBackupUsecase) createTempPgpassFile(
return "", fmt.Errorf("failed to create temporary directory: %w", err)
}
if err := os.Chmod(tempDir, 0700); err != nil {
if err := os.Chmod(tempDir, 0o700); err != nil {
_ = os.RemoveAll(tempDir)
return "", fmt.Errorf("failed to set temporary directory permissions: %w", err)
}
pgpassFile := filepath.Join(tempDir, ".pgpass")
err = os.WriteFile(pgpassFile, []byte(pgpassContent), 0600)
err = os.WriteFile(pgpassFile, []byte(pgpassContent), 0o600)
if err != nil {
_ = os.RemoveAll(tempDir)
return "", fmt.Errorf("failed to write temporary .pgpass file: %w", err)

View File

@@ -4,10 +4,10 @@ import (
"errors"
"net/http"
users_middleware "databasus-backend/internal/features/users/middleware"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
users_middleware "databasus-backend/internal/features/users/middleware"
)
type BackupConfigController struct {

View File

@@ -12,16 +12,19 @@ import (
"databasus-backend/internal/util/logger"
)
var backupConfigRepository = &BackupConfigRepository{}
var backupConfigService = &BackupConfigService{
backupConfigRepository,
databases.GetDatabaseService(),
storages.GetStorageService(),
notifiers.GetNotifierService(),
workspaces_services.GetWorkspaceService(),
plans.GetDatabasePlanService(),
nil,
}
var (
backupConfigRepository = &BackupConfigRepository{}
backupConfigService = &BackupConfigService{
backupConfigRepository,
databases.GetDatabaseService(),
storages.GetStorageService(),
notifiers.GetNotifierService(),
workspaces_services.GetWorkspaceService(),
plans.GetDatabasePlanService(),
nil,
}
)
var backupConfigController = &BackupConfigController{
backupConfigService,
}

View File

@@ -1,16 +1,17 @@
package backups_config
import (
"databasus-backend/internal/config"
"databasus-backend/internal/features/intervals"
plans "databasus-backend/internal/features/plan"
"databasus-backend/internal/features/storages"
"databasus-backend/internal/util/period"
"errors"
"strings"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/config"
"databasus-backend/internal/features/intervals"
plans "databasus-backend/internal/features/plan"
"databasus-backend/internal/features/storages"
"databasus-backend/internal/util/period"
)
type BackupConfig struct {
@@ -43,7 +44,7 @@ type BackupConfig struct {
Encryption BackupEncryption `json:"encryption" gorm:"column:encryption;type:text;not null;default:'NONE'"`
// MaxBackupSizeMB limits individual backup size. 0 = unlimited.
MaxBackupSizeMB int64 `json:"maxBackupSizeMb" gorm:"column:max_backup_size_mb;type:int;not null"`
MaxBackupSizeMB int64 `json:"maxBackupSizeMb" gorm:"column:max_backup_size_mb;type:int;not null"`
// MaxBackupsTotalSizeMB limits total size of all backups. 0 = unlimited.
MaxBackupsTotalSizeMB int64 `json:"maxBackupsTotalSizeMb" gorm:"column:max_backups_total_size_mb;type:int;not null"`
}

View File

@@ -3,12 +3,12 @@ package backups_config
import (
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"databasus-backend/internal/features/intervals"
plans "databasus-backend/internal/features/plan"
"databasus-backend/internal/util/period"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
func Test_Validate_WhenRetentionTimePeriodIsWeekAndPlanAllowsMonth_ValidationPasses(t *testing.T) {

View File

@@ -1,11 +1,12 @@
package backups_config
import (
"databasus-backend/internal/storage"
"errors"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/storage"
)
type BackupConfigRepository struct{}
@@ -47,7 +48,6 @@ func (r *BackupConfigRepository) Save(
return nil
})
if err != nil {
return nil, err
}

View File

@@ -3,6 +3,8 @@ package backups_config
import (
"errors"
"github.com/google/uuid"
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/intervals"
"databasus-backend/internal/features/notifiers"
@@ -10,8 +12,6 @@ import (
"databasus-backend/internal/features/storages"
users_models "databasus-backend/internal/features/users/models"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"github.com/google/uuid"
)
type BackupConfigService struct {
@@ -214,39 +214,6 @@ func (s *BackupConfigService) CreateDisabledBackupConfig(databaseID uuid.UUID) e
return s.initializeDefaultConfig(databaseID)
}
func (s *BackupConfigService) initializeDefaultConfig(
databaseID uuid.UUID,
) error {
plan, err := s.databasePlanService.GetDatabasePlan(databaseID)
if err != nil {
return err
}
timeOfDay := "04:00"
_, err = s.backupConfigRepository.Save(&BackupConfig{
DatabaseID: databaseID,
IsBackupsEnabled: false,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: plan.MaxStoragePeriod,
MaxBackupSizeMB: plan.MaxBackupSizeMB,
MaxBackupsTotalSizeMB: plan.MaxBackupsTotalSizeMB,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
},
SendNotificationsOn: []BackupNotificationType{
NotificationBackupFailed,
NotificationBackupSuccess,
},
IsRetryIfFailed: true,
MaxFailedTriesCount: 3,
Encryption: BackupEncryptionNone,
})
return err
}
func (s *BackupConfigService) TransferDatabaseToWorkspace(
user *users_models.User,
databaseID uuid.UUID,
@@ -290,7 +257,8 @@ func (s *BackupConfigService) TransferDatabaseToWorkspace(
s.transferNotifiers(user, database, request.TargetWorkspaceID)
}
if request.IsTransferWithStorage {
switch {
case request.IsTransferWithStorage:
if backupConfig.StorageID == nil {
return ErrDatabaseHasNoStorage
}
@@ -315,7 +283,7 @@ func (s *BackupConfigService) TransferDatabaseToWorkspace(
if err != nil {
return err
}
} else if request.TargetStorageID != nil {
case request.TargetStorageID != nil:
targetStorage, err := s.storageService.GetStorageByID(*request.TargetStorageID)
if err != nil {
return err
@@ -332,7 +300,7 @@ func (s *BackupConfigService) TransferDatabaseToWorkspace(
if err != nil {
return err
}
} else {
default:
return ErrTargetStorageNotSpecified
}
@@ -351,6 +319,39 @@ func (s *BackupConfigService) TransferDatabaseToWorkspace(
return nil
}
func (s *BackupConfigService) initializeDefaultConfig(
databaseID uuid.UUID,
) error {
plan, err := s.databasePlanService.GetDatabasePlan(databaseID)
if err != nil {
return err
}
timeOfDay := "04:00"
_, err = s.backupConfigRepository.Save(&BackupConfig{
DatabaseID: databaseID,
IsBackupsEnabled: false,
RetentionPolicyType: RetentionPolicyTypeTimePeriod,
RetentionTimePeriod: plan.MaxStoragePeriod,
MaxBackupSizeMB: plan.MaxBackupSizeMB,
MaxBackupsTotalSizeMB: plan.MaxBackupsTotalSizeMB,
BackupInterval: &intervals.Interval{
Interval: intervals.IntervalDaily,
TimeOfDay: &timeOfDay,
},
SendNotificationsOn: []BackupNotificationType{
NotificationBackupFailed,
NotificationBackupSuccess,
},
IsRetryIfFailed: true,
MaxFailedTriesCount: 3,
Encryption: BackupEncryptionNone,
})
return err
}
func (s *BackupConfigService) transferNotifiers(
user *users_models.User,
database *databases.Database,

View File

@@ -1,11 +1,11 @@
package backups_config
import (
"github.com/google/uuid"
"databasus-backend/internal/features/intervals"
"databasus-backend/internal/features/storages"
"databasus-backend/internal/util/period"
"github.com/google/uuid"
)
func EnableBackupsForTestDatabase(

View File

@@ -1,13 +1,14 @@
package databases
import (
users_middleware "databasus-backend/internal/features/users/middleware"
users_services "databasus-backend/internal/features/users/services"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"net/http"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
users_middleware "databasus-backend/internal/features/users/middleware"
users_services "databasus-backend/internal/features/users/services"
workspaces_services "databasus-backend/internal/features/workspaces/services"
)
type DatabaseController struct {

View File

@@ -862,7 +862,7 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
name string
databaseType DatabaseType
createDatabase func(workspaceID uuid.UUID) *Database
updateDatabase func(workspaceID uuid.UUID, databaseID uuid.UUID) *Database
updateDatabase func(workspaceID, databaseID uuid.UUID) *Database
verifySensitiveData func(t *testing.T, database *Database)
verifyHiddenData func(t *testing.T, database *Database)
}{
@@ -878,7 +878,7 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
Postgresql: pgConfig,
}
},
updateDatabase: func(workspaceID uuid.UUID, databaseID uuid.UUID) *Database {
updateDatabase: func(workspaceID, databaseID uuid.UUID) *Database {
pgConfig := getTestPostgresConfig()
pgConfig.Password = ""
return &Database{
@@ -914,7 +914,7 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
Mariadb: mariaConfig,
}
},
updateDatabase: func(workspaceID uuid.UUID, databaseID uuid.UUID) *Database {
updateDatabase: func(workspaceID, databaseID uuid.UUID) *Database {
mariaConfig := getTestMariadbConfig()
mariaConfig.Password = ""
return &Database{
@@ -950,7 +950,7 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
Mongodb: mongoConfig,
}
},
updateDatabase: func(workspaceID uuid.UUID, databaseID uuid.UUID) *Database {
updateDatabase: func(workspaceID, databaseID uuid.UUID) *Database {
mongoConfig := getTestMongodbConfig()
mongoConfig.Password = ""
return &Database{

View File

@@ -12,11 +12,11 @@ import (
"strings"
"time"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
"github.com/go-sql-driver/mysql"
"github.com/google/uuid"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
)
type MariadbDatabase struct {
@@ -391,7 +391,7 @@ func (m *MariadbDatabase) HasPrivilege(priv string) bool {
}
func HasPrivilege(privileges, priv string) bool {
for _, p := range strings.Split(privileges, ",") {
for p := range strings.SplitSeq(privileges, ",") {
if strings.TrimSpace(p) == priv {
return true
}
@@ -399,7 +399,7 @@ func HasPrivilege(privileges, priv string) bool {
return false
}
func (m *MariadbDatabase) buildDSN(password string, database string) string {
func (m *MariadbDatabase) buildDSN(password, database string) string {
tlsConfig := "false"
if m.IsHttps {

View File

@@ -10,13 +10,13 @@ import (
"strings"
"time"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
"github.com/google/uuid"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
)
type MongodbDatabase struct {
@@ -434,7 +434,6 @@ func (m *MongodbDatabase) CreateReadOnlyUser(
},
}},
}).Err()
if err != nil {
if attempt < maxRetries-1 {
continue
@@ -452,6 +451,48 @@ func (m *MongodbDatabase) CreateReadOnlyUser(
return "", "", errors.New("failed to generate unique username after 3 attempts")
}
// BuildMongodumpURI builds a URI suitable for mongodump (without database in path)
func (m *MongodbDatabase) BuildMongodumpURI(password string) string {
authDB := m.AuthDatabase
if authDB == "" {
authDB = "admin"
}
extraParams := ""
if m.IsHttps {
extraParams += "&tls=true&tlsInsecure=true"
}
if m.IsDirectConnection {
extraParams += "&directConnection=true"
}
if m.IsSrv {
return fmt.Sprintf(
"mongodb+srv://%s:%s@%s/?authSource=%s&connectTimeoutMS=15000%s",
url.QueryEscape(m.Username),
url.QueryEscape(password),
m.Host,
authDB,
extraParams,
)
}
port := 27017
if m.Port != nil {
port = *m.Port
}
return fmt.Sprintf(
"mongodb://%s:%s@%s:%d/?authSource=%s&connectTimeoutMS=15000%s",
url.QueryEscape(m.Username),
url.QueryEscape(password),
m.Host,
port,
authDB,
extraParams,
)
}
// buildConnectionURI builds a MongoDB connection URI
func (m *MongodbDatabase) buildConnectionURI(password string) string {
authDB := m.AuthDatabase
@@ -496,48 +537,6 @@ func (m *MongodbDatabase) buildConnectionURI(password string) string {
)
}
// BuildMongodumpURI builds a URI suitable for mongodump (without database in path)
func (m *MongodbDatabase) BuildMongodumpURI(password string) string {
authDB := m.AuthDatabase
if authDB == "" {
authDB = "admin"
}
extraParams := ""
if m.IsHttps {
extraParams += "&tls=true&tlsInsecure=true"
}
if m.IsDirectConnection {
extraParams += "&directConnection=true"
}
if m.IsSrv {
return fmt.Sprintf(
"mongodb+srv://%s:%s@%s/?authSource=%s&connectTimeoutMS=15000%s",
url.QueryEscape(m.Username),
url.QueryEscape(password),
m.Host,
authDB,
extraParams,
)
}
port := 27017
if m.Port != nil {
port = *m.Port
}
return fmt.Sprintf(
"mongodb://%s:%s@%s:%d/?authSource=%s&connectTimeoutMS=15000%s",
url.QueryEscape(m.Username),
url.QueryEscape(password),
m.Host,
port,
authDB,
extraParams,
)
}
// detectMongodbVersion gets MongoDB server version from buildInfo command
func detectMongodbVersion(ctx context.Context, client *mongo.Client) (tools.MongodbVersion, error) {
adminDB := client.Database("admin")

View File

@@ -12,11 +12,11 @@ import (
"strings"
"time"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
"github.com/go-sql-driver/mysql"
"github.com/google/uuid"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
)
type MysqlDatabase struct {
@@ -25,13 +25,14 @@ type MysqlDatabase struct {
Version tools.MysqlVersion `json:"version" gorm:"type:text;not null"`
Host string `json:"host" gorm:"type:text;not null"`
Port int `json:"port" gorm:"type:int;not null"`
Username string `json:"username" gorm:"type:text;not null"`
Password string `json:"password" gorm:"type:text;not null"`
Database *string `json:"database" gorm:"type:text"`
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
Privileges string `json:"privileges" gorm:"column:privileges;type:text;not null;default:''"`
Host string `json:"host" gorm:"type:text;not null"`
Port int `json:"port" gorm:"type:int;not null"`
Username string `json:"username" gorm:"type:text;not null"`
Password string `json:"password" gorm:"type:text;not null"`
Database *string `json:"database" gorm:"type:text"`
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
Privileges string `json:"privileges" gorm:"column:privileges;type:text;not null;default:''"`
IsZstdSupported bool `json:"isZstdSupported" gorm:"column:is_zstd_supported;type:boolean;not null;default:true"`
}
func (m *MysqlDatabase) TableName() string {
@@ -102,6 +103,7 @@ func (m *MysqlDatabase) TestConnection(
return err
}
m.Privileges = privileges
m.IsZstdSupported = detectZstdSupport(ctx, db)
if err := checkBackupPermissions(m.Privileges); err != nil {
return err
@@ -125,6 +127,7 @@ func (m *MysqlDatabase) Update(incoming *MysqlDatabase) {
m.Database = incoming.Database
m.IsHttps = incoming.IsHttps
m.Privileges = incoming.Privileges
m.IsZstdSupported = incoming.IsZstdSupported
if incoming.Password != "" {
m.Password = incoming.Password
@@ -185,6 +188,7 @@ func (m *MysqlDatabase) PopulateDbData(
return err
}
m.Privileges = privileges
m.IsZstdSupported = detectZstdSupport(ctx, db)
return nil
}
@@ -223,6 +227,7 @@ func (m *MysqlDatabase) PopulateVersion(
return err
}
m.Version = detectedVersion
m.IsZstdSupported = detectZstdSupport(ctx, db)
return nil
}
@@ -398,7 +403,7 @@ func HasPrivilege(privileges, priv string) bool {
return false
}
func (m *MysqlDatabase) buildDSN(password string, database string) string {
func (m *MysqlDatabase) buildDSN(password, database string) string {
tlsConfig := "false"
allowCleartext := ""
@@ -575,6 +580,22 @@ func checkBackupPermissions(privileges string) error {
return nil
}
// detectZstdSupport checks if the MySQL server supports zstd network compression.
// The protocol_compression_algorithms variable was introduced in MySQL 8.0.18.
// Managed MySQL providers (e.g. PlanetScale) may not support zstd even on 8.0+.
func detectZstdSupport(ctx context.Context, db *sql.DB) bool {
var varName, value string
err := db.QueryRowContext(ctx,
"SHOW VARIABLES LIKE 'protocol_compression_algorithms'",
).Scan(&varName, &value)
if err != nil {
return false
}
return strings.Contains(strings.ToLower(value), "zstd")
}
func decryptPasswordIfNeeded(
password string,
encryptor encryption.FieldEncryptor,

View File

@@ -177,6 +177,38 @@ func Test_TestConnection_SufficientPermissions_Success(t *testing.T) {
}
}
func Test_TestConnection_DetectsZstdSupport(t *testing.T) {
env := config.GetEnv()
cases := []struct {
name string
version tools.MysqlVersion
port string
isExpectZstd bool
}{
{"MySQL 5.7", tools.MysqlVersion57, env.TestMysql57Port, false},
{"MySQL 8.0", tools.MysqlVersion80, env.TestMysql80Port, true},
{"MySQL 8.4", tools.MysqlVersion84, env.TestMysql84Port, true},
{"MySQL 9", tools.MysqlVersion9, env.TestMysql90Port, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
container := connectToMysqlContainer(t, tc.port, tc.version)
defer container.DB.Close()
mysqlModel := createMysqlModel(container)
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
err := mysqlModel.TestConnection(logger, nil, uuid.New())
assert.NoError(t, err)
assert.Equal(t, tc.isExpectZstd, mysqlModel.IsZstdSupported,
"IsZstdSupported mismatch for %s", tc.name)
})
}
}
func Test_IsUserReadOnly_AdminUser_ReturnsFalse(t *testing.T) {
env := config.GetEnv()
cases := []struct {

View File

@@ -2,9 +2,6 @@ package postgresql
import (
"context"
"databasus-backend/internal/config"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
"errors"
"fmt"
"log/slog"
@@ -16,6 +13,10 @@ import (
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"gorm.io/gorm"
"databasus-backend/internal/config"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/tools"
)
type PostgresBackupType string
@@ -1112,7 +1113,7 @@ func checkBackupPermissions(
}
// buildConnectionStringForDB builds connection string for specific database
func buildConnectionStringForDB(p *PostgresqlDatabase, dbName string, password string) string {
func buildConnectionStringForDB(p *PostgresqlDatabase, dbName, password string) string {
sslMode := "disable"
if p.IsHttps {
sslMode = "require"
@@ -1152,8 +1153,8 @@ func isSupabaseConnection(host, username string) bool {
}
func extractSupabaseProjectID(username string) string {
if idx := strings.Index(username, "."); idx != -1 {
return username[idx+1:]
if _, after, found := strings.Cut(username, "."); found {
return after
}
return ""
}

View File

@@ -1,10 +1,11 @@
package databases
import (
"databasus-backend/internal/util/encryption"
"log/slog"
"github.com/google/uuid"
"databasus-backend/internal/util/encryption"
)
type DatabaseValidator interface {

View File

@@ -2,17 +2,18 @@ package databases
import (
"context"
"errors"
"log/slog"
"time"
"github.com/google/uuid"
"databasus-backend/internal/features/databases/databases/mariadb"
"databasus-backend/internal/features/databases/databases/mongodb"
"databasus-backend/internal/features/databases/databases/mysql"
"databasus-backend/internal/features/databases/databases/postgresql"
"databasus-backend/internal/features/notifiers"
"databasus-backend/internal/util/encryption"
"errors"
"log/slog"
"time"
"github.com/google/uuid"
)
type Database struct {

View File

@@ -1,15 +1,16 @@
package databases
import (
"errors"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/features/databases/databases/mariadb"
"databasus-backend/internal/features/databases/databases/mongodb"
"databasus-backend/internal/features/databases/databases/mysql"
"databasus-backend/internal/features/databases/databases/postgresql"
"databasus-backend/internal/storage"
"errors"
"github.com/google/uuid"
"gorm.io/gorm"
)
type DatabaseRepository struct{}
@@ -120,7 +121,6 @@ func (r *DatabaseRepository) Save(database *Database) (*Database, error) {
return nil
})
if err != nil {
return nil, err
}

View File

@@ -9,6 +9,8 @@ import (
"strings"
"time"
"github.com/google/uuid"
"databasus-backend/internal/config"
audit_logs "databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/databases/databases/mariadb"
@@ -19,8 +21,6 @@ import (
users_models "databasus-backend/internal/features/users/models"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"databasus-backend/internal/util/encryption"
"github.com/google/uuid"
)
type DatabaseService struct {

View File

@@ -4,6 +4,8 @@ import (
"fmt"
"strconv"
"github.com/google/uuid"
"databasus-backend/internal/config"
"databasus-backend/internal/features/databases/databases/mariadb"
"databasus-backend/internal/features/databases/databases/mongodb"
@@ -12,8 +14,6 @@ import (
"databasus-backend/internal/features/storages"
"databasus-backend/internal/storage"
"databasus-backend/internal/util/tools"
"github.com/google/uuid"
)
func GetTestPostgresConfig() *postgresql.PostgresqlDatabase {

View File

@@ -1,12 +1,13 @@
package disk
import (
"databasus-backend/internal/config"
"fmt"
"path/filepath"
"runtime"
"github.com/shirou/gopsutil/v4/disk"
"databasus-backend/internal/config"
)
type DiskService struct{}

View File

@@ -5,8 +5,10 @@ import (
"databasus-backend/internal/util/logger"
)
var env = config.GetEnv()
var log = logger.GetLogger()
var (
env = config.GetEnv()
log = logger.GetLogger()
)
var emailSMTPSender = &EmailSMTPSender{
log,

View File

@@ -1,6 +1,7 @@
package email
import (
"context"
"crypto/tls"
"fmt"
"log/slog"
@@ -114,7 +115,7 @@ func (s *EmailSMTPSender) createImplicitTLSClient() (*smtp.Client, func(), error
tlsConfig := &tls.Config{ServerName: s.smtpHost}
dialer := &net.Dialer{Timeout: DefaultTimeout}
conn, err := tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
conn, err := (&tls.Dialer{NetDialer: dialer, Config: tlsConfig}).DialContext(context.Background(), "tcp", addr)
if err != nil {
return nil, nil, fmt.Errorf("failed to connect to SMTP server: %w", err)
}

View File

@@ -5,12 +5,12 @@ import (
"fmt"
"os"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/config"
user_models "databasus-backend/internal/features/users/models"
"databasus-backend/internal/storage"
"github.com/google/uuid"
"gorm.io/gorm"
)
type SecretKeyService struct {
@@ -33,7 +33,7 @@ func (s *SecretKeyService) MigrateKeyFromDbToFileIfExist() error {
}
secretKeyPath := config.GetEnv().SecretKeyPath
if err := os.WriteFile(secretKeyPath, []byte(secretKey.Secret), 0600); err != nil {
if err := os.WriteFile(secretKeyPath, []byte(secretKey.Secret), 0o600); err != nil {
return fmt.Errorf("failed to write secret key to file: %w", err)
}
@@ -54,7 +54,7 @@ func (s *SecretKeyService) GetSecretKey() (string, error) {
if err != nil {
if os.IsNotExist(err) {
newKey := s.generateNewSecretKey()
if err := os.WriteFile(secretKeyPath, []byte(newKey), 0600); err != nil {
if err := os.WriteFile(secretKeyPath, []byte(newKey), 0o600); err != nil {
return "", fmt.Errorf("failed to write new secret key: %w", err)
}
s.cachedKey = &newKey

View File

@@ -1,9 +1,6 @@
package healthcheck_attempt
import (
"databasus-backend/internal/features/databases"
healthcheck_config "databasus-backend/internal/features/healthcheck/config"
"databasus-backend/internal/util/logger"
"errors"
"fmt"
"log/slog"
@@ -11,6 +8,10 @@ import (
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/features/databases"
healthcheck_config "databasus-backend/internal/features/healthcheck/config"
"databasus-backend/internal/util/logger"
)
type CheckDatabaseHealthUseCase struct {
@@ -251,5 +252,4 @@ func (uc *CheckDatabaseHealthUseCase) sendDbStatusNotification(
messageBody,
)
}
}

View File

@@ -6,6 +6,9 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"databasus-backend/internal/features/databases"
healthcheck_config "databasus-backend/internal/features/healthcheck/config"
"databasus-backend/internal/features/notifiers"
@@ -13,9 +16,6 @@ import (
users_enums "databasus-backend/internal/features/users/enums"
users_testing "databasus-backend/internal/features/users/testing"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
func Test_CheckDatabaseHealthUseCase(t *testing.T) {

View File

@@ -1,12 +1,13 @@
package healthcheck_attempt
import (
users_middleware "databasus-backend/internal/features/users/middleware"
"net/http"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
users_middleware "databasus-backend/internal/features/users/middleware"
)
type HealthcheckAttemptController struct {

View File

@@ -11,12 +11,14 @@ import (
"databasus-backend/internal/util/logger"
)
var healthcheckAttemptRepository = &HealthcheckAttemptRepository{}
var healthcheckAttemptService = &HealthcheckAttemptService{
healthcheckAttemptRepository,
databases.GetDatabaseService(),
workspaces_services.GetWorkspaceService(),
}
var (
healthcheckAttemptRepository = &HealthcheckAttemptRepository{}
healthcheckAttemptService = &HealthcheckAttemptService{
healthcheckAttemptRepository,
databases.GetDatabaseService(),
workspaces_services.GetWorkspaceService(),
}
)
var checkDatabaseHealthUseCase = &CheckDatabaseHealthUseCase{
healthcheckAttemptRepository,
@@ -31,6 +33,7 @@ var healthcheckAttemptBackgroundService = &HealthcheckAttemptBackgroundService{
runOnce: sync.Once{},
hasRun: atomic.Bool{},
}
var healthcheckAttemptController = &HealthcheckAttemptController{
healthcheckAttemptService,
}

View File

@@ -1,10 +1,10 @@
package healthcheck_attempt
import (
"github.com/google/uuid"
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/notifiers"
"github.com/google/uuid"
)
type HealthcheckAttemptSender interface {

View File

@@ -1,11 +1,11 @@
package healthcheck_attempt
import (
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/notifiers"
"github.com/google/uuid"
"github.com/stretchr/testify/mock"
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/notifiers"
)
type MockHealthcheckAttemptSender struct {

View File

@@ -1,10 +1,11 @@
package healthcheck_attempt
import (
"databasus-backend/internal/features/databases"
"time"
"github.com/google/uuid"
"databasus-backend/internal/features/databases"
)
type HealthcheckAttempt struct {

View File

@@ -1,10 +1,11 @@
package healthcheck_attempt
import (
"databasus-backend/internal/storage"
"time"
"github.com/google/uuid"
"databasus-backend/internal/storage"
)
type HealthcheckAttemptRepository struct{}

View File

@@ -1,13 +1,14 @@
package healthcheck_attempt
import (
"databasus-backend/internal/features/databases"
users_models "databasus-backend/internal/features/users/models"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"errors"
"time"
"github.com/google/uuid"
"databasus-backend/internal/features/databases"
users_models "databasus-backend/internal/features/users/models"
workspaces_services "databasus-backend/internal/features/workspaces/services"
)
type HealthcheckAttemptService struct {

View File

@@ -1,11 +1,12 @@
package healthcheck_config
import (
users_middleware "databasus-backend/internal/features/users/middleware"
"net/http"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
users_middleware "databasus-backend/internal/features/users/middleware"
)
type HealthcheckConfigController struct {

View File

@@ -10,14 +10,17 @@ import (
"databasus-backend/internal/util/logger"
)
var healthcheckConfigRepository = &HealthcheckConfigRepository{}
var healthcheckConfigService = &HealthcheckConfigService{
databases.GetDatabaseService(),
healthcheckConfigRepository,
workspaces_services.GetWorkspaceService(),
audit_logs.GetAuditLogService(),
logger.GetLogger(),
}
var (
healthcheckConfigRepository = &HealthcheckConfigRepository{}
healthcheckConfigService = &HealthcheckConfigService{
databases.GetDatabaseService(),
healthcheckConfigRepository,
workspaces_services.GetWorkspaceService(),
audit_logs.GetAuditLogService(),
logger.GetLogger(),
}
)
var healthcheckConfigController = &HealthcheckConfigController{
healthcheckConfigService,
}

View File

@@ -1,11 +1,12 @@
package healthcheck_config
import (
"databasus-backend/internal/storage"
"errors"
"github.com/google/uuid"
"gorm.io/gorm"
"databasus-backend/internal/storage"
)
type HealthcheckConfigRepository struct{}

View File

@@ -1,15 +1,16 @@
package healthcheck_config
import (
"databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/databases"
users_models "databasus-backend/internal/features/users/models"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"errors"
"fmt"
"log/slog"
"github.com/google/uuid"
"databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/databases"
users_models "databasus-backend/internal/features/users/models"
workspaces_services "databasus-backend/internal/features/workspaces/services"
)
type HealthcheckConfigService struct {

View File

@@ -13,11 +13,11 @@ type Interval struct {
ID uuid.UUID `json:"id" gorm:"primaryKey;type:uuid;default:gen_random_uuid()"`
Interval IntervalType `json:"interval" gorm:"type:text;not null"`
TimeOfDay *string `json:"timeOfDay" gorm:"type:text;"`
TimeOfDay *string `json:"timeOfDay" gorm:"type:text;"`
// only for WEEKLY
Weekday *int `json:"weekday,omitempty" gorm:"type:int"`
Weekday *int `json:"weekday,omitempty" gorm:"type:int"`
// only for MONTHLY
DayOfMonth *int `json:"dayOfMonth,omitempty" gorm:"type:int"`
DayOfMonth *int `json:"dayOfMonth,omitempty" gorm:"type:int"`
// only for CRON
CronExpression *string `json:"cronExpression,omitempty" gorm:"type:text"`
}

View File

@@ -2,13 +2,13 @@ package notifiers
import (
"errors"
users_middleware "databasus-backend/internal/features/users/middleware"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"net/http"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
users_middleware "databasus-backend/internal/features/users/middleware"
workspaces_services "databasus-backend/internal/features/workspaces/services"
)
type NotifierController struct {

View File

@@ -5,6 +5,10 @@ import (
"net/http"
"testing"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"databasus-backend/internal/config"
audit_logs "databasus-backend/internal/features/audit_logs"
discord_notifier "databasus-backend/internal/features/notifiers/models/discord"
@@ -20,10 +24,6 @@ import (
workspaces_controllers "databasus-backend/internal/features/workspaces/controllers"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
test_utils "databasus-backend/internal/util/testing"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
func Test_SaveNewNotifier_NotifierReturnedViaGet(t *testing.T) {
@@ -455,7 +455,7 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
name string
notifierType NotifierType
createNotifier func(workspaceID uuid.UUID) *Notifier
updateNotifier func(workspaceID uuid.UUID, notifierID uuid.UUID) *Notifier
updateNotifier func(workspaceID, notifierID uuid.UUID) *Notifier
verifySensitiveData func(t *testing.T, notifier *Notifier)
verifyHiddenData func(t *testing.T, notifier *Notifier)
}{
@@ -473,7 +473,7 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
},
}
},
updateNotifier: func(workspaceID uuid.UUID, notifierID uuid.UUID) *Notifier {
updateNotifier: func(workspaceID, notifierID uuid.UUID) *Notifier {
return &Notifier{
ID: notifierID,
WorkspaceID: workspaceID,
@@ -515,7 +515,7 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
},
}
},
updateNotifier: func(workspaceID uuid.UUID, notifierID uuid.UUID) *Notifier {
updateNotifier: func(workspaceID, notifierID uuid.UUID) *Notifier {
return &Notifier{
ID: notifierID,
WorkspaceID: workspaceID,
@@ -557,7 +557,7 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
},
}
},
updateNotifier: func(workspaceID uuid.UUID, notifierID uuid.UUID) *Notifier {
updateNotifier: func(workspaceID, notifierID uuid.UUID) *Notifier {
return &Notifier{
ID: notifierID,
WorkspaceID: workspaceID,
@@ -595,7 +595,7 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
},
}
},
updateNotifier: func(workspaceID uuid.UUID, notifierID uuid.UUID) *Notifier {
updateNotifier: func(workspaceID, notifierID uuid.UUID) *Notifier {
return &Notifier{
ID: notifierID,
WorkspaceID: workspaceID,
@@ -636,7 +636,7 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
},
}
},
updateNotifier: func(workspaceID uuid.UUID, notifierID uuid.UUID) *Notifier {
updateNotifier: func(workspaceID, notifierID uuid.UUID) *Notifier {
return &Notifier{
ID: notifierID,
WorkspaceID: workspaceID,
@@ -682,7 +682,7 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
},
}
},
updateNotifier: func(workspaceID uuid.UUID, notifierID uuid.UUID) *Notifier {
updateNotifier: func(workspaceID, notifierID uuid.UUID) *Notifier {
return &Notifier{
ID: notifierID,
WorkspaceID: workspaceID,
@@ -1265,7 +1265,7 @@ func createTelegramNotifier(workspaceID uuid.UUID) *Notifier {
}
}
func verifyNotifierData(t *testing.T, expected *Notifier, actual *Notifier) {
func verifyNotifierData(t *testing.T, expected, actual *Notifier) {
assert.Equal(t, expected.Name, actual.Name)
assert.Equal(t, expected.NotifierType, actual.NotifierType)
assert.Equal(t, expected.WorkspaceID, actual.WorkspaceID)

View File

@@ -10,15 +10,18 @@ import (
"databasus-backend/internal/util/logger"
)
var notifierRepository = &NotifierRepository{}
var notifierService = &NotifierService{
notifierRepository,
logger.GetLogger(),
workspaces_services.GetWorkspaceService(),
audit_logs.GetAuditLogService(),
encryption.GetFieldEncryptor(),
nil,
}
var (
notifierRepository = &NotifierRepository{}
notifierService = &NotifierService{
notifierRepository,
logger.GetLogger(),
workspaces_services.GetWorkspaceService(),
audit_logs.GetAuditLogService(),
encryption.GetFieldEncryptor(),
nil,
}
)
var notifierController = &NotifierController{
notifierService,
workspaces_services.GetWorkspaceService(),

View File

@@ -1,10 +1,11 @@
package notifiers
import (
"databasus-backend/internal/util/encryption"
"log/slog"
"github.com/google/uuid"
"databasus-backend/internal/util/encryption"
)
type NotificationSender interface {

Some files were not shown because too many files have changed in this diff Show More