mirror of
https://github.com/databasus/databasus.git
synced 2026-04-06 00:32:03 +02:00
Compare commits
42 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
49115684a7 | ||
|
|
58ae86ff7a | ||
|
|
82939bb079 | ||
|
|
1697bfbae8 | ||
|
|
205cb1ec02 | ||
|
|
b9668875ef | ||
|
|
ca3f0281a3 | ||
|
|
1b8d783d4e | ||
|
|
75b0477874 | ||
|
|
19533514c2 | ||
|
|
b3c3ef136f | ||
|
|
4a2ada384e | ||
|
|
b4fc0cfb56 | ||
|
|
a8fca1943b | ||
|
|
880b635827 | ||
|
|
67c14cfa89 | ||
|
|
428a87ae84 | ||
|
|
1f1e22e69c | ||
|
|
c325d42b89 | ||
|
|
04a19cead1 | ||
|
|
648c315312 | ||
|
|
3a205c2f1d | ||
|
|
49ebb01ffd | ||
|
|
e957fb67dd | ||
|
|
7cda83122a | ||
|
|
11195d9078 | ||
|
|
64d7a12f9f | ||
|
|
9853ac425a | ||
|
|
6ad38228ce | ||
|
|
7d576b50a9 | ||
|
|
db3bd98425 | ||
|
|
7d8d0846cb | ||
|
|
05540a8d8d | ||
|
|
8250db9ce5 | ||
|
|
1e8cc46672 | ||
|
|
9d30406d83 | ||
|
|
22e9c605da | ||
|
|
60fe0322f1 | ||
|
|
0ab734f947 | ||
|
|
908fe337d4 | ||
|
|
2364b78e18 | ||
|
|
94fe41f66f |
@@ -32,5 +32,5 @@ keywords:
|
||||
- mongodb
|
||||
- mariadb
|
||||
license: Apache-2.0
|
||||
version: 2.16.1
|
||||
date-released: "2025-12-25"
|
||||
version: 2.19.0
|
||||
date-released: "2026-01-02"
|
||||
|
||||
92
Dockerfile
92
Dockerfile
@@ -172,19 +172,23 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
|
||||
|
||||
# ========= Install MongoDB Database Tools =========
|
||||
# Note: MongoDB Database Tools are backward compatible - single version supports all server versions (4.0-8.0)
|
||||
# Use dpkg with apt-get -f install to handle dependencies
|
||||
# Note: For ARM64, we use Ubuntu 22.04 package as MongoDB doesn't provide Debian 12 ARM64 packages
|
||||
RUN apt-get update && \
|
||||
if [ "$TARGETARCH" = "amd64" ]; then \
|
||||
wget -q https://fastdl.mongodb.org/tools/db/mongodb-database-tools-debian12-x86_64-100.10.0.deb -O /tmp/mongodb-database-tools.deb; \
|
||||
elif [ "$TARGETARCH" = "arm64" ]; then \
|
||||
wget -q https://fastdl.mongodb.org/tools/db/mongodb-database-tools-debian12-aarch64-100.10.0.deb -O /tmp/mongodb-database-tools.deb; \
|
||||
wget -q https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-arm64-100.10.0.deb -O /tmp/mongodb-database-tools.deb; \
|
||||
fi && \
|
||||
dpkg -i /tmp/mongodb-database-tools.deb || true && \
|
||||
apt-get install -f -y --no-install-recommends && \
|
||||
rm /tmp/mongodb-database-tools.deb && \
|
||||
dpkg -i /tmp/mongodb-database-tools.deb || apt-get install -f -y --no-install-recommends && \
|
||||
rm -f /tmp/mongodb-database-tools.deb && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
ln -sf /usr/bin/mongodump /usr/local/mongodb-database-tools/bin/mongodump && \
|
||||
ln -sf /usr/bin/mongorestore /usr/local/mongodb-database-tools/bin/mongorestore
|
||||
mkdir -p /usr/local/mongodb-database-tools/bin && \
|
||||
if [ -f /usr/bin/mongodump ]; then \
|
||||
ln -sf /usr/bin/mongodump /usr/local/mongodb-database-tools/bin/mongodump; \
|
||||
fi && \
|
||||
if [ -f /usr/bin/mongorestore ]; then \
|
||||
ln -sf /usr/bin/mongorestore /usr/local/mongodb-database-tools/bin/mongorestore; \
|
||||
fi
|
||||
|
||||
# Create postgres user and set up directories
|
||||
RUN useradd -m -s /bin/bash postgres || true && \
|
||||
@@ -257,24 +261,68 @@ if [ ! -s "/databasus-data/pgdata/PG_VERSION" ]; then
|
||||
echo "max_connections = 100" >> /databasus-data/pgdata/postgresql.conf
|
||||
fi
|
||||
|
||||
# Start PostgreSQL in background
|
||||
echo "Starting PostgreSQL..."
|
||||
gosu postgres \$PG_BIN/postgres -D /databasus-data/pgdata -p 5437 &
|
||||
POSTGRES_PID=\$!
|
||||
# Function to start PostgreSQL and wait for it to be ready
|
||||
start_postgres() {
|
||||
echo "Starting PostgreSQL..."
|
||||
gosu postgres \$PG_BIN/postgres -D /databasus-data/pgdata -p 5437 &
|
||||
POSTGRES_PID=\$!
|
||||
|
||||
echo "Waiting for PostgreSQL to be ready..."
|
||||
for i in {1..30}; do
|
||||
if gosu postgres \$PG_BIN/pg_isready -p 5437 -h localhost >/dev/null 2>&1; then
|
||||
echo "PostgreSQL is ready!"
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Wait for PostgreSQL to be ready
|
||||
echo "Waiting for PostgreSQL to be ready..."
|
||||
for i in {1..30}; do
|
||||
if gosu postgres \$PG_BIN/pg_isready -p 5437 -h localhost >/dev/null 2>&1; then
|
||||
echo "PostgreSQL is ready!"
|
||||
break
|
||||
fi
|
||||
if [ \$i -eq 30 ]; then
|
||||
echo "PostgreSQL failed to start"
|
||||
# Try to start PostgreSQL
|
||||
if ! start_postgres; then
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "PostgreSQL failed to start. Attempting WAL reset recovery..."
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
# Kill any remaining postgres processes
|
||||
pkill -9 postgres 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
# Attempt pg_resetwal to recover from WAL corruption
|
||||
echo "Running pg_resetwal to reset WAL..."
|
||||
if gosu postgres \$PG_BIN/pg_resetwal -f /databasus-data/pgdata; then
|
||||
echo "WAL reset successful. Restarting PostgreSQL..."
|
||||
|
||||
# Try starting PostgreSQL again after WAL reset
|
||||
if start_postgres; then
|
||||
echo "PostgreSQL recovered successfully after WAL reset!"
|
||||
else
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "ERROR: PostgreSQL failed to start even after WAL reset."
|
||||
echo "The database may be severely corrupted."
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " 1. Delete the volume and start fresh (data loss)"
|
||||
echo " 2. Manually inspect /databasus-data/pgdata for issues"
|
||||
echo "=========================================="
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "ERROR: pg_resetwal failed."
|
||||
echo "The database may be severely corrupted."
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " 1. Delete the volume and start fresh (data loss)"
|
||||
echo " 2. Manually inspect /databasus-data/pgdata for issues"
|
||||
echo "=========================================="
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
|
||||
# Create database and set password for postgres user
|
||||
echo "Setting up database and user..."
|
||||
|
||||
40
README.md
40
README.md
@@ -114,7 +114,7 @@ You have four ways to install Databasus:
|
||||
|
||||
## 📦 Installation
|
||||
|
||||
You have three ways to install Databasus: automated script (recommended), simple Docker run, or Docker Compose setup.
|
||||
You have four ways to install Databasus: automated script (recommended), simple Docker run, or Docker Compose setup.
|
||||
|
||||
### Option 1: Automated installation script (recommended, Linux only)
|
||||
|
||||
@@ -245,6 +245,8 @@ This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENS
|
||||
|
||||
Contributions are welcome! Read the <a href="https://databasus.com/contribute">contributing guide</a> for more details, priorities and rules. If you want to contribute but don't know where to start, message me on Telegram [@rostislav_dugin](https://t.me/rostislav_dugin)
|
||||
|
||||
Also you can join our large community of developers, DBAs and DevOps engineers on Telegram [@databasus_community](https://t.me/databasus_community).
|
||||
|
||||
--
|
||||
|
||||
## 📖 Migration guide
|
||||
@@ -271,10 +273,44 @@ Then manually move databases from Postgresus to Databasus.
|
||||
|
||||
### Why was Postgresus renamed to Databasus?
|
||||
|
||||
Databasus has been developed since 2023. It was internal tool to backup production and home projects databases. In start of 2025 it was released as open source project on GitHub. By the end of 2025 it became popular and the time for renaming has come in December 2025.
|
||||
|
||||
It was an important step for the project to grow. Actually, there are a couple of reasons:
|
||||
|
||||
1. Postgresus is no longer a little tool that just adds UI for pg_dump for little projects. It became a tool both for individual users, DevOps, DBAs,teams, companies and even large enterprises. Tens of thousands of users use Postgresus every day. Postgresus grew into a reliable backup management tool. Initial positioning is no longer suitable: the project is not just a UI wrapper, it's a solid backup management system now (despite it's still easy to use).
|
||||
1. Postgresus is no longer a little tool that just adds UI for pg_dump for little projects. It became a tool both for individual users, DevOps, DBAs, teams, companies and even large enterprises. Tens of thousands of users use Postgresus every day. Postgresus grew into a reliable backup management tool. Initial positioning is no longer suitable: the project is not just a UI wrapper, it's a solid backup management system now (despite it's still easy to use).
|
||||
|
||||
2. New databases are supported: although the primary focus is PostgreSQL (with 100% support in the most efficient way) and always will be, Databasus added support for MySQL, MariaDB and MongoDB. Later more databases will be supported.
|
||||
|
||||
3. Trademark issue: "postgres" is a trademark of PostgreSQL Inc. and cannot be used in the project name. So for safety and legal reasons, we had to rename the project.
|
||||
|
||||
## AI disclaimer
|
||||
|
||||
There have been questions about AI usage in project development in issues and discussions. As the project focuses on security, reliability and production usage, it's important to explain how AI is used in the development process.
|
||||
|
||||
AI is used as a helper for:
|
||||
|
||||
- verification of code quality and searching for vulnerabilities
|
||||
- cleaning up and improving documentation, comments and code
|
||||
- assistance during development
|
||||
- double-checking PRs and commits after human review
|
||||
|
||||
AI is not used for:
|
||||
|
||||
- writing entire code
|
||||
- "vibe code" approach
|
||||
- code without line-by-line verification by a human
|
||||
- code without tests
|
||||
|
||||
The project has:
|
||||
|
||||
- solid test coverage (both unit and integration tests)
|
||||
- CI/CD pipeline automation with tests and linting to ensure code quality
|
||||
- verification by experienced developers with experience in large and secure projects
|
||||
|
||||
So AI is just an assistant and a tool for developers to increase productivity and ensure code quality. The work is done by developers.
|
||||
|
||||
Moreover, it's important to note that we do not differentiate between bad human code and AI vibe code. There are strict requirements for any code to be merged to keep the codebase maintainable.
|
||||
|
||||
Even if code is written manually by a human, it's not guaranteed to be merged. Vibe code is not allowed at all and all such PRs are rejected by default (see [contributing guide](https://databasus.com/contribute)).
|
||||
|
||||
We also draw attention to fast issue resolution and security [vulnerability reporting](https://github.com/databasus/databasus?tab=security-ov-file#readme).
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 767 KiB After Width: | Height: | Size: 766 KiB |
BIN
assets/logo-square.png
Normal file
BIN
assets/logo-square.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 3.2 KiB |
12
assets/logo-square.svg
Normal file
12
assets/logo-square.svg
Normal file
@@ -0,0 +1,12 @@
|
||||
<svg width="128" height="128" viewBox="0 0 128 128" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g clip-path="url(#clip0_287_1020)">
|
||||
<path d="M50.1522 115.189C50.1522 121.189 57.1564 121.193 59 118C60.1547 116 61 114 61 108C61 102 58.1044 96.9536 55.3194 91.5175C54.6026 90.1184 53.8323 88.6149 53.0128 86.9234C51.6073 84.0225 49.8868 81.3469 47.3885 79.2139C47.0053 78.8867 46.8935 78.0093 46.9624 77.422C47.2351 75.1036 47.5317 72.7876 47.8283 70.4718C48.3186 66.6436 48.8088 62.8156 49.1909 58.9766C49.459 56.2872 49.4542 53.5119 49.1156 50.8329C48.3833 45.0344 45.1292 40.7783 40.1351 37.9114C38.6818 37.0771 38.2533 36.1455 38.4347 34.5853C38.9402 30.2473 40.6551 26.3306 42.8342 22.6642C44.8356 19.297 47.1037 16.0858 49.3676 12.8804C49.6576 12.4699 49.9475 12.0594 50.2367 11.6488C50.6069 11.1231 51.5231 10.7245 52.1971 10.7075C60.4129 10.5017 68.6303 10.3648 76.8477 10.2636C77.4123 10.2563 78.1584 10.5196 78.5221 10.9246C83.6483 16.634 88.2284 22.712 90.9778 29.9784C91.1658 30.4758 91.3221 30.9869 91.4655 31.4997C92.4976 35.1683 92.4804 35.1803 89.5401 37.2499L89.4071 37.3436C83.8702 41.2433 81.8458 46.8198 82.0921 53.349C82.374 60.8552 84.0622 68.1313 85.9869 75.3539C86.3782 76.8218 86.6318 77.9073 85.2206 79.2609C82.3951 81.9698 81.2196 85.6872 80.6575 89.4687C80.0724 93.4081 79.599 97.3637 79.1254 101.32C78.8627 103.515 78.8497 105.368 78.318 107.904C76.2819 117.611 71 128 63 128H50.1522C45 128 41 123.189 41 115.189H50.1522Z" fill="#155DFC"/>
|
||||
<path d="M46.2429 6.56033C43.3387 11.1 40.3642 15.4031 37.7614 19.9209C35.413 23.9964 33.8487 28.4226 33.0913 33.1211C32.0998 39.2728 33.694 44.7189 38.0765 48.9775C41.6846 52.4835 42.6153 56.4472 42.152 61.1675C41.1426 71.4587 39.1174 81.5401 36.2052 91.4522C36.1769 91.5477 36.0886 91.6255 35.8974 91.8977C34.1517 91.3525 32.3161 90.8446 30.5266 90.2095C5.53011 81.3376 -12.7225 64.953 -24.1842 41.0298C-25.175 38.9625 -26.079 36.8498 -26.9263 34.7202C-27.0875 34.3151 -26.9749 33.5294 -26.6785 33.2531C-17.1479 24.3723 -7.64007 15.4647 2.00468 6.70938C8.64568 0.681612 16.5812 -1.21558 25.2457 0.739942C31.9378 2.24992 38.5131 4.27834 45.1363 6.09048C45.5843 6.2128 45.9998 6.45502 46.2429 6.56033Z" fill="#155DFC"/>
|
||||
<path d="M96.9586 89.3257C95.5888 84.7456 94.0796 80.4011 93.0111 75.9514C91.6065 70.0978 90.4683 64.1753 89.3739 58.2529C88.755 54.9056 89.3998 51.8176 91.89 49.2108C98.2669 42.5358 98.3933 34.7971 95.3312 26.7037C92.7471 19.8739 88.593 13.9904 83.7026 8.60904C83.1298 7.9788 82.5693 7.33641 81.918 6.60491C82.2874 6.40239 82.5709 6.18773 82.8909 6.07999C90.1281 3.64085 97.4495 1.54842 105.041 0.488845C112.781 -0.591795 119.379 1.81818 125.045 6.97592C130.017 11.5018 134.805 16.2327 139.812 20.7188C143.822 24.3115 148.013 27.7066 152.19 31.1073C152.945 31.7205 153.137 32.2154 152.913 33.1041C149.059 48.4591 141.312 61.4883 129.457 71.9877C120.113 80.2626 109.35 85.9785 96.9586 89.3265V89.3257Z" fill="#155DFC"/>
|
||||
</g>
|
||||
<defs>
|
||||
<clipPath id="clip0_287_1020">
|
||||
<rect width="128" height="128" rx="6" fill="white"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.0 KiB |
17
assets/tools/README.md
Normal file
17
assets/tools/README.md
Normal file
@@ -0,0 +1,17 @@
|
||||
We keep binaries here to speed up CI \ CD tasks and building.
|
||||
|
||||
Docker image needs:
|
||||
- PostgreSQL client tools (versions 12-18)
|
||||
- MySQL client tools (versions 5.7, 8.0, 8.4, 9)
|
||||
- MariaDB client tools (versions 10.6, 12.1)
|
||||
- MongoDB Database Tools (latest)
|
||||
|
||||
For the most of tools, we need a couple of binaries for each version. However, if we download them on each run, it will download a couple of GBs each time.
|
||||
|
||||
So, for speed up we keep only required executables (like pg_dump, mysqldump, mariadb-dump, mongodump, etc.).
|
||||
|
||||
It takes:
|
||||
- ~ 100MB for ARM
|
||||
- ~ 100MB for x64
|
||||
|
||||
Instead of GBs. See Dockefile for usage details.
|
||||
3
backend/.gitignore
vendored
3
backend/.gitignore
vendored
@@ -16,4 +16,5 @@ databasus-backend.exe
|
||||
ui/build/*
|
||||
pgdata-for-restore/
|
||||
temp/
|
||||
cmd.exe
|
||||
cmd.exe
|
||||
temp/
|
||||
@@ -238,6 +238,10 @@ func runBackgroundTasks(log *slog.Logger) {
|
||||
go runWithPanicLogging(log, "healthcheck attempt background service", func() {
|
||||
healthcheck_attempt.GetHealthcheckAttemptBackgroundService().Run()
|
||||
})
|
||||
|
||||
go runWithPanicLogging(log, "audit log cleanup background service", func() {
|
||||
audit_logs.GetAuditLogBackgroundService().Run()
|
||||
})
|
||||
}
|
||||
|
||||
func runWithPanicLogging(log *slog.Logger, serviceName string, fn func()) {
|
||||
|
||||
36
backend/internal/features/audit_logs/background_service.go
Normal file
36
backend/internal/features/audit_logs/background_service.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package audit_logs
|
||||
|
||||
import (
|
||||
"databasus-backend/internal/config"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
type AuditLogBackgroundService struct {
|
||||
auditLogService *AuditLogService
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func (s *AuditLogBackgroundService) Run() {
|
||||
s.logger.Info("Starting audit log cleanup background service")
|
||||
|
||||
if config.IsShouldShutdown() {
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
if config.IsShouldShutdown() {
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.cleanOldAuditLogs(); err != nil {
|
||||
s.logger.Error("Failed to clean old audit logs", "error", err)
|
||||
}
|
||||
|
||||
time.Sleep(1 * time.Hour)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *AuditLogBackgroundService) cleanOldAuditLogs() error {
|
||||
return s.auditLogService.CleanOldAuditLogs()
|
||||
}
|
||||
141
backend/internal/features/audit_logs/background_service_test.go
Normal file
141
backend/internal/features/audit_logs/background_service_test.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package audit_logs
|
||||
|
||||
import (
|
||||
"databasus-backend/internal/storage"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
user_enums "databasus-backend/internal/features/users/enums"
|
||||
users_testing "databasus-backend/internal/features/users/testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func Test_CleanOldAuditLogs_DeletesLogsOlderThanOneYear(t *testing.T) {
|
||||
service := GetAuditLogService()
|
||||
user := users_testing.CreateTestUser(user_enums.UserRoleMember)
|
||||
db := storage.GetDb()
|
||||
baseTime := time.Now().UTC()
|
||||
|
||||
// Create old logs (more than 1 year old)
|
||||
createTimedAuditLog(db, &user.UserID, "Old log 1", baseTime.Add(-400*24*time.Hour))
|
||||
createTimedAuditLog(db, &user.UserID, "Old log 2", baseTime.Add(-370*24*time.Hour))
|
||||
|
||||
// Create recent logs (less than 1 year old)
|
||||
createAuditLog(service, "Recent log 1", &user.UserID, nil)
|
||||
createAuditLog(service, "Recent log 2", &user.UserID, nil)
|
||||
|
||||
// Run cleanup
|
||||
err := service.CleanOldAuditLogs()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify old logs were deleted
|
||||
oneYearAgo := baseTime.Add(-365 * 24 * time.Hour)
|
||||
var oldLogs []*AuditLog
|
||||
db.Where("created_at < ?", oneYearAgo).Find(&oldLogs)
|
||||
assert.Equal(t, 0, len(oldLogs), "All logs older than 1 year should be deleted")
|
||||
|
||||
// Verify recent logs still exist
|
||||
var recentLogs []*AuditLog
|
||||
db.Where("created_at >= ?", oneYearAgo).Find(&recentLogs)
|
||||
assert.GreaterOrEqual(t, len(recentLogs), 2, "Recent logs should not be deleted")
|
||||
}
|
||||
|
||||
func Test_CleanOldAuditLogs_PreservesLogsNewerThanOneYear(t *testing.T) {
|
||||
service := GetAuditLogService()
|
||||
user := users_testing.CreateTestUser(user_enums.UserRoleMember)
|
||||
db := storage.GetDb()
|
||||
baseTime := time.Now().UTC()
|
||||
|
||||
// Create logs exactly at boundary (1 year old)
|
||||
boundaryTime := baseTime.Add(-365 * 24 * time.Hour)
|
||||
createTimedAuditLog(db, &user.UserID, "Boundary log", boundaryTime)
|
||||
|
||||
// Create recent logs
|
||||
createTimedAuditLog(db, &user.UserID, "Recent log 1", baseTime.Add(-364*24*time.Hour))
|
||||
createTimedAuditLog(db, &user.UserID, "Recent log 2", baseTime.Add(-100*24*time.Hour))
|
||||
createAuditLog(service, "Current log", &user.UserID, nil)
|
||||
|
||||
// Get count before cleanup
|
||||
var countBefore int64
|
||||
db.Model(&AuditLog{}).Count(&countBefore)
|
||||
|
||||
// Run cleanup
|
||||
err := service.CleanOldAuditLogs()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Get count after cleanup
|
||||
var countAfter int64
|
||||
db.Model(&AuditLog{}).Count(&countAfter)
|
||||
|
||||
// Verify logs newer than 1 year are preserved
|
||||
oneYearAgo := baseTime.Add(-365 * 24 * time.Hour)
|
||||
var recentLogs []*AuditLog
|
||||
db.Where("created_at >= ?", oneYearAgo).Find(&recentLogs)
|
||||
|
||||
messages := make([]string, len(recentLogs))
|
||||
for i, log := range recentLogs {
|
||||
messages[i] = log.Message
|
||||
}
|
||||
|
||||
assert.Contains(t, messages, "Recent log 1")
|
||||
assert.Contains(t, messages, "Recent log 2")
|
||||
assert.Contains(t, messages, "Current log")
|
||||
}
|
||||
|
||||
func Test_CleanOldAuditLogs_HandlesEmptyDatabase(t *testing.T) {
|
||||
service := GetAuditLogService()
|
||||
|
||||
// Run cleanup on database that may have no old logs
|
||||
err := service.CleanOldAuditLogs()
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func Test_CleanOldAuditLogs_DeletesMultipleOldLogs(t *testing.T) {
|
||||
service := GetAuditLogService()
|
||||
user := users_testing.CreateTestUser(user_enums.UserRoleMember)
|
||||
db := storage.GetDb()
|
||||
baseTime := time.Now().UTC()
|
||||
|
||||
// Create many old logs with specific UUIDs to track them
|
||||
testLogIDs := make([]uuid.UUID, 5)
|
||||
for i := 0; i < 5; i++ {
|
||||
testLogIDs[i] = uuid.New()
|
||||
daysAgo := 400 + (i * 10)
|
||||
log := &AuditLog{
|
||||
ID: testLogIDs[i],
|
||||
UserID: &user.UserID,
|
||||
Message: fmt.Sprintf("Test old log %d", i),
|
||||
CreatedAt: baseTime.Add(-time.Duration(daysAgo) * 24 * time.Hour),
|
||||
}
|
||||
result := db.Create(log)
|
||||
assert.NoError(t, result.Error)
|
||||
}
|
||||
|
||||
// Verify logs exist before cleanup
|
||||
var logsBeforeCleanup []*AuditLog
|
||||
db.Where("id IN ?", testLogIDs).Find(&logsBeforeCleanup)
|
||||
assert.Equal(t, 5, len(logsBeforeCleanup), "All test logs should exist before cleanup")
|
||||
|
||||
// Run cleanup
|
||||
err := service.CleanOldAuditLogs()
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify test logs were deleted
|
||||
var logsAfterCleanup []*AuditLog
|
||||
db.Where("id IN ?", testLogIDs).Find(&logsAfterCleanup)
|
||||
assert.Equal(t, 0, len(logsAfterCleanup), "All old test logs should be deleted")
|
||||
}
|
||||
|
||||
func createTimedAuditLog(db *gorm.DB, userID *uuid.UUID, message string, createdAt time.Time) {
|
||||
log := &AuditLog{
|
||||
ID: uuid.New(),
|
||||
UserID: userID,
|
||||
Message: message,
|
||||
CreatedAt: createdAt,
|
||||
}
|
||||
db.Create(log)
|
||||
}
|
||||
@@ -7,11 +7,15 @@ import (
|
||||
|
||||
var auditLogRepository = &AuditLogRepository{}
|
||||
var auditLogService = &AuditLogService{
|
||||
auditLogRepository: auditLogRepository,
|
||||
logger: logger.GetLogger(),
|
||||
auditLogRepository,
|
||||
logger.GetLogger(),
|
||||
}
|
||||
var auditLogController = &AuditLogController{
|
||||
auditLogService: auditLogService,
|
||||
auditLogService,
|
||||
}
|
||||
var auditLogBackgroundService = &AuditLogBackgroundService{
|
||||
auditLogService,
|
||||
logger.GetLogger(),
|
||||
}
|
||||
|
||||
func GetAuditLogService() *AuditLogService {
|
||||
@@ -22,6 +26,10 @@ func GetAuditLogController() *AuditLogController {
|
||||
return auditLogController
|
||||
}
|
||||
|
||||
func GetAuditLogBackgroundService() *AuditLogBackgroundService {
|
||||
return auditLogBackgroundService
|
||||
}
|
||||
|
||||
func SetupDependencies() {
|
||||
users_services.GetUserService().SetAuditLogWriter(auditLogService)
|
||||
users_services.GetSettingsService().SetAuditLogWriter(auditLogService)
|
||||
|
||||
@@ -137,3 +137,15 @@ func (r *AuditLogRepository) CountGlobal(beforeDate *time.Time) (int64, error) {
|
||||
err := query.Count(&count).Error
|
||||
return count, err
|
||||
}
|
||||
|
||||
func (r *AuditLogRepository) DeleteOlderThan(beforeDate time.Time) (int64, error) {
|
||||
result := storage.GetDb().
|
||||
Where("created_at < ?", beforeDate).
|
||||
Delete(&AuditLog{})
|
||||
|
||||
if result.Error != nil {
|
||||
return 0, result.Error
|
||||
}
|
||||
|
||||
return result.RowsAffected, nil
|
||||
}
|
||||
|
||||
@@ -135,3 +135,19 @@ func (s *AuditLogService) GetWorkspaceAuditLogs(
|
||||
Offset: offset,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *AuditLogService) CleanOldAuditLogs() error {
|
||||
oneYearAgo := time.Now().UTC().Add(-365 * 24 * time.Hour)
|
||||
|
||||
deletedCount, err := s.auditLogRepository.DeleteOlderThan(oneYearAgo)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to delete old audit logs", "error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if deletedCount > 0 {
|
||||
s.logger.Info("Deleted old audit logs", "count", deletedCount, "olderThan", oneYearAgo)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
17
backend/internal/features/backups/backups/common/dto.go
Normal file
17
backend/internal/features/backups/backups/common/dto.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package common
|
||||
|
||||
import backups_config "databasus-backend/internal/features/backups/config"
|
||||
|
||||
type BackupType string
|
||||
|
||||
const (
|
||||
BackupTypeDefault BackupType = "DEFAULT" // For MySQL, MongoDB, PostgreSQL legacy (-Fc)
|
||||
BackupTypeDirectory BackupType = "DIRECTORY" // PostgreSQL directory type (-Fd)
|
||||
)
|
||||
|
||||
type BackupMetadata struct {
|
||||
EncryptionSalt *string
|
||||
EncryptionIV *string
|
||||
Encryption backups_config.BackupEncryption
|
||||
Type BackupType
|
||||
}
|
||||
@@ -182,7 +182,7 @@ func (c *BackupController) GetFile(ctx *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
fileReader, dbType, err := c.backupService.GetBackupFile(user, id)
|
||||
fileReader, backup, database, err := c.backupService.GetBackupFile(user, id)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
@@ -193,15 +193,12 @@ func (c *BackupController) GetFile(ctx *gin.Context) {
|
||||
}
|
||||
}()
|
||||
|
||||
extension := ".dump.zst"
|
||||
if dbType == databases.DatabaseTypeMysql {
|
||||
extension = ".sql.zst"
|
||||
}
|
||||
filename := c.generateBackupFilename(backup, database)
|
||||
|
||||
ctx.Header("Content-Type", "application/octet-stream")
|
||||
ctx.Header(
|
||||
"Content-Disposition",
|
||||
fmt.Sprintf("attachment; filename=\"backup_%s%s\"", id.String(), extension),
|
||||
fmt.Sprintf("attachment; filename=\"%s\"", filename),
|
||||
)
|
||||
|
||||
_, err = io.Copy(ctx.Writer, fileReader)
|
||||
@@ -214,3 +211,62 @@ func (c *BackupController) GetFile(ctx *gin.Context) {
|
||||
type MakeBackupRequest struct {
|
||||
DatabaseID uuid.UUID `json:"database_id" binding:"required"`
|
||||
}
|
||||
|
||||
func (c *BackupController) generateBackupFilename(
|
||||
backup *Backup,
|
||||
database *databases.Database,
|
||||
) string {
|
||||
// Format timestamp as YYYY-MM-DD_HH-mm-ss
|
||||
timestamp := backup.CreatedAt.Format("2006-01-02_15-04-05")
|
||||
|
||||
// Sanitize database name for filename (replace spaces and special chars)
|
||||
safeName := sanitizeFilename(database.Name)
|
||||
|
||||
// Determine extension based on database type
|
||||
extension := c.getBackupExtension(database.Type)
|
||||
|
||||
return fmt.Sprintf("%s_backup_%s%s", safeName, timestamp, extension)
|
||||
}
|
||||
|
||||
func (c *BackupController) getBackupExtension(
|
||||
dbType databases.DatabaseType,
|
||||
) string {
|
||||
switch dbType {
|
||||
case databases.DatabaseTypeMysql, databases.DatabaseTypeMariadb:
|
||||
return ".sql.zst"
|
||||
case databases.DatabaseTypePostgres:
|
||||
// PostgreSQL custom format
|
||||
return ".dump"
|
||||
case databases.DatabaseTypeMongodb:
|
||||
return ".archive"
|
||||
default:
|
||||
return ".backup"
|
||||
}
|
||||
}
|
||||
|
||||
func sanitizeFilename(name string) string {
|
||||
// Replace characters that are invalid in filenames
|
||||
replacer := map[rune]rune{
|
||||
' ': '_',
|
||||
'/': '-',
|
||||
'\\': '-',
|
||||
':': '-',
|
||||
'*': '-',
|
||||
'?': '-',
|
||||
'"': '-',
|
||||
'<': '-',
|
||||
'>': '-',
|
||||
'|': '-',
|
||||
}
|
||||
|
||||
result := make([]rune, 0, len(name))
|
||||
for _, char := range name {
|
||||
if replacement, exists := replacer[char]; exists {
|
||||
result = append(result, replacement)
|
||||
} else {
|
||||
result = append(result, char)
|
||||
}
|
||||
}
|
||||
|
||||
return string(result)
|
||||
}
|
||||
|
||||
@@ -494,6 +494,112 @@ func Test_DownloadBackup_AuditLogWritten(t *testing.T) {
|
||||
assert.True(t, found, "Audit log for backup download not found")
|
||||
}
|
||||
|
||||
func Test_DownloadBackup_ProperFilenameForPostgreSQL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
databaseName string
|
||||
expectedExt string
|
||||
expectedInName string
|
||||
}{
|
||||
{
|
||||
name: "PostgreSQL database",
|
||||
databaseName: "my_postgres_db",
|
||||
expectedExt: ".dump",
|
||||
expectedInName: "my_postgres_db_backup_",
|
||||
},
|
||||
{
|
||||
name: "Database name with spaces",
|
||||
databaseName: "my test db",
|
||||
expectedExt: ".dump",
|
||||
expectedInName: "my_test_db_backup_",
|
||||
},
|
||||
{
|
||||
name: "Database name with special characters",
|
||||
databaseName: "my:db/test",
|
||||
expectedExt: ".dump",
|
||||
expectedInName: "my-db-test_backup_",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
|
||||
database := createTestDatabase(tt.databaseName, workspace.ID, owner.Token, router)
|
||||
storage := createTestStorage(workspace.ID)
|
||||
|
||||
configService := backups_config.GetBackupConfigService()
|
||||
config, err := configService.GetBackupConfigByDbId(database.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
config.IsBackupsEnabled = true
|
||||
config.StorageID = &storage.ID
|
||||
config.Storage = storage
|
||||
_, err = configService.SaveBackupConfig(config)
|
||||
assert.NoError(t, err)
|
||||
|
||||
backup := createTestBackup(database, owner)
|
||||
|
||||
resp := test_utils.MakeGetRequest(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/backups/%s/file", backup.ID.String()),
|
||||
"Bearer "+owner.Token,
|
||||
http.StatusOK,
|
||||
)
|
||||
|
||||
contentDisposition := resp.Headers.Get("Content-Disposition")
|
||||
assert.NotEmpty(t, contentDisposition, "Content-Disposition header should be present")
|
||||
|
||||
// Verify the filename contains expected parts
|
||||
assert.Contains(
|
||||
t,
|
||||
contentDisposition,
|
||||
tt.expectedInName,
|
||||
"Filename should contain sanitized database name",
|
||||
)
|
||||
assert.Contains(
|
||||
t,
|
||||
contentDisposition,
|
||||
tt.expectedExt,
|
||||
"Filename should have correct extension",
|
||||
)
|
||||
assert.Contains(t, contentDisposition, "attachment", "Should be an attachment")
|
||||
|
||||
// Verify timestamp format (YYYY-MM-DD_HH-mm-ss)
|
||||
assert.Regexp(
|
||||
t,
|
||||
`\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}`,
|
||||
contentDisposition,
|
||||
"Filename should contain timestamp",
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_SanitizeFilename(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{input: "simple_name", expected: "simple_name"},
|
||||
{input: "name with spaces", expected: "name_with_spaces"},
|
||||
{input: "name/with\\slashes", expected: "name-with-slashes"},
|
||||
{input: "name:with*special?chars", expected: "name-with-special-chars"},
|
||||
{input: "name<with>pipes|", expected: "name-with-pipes-"},
|
||||
{input: `name"with"quotes`, expected: "name-with-quotes"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
result := sanitizeFilename(tt.input)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_CancelBackup_InProgressBackup_SuccessfullyCancelled(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
@@ -585,6 +691,7 @@ func createTestDatabase(
|
||||
Username: "postgres",
|
||||
Password: "postgres",
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ package backups
|
||||
import (
|
||||
"context"
|
||||
|
||||
usecases_common "databasus-backend/internal/features/backups/backups/usecases/common"
|
||||
usecases_common "databasus-backend/internal/features/backups/backups/common"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/notifiers"
|
||||
|
||||
@@ -502,19 +502,19 @@ func (s *BackupService) CancelBackup(
|
||||
func (s *BackupService) GetBackupFile(
|
||||
user *users_models.User,
|
||||
backupID uuid.UUID,
|
||||
) (io.ReadCloser, databases.DatabaseType, error) {
|
||||
) (io.ReadCloser, *Backup, *databases.Database, error) {
|
||||
backup, err := s.backupRepository.FindByID(backupID)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
database, err := s.databaseService.GetDatabaseByID(backup.DatabaseID)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
if database.WorkspaceID == nil {
|
||||
return nil, "", errors.New("cannot download backup for database without workspace")
|
||||
return nil, nil, nil, errors.New("cannot download backup for database without workspace")
|
||||
}
|
||||
|
||||
canAccess, _, err := s.workspaceService.CanUserAccessWorkspace(
|
||||
@@ -522,10 +522,12 @@ func (s *BackupService) GetBackupFile(
|
||||
user,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
if !canAccess {
|
||||
return nil, "", errors.New("insufficient permissions to download backup for this database")
|
||||
return nil, nil, nil, errors.New(
|
||||
"insufficient permissions to download backup for this database",
|
||||
)
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
@@ -540,10 +542,10 @@ func (s *BackupService) GetBackupFile(
|
||||
|
||||
reader, err := s.getBackupReader(backupID)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
return reader, database.Type, nil
|
||||
return reader, backup, database, nil
|
||||
}
|
||||
|
||||
func (s *BackupService) deleteBackup(backup *Backup) error {
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"databasus-backend/internal/features/backups/backups/usecases/common"
|
||||
common "databasus-backend/internal/features/backups/backups/common"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
encryption_secrets "databasus-backend/internal/features/encryption/secrets"
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
package common
|
||||
|
||||
import backups_config "databasus-backend/internal/features/backups/config"
|
||||
|
||||
type BackupMetadata struct {
|
||||
EncryptionSalt *string
|
||||
EncryptionIV *string
|
||||
Encryption backups_config.BackupEncryption
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
usecases_common "databasus-backend/internal/features/backups/backups/usecases/common"
|
||||
common "databasus-backend/internal/features/backups/backups/common"
|
||||
usecases_mariadb "databasus-backend/internal/features/backups/backups/usecases/mariadb"
|
||||
usecases_mongodb "databasus-backend/internal/features/backups/backups/usecases/mongodb"
|
||||
usecases_mysql "databasus-backend/internal/features/backups/backups/usecases/mysql"
|
||||
@@ -30,7 +30,7 @@ func (uc *CreateBackupUsecase) Execute(
|
||||
database *databases.Database,
|
||||
storage *storages.Storage,
|
||||
backupProgressListener func(completedMBs float64),
|
||||
) (*usecases_common.BackupMetadata, error) {
|
||||
) (*common.BackupMetadata, error) {
|
||||
switch database.Type {
|
||||
case databases.DatabaseTypePostgres:
|
||||
return uc.CreatePostgresqlBackupUsecase.Execute(
|
||||
|
||||
@@ -18,8 +18,8 @@ import (
|
||||
"github.com/klauspost/compress/zstd"
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
common "databasus-backend/internal/features/backups/backups/common"
|
||||
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
|
||||
usecases_common "databasus-backend/internal/features/backups/backups/usecases/common"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
mariadbtypes "databasus-backend/internal/features/databases/databases/mariadb"
|
||||
@@ -57,7 +57,7 @@ func (uc *CreateMariadbBackupUsecase) Execute(
|
||||
db *databases.Database,
|
||||
storage *storages.Storage,
|
||||
backupProgressListener func(completedMBs float64),
|
||||
) (*usecases_common.BackupMetadata, error) {
|
||||
) (*common.BackupMetadata, error) {
|
||||
uc.logger.Info(
|
||||
"Creating MariaDB backup via mariadb-dump",
|
||||
"databaseId", db.ID,
|
||||
@@ -140,7 +140,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
|
||||
storage *storages.Storage,
|
||||
backupProgressListener func(completedMBs float64),
|
||||
mdbConfig *mariadbtypes.MariadbDatabase,
|
||||
) (*usecases_common.BackupMetadata, error) {
|
||||
) (*common.BackupMetadata, error) {
|
||||
uc.logger.Info("Streaming MariaDB backup to storage", "mariadbBin", mariadbBin)
|
||||
|
||||
ctx, cancel := uc.createBackupContext(parentCtx)
|
||||
@@ -196,7 +196,7 @@ func (uc *CreateMariadbBackupUsecase) streamToStorage(
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create zstd writer: %w", err)
|
||||
}
|
||||
countingWriter := usecases_common.NewCountingWriter(zstdWriter)
|
||||
countingWriter := common.NewCountingWriter(zstdWriter)
|
||||
|
||||
saveErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
@@ -264,7 +264,7 @@ func (uc *CreateMariadbBackupUsecase) createTempMyCnfFile(
|
||||
mdbConfig *mariadbtypes.MariadbDatabase,
|
||||
password string,
|
||||
) (string, error) {
|
||||
tempDir, err := os.MkdirTemp("", "mycnf")
|
||||
tempDir, err := os.MkdirTemp(config.GetEnv().TempFolder, "mycnf_"+uuid.New().String())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp directory: %w", err)
|
||||
}
|
||||
@@ -401,8 +401,8 @@ func (uc *CreateMariadbBackupUsecase) setupBackupEncryption(
|
||||
backupID uuid.UUID,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
storageWriter io.WriteCloser,
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, usecases_common.BackupMetadata, error) {
|
||||
metadata := usecases_common.BackupMetadata{}
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, common.BackupMetadata, error) {
|
||||
metadata := common.BackupMetadata{}
|
||||
|
||||
if backupConfig.Encryption != backups_config.BackupEncryptionEncrypted {
|
||||
metadata.Encryption = backups_config.BackupEncryptionNone
|
||||
|
||||
@@ -15,8 +15,8 @@ import (
|
||||
"github.com/google/uuid"
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
common "databasus-backend/internal/features/backups/backups/common"
|
||||
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
|
||||
usecases_common "databasus-backend/internal/features/backups/backups/usecases/common"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
mongodbtypes "databasus-backend/internal/features/databases/databases/mongodb"
|
||||
@@ -51,7 +51,7 @@ func (uc *CreateMongodbBackupUsecase) Execute(
|
||||
db *databases.Database,
|
||||
storage *storages.Storage,
|
||||
backupProgressListener func(completedMBs float64),
|
||||
) (*usecases_common.BackupMetadata, error) {
|
||||
) (*common.BackupMetadata, error) {
|
||||
uc.logger.Info(
|
||||
"Creating MongoDB backup via mongodump",
|
||||
"databaseId", db.ID,
|
||||
@@ -106,6 +106,13 @@ func (uc *CreateMongodbBackupUsecase) buildMongodumpArgs(
|
||||
"--gzip",
|
||||
}
|
||||
|
||||
// Use numParallelCollections based on CPU count
|
||||
// Cap between 1 and 16 to balance performance and resource usage
|
||||
parallelCollections := max(1, min(mdb.CpuCount, 16))
|
||||
if parallelCollections > 1 {
|
||||
args = append(args, "--numParallelCollections="+fmt.Sprintf("%d", parallelCollections))
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
@@ -117,7 +124,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
|
||||
args []string,
|
||||
storage *storages.Storage,
|
||||
backupProgressListener func(completedMBs float64),
|
||||
) (*usecases_common.BackupMetadata, error) {
|
||||
) (*common.BackupMetadata, error) {
|
||||
uc.logger.Info("Streaming MongoDB backup to storage", "mongodumpBin", mongodumpBin)
|
||||
|
||||
ctx, cancel := uc.createBackupContext(parentCtx)
|
||||
@@ -168,7 +175,7 @@ func (uc *CreateMongodbBackupUsecase) streamToStorage(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
countingWriter := usecases_common.NewCountingWriter(finalWriter)
|
||||
countingWriter := common.NewCountingWriter(finalWriter)
|
||||
|
||||
saveErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
@@ -257,8 +264,8 @@ func (uc *CreateMongodbBackupUsecase) setupBackupEncryption(
|
||||
backupID uuid.UUID,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
storageWriter io.WriteCloser,
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, usecases_common.BackupMetadata, error) {
|
||||
backupMetadata := usecases_common.BackupMetadata{
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, common.BackupMetadata, error) {
|
||||
backupMetadata := common.BackupMetadata{
|
||||
Encryption: backups_config.BackupEncryptionNone,
|
||||
}
|
||||
|
||||
|
||||
@@ -18,8 +18,8 @@ import (
|
||||
"github.com/klauspost/compress/zstd"
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
common "databasus-backend/internal/features/backups/backups/common"
|
||||
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
|
||||
usecases_common "databasus-backend/internal/features/backups/backups/usecases/common"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
mysqltypes "databasus-backend/internal/features/databases/databases/mysql"
|
||||
@@ -57,7 +57,7 @@ func (uc *CreateMysqlBackupUsecase) Execute(
|
||||
db *databases.Database,
|
||||
storage *storages.Storage,
|
||||
backupProgressListener func(completedMBs float64),
|
||||
) (*usecases_common.BackupMetadata, error) {
|
||||
) (*common.BackupMetadata, error) {
|
||||
uc.logger.Info(
|
||||
"Creating MySQL backup via mysqldump",
|
||||
"databaseId", db.ID,
|
||||
@@ -155,7 +155,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
|
||||
storage *storages.Storage,
|
||||
backupProgressListener func(completedMBs float64),
|
||||
myConfig *mysqltypes.MysqlDatabase,
|
||||
) (*usecases_common.BackupMetadata, error) {
|
||||
) (*common.BackupMetadata, error) {
|
||||
uc.logger.Info("Streaming MySQL backup to storage", "mysqlBin", mysqlBin)
|
||||
|
||||
ctx, cancel := uc.createBackupContext(parentCtx)
|
||||
@@ -211,7 +211,7 @@ func (uc *CreateMysqlBackupUsecase) streamToStorage(
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create zstd writer: %w", err)
|
||||
}
|
||||
countingWriter := usecases_common.NewCountingWriter(zstdWriter)
|
||||
countingWriter := common.NewCountingWriter(zstdWriter)
|
||||
|
||||
saveErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
@@ -279,7 +279,7 @@ func (uc *CreateMysqlBackupUsecase) createTempMyCnfFile(
|
||||
myConfig *mysqltypes.MysqlDatabase,
|
||||
password string,
|
||||
) (string, error) {
|
||||
tempDir, err := os.MkdirTemp("", "mycnf")
|
||||
tempDir, err := os.MkdirTemp(config.GetEnv().TempFolder, "mycnf_"+uuid.New().String())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp directory: %w", err)
|
||||
}
|
||||
@@ -414,8 +414,8 @@ func (uc *CreateMysqlBackupUsecase) setupBackupEncryption(
|
||||
backupID uuid.UUID,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
storageWriter io.WriteCloser,
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, usecases_common.BackupMetadata, error) {
|
||||
metadata := usecases_common.BackupMetadata{}
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, common.BackupMetadata, error) {
|
||||
metadata := common.BackupMetadata{}
|
||||
|
||||
if backupConfig.Encryption != backups_config.BackupEncryptionEncrypted {
|
||||
metadata.Encryption = backups_config.BackupEncryptionNone
|
||||
|
||||
@@ -15,8 +15,8 @@ import (
|
||||
"time"
|
||||
|
||||
"databasus-backend/internal/config"
|
||||
common "databasus-backend/internal/features/backups/backups/common"
|
||||
backup_encryption "databasus-backend/internal/features/backups/backups/encryption"
|
||||
usecases_common "databasus-backend/internal/features/backups/backups/usecases/common"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
pgtypes "databasus-backend/internal/features/databases/databases/postgresql"
|
||||
@@ -60,7 +60,7 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(
|
||||
backupProgressListener func(
|
||||
completedMBs float64,
|
||||
),
|
||||
) (*usecases_common.BackupMetadata, error) {
|
||||
) (*common.BackupMetadata, error) {
|
||||
uc.logger.Info(
|
||||
"Creating PostgreSQL backup via pg_dump custom format",
|
||||
"databaseId",
|
||||
@@ -119,7 +119,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
|
||||
storage *storages.Storage,
|
||||
db *databases.Database,
|
||||
backupProgressListener func(completedMBs float64),
|
||||
) (*usecases_common.BackupMetadata, error) {
|
||||
) (*common.BackupMetadata, error) {
|
||||
uc.logger.Info("Streaming PostgreSQL backup to storage", "pgBin", pgBin, "args", args)
|
||||
|
||||
ctx, cancel := uc.createBackupContext(parentCtx)
|
||||
@@ -139,7 +139,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
|
||||
cmd := exec.CommandContext(ctx, pgBin, args...)
|
||||
uc.logger.Info("Executing PostgreSQL backup command", "command", cmd.String())
|
||||
|
||||
if err := uc.setupPgEnvironment(cmd, pgpassFile, db.Postgresql.IsHttps, password, backupConfig.CpuCount, pgBin); err != nil {
|
||||
if err := uc.setupPgEnvironment(cmd, pgpassFile, db.Postgresql.IsHttps, password, db.Postgresql.CpuCount, pgBin); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -171,7 +171,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
countingWriter := usecases_common.NewCountingWriter(finalWriter)
|
||||
countingWriter := common.NewCountingWriter(finalWriter)
|
||||
|
||||
// The backup ID becomes the object key / filename in storage
|
||||
|
||||
@@ -335,6 +335,11 @@ func (uc *CreatePostgresqlBackupUsecase) buildPgDumpArgs(pg *pgtypes.PostgresqlD
|
||||
"--verbose",
|
||||
}
|
||||
|
||||
// Add parallel jobs based on CPU count
|
||||
if pg.CpuCount > 1 {
|
||||
args = append(args, "-j", strconv.Itoa(pg.CpuCount))
|
||||
}
|
||||
|
||||
for _, schema := range pg.IncludeSchemas {
|
||||
args = append(args, "-n", schema)
|
||||
}
|
||||
@@ -471,8 +476,8 @@ func (uc *CreatePostgresqlBackupUsecase) setupBackupEncryption(
|
||||
backupID uuid.UUID,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
storageWriter io.WriteCloser,
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, usecases_common.BackupMetadata, error) {
|
||||
metadata := usecases_common.BackupMetadata{}
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, common.BackupMetadata, error) {
|
||||
metadata := common.BackupMetadata{}
|
||||
|
||||
if backupConfig.Encryption != backups_config.BackupEncryptionEncrypted {
|
||||
metadata.Encryption = backups_config.BackupEncryptionNone
|
||||
|
||||
@@ -109,7 +109,6 @@ func Test_SaveBackupConfig_PermissionsEnforced(t *testing.T) {
|
||||
SendNotificationsOn: []BackupNotificationType{
|
||||
NotificationBackupFailed,
|
||||
},
|
||||
CpuCount: 2,
|
||||
IsRetryIfFailed: true,
|
||||
MaxFailedTriesCount: 3,
|
||||
}
|
||||
@@ -129,7 +128,6 @@ func Test_SaveBackupConfig_PermissionsEnforced(t *testing.T) {
|
||||
assert.Equal(t, database.ID, response.DatabaseID)
|
||||
assert.True(t, response.IsBackupsEnabled)
|
||||
assert.Equal(t, period.PeriodWeek, response.StorePeriod)
|
||||
assert.Equal(t, 2, response.CpuCount)
|
||||
} else {
|
||||
assert.Contains(t, string(testResp.Body), "insufficient permissions")
|
||||
}
|
||||
@@ -158,7 +156,6 @@ func Test_SaveBackupConfig_WhenUserIsNotWorkspaceMember_ReturnsForbidden(t *test
|
||||
SendNotificationsOn: []BackupNotificationType{
|
||||
NotificationBackupFailed,
|
||||
},
|
||||
CpuCount: 2,
|
||||
IsRetryIfFailed: true,
|
||||
MaxFailedTriesCount: 3,
|
||||
}
|
||||
@@ -290,7 +287,6 @@ func Test_GetBackupConfigByDbID_ReturnsDefaultConfigForNewDatabase(t *testing.T)
|
||||
assert.Equal(t, database.ID, response.DatabaseID)
|
||||
assert.False(t, response.IsBackupsEnabled)
|
||||
assert.Equal(t, period.PeriodWeek, response.StorePeriod)
|
||||
assert.Equal(t, 1, response.CpuCount)
|
||||
assert.True(t, response.IsRetryIfFailed)
|
||||
assert.Equal(t, 3, response.MaxFailedTriesCount)
|
||||
assert.NotNil(t, response.BackupInterval)
|
||||
@@ -387,7 +383,6 @@ func Test_SaveBackupConfig_WithEncryptionNone_ConfigSaved(t *testing.T) {
|
||||
SendNotificationsOn: []BackupNotificationType{
|
||||
NotificationBackupFailed,
|
||||
},
|
||||
CpuCount: 2,
|
||||
IsRetryIfFailed: true,
|
||||
MaxFailedTriesCount: 3,
|
||||
Encryption: BackupEncryptionNone,
|
||||
@@ -427,7 +422,6 @@ func Test_SaveBackupConfig_WithEncryptionEncrypted_ConfigSaved(t *testing.T) {
|
||||
SendNotificationsOn: []BackupNotificationType{
|
||||
NotificationBackupFailed,
|
||||
},
|
||||
CpuCount: 2,
|
||||
IsRetryIfFailed: true,
|
||||
MaxFailedTriesCount: 3,
|
||||
Encryption: BackupEncryptionEncrypted,
|
||||
@@ -466,6 +460,7 @@ func createTestDatabaseViaAPI(
|
||||
Username: "postgres",
|
||||
Password: "postgres",
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -30,8 +30,6 @@ type BackupConfig struct {
|
||||
IsRetryIfFailed bool `json:"isRetryIfFailed" gorm:"column:is_retry_if_failed;type:boolean;not null"`
|
||||
MaxFailedTriesCount int `json:"maxFailedTriesCount" gorm:"column:max_failed_tries_count;type:int;not null"`
|
||||
|
||||
CpuCount int `json:"cpuCount" gorm:"type:int;not null"`
|
||||
|
||||
Encryption BackupEncryption `json:"encryption" gorm:"column:encryption;type:text;not null;default:'NONE'"`
|
||||
}
|
||||
|
||||
@@ -82,10 +80,6 @@ func (b *BackupConfig) Validate() error {
|
||||
return errors.New("store period is required")
|
||||
}
|
||||
|
||||
if b.CpuCount == 0 {
|
||||
return errors.New("cpu count is required")
|
||||
}
|
||||
|
||||
if b.IsRetryIfFailed && b.MaxFailedTriesCount <= 0 {
|
||||
return errors.New("max failed tries count must be greater than 0")
|
||||
}
|
||||
@@ -109,7 +103,6 @@ func (b *BackupConfig) Copy(newDatabaseID uuid.UUID) *BackupConfig {
|
||||
SendNotificationsOn: b.SendNotificationsOn,
|
||||
IsRetryIfFailed: b.IsRetryIfFailed,
|
||||
MaxFailedTriesCount: b.MaxFailedTriesCount,
|
||||
CpuCount: b.CpuCount,
|
||||
Encryption: b.Encryption,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,7 +168,6 @@ func (s *BackupConfigService) initializeDefaultConfig(
|
||||
NotificationBackupFailed,
|
||||
NotificationBackupSuccess,
|
||||
},
|
||||
CpuCount: 1,
|
||||
IsRetryIfFailed: true,
|
||||
MaxFailedTriesCount: 3,
|
||||
Encryption: BackupEncryptionNone,
|
||||
|
||||
@@ -28,7 +28,6 @@ func EnableBackupsForTestDatabase(
|
||||
NotificationBackupFailed,
|
||||
NotificationBackupSuccess,
|
||||
},
|
||||
CpuCount: 1,
|
||||
}
|
||||
|
||||
backupConfig, err := GetBackupConfigService().SaveBackupConfig(backupConfig)
|
||||
|
||||
@@ -100,6 +100,7 @@ func Test_CreateDatabase_PermissionsEnforced(t *testing.T) {
|
||||
Username: "postgres",
|
||||
Password: "postgres",
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -143,6 +144,7 @@ func Test_CreateDatabase_WhenUserIsNotWorkspaceMember_ReturnsForbidden(t *testin
|
||||
Username: "postgres",
|
||||
Password: "postgres",
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -747,6 +749,7 @@ func createTestDatabaseViaAPI(
|
||||
Username: "postgres",
|
||||
Password: "postgres",
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -790,6 +793,7 @@ func Test_CreateDatabase_PasswordIsEncryptedInDB(t *testing.T) {
|
||||
Username: "postgres",
|
||||
Password: plainPassword,
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -862,6 +866,7 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
Username: "postgres",
|
||||
Password: "original-password-secret",
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
},
|
||||
@@ -879,6 +884,7 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
Username: "updated_user",
|
||||
Password: "",
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
},
|
||||
@@ -961,6 +967,7 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
Database: "test_db",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
},
|
||||
@@ -979,6 +986,7 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
Database: "updated_test_db",
|
||||
AuthDatabase: "admin",
|
||||
IsHttps: false,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
@@ -30,6 +30,7 @@ type MongodbDatabase struct {
|
||||
Database string `json:"database" gorm:"type:text;not null"`
|
||||
AuthDatabase string `json:"authDatabase" gorm:"type:text;not null;default:'admin'"`
|
||||
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
|
||||
CpuCount int `json:"cpuCount" gorm:"column:cpu_count;type:int;not null;default:1"`
|
||||
}
|
||||
|
||||
func (m *MongodbDatabase) TableName() string {
|
||||
@@ -52,6 +53,9 @@ func (m *MongodbDatabase) Validate() error {
|
||||
if m.Database == "" {
|
||||
return errors.New("database is required")
|
||||
}
|
||||
if m.CpuCount <= 0 {
|
||||
return errors.New("cpu count must be greater than 0")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -109,6 +113,7 @@ func (m *MongodbDatabase) Update(incoming *MongodbDatabase) {
|
||||
m.Database = incoming.Database
|
||||
m.AuthDatabase = incoming.AuthDatabase
|
||||
m.IsHttps = incoming.IsHttps
|
||||
m.CpuCount = incoming.CpuCount
|
||||
|
||||
if incoming.Password != "" {
|
||||
m.Password = incoming.Password
|
||||
|
||||
@@ -34,6 +34,7 @@ type PostgresqlDatabase struct {
|
||||
// backup settings
|
||||
IncludeSchemas []string `json:"includeSchemas" gorm:"-"`
|
||||
IncludeSchemasString string `json:"-" gorm:"column:include_schemas;type:text;not null;default:''"`
|
||||
CpuCount int `json:"cpuCount" gorm:"column:cpu_count;type:int;not null;default:1"`
|
||||
|
||||
// restore settings (not saved to DB)
|
||||
IsExcludeExtensions bool `json:"isExcludeExtensions" gorm:"-"`
|
||||
@@ -80,6 +81,10 @@ func (p *PostgresqlDatabase) Validate() error {
|
||||
return errors.New("password is required")
|
||||
}
|
||||
|
||||
if p.CpuCount <= 0 {
|
||||
return errors.New("cpu count must be greater than 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -110,6 +115,7 @@ func (p *PostgresqlDatabase) Update(incoming *PostgresqlDatabase) {
|
||||
p.Database = incoming.Database
|
||||
p.IsHttps = incoming.IsHttps
|
||||
p.IncludeSchemas = incoming.IncludeSchemas
|
||||
p.CpuCount = incoming.CpuCount
|
||||
|
||||
if incoming.Password != "" {
|
||||
p.Password = incoming.Password
|
||||
|
||||
@@ -396,15 +396,17 @@ func (s *DatabaseService) CopyDatabase(
|
||||
case DatabaseTypePostgres:
|
||||
if existingDatabase.Postgresql != nil {
|
||||
newDatabase.Postgresql = &postgresql.PostgresqlDatabase{
|
||||
ID: uuid.Nil,
|
||||
DatabaseID: nil,
|
||||
Version: existingDatabase.Postgresql.Version,
|
||||
Host: existingDatabase.Postgresql.Host,
|
||||
Port: existingDatabase.Postgresql.Port,
|
||||
Username: existingDatabase.Postgresql.Username,
|
||||
Password: existingDatabase.Postgresql.Password,
|
||||
Database: existingDatabase.Postgresql.Database,
|
||||
IsHttps: existingDatabase.Postgresql.IsHttps,
|
||||
ID: uuid.Nil,
|
||||
DatabaseID: nil,
|
||||
Version: existingDatabase.Postgresql.Version,
|
||||
Host: existingDatabase.Postgresql.Host,
|
||||
Port: existingDatabase.Postgresql.Port,
|
||||
Username: existingDatabase.Postgresql.Username,
|
||||
Password: existingDatabase.Postgresql.Password,
|
||||
Database: existingDatabase.Postgresql.Database,
|
||||
IsHttps: existingDatabase.Postgresql.IsHttps,
|
||||
IncludeSchemas: existingDatabase.Postgresql.IncludeSchemas,
|
||||
CpuCount: existingDatabase.Postgresql.CpuCount,
|
||||
}
|
||||
}
|
||||
case DatabaseTypeMysql:
|
||||
@@ -448,6 +450,7 @@ func (s *DatabaseService) CopyDatabase(
|
||||
Database: existingDatabase.Mongodb.Database,
|
||||
AuthDatabase: existingDatabase.Mongodb.AuthDatabase,
|
||||
IsHttps: existingDatabase.Mongodb.IsHttps,
|
||||
CpuCount: existingDatabase.Mongodb.CpuCount,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ func CreateTestDatabase(
|
||||
Port: 5432,
|
||||
Username: "postgres",
|
||||
Password: "postgres",
|
||||
CpuCount: 1,
|
||||
},
|
||||
|
||||
Notifiers: []notifiers.Notifier{
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package disk
|
||||
|
||||
import (
|
||||
"databasus-backend/internal/config"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
|
||||
"github.com/shirou/gopsutil/v4/disk"
|
||||
@@ -12,10 +14,14 @@ type DiskService struct{}
|
||||
func (s *DiskService) GetDiskUsage() (*DiskUsage, error) {
|
||||
platform := s.detectPlatform()
|
||||
|
||||
// Set path based on platform
|
||||
path := "/"
|
||||
var path string
|
||||
|
||||
if platform == PlatformWindows {
|
||||
path = "C:\\"
|
||||
} else {
|
||||
// Use databasus-data folder location for Linux (Docker)
|
||||
cfg := config.GetEnv()
|
||||
path = filepath.Dir(cfg.DataFolder) // Gets /databasus-data from /databasus-data/backups
|
||||
}
|
||||
|
||||
diskUsage, err := disk.Usage(path)
|
||||
|
||||
@@ -217,6 +217,7 @@ func createTestDatabaseViaAPI(
|
||||
Username: "postgres",
|
||||
Password: "postgres",
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -305,6 +305,7 @@ func createTestDatabaseViaAPI(
|
||||
Username: "postgres",
|
||||
Password: "postgres",
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -250,6 +250,44 @@ func Test_RestoreBackup_AuditLogWritten(t *testing.T) {
|
||||
assert.True(t, found, "Audit log for restore not found")
|
||||
}
|
||||
|
||||
func Test_RestoreBackup_InsufficientDiskSpace_ReturnsError(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
|
||||
_, backup := createTestDatabaseWithBackupForRestore(workspace, owner, router)
|
||||
|
||||
// Update backup size to 10 TB via repository
|
||||
repo := &backups.BackupRepository{}
|
||||
backup.BackupSizeMb = 10485760.0 // 10 TB in MB
|
||||
err := repo.Save(backup)
|
||||
assert.NoError(t, err)
|
||||
|
||||
request := RestoreBackupRequest{
|
||||
PostgresqlDatabase: &postgresql.PostgresqlDatabase{
|
||||
Version: tools.PostgresqlVersion16,
|
||||
Host: "localhost",
|
||||
Port: 5432,
|
||||
Username: "postgres",
|
||||
Password: "postgres",
|
||||
},
|
||||
}
|
||||
|
||||
testResp := test_utils.MakePostRequest(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/restores/%s/restore", backup.ID.String()),
|
||||
"Bearer "+owner.Token,
|
||||
request,
|
||||
http.StatusBadRequest,
|
||||
)
|
||||
|
||||
bodyStr := string(testResp.Body)
|
||||
assert.Contains(t, bodyStr, "is required")
|
||||
assert.Contains(t, bodyStr, "is available")
|
||||
assert.Contains(t, bodyStr, "disk space")
|
||||
}
|
||||
|
||||
func createTestDatabaseWithBackupForRestore(
|
||||
workspace *workspaces_models.Workspace,
|
||||
owner *users_dto.SignInResponseDTO,
|
||||
@@ -295,6 +333,7 @@ func createTestDatabase(
|
||||
Username: "postgres",
|
||||
Password: "postgres",
|
||||
Database: &testDbName,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/disk"
|
||||
"databasus-backend/internal/features/restores/usecases"
|
||||
"databasus-backend/internal/features/storages"
|
||||
workspaces_services "databasus-backend/internal/features/workspaces/services"
|
||||
@@ -24,6 +25,7 @@ var restoreService = &RestoreService{
|
||||
workspaces_services.GetWorkspaceService(),
|
||||
audit_logs.GetAuditLogService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
disk.GetDiskService(),
|
||||
}
|
||||
var restoreController = &RestoreController{
|
||||
restoreService,
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"databasus-backend/internal/features/backups/backups"
|
||||
backups_config "databasus-backend/internal/features/backups/config"
|
||||
"databasus-backend/internal/features/databases"
|
||||
"databasus-backend/internal/features/disk"
|
||||
"databasus-backend/internal/features/restores/enums"
|
||||
"databasus-backend/internal/features/restores/models"
|
||||
"databasus-backend/internal/features/restores/usecases"
|
||||
@@ -32,6 +33,7 @@ type RestoreService struct {
|
||||
workspaceService *workspaces_services.WorkspaceService
|
||||
auditLogService *audit_logs.AuditLogService
|
||||
fieldEncryptor encryption.FieldEncryptor
|
||||
diskService *disk.DiskService
|
||||
}
|
||||
|
||||
func (s *RestoreService) OnBeforeBackupRemove(backup *backups.Backup) error {
|
||||
@@ -126,6 +128,11 @@ func (s *RestoreService) RestoreBackupWithAuth(
|
||||
return err
|
||||
}
|
||||
|
||||
// Validate disk space before starting restore
|
||||
if err := s.validateDiskSpace(backup); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
go func() {
|
||||
if err := s.RestoreBackup(backup, requestDTO); err != nil {
|
||||
s.logger.Error("Failed to restore backup", "error", err)
|
||||
@@ -361,3 +368,41 @@ func (s *RestoreService) validateVersionCompatibility(
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *RestoreService) validateDiskSpace(backup *backups.Backup) error {
|
||||
diskUsage, err := s.diskService.GetDiskUsage()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check disk space: %w", err)
|
||||
}
|
||||
|
||||
// Convert backup size from MB to bytes
|
||||
backupSizeBytes := int64(backup.BackupSizeMb * 1024 * 1024)
|
||||
|
||||
// Calculate required space: backup size + 10% buffer
|
||||
bufferBytes := int64(float64(backupSizeBytes) * 0.1)
|
||||
requiredBytes := backupSizeBytes + bufferBytes
|
||||
|
||||
// Ensure minimum of 1 GB total (even if backup is small)
|
||||
minRequiredBytes := int64(1024 * 1024 * 1024) // 1 GB
|
||||
if requiredBytes < minRequiredBytes {
|
||||
requiredBytes = minRequiredBytes
|
||||
}
|
||||
|
||||
// Check if there's enough free space
|
||||
if diskUsage.FreeSpaceBytes < requiredBytes {
|
||||
backupSizeGB := float64(backupSizeBytes) / (1024 * 1024 * 1024)
|
||||
bufferSizeGB := float64(bufferBytes) / (1024 * 1024 * 1024)
|
||||
requiredGB := float64(requiredBytes) / (1024 * 1024 * 1024)
|
||||
availableGB := float64(diskUsage.FreeSpaceBytes) / (1024 * 1024 * 1024)
|
||||
|
||||
return fmt.Errorf(
|
||||
"to restore this backup, %.1f GB (%.1f GB backup + %.1f GB buffer) is required, but only %.1f GB is available. Please free up disk space before restoring",
|
||||
requiredGB,
|
||||
backupSizeGB,
|
||||
bufferSizeGB,
|
||||
availableGB,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -330,7 +330,7 @@ func (uc *RestoreMariadbBackupUsecase) createTempMyCnfFile(
|
||||
mdbConfig *mariadbtypes.MariadbDatabase,
|
||||
password string,
|
||||
) (string, error) {
|
||||
tempDir, err := os.MkdirTemp("", "mycnf")
|
||||
tempDir, err := os.MkdirTemp(config.GetEnv().TempFolder, "mycnf_"+uuid.New().String())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp directory: %w", err)
|
||||
}
|
||||
|
||||
@@ -111,6 +111,16 @@ func (uc *RestoreMongodbBackupUsecase) buildMongorestoreArgs(
|
||||
args = append(args, "--nsInclude="+mdb.Database+".*")
|
||||
}
|
||||
|
||||
// Use numInsertionWorkersPerCollection based on CPU count
|
||||
// Cap between 1 and 16 to balance performance and resource usage
|
||||
parallelWorkers := max(1, min(mdb.CpuCount, 16))
|
||||
if parallelWorkers > 1 {
|
||||
args = append(
|
||||
args,
|
||||
"--numInsertionWorkersPerCollection="+fmt.Sprintf("%d", parallelWorkers),
|
||||
)
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
|
||||
@@ -322,7 +322,7 @@ func (uc *RestoreMysqlBackupUsecase) createTempMyCnfFile(
|
||||
myConfig *mysqltypes.MysqlDatabase,
|
||||
password string,
|
||||
) (string, error) {
|
||||
tempDir, err := os.MkdirTemp("", "mycnf")
|
||||
tempDir, err := os.MkdirTemp(config.GetEnv().TempFolder, "mycnf_"+uuid.New().String())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp directory: %w", err)
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"databasus-backend/internal/features/restores/models"
|
||||
"databasus-backend/internal/features/storages"
|
||||
util_encryption "databasus-backend/internal/util/encryption"
|
||||
files_utils "databasus-backend/internal/util/files"
|
||||
"databasus-backend/internal/util/tools"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -65,33 +64,57 @@ func (uc *RestorePostgresqlBackupUsecase) Execute(
|
||||
return fmt.Errorf("target database name is required for pg_restore")
|
||||
}
|
||||
|
||||
// Use parallel jobs based on CPU count (same as backup)
|
||||
pgBin := tools.GetPostgresqlExecutable(
|
||||
pg.Version,
|
||||
"pg_restore",
|
||||
config.GetEnv().EnvMode,
|
||||
config.GetEnv().PostgresesInstallDir,
|
||||
)
|
||||
|
||||
// All PostgreSQL backups are now custom format (-Fc)
|
||||
return uc.restoreCustomType(
|
||||
originalDB,
|
||||
pgBin,
|
||||
backup,
|
||||
storage,
|
||||
pg,
|
||||
isExcludeExtensions,
|
||||
)
|
||||
}
|
||||
|
||||
// restoreCustomType restores a backup in custom type (-Fc) - legacy type
|
||||
func (uc *RestorePostgresqlBackupUsecase) restoreCustomType(
|
||||
originalDB *databases.Database,
|
||||
pgBin string,
|
||||
backup *backups.Backup,
|
||||
storage *storages.Storage,
|
||||
pg *pgtypes.PostgresqlDatabase,
|
||||
isExcludeExtensions bool,
|
||||
) error {
|
||||
uc.logger.Info("Restoring backup in custom type (-Fc)", "backupId", backup.ID)
|
||||
|
||||
// Use parallel jobs based on CPU count
|
||||
// Cap between 1 and 8 to avoid overwhelming the server
|
||||
parallelJobs := max(1, min(backupConfig.CpuCount, 8))
|
||||
parallelJobs := max(1, min(pg.CpuCount, 8))
|
||||
|
||||
args := []string{
|
||||
"-Fc", // expect custom format (same as backup)
|
||||
"-Fc", // expect custom type
|
||||
"-j", strconv.Itoa(parallelJobs), // parallel jobs based on CPU count
|
||||
"--no-password", // Use environment variable for password, prevent prompts
|
||||
"--no-password",
|
||||
"-h", pg.Host,
|
||||
"-p", strconv.Itoa(pg.Port),
|
||||
"-U", pg.Username,
|
||||
"-d", *pg.Database,
|
||||
"--verbose", // Add verbose output to help with debugging
|
||||
"--clean", // Clean (drop) database objects before recreating them
|
||||
"--if-exists", // Use IF EXISTS when dropping objects
|
||||
"--no-owner", // Skip restoring ownership
|
||||
"--no-acl", // Skip restoring access privileges (GRANT/REVOKE commands)
|
||||
"--verbose",
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
}
|
||||
|
||||
return uc.restoreFromStorage(
|
||||
originalDB,
|
||||
tools.GetPostgresqlExecutable(
|
||||
pg.Version,
|
||||
"pg_restore",
|
||||
config.GetEnv().EnvMode,
|
||||
config.GetEnv().PostgresesInstallDir,
|
||||
),
|
||||
pgBin,
|
||||
args,
|
||||
pg.Password,
|
||||
backup,
|
||||
@@ -150,7 +173,7 @@ func (uc *RestorePostgresqlBackupUsecase) restoreFromStorage(
|
||||
}
|
||||
defer func() {
|
||||
if pgpassFile != "" {
|
||||
_ = os.Remove(pgpassFile)
|
||||
_ = os.RemoveAll(filepath.Dir(pgpassFile))
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -208,13 +231,6 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(
|
||||
backup *backups.Backup,
|
||||
storage *storages.Storage,
|
||||
) (string, func(), error) {
|
||||
err := files_utils.EnsureDirectories([]string{
|
||||
config.GetEnv().TempFolder,
|
||||
})
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("failed to ensure directories: %w", err)
|
||||
}
|
||||
|
||||
// Create temporary directory for backup data
|
||||
tempDir, err := os.MkdirTemp(config.GetEnv().TempFolder, "restore_"+uuid.New().String())
|
||||
if err != nil {
|
||||
@@ -621,7 +637,7 @@ func (uc *RestorePostgresqlBackupUsecase) generateFilteredTocList(
|
||||
}
|
||||
|
||||
// Write filtered TOC to temporary file
|
||||
tocFile, err := os.CreateTemp("", "pg_restore_toc_*.list")
|
||||
tocFile, err := os.CreateTemp(config.GetEnv().TempFolder, "pg_restore_toc_*.list")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create TOC list file: %w", err)
|
||||
}
|
||||
@@ -668,7 +684,7 @@ func (uc *RestorePostgresqlBackupUsecase) createTempPgpassFile(
|
||||
escapedPassword,
|
||||
)
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "pgpass")
|
||||
tempDir, err := os.MkdirTemp(config.GetEnv().TempFolder, "pgpass_"+uuid.New().String())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temporary directory: %w", err)
|
||||
}
|
||||
|
||||
@@ -3,8 +3,10 @@ package s3_storage
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/tls"
|
||||
"databasus-backend/internal/util/encryption"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -101,15 +103,21 @@ func (s *S3Storage) SaveFile(
|
||||
return fmt.Errorf("read error: %w", readErr)
|
||||
}
|
||||
|
||||
partData := buf[:n]
|
||||
hash := md5.Sum(partData)
|
||||
md5Base64 := base64.StdEncoding.EncodeToString(hash[:])
|
||||
|
||||
part, err := coreClient.PutObjectPart(
|
||||
ctx,
|
||||
s.S3Bucket,
|
||||
objectKey,
|
||||
uploadID,
|
||||
partNumber,
|
||||
bytes.NewReader(buf[:n]),
|
||||
bytes.NewReader(partData),
|
||||
int64(n),
|
||||
minio.PutObjectPartOptions{},
|
||||
minio.PutObjectPartOptions{
|
||||
Md5Base64: md5Base64,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
_ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)
|
||||
@@ -147,7 +155,9 @@ func (s *S3Storage) SaveFile(
|
||||
objectKey,
|
||||
bytes.NewReader([]byte{}),
|
||||
0,
|
||||
minio.PutObjectOptions{},
|
||||
minio.PutObjectOptions{
|
||||
SendContentMd5: true,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload empty file: %w", err)
|
||||
@@ -283,7 +293,9 @@ func (s *S3Storage) TestConnection(encryptor encryption.FieldEncryptor) error {
|
||||
testObjectKey,
|
||||
testReader,
|
||||
int64(len(testData)),
|
||||
minio.PutObjectOptions{},
|
||||
minio.PutObjectOptions{
|
||||
SendContentMd5: true,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload test file to S3: %w", err)
|
||||
|
||||
@@ -394,6 +394,7 @@ func createMongodbDatabaseViaAPI(
|
||||
AuthDatabase: authDatabase,
|
||||
Version: version,
|
||||
IsHttps: false,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -440,6 +441,7 @@ func createMongodbRestoreViaAPI(
|
||||
AuthDatabase: authDatabase,
|
||||
Version: version,
|
||||
IsHttps: false,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -1269,6 +1269,7 @@ func createDatabaseViaAPI(
|
||||
Username: username,
|
||||
Password: password,
|
||||
Database: &database,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1387,6 +1388,7 @@ func createRestoreWithOptionsViaAPI(
|
||||
Password: password,
|
||||
Database: &database,
|
||||
IsExcludeExtensions: isExcludeExtensions,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1424,6 +1426,7 @@ func createDatabaseWithSchemasViaAPI(
|
||||
Password: password,
|
||||
Database: &database,
|
||||
IncludeSchemas: includeSchemas,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1476,6 +1479,7 @@ func createSupabaseDatabaseViaAPI(
|
||||
Database: &database,
|
||||
IsHttps: true,
|
||||
IncludeSchemas: includeSchemas,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1522,6 +1526,7 @@ func createSupabaseRestoreViaAPI(
|
||||
Password: password,
|
||||
Database: &database,
|
||||
IsHttps: true,
|
||||
CpuCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE backup_configs DROP COLUMN cpu_count;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE postgresql_databases ADD COLUMN cpu_count INT NOT NULL DEFAULT 1;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE mongodb_databases ADD COLUMN cpu_count INT NOT NULL DEFAULT 1;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE backup_configs ADD COLUMN cpu_count INT NOT NULL DEFAULT 1;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE postgresql_databases DROP COLUMN cpu_count;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE mongodb_databases DROP COLUMN cpu_count;
|
||||
-- +goose StatementEnd
|
||||
9
backend/migrations/20260101200438_add_backup_format.sql
Normal file
9
backend/migrations/20260101200438_add_backup_format.sql
Normal file
@@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE backups ADD COLUMN type TEXT NOT NULL DEFAULT 'DEFAULT';
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE backups DROP COLUMN type;
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,9 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE backups DROP COLUMN type;
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
ALTER TABLE backups ADD COLUMN type TEXT NOT NULL DEFAULT 'DEFAULT';
|
||||
-- +goose StatementEnd
|
||||
Binary file not shown.
@@ -1 +0,0 @@
|
||||
This is test data for storage testing
|
||||
Binary file not shown.
@@ -1 +0,0 @@
|
||||
This is test data for storage testing
|
||||
Binary file not shown.
@@ -29,8 +29,23 @@ export const backupsApi = {
|
||||
return apiHelper.fetchDeleteRaw(`${getApplicationServer()}/api/v1/backups/${id}`);
|
||||
},
|
||||
|
||||
async downloadBackup(id: string): Promise<Blob> {
|
||||
return apiHelper.fetchGetBlob(`${getApplicationServer()}/api/v1/backups/${id}/file`);
|
||||
async downloadBackup(id: string): Promise<{ blob: Blob; filename: string }> {
|
||||
const result = await apiHelper.fetchGetBlobWithHeaders(
|
||||
`${getApplicationServer()}/api/v1/backups/${id}/file`,
|
||||
);
|
||||
|
||||
// Extract filename from Content-Disposition header
|
||||
const contentDisposition = result.headers.get('Content-Disposition');
|
||||
let filename = `backup_${id}.backup`; // fallback filename
|
||||
|
||||
if (contentDisposition) {
|
||||
const filenameMatch = contentDisposition.match(/filename="?(.+?)"?$/);
|
||||
if (filenameMatch && filenameMatch[1]) {
|
||||
filename = filenameMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
return { blob: result.blob, filename };
|
||||
},
|
||||
|
||||
async cancelBackup(id: string) {
|
||||
|
||||
@@ -12,7 +12,6 @@ export interface BackupConfig {
|
||||
backupInterval?: Interval;
|
||||
storage?: Storage;
|
||||
sendNotificationsOn: BackupNotificationType[];
|
||||
cpuCount: number;
|
||||
isRetryIfFailed: boolean;
|
||||
maxFailedTriesCount: number;
|
||||
encryption: BackupEncryption;
|
||||
|
||||
@@ -9,5 +9,6 @@ export interface MongodbDatabase {
|
||||
password: string;
|
||||
database: string;
|
||||
authDatabase: string;
|
||||
useTls: boolean;
|
||||
isHttps: boolean;
|
||||
cpuCount: number;
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ export interface PostgresqlDatabase {
|
||||
|
||||
// backup settings
|
||||
includeSchemas?: string[];
|
||||
cpuCount: number;
|
||||
|
||||
// restore settings (not saved to DB)
|
||||
isExcludeExtensions?: boolean;
|
||||
|
||||
@@ -64,18 +64,13 @@ export const BackupsComponent = ({ database, isCanManageDBs, scrollContainerRef
|
||||
|
||||
const downloadBackup = async (backupId: string) => {
|
||||
try {
|
||||
const blob = await backupsApi.downloadBackup(backupId);
|
||||
const { blob, filename } = await backupsApi.downloadBackup(backupId);
|
||||
|
||||
// Create a download link
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
const link = document.createElement('a');
|
||||
link.href = url;
|
||||
|
||||
// Find the backup to get a meaningful filename
|
||||
const backup = backups.find((b) => b.id === backupId);
|
||||
const createdAt = backup ? dayjs(backup.createdAt).format('YYYY-MM-DD_HH-mm-ss') : 'backup';
|
||||
const extension = database.type === DatabaseType.MYSQL ? '.sql.zst' : '.dump.zst';
|
||||
link.download = `${database.name}_backup_${createdAt}${extension}`;
|
||||
link.download = filename;
|
||||
|
||||
// Trigger download
|
||||
document.body.appendChild(link);
|
||||
@@ -541,7 +536,7 @@ export const BackupsComponent = ({ database, isCanManageDBs, scrollContainerRef
|
||||
<h2 className="text-lg font-bold md:text-xl dark:text-white">Backups</h2>
|
||||
|
||||
{!isBackupConfigLoading && !backupConfig?.isBackupsEnabled && (
|
||||
<div className="text-sm text-red-600 md:text-base">
|
||||
<div className="text-sm text-red-600">
|
||||
Scheduled backups are disabled (you can enable it back in the backup configuration)
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -154,7 +154,6 @@ export const EditBackupConfigComponent = ({
|
||||
timeOfDay: '00:00',
|
||||
},
|
||||
storage: undefined,
|
||||
cpuCount: 1,
|
||||
storePeriod: Period.THREE_MONTH,
|
||||
sendNotificationsOn: [],
|
||||
isRetryIfFailed: true,
|
||||
@@ -201,7 +200,6 @@ export const EditBackupConfigComponent = ({
|
||||
!backupConfig.isBackupsEnabled ||
|
||||
(Boolean(backupConfig.storePeriod) &&
|
||||
Boolean(backupConfig.storage?.id) &&
|
||||
Boolean(backupConfig.cpuCount) &&
|
||||
Boolean(backupConfig.encryption) &&
|
||||
Boolean(backupInterval?.interval) &&
|
||||
(!backupInterval ||
|
||||
@@ -211,16 +209,18 @@ export const EditBackupConfigComponent = ({
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div className="mb-1 flex w-full flex-col items-start sm:flex-row sm:items-center">
|
||||
<div className="mb-1 min-w-[150px] sm:mb-0">Backups enabled</div>
|
||||
<Switch
|
||||
checked={backupConfig.isBackupsEnabled}
|
||||
onChange={(checked) => {
|
||||
updateBackupConfig({ isBackupsEnabled: checked });
|
||||
}}
|
||||
size="small"
|
||||
/>
|
||||
</div>
|
||||
{database.id && (
|
||||
<div className="mb-1 flex w-full flex-col items-start sm:flex-row sm:items-center">
|
||||
<div className="mb-1 min-w-[150px] sm:mb-0">Backups enabled</div>
|
||||
<Switch
|
||||
checked={backupConfig.isBackupsEnabled}
|
||||
onChange={(checked) => {
|
||||
updateBackupConfig({ isBackupsEnabled: checked });
|
||||
}}
|
||||
size="small"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{backupConfig.isBackupsEnabled && (
|
||||
<>
|
||||
@@ -404,27 +404,6 @@ export const EditBackupConfigComponent = ({
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="mt-5 mb-1 flex w-full flex-col items-start sm:flex-row sm:items-center">
|
||||
<div className="mb-1 min-w-[150px] sm:mb-0">CPU count</div>
|
||||
<div className="flex items-center">
|
||||
<InputNumber
|
||||
min={1}
|
||||
max={16}
|
||||
value={backupConfig.cpuCount}
|
||||
onChange={(value) => updateBackupConfig({ cpuCount: value || 1 })}
|
||||
size="small"
|
||||
className="w-full max-w-[200px] grow"
|
||||
/>
|
||||
|
||||
<Tooltip
|
||||
className="cursor-pointer"
|
||||
title="Number of CPU cores to use for restore processing. Higher values may speed up restores, but use more resources."
|
||||
>
|
||||
<InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />
|
||||
</Tooltip>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="mb-1 flex w-full flex-col items-start sm:flex-row sm:items-center">
|
||||
<div className="mb-1 min-w-[150px] sm:mb-0">Store period</div>
|
||||
<div className="flex items-center">
|
||||
|
||||
@@ -31,8 +31,6 @@ const createInitialDatabase = (workspaceId: string): Database =>
|
||||
workspaceId,
|
||||
storePeriod: Period.MONTH,
|
||||
|
||||
postgresql: {} as PostgresqlDatabase,
|
||||
|
||||
type: DatabaseType.POSTGRES,
|
||||
|
||||
storage: {} as unknown as Storage,
|
||||
@@ -52,13 +50,13 @@ const initializeDatabaseTypeData = (db: Database): Database => {
|
||||
|
||||
switch (db.type) {
|
||||
case DatabaseType.POSTGRES:
|
||||
return { ...base, postgresql: db.postgresql ?? ({} as PostgresqlDatabase) };
|
||||
return { ...base, postgresql: db.postgresql ?? ({ cpuCount: 4 } as PostgresqlDatabase) };
|
||||
case DatabaseType.MYSQL:
|
||||
return { ...base, mysql: db.mysql ?? ({} as MysqlDatabase) };
|
||||
case DatabaseType.MARIADB:
|
||||
return { ...base, mariadb: db.mariadb ?? ({} as MariadbDatabase) };
|
||||
case DatabaseType.MONGODB:
|
||||
return { ...base, mongodb: db.mongodb ?? ({} as MongodbDatabase) };
|
||||
return { ...base, mongodb: db.mongodb ?? ({ cpuCount: 4 } as MongodbDatabase) };
|
||||
default:
|
||||
return db;
|
||||
}
|
||||
|
||||
@@ -23,7 +23,17 @@ export const CreateReadOnlyComponent = ({
|
||||
|
||||
const isPostgres = database.type === DatabaseType.POSTGRES;
|
||||
const isMysql = database.type === DatabaseType.MYSQL;
|
||||
const databaseTypeName = isPostgres ? 'PostgreSQL' : isMysql ? 'MySQL' : 'database';
|
||||
const isMariadb = database.type === DatabaseType.MARIADB;
|
||||
const isMongodb = database.type === DatabaseType.MONGODB;
|
||||
const databaseTypeName = isPostgres
|
||||
? 'PostgreSQL'
|
||||
: isMysql
|
||||
? 'MySQL'
|
||||
: isMariadb
|
||||
? 'MariaDB'
|
||||
: isMongodb
|
||||
? 'MongoDB'
|
||||
: 'database';
|
||||
|
||||
const checkReadOnlyUser = async (): Promise<boolean> => {
|
||||
try {
|
||||
@@ -47,6 +57,12 @@ export const CreateReadOnlyComponent = ({
|
||||
} else if (isMysql && database.mysql) {
|
||||
database.mysql.username = response.username;
|
||||
database.mysql.password = response.password;
|
||||
} else if (isMariadb && database.mariadb) {
|
||||
database.mariadb.username = response.username;
|
||||
database.mariadb.password = response.password;
|
||||
} else if (isMongodb && database.mongodb) {
|
||||
database.mongodb.username = response.username;
|
||||
database.mongodb.password = response.password;
|
||||
}
|
||||
|
||||
onReadOnlyUserUpdated(database);
|
||||
|
||||
@@ -65,7 +65,8 @@ export const EditDatabaseBaseInfoComponent = ({
|
||||
|
||||
switch (newType) {
|
||||
case DatabaseType.POSTGRES:
|
||||
updatedDatabase.postgresql = editingDatabase.postgresql ?? ({} as PostgresqlDatabase);
|
||||
updatedDatabase.postgresql =
|
||||
editingDatabase.postgresql ?? ({ cpuCount: 1 } as PostgresqlDatabase);
|
||||
break;
|
||||
case DatabaseType.MYSQL:
|
||||
updatedDatabase.mysql = editingDatabase.mysql ?? ({} as MysqlDatabase);
|
||||
@@ -74,7 +75,7 @@ export const EditDatabaseBaseInfoComponent = ({
|
||||
updatedDatabase.mariadb = editingDatabase.mariadb ?? ({} as MariadbDatabase);
|
||||
break;
|
||||
case DatabaseType.MONGODB:
|
||||
updatedDatabase.mongodb = editingDatabase.mongodb ?? ({} as MongodbDatabase);
|
||||
updatedDatabase.mongodb = editingDatabase.mongodb ?? ({ cpuCount: 1 } as MongodbDatabase);
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { CopyOutlined, DownOutlined, UpOutlined } from '@ant-design/icons';
|
||||
import { App, Button, Input, InputNumber, Switch } from 'antd';
|
||||
import { CopyOutlined, DownOutlined, InfoCircleOutlined, UpOutlined } from '@ant-design/icons';
|
||||
import { App, Button, Input, InputNumber, Switch, Tooltip } from 'antd';
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
import { type Database, databaseApi } from '../../../../entity/databases';
|
||||
@@ -77,7 +77,8 @@ export const EditMongoDbSpecificDataComponent = ({
|
||||
password: result.password,
|
||||
database: result.database,
|
||||
authDatabase: result.authDatabase,
|
||||
useTls: result.useTls,
|
||||
isHttps: result.useTls,
|
||||
cpuCount: 4,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -285,16 +286,16 @@ export const EditMongoDbSpecificDataComponent = ({
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="mb-3 flex w-full items-center">
|
||||
<div className="min-w-[150px]">Use TLS</div>
|
||||
<div className="mb-1 flex w-full items-center">
|
||||
<div className="min-w-[150px]">Use HTTPS</div>
|
||||
<Switch
|
||||
checked={editingDatabase.mongodb?.useTls}
|
||||
checked={editingDatabase.mongodb?.isHttps}
|
||||
onChange={(checked) => {
|
||||
if (!editingDatabase.mongodb) return;
|
||||
|
||||
setEditingDatabase({
|
||||
...editingDatabase,
|
||||
mongodb: { ...editingDatabase.mongodb, useTls: checked },
|
||||
mongodb: { ...editingDatabase.mongodb, isHttps: checked },
|
||||
});
|
||||
setIsConnectionTested(false);
|
||||
}}
|
||||
@@ -302,7 +303,36 @@ export const EditMongoDbSpecificDataComponent = ({
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="mt-4 mb-3 flex items-center">
|
||||
<div className="mb-5 flex w-full items-center">
|
||||
<div className="min-w-[150px]">CPU count</div>
|
||||
<div className="flex items-center">
|
||||
<InputNumber
|
||||
min={1}
|
||||
max={16}
|
||||
value={editingDatabase.mongodb?.cpuCount}
|
||||
onChange={(value) => {
|
||||
if (!editingDatabase.mongodb) return;
|
||||
|
||||
setEditingDatabase({
|
||||
...editingDatabase,
|
||||
mongodb: { ...editingDatabase.mongodb, cpuCount: value || 1 },
|
||||
});
|
||||
setIsConnectionTested(false);
|
||||
}}
|
||||
size="small"
|
||||
className="max-w-[200px] grow"
|
||||
/>
|
||||
|
||||
<Tooltip
|
||||
className="cursor-pointer"
|
||||
title="Number of CPU cores to use for backup and restore operations. Higher values may speed up operations but use more resources."
|
||||
>
|
||||
<InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />
|
||||
</Tooltip>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="mt-4 mb-1 flex items-center">
|
||||
<div
|
||||
className="flex cursor-pointer items-center text-sm text-blue-600 hover:text-blue-800"
|
||||
onClick={() => setShowAdvanced(!isShowAdvanced)}
|
||||
@@ -318,24 +348,26 @@ export const EditMongoDbSpecificDataComponent = ({
|
||||
</div>
|
||||
|
||||
{isShowAdvanced && (
|
||||
<div className="mb-1 flex w-full items-center">
|
||||
<div className="min-w-[150px]">Auth database</div>
|
||||
<Input
|
||||
value={editingDatabase.mongodb?.authDatabase}
|
||||
onChange={(e) => {
|
||||
if (!editingDatabase.mongodb) return;
|
||||
<>
|
||||
<div className="mb-1 flex w-full items-center">
|
||||
<div className="min-w-[150px]">Auth database</div>
|
||||
<Input
|
||||
value={editingDatabase.mongodb?.authDatabase}
|
||||
onChange={(e) => {
|
||||
if (!editingDatabase.mongodb) return;
|
||||
|
||||
setEditingDatabase({
|
||||
...editingDatabase,
|
||||
mongodb: { ...editingDatabase.mongodb, authDatabase: e.target.value.trim() },
|
||||
});
|
||||
setIsConnectionTested(false);
|
||||
}}
|
||||
size="small"
|
||||
className="max-w-[200px] grow"
|
||||
placeholder="admin"
|
||||
/>
|
||||
</div>
|
||||
setEditingDatabase({
|
||||
...editingDatabase,
|
||||
mongodb: { ...editingDatabase.mongodb, authDatabase: e.target.value.trim() },
|
||||
});
|
||||
setIsConnectionTested(false);
|
||||
}}
|
||||
size="small"
|
||||
className="max-w-[200px] grow"
|
||||
placeholder="admin"
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
<div className="mt-5 flex">
|
||||
|
||||
@@ -82,6 +82,7 @@ export const EditPostgreSqlSpecificDataComponent = ({
|
||||
password: result.password,
|
||||
database: result.database,
|
||||
isHttps: result.isHttps,
|
||||
cpuCount: 4,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -338,7 +339,7 @@ export const EditPostgreSqlSpecificDataComponent = ({
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="mb-3 flex w-full items-center">
|
||||
<div className="mb-1 flex w-full items-center">
|
||||
<div className="min-w-[150px]">Use HTTPS</div>
|
||||
<Switch
|
||||
checked={editingDatabase.postgresql?.isHttps}
|
||||
@@ -355,7 +356,36 @@ export const EditPostgreSqlSpecificDataComponent = ({
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="mt-4 mb-3 flex items-center">
|
||||
<div className="mb-5 flex w-full items-center">
|
||||
<div className="min-w-[150px]">CPU count</div>
|
||||
<div className="flex items-center">
|
||||
<InputNumber
|
||||
min={1}
|
||||
max={128}
|
||||
value={editingDatabase.postgresql?.cpuCount}
|
||||
onChange={(value) => {
|
||||
if (!editingDatabase.postgresql) return;
|
||||
|
||||
setEditingDatabase({
|
||||
...editingDatabase,
|
||||
postgresql: { ...editingDatabase.postgresql, cpuCount: value || 1 },
|
||||
});
|
||||
setIsConnectionTested(false);
|
||||
}}
|
||||
size="small"
|
||||
className="max-w-[75px] grow"
|
||||
/>
|
||||
|
||||
<Tooltip
|
||||
className="cursor-pointer"
|
||||
title="Number of CPU cores to use for backup and restore operations. Higher values may speed up operations but use more resources."
|
||||
>
|
||||
<InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />
|
||||
</Tooltip>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="mt-4 mb-1 flex items-center">
|
||||
<div
|
||||
className="flex cursor-pointer items-center text-sm text-blue-600 hover:text-blue-800"
|
||||
onClick={() => setShowAdvanced(!isShowAdvanced)}
|
||||
|
||||
@@ -33,8 +33,13 @@ export const ShowMongoDbSpecificDataComponent = ({ database }: Props) => {
|
||||
</div>
|
||||
|
||||
<div className="mb-1 flex w-full items-center">
|
||||
<div className="min-w-[150px]">Use TLS</div>
|
||||
<div>{database.mongodb?.useTls ? 'Yes' : 'No'}</div>
|
||||
<div className="min-w-[150px]">Use HTTPS</div>
|
||||
<div>{database.mongodb?.isHttps ? 'Yes' : 'No'}</div>
|
||||
</div>
|
||||
|
||||
<div className="mb-1 flex w-full items-center">
|
||||
<div className="min-w-[150px]">CPU count</div>
|
||||
<div>{database.mongodb?.cpuCount}</div>
|
||||
</div>
|
||||
|
||||
{database.mongodb?.authDatabase && (
|
||||
|
||||
@@ -54,6 +54,11 @@ export const ShowPostgreSqlSpecificDataComponent = ({ database }: Props) => {
|
||||
<div>{database.postgresql?.isHttps ? 'Yes' : 'No'}</div>
|
||||
</div>
|
||||
|
||||
<div className="mb-1 flex w-full items-center">
|
||||
<div className="min-w-[150px]">CPU count</div>
|
||||
<div>{database.postgresql?.cpuCount}</div>
|
||||
</div>
|
||||
|
||||
{!!database.postgresql?.includeSchemas?.length && (
|
||||
<div className="mb-1 flex w-full items-center">
|
||||
<div className="min-w-[150px]">Include schemas</div>
|
||||
|
||||
@@ -14,7 +14,8 @@ export function EditSFTPStorageComponent({ storage, setStorage, setUnsaved }: Pr
|
||||
const hasAdvancedValues = !!storage?.sftpStorage?.skipHostKeyVerify;
|
||||
const [showAdvanced, setShowAdvanced] = useState(hasAdvancedValues);
|
||||
|
||||
const authMethod = storage?.sftpStorage?.privateKey ? 'privateKey' : 'password';
|
||||
const initialAuthMethod = storage?.sftpStorage?.privateKey ? 'privateKey' : 'password';
|
||||
const [authMethod, setAuthMethod] = useState<'password' | 'privateKey'>(initialAuthMethod);
|
||||
|
||||
return (
|
||||
<>
|
||||
@@ -93,7 +94,10 @@ export function EditSFTPStorageComponent({ storage, setStorage, setUnsaved }: Pr
|
||||
onChange={(e) => {
|
||||
if (!storage?.sftpStorage) return;
|
||||
|
||||
if (e.target.value === 'password') {
|
||||
const newMethod = e.target.value as 'password' | 'privateKey';
|
||||
setAuthMethod(newMethod);
|
||||
|
||||
if (newMethod === 'password') {
|
||||
setStorage({
|
||||
...storage,
|
||||
sftpStorage: {
|
||||
|
||||
@@ -130,7 +130,7 @@ export function AdminPasswordComponent({
|
||||
}}
|
||||
type="primary"
|
||||
>
|
||||
Set Password
|
||||
Set password
|
||||
</Button>
|
||||
|
||||
{adminPasswordError && (
|
||||
|
||||
@@ -7,7 +7,7 @@ export function AuthNavbarComponent() {
|
||||
<div className="flex h-[65px] items-center justify-center px-5 pt-5 sm:justify-start">
|
||||
<div className="flex items-center gap-3 hover:opacity-80">
|
||||
<a href="https://databasus.com" target="_blank" rel="noreferrer">
|
||||
<img className="padding-[2.5px] h-[45px] w-[45px]" src="/logo.svg" />
|
||||
<img className="h-[45px] w-[45px] p-1" src="/logo.svg" />
|
||||
</a>
|
||||
|
||||
<div className="text-xl font-bold">
|
||||
|
||||
@@ -176,6 +176,25 @@ export const apiHelper = {
|
||||
return response.blob();
|
||||
},
|
||||
|
||||
fetchGetBlobWithHeaders: async (
|
||||
url: string,
|
||||
requestOptions?: RequestOptions,
|
||||
isRetryOnError = false,
|
||||
): Promise<{ blob: Blob; headers: Headers }> => {
|
||||
const optionsWrapper = (requestOptions ?? new RequestOptions())
|
||||
.addHeader('Access-Control-Allow-Methods', 'GET')
|
||||
.addHeader('Authorization', accessTokenHelper.getAccessToken());
|
||||
|
||||
const response = await makeRequest(
|
||||
url,
|
||||
optionsWrapper,
|
||||
isRetryOnError ? 0 : REPEAT_TRIES_COUNT,
|
||||
);
|
||||
|
||||
const blob = await response.blob();
|
||||
return { blob, headers: response.headers };
|
||||
},
|
||||
|
||||
fetchPutJson: async <T>(
|
||||
url: string,
|
||||
requestOptions?: RequestOptions,
|
||||
|
||||
@@ -193,10 +193,7 @@ export const MainScreenComponent = () => {
|
||||
<div className="mb-2 flex h-[50px] items-center rounded bg-white px-2 py-2 shadow md:mb-3 md:h-[60px] md:p-3 dark:bg-gray-800">
|
||||
<div className="flex items-center gap-2 hover:opacity-80 md:gap-3">
|
||||
<a href="https://databasus.com" target="_blank" rel="noreferrer">
|
||||
<img
|
||||
className="padding-[2.5px] h-[30px] w-[30px] md:h-[40px] md:w-[40px]"
|
||||
src="/logo.svg"
|
||||
/>
|
||||
<img className="h-[30px] w-[30px] p-1 md:h-[40px] md:w-[40px]" src="/logo.svg" />
|
||||
</a>
|
||||
</div>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user