Compare commits

...

2 Commits

Author SHA1 Message Date
Rostislav Dugin
244a56d1bb FEATURE (secrets): Move secrets to the secret.key file instead of DB 2025-11-19 18:53:58 +03:00
Rostislav Dugin
95c833b619 FIX (backups): Fix passing encypted password to .pgpass 2025-11-19 17:10:19 +03:00
20 changed files with 531 additions and 318 deletions

View File

@@ -18,6 +18,7 @@ import (
backups_config "postgresus-backend/internal/features/backups/config"
"postgresus-backend/internal/features/databases"
"postgresus-backend/internal/features/disk"
"postgresus-backend/internal/features/encryption/secrets"
healthcheck_attempt "postgresus-backend/internal/features/healthcheck/attempt"
healthcheck_config "postgresus-backend/internal/features/healthcheck/config"
"postgresus-backend/internal/features/notifiers"
@@ -64,6 +65,12 @@ func main() {
os.Exit(1)
}
err = secrets.GetSecretKeyService().MigrateKeyFromDbToFileIfExist()
if err != nil {
log.Error("Failed to migrate secret key from database to file", "error", err)
os.Exit(1)
}
err = users_services.GetUserService().CreateInitialAdmin()
if err != nil {
log.Error("Failed to create initial admin", "error", err)

View File

@@ -26,8 +26,9 @@ type EnvVariables struct {
EnvMode env_utils.EnvMode `env:"ENV_MODE" required:"true"`
PostgresesInstallDir string `env:"POSTGRES_INSTALL_DIR"`
DataFolder string
TempFolder string
DataFolder string
TempFolder string
SecretKeyPath string
TestGoogleDriveClientID string `env:"TEST_GOOGLE_DRIVE_CLIENT_ID"`
TestGoogleDriveClientSecret string `env:"TEST_GOOGLE_DRIVE_CLIENT_SECRET"`
@@ -146,6 +147,7 @@ func loadEnvVariables() {
// (projectRoot/postgresus-data -> /postgresus-data)
env.DataFolder = filepath.Join(filepath.Dir(backendRoot), "postgresus-data", "backups")
env.TempFolder = filepath.Join(filepath.Dir(backendRoot), "postgresus-data", "temp")
env.SecretKeyPath = filepath.Join(filepath.Dir(backendRoot), "postgresus-data", "secret.key")
if env.IsTesting {
if env.TestPostgres12Port == "" {

View File

@@ -1,17 +1,18 @@
package backups
import (
"time"
audit_logs "postgresus-backend/internal/features/audit_logs"
"postgresus-backend/internal/features/backups/backups/usecases"
backups_config "postgresus-backend/internal/features/backups/config"
"postgresus-backend/internal/features/databases"
encryption_secrets "postgresus-backend/internal/features/encryption/secrets"
"postgresus-backend/internal/features/notifiers"
"postgresus-backend/internal/features/storages"
users_repositories "postgresus-backend/internal/features/users/repositories"
workspaces_services "postgresus-backend/internal/features/workspaces/services"
"postgresus-backend/internal/util/encryption"
"postgresus-backend/internal/util/logger"
"time"
)
var backupRepository = &BackupRepository{}
@@ -25,7 +26,7 @@ var backupService = &BackupService{
notifiers.GetNotifierService(),
notifiers.GetNotifierService(),
backups_config.GetBackupConfigService(),
users_repositories.GetSecretKeyRepository(),
encryption_secrets.GetSecretKeyService(),
encryption.GetFieldEncryptor(),
usecases.GetCreateBackupUsecase(),
logger.GetLogger(),

View File

@@ -7,19 +7,20 @@ import (
"fmt"
"io"
"log/slog"
"slices"
"strings"
"time"
audit_logs "postgresus-backend/internal/features/audit_logs"
"postgresus-backend/internal/features/backups/backups/encryption"
backups_config "postgresus-backend/internal/features/backups/config"
"postgresus-backend/internal/features/databases"
encryption_secrets "postgresus-backend/internal/features/encryption/secrets"
"postgresus-backend/internal/features/notifiers"
"postgresus-backend/internal/features/storages"
users_models "postgresus-backend/internal/features/users/models"
users_repositories "postgresus-backend/internal/features/users/repositories"
workspaces_services "postgresus-backend/internal/features/workspaces/services"
util_encryption "postgresus-backend/internal/util/encryption"
"slices"
"strings"
"time"
"github.com/google/uuid"
)
@@ -31,7 +32,7 @@ type BackupService struct {
notifierService *notifiers.NotifierService
notificationSender NotificationSender
backupConfigService *backups_config.BackupConfigService
secretKeyRepo *users_repositories.SecretKeyRepository
secretKeyService *encryption_secrets.SecretKeyService
fieldEncryptor util_encryption.FieldEncryptor
createBackupUseCase CreateBackupUsecase
@@ -628,7 +629,7 @@ func (s *BackupService) getBackupReader(backupID uuid.UUID) (io.ReadCloser, erro
}
// Get master key
masterKey, err := s.secretKeyRepo.GetSecretKey()
masterKey, err := s.secretKeyService.GetSecretKey()
if err != nil {
if closeErr := fileReader.Close(); closeErr != nil {
s.logger.Error("Failed to close file reader", "error", closeErr)

View File

@@ -3,21 +3,22 @@ package backups
import (
"context"
"errors"
"strings"
"testing"
"time"
usecases_postgresql "postgresus-backend/internal/features/backups/backups/usecases/postgresql"
backups_config "postgresus-backend/internal/features/backups/config"
"postgresus-backend/internal/features/databases"
encryption_secrets "postgresus-backend/internal/features/encryption/secrets"
"postgresus-backend/internal/features/notifiers"
"postgresus-backend/internal/features/storages"
users_enums "postgresus-backend/internal/features/users/enums"
users_repositories "postgresus-backend/internal/features/users/repositories"
users_testing "postgresus-backend/internal/features/users/testing"
workspaces_services "postgresus-backend/internal/features/workspaces/services"
workspaces_testing "postgresus-backend/internal/features/workspaces/testing"
"postgresus-backend/internal/util/encryption"
"postgresus-backend/internal/util/logger"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
@@ -56,7 +57,7 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {
notifiers.GetNotifierService(),
mockNotificationSender,
backups_config.GetBackupConfigService(),
users_repositories.GetSecretKeyRepository(),
encryption_secrets.GetSecretKeyService(),
encryption.GetFieldEncryptor(),
&CreateFailedBackupUsecase{},
logger.GetLogger(),
@@ -104,7 +105,7 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {
notifiers.GetNotifierService(),
mockNotificationSender,
backups_config.GetBackupConfigService(),
users_repositories.GetSecretKeyRepository(),
encryption_secrets.GetSecretKeyService(),
encryption.GetFieldEncryptor(),
&CreateSuccessBackupUsecase{},
logger.GetLogger(),
@@ -129,7 +130,7 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {
notifiers.GetNotifierService(),
mockNotificationSender,
backups_config.GetBackupConfigService(),
users_repositories.GetSecretKeyRepository(),
encryption_secrets.GetSecretKeyService(),
encryption.GetFieldEncryptor(),
&CreateSuccessBackupUsecase{},
logger.GetLogger(),

View File

@@ -19,8 +19,8 @@ import (
backups_config "postgresus-backend/internal/features/backups/config"
"postgresus-backend/internal/features/databases"
pgtypes "postgresus-backend/internal/features/databases/databases/postgresql"
encryption_secrets "postgresus-backend/internal/features/encryption/secrets"
"postgresus-backend/internal/features/storages"
users_repositories "postgresus-backend/internal/features/users/repositories"
"postgresus-backend/internal/util/encryption"
"postgresus-backend/internal/util/tools"
@@ -34,16 +34,15 @@ const (
progressReportIntervalMB = 1.0
pgConnectTimeout = 30
compressionLevel = 5
defaultBackupLimit = 1000
exitCodeAccessViolation = -1073741819
exitCodeGenericError = 1
exitCodeConnectionError = 2
)
type CreatePostgresqlBackupUsecase struct {
logger *slog.Logger
secretKeyRepo *users_repositories.SecretKeyRepository
fieldEncryptor encryption.FieldEncryptor
logger *slog.Logger
secretKeyService *encryption_secrets.SecretKeyService
fieldEncryptor encryption.FieldEncryptor
}
// Execute creates a backup of the database
@@ -81,6 +80,11 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(
args := uc.buildPgDumpArgs(pg)
decryptedPassword, err := uc.fieldEncryptor.Decrypt(db.ID, pg.Password)
if err != nil {
return nil, fmt.Errorf("failed to decrypt database password: %w", err)
}
return uc.streamToStorage(
ctx,
backupID,
@@ -92,7 +96,7 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(
config.GetEnv().PostgresesInstallDir,
),
args,
pg.Password,
decryptedPassword,
storage,
db,
backupProgressListener,
@@ -461,7 +465,7 @@ func (uc *CreatePostgresqlBackupUsecase) setupBackupEncryption(
return nil, nil, metadata, fmt.Errorf("failed to generate nonce: %w", err)
}
masterKey, err := uc.secretKeyRepo.GetSecretKey()
masterKey, err := uc.secretKeyService.GetSecretKey()
if err != nil {
return nil, nil, metadata, fmt.Errorf("failed to get master key: %w", err)
}

View File

@@ -1,14 +1,14 @@
package usecases_postgresql
import (
users_repositories "postgresus-backend/internal/features/users/repositories"
"postgresus-backend/internal/features/encryption/secrets"
"postgresus-backend/internal/util/encryption"
"postgresus-backend/internal/util/logger"
)
var createPostgresqlBackupUsecase = &CreatePostgresqlBackupUsecase{
logger.GetLogger(),
users_repositories.GetSecretKeyRepository(),
secrets.GetSecretKeyService(),
encryption.GetFieldEncryptor(),
}

View File

@@ -0,0 +1,9 @@
package secrets
var secretKeyService = &SecretKeyService{
nil,
}
func GetSecretKeyService() *SecretKeyService {
return secretKeyService
}

View File

@@ -0,0 +1 @@
package secrets

View File

@@ -0,0 +1,73 @@
package secrets
import (
"errors"
"fmt"
"os"
"postgresus-backend/internal/config"
user_models "postgresus-backend/internal/features/users/models"
"postgresus-backend/internal/storage"
"github.com/google/uuid"
"gorm.io/gorm"
)
type SecretKeyService struct {
cachedKey *string
}
func (s *SecretKeyService) MigrateKeyFromDbToFileIfExist() error {
var secretKey user_models.SecretKey
err := storage.GetDb().First(&secretKey).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil
}
return fmt.Errorf("failed to check for secret key in database: %w", err)
}
if secretKey.Secret == "" {
return nil
}
secretKeyPath := config.GetEnv().SecretKeyPath
if err := os.WriteFile(secretKeyPath, []byte(secretKey.Secret), 0600); err != nil {
return fmt.Errorf("failed to write secret key to file: %w", err)
}
if err := storage.GetDb().Exec("DELETE FROM secret_keys").Error; err != nil {
return fmt.Errorf("failed to delete secret key from database: %w", err)
}
return nil
}
func (s *SecretKeyService) GetSecretKey() (string, error) {
if s.cachedKey != nil {
return *s.cachedKey, nil
}
secretKeyPath := config.GetEnv().SecretKeyPath
data, err := os.ReadFile(secretKeyPath)
if err != nil {
if os.IsNotExist(err) {
newKey := s.generateNewSecretKey()
if err := os.WriteFile(secretKeyPath, []byte(newKey), 0600); err != nil {
return "", fmt.Errorf("failed to write new secret key: %w", err)
}
s.cachedKey = &newKey
return newKey, nil
}
return "", fmt.Errorf("failed to read secret key file: %w", err)
}
key := string(data)
s.cachedKey = &key
return key, nil
}
func (s *SecretKeyService) generateNewSecretKey() string {
return uuid.New().String() + uuid.New().String()
}

View File

@@ -1,13 +1,13 @@
package usecases_postgresql
import (
users_repositories "postgresus-backend/internal/features/users/repositories"
"postgresus-backend/internal/features/encryption/secrets"
"postgresus-backend/internal/util/logger"
)
var restorePostgresqlBackupUsecase = &RestorePostgresqlBackupUsecase{
logger.GetLogger(),
users_repositories.GetSecretKeyRepository(),
secrets.GetSecretKeyService(),
}
func GetRestorePostgresqlBackupUsecase() *RestorePostgresqlBackupUsecase {

View File

@@ -20,9 +20,9 @@ import (
backups_config "postgresus-backend/internal/features/backups/config"
"postgresus-backend/internal/features/databases"
pgtypes "postgresus-backend/internal/features/databases/databases/postgresql"
encryption_secrets "postgresus-backend/internal/features/encryption/secrets"
"postgresus-backend/internal/features/restores/models"
"postgresus-backend/internal/features/storages"
users_repositories "postgresus-backend/internal/features/users/repositories"
util_encryption "postgresus-backend/internal/util/encryption"
files_utils "postgresus-backend/internal/util/files"
"postgresus-backend/internal/util/tools"
@@ -31,8 +31,8 @@ import (
)
type RestorePostgresqlBackupUsecase struct {
logger *slog.Logger
secretKeyRepo *users_repositories.SecretKeyRepository
logger *slog.Logger
secretKeyService *encryption_secrets.SecretKeyService
}
func (uc *RestorePostgresqlBackupUsecase) Execute(
@@ -232,7 +232,7 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(
}
// Get master key
masterKey, err := uc.secretKeyRepo.GetSecretKey()
masterKey, err := uc.secretKeyService.GetSecretKey()
if err != nil {
cleanupFunc()
return "", nil, fmt.Errorf("failed to get master key for decryption: %w", err)

View File

@@ -1,31 +1,36 @@
package tests
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"
"path/filepath"
"postgresus-backend/internal/config"
"postgresus-backend/internal/features/backups/backups"
usecases_postgresql_backup "postgresus-backend/internal/features/backups/backups/usecases/postgresql"
backups_config "postgresus-backend/internal/features/backups/config"
"postgresus-backend/internal/features/databases"
pgtypes "postgresus-backend/internal/features/databases/databases/postgresql"
"postgresus-backend/internal/features/intervals"
"postgresus-backend/internal/features/restores/models"
usecases_postgresql_restore "postgresus-backend/internal/features/restores/usecases/postgresql"
"postgresus-backend/internal/features/storages"
local_storage "postgresus-backend/internal/features/storages/models/local"
"postgresus-backend/internal/util/period"
"postgresus-backend/internal/util/tools"
"strconv"
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
_ "github.com/lib/pq"
"github.com/stretchr/testify/assert"
"postgresus-backend/internal/config"
"postgresus-backend/internal/features/backups/backups"
backups_config "postgresus-backend/internal/features/backups/config"
"postgresus-backend/internal/features/databases"
pgtypes "postgresus-backend/internal/features/databases/databases/postgresql"
"postgresus-backend/internal/features/restores"
restores_enums "postgresus-backend/internal/features/restores/enums"
restores_models "postgresus-backend/internal/features/restores/models"
"postgresus-backend/internal/features/storages"
users_enums "postgresus-backend/internal/features/users/enums"
users_testing "postgresus-backend/internal/features/users/testing"
workspaces_controllers "postgresus-backend/internal/features/workspaces/controllers"
workspaces_testing "postgresus-backend/internal/features/workspaces/testing"
test_utils "postgresus-backend/internal/util/testing"
"postgresus-backend/internal/util/tools"
)
const createAndFillTableQuery = `
@@ -61,7 +66,6 @@ type TestDataItem struct {
CreatedAt time.Time `db:"created_at"`
}
// Main test functions for each PostgreSQL version
func Test_BackupAndRestorePostgresql_RestoreIsSuccesful(t *testing.T) {
env := config.GetEnv()
cases := []struct {
@@ -110,143 +114,7 @@ func Test_BackupAndRestorePostgresqlWithEncryption_RestoreIsSuccessful(t *testin
}
}
func testBackupRestoreWithEncryptionForVersion(t *testing.T, pgVersion string, port string) {
// Connect to pre-configured PostgreSQL container
container, err := connectToPostgresContainer(pgVersion, port)
assert.NoError(t, err)
defer func() {
if container.DB != nil {
container.DB.Close()
}
}()
_, err = container.DB.Exec(createAndFillTableQuery)
assert.NoError(t, err)
// Prepare data for backup
backupID := uuid.New()
pgVersionEnum := tools.GetPostgresqlVersionEnum(pgVersion)
backupDb := &databases.Database{
ID: uuid.New(),
Type: databases.DatabaseTypePostgres,
Name: "Test Database",
Postgresql: &pgtypes.PostgresqlDatabase{
Version: pgVersionEnum,
Host: container.Host,
Port: container.Port,
Username: container.Username,
Password: container.Password,
Database: &container.Database,
IsHttps: false,
},
}
storageID := uuid.New()
backupConfig := &backups_config.BackupConfig{
DatabaseID: backupDb.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodDay,
BackupInterval: &intervals.Interval{Interval: intervals.IntervalDaily},
StorageID: &storageID,
CpuCount: 1,
Encryption: backups_config.BackupEncryptionEncrypted,
}
storage := &storages.Storage{
WorkspaceID: uuid.New(),
Type: storages.StorageTypeLocal,
Name: "Test Storage",
LocalStorage: &local_storage.LocalStorage{},
}
// Make backup
progressTracker := func(completedMBs float64) {}
metadata, err := usecases_postgresql_backup.GetCreatePostgresqlBackupUsecase().Execute(
context.Background(),
backupID,
backupConfig,
backupDb,
storage,
progressTracker,
)
assert.NoError(t, err)
assert.NotNil(t, metadata)
// Verify encryption metadata is set
assert.Equal(t, backups_config.BackupEncryptionEncrypted, metadata.Encryption)
assert.NotNil(t, metadata.EncryptionSalt)
assert.NotNil(t, metadata.EncryptionIV)
// Create new database
newDBName := "restoreddb_encrypted"
_, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName))
assert.NoError(t, err)
_, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName))
assert.NoError(t, err)
// Connect to the new database
newDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
container.Host, container.Port, container.Username, container.Password, newDBName)
newDB, err := sqlx.Connect("postgres", newDSN)
assert.NoError(t, err)
defer newDB.Close()
// Setup data for restore with encryption metadata
completedBackup := &backups.Backup{
ID: backupID,
DatabaseID: backupDb.ID,
StorageID: storage.ID,
Status: backups.BackupStatusCompleted,
CreatedAt: time.Now().UTC(),
EncryptionSalt: metadata.EncryptionSalt,
EncryptionIV: metadata.EncryptionIV,
Encryption: metadata.Encryption,
}
restoreID := uuid.New()
restore := models.Restore{
ID: restoreID,
Backup: completedBackup,
Postgresql: &pgtypes.PostgresqlDatabase{
Version: pgVersionEnum,
Host: container.Host,
Port: container.Port,
Username: container.Username,
Password: container.Password,
Database: &newDBName,
IsHttps: false,
},
}
// Restore the encrypted backup
restoreBackupUC := usecases_postgresql_restore.GetRestorePostgresqlBackupUsecase()
err = restoreBackupUC.Execute(backupDb, backupConfig, restore, completedBackup, storage)
assert.NoError(t, err)
// Verify restored table exists
var tableExists bool
err = newDB.Get(
&tableExists,
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'test_data')",
)
assert.NoError(t, err)
assert.True(t, tableExists, "Table 'test_data' should exist in restored database")
// Verify data integrity
verifyDataIntegrity(t, container.DB, newDB)
// Clean up the backup file after the test
err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backupID.String()))
if err != nil {
t.Logf("Warning: Failed to delete backup file: %v", err)
}
}
// Run a test for a specific PostgreSQL version
func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {
// Connect to pre-configured PostgreSQL container
container, err := connectToPostgresContainer(pgVersion, port)
assert.NoError(t, err)
defer func() {
@@ -258,55 +126,30 @@ func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {
_, err = container.DB.Exec(createAndFillTableQuery)
assert.NoError(t, err)
// Prepare data for backup
backupID := uuid.New()
router := createTestRouter()
user := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", user, router)
storage := storages.CreateTestStorage(workspace.ID)
pgVersionEnum := tools.GetPostgresqlVersionEnum(pgVersion)
backupDb := &databases.Database{
ID: uuid.New(),
Type: databases.DatabaseTypePostgres,
Name: "Test Database",
Postgresql: &pgtypes.PostgresqlDatabase{
Version: pgVersionEnum,
Host: container.Host,
Port: container.Port,
Username: container.Username,
Password: container.Password,
Database: &container.Database,
IsHttps: false,
},
}
storageID := uuid.New()
backupConfig := &backups_config.BackupConfig{
DatabaseID: backupDb.ID,
IsBackupsEnabled: true,
StorePeriod: period.PeriodDay,
BackupInterval: &intervals.Interval{Interval: intervals.IntervalDaily},
StorageID: &storageID,
CpuCount: 1,
}
storage := &storages.Storage{
WorkspaceID: uuid.New(),
Type: storages.StorageTypeLocal,
Name: "Test Storage",
LocalStorage: &local_storage.LocalStorage{},
}
// Make backup
progressTracker := func(completedMBs float64) {}
_, err = usecases_postgresql_backup.GetCreatePostgresqlBackupUsecase().Execute(
context.Background(),
backupID,
backupConfig,
backupDb,
storage,
progressTracker,
database := createDatabaseViaAPI(
t, router, "Test Database", workspace.ID,
pgVersionEnum, container.Host, container.Port,
container.Username, container.Password, container.Database,
user.Token,
)
assert.NoError(t, err)
// Create new database
enableBackupsViaAPI(
t, router, database.ID, storage.ID,
backups_config.BackupEncryptionNone, user.Token,
)
createBackupViaAPI(t, router, database.ID, user.Token)
backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute)
assert.Equal(t, backups.BackupStatusCompleted, backup.Status)
newDBName := "restoreddb"
_, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName))
assert.NoError(t, err)
@@ -314,43 +157,22 @@ func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {
_, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName))
assert.NoError(t, err)
// Connect to the new database
newDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
container.Host, container.Port, container.Username, container.Password, newDBName)
newDB, err := sqlx.Connect("postgres", newDSN)
assert.NoError(t, err)
defer newDB.Close()
// Setup data for restore
completedBackup := &backups.Backup{
ID: backupID,
DatabaseID: backupDb.ID,
StorageID: storage.ID,
Status: backups.BackupStatusCompleted,
CreatedAt: time.Now().UTC(),
}
createRestoreViaAPI(
t, router, backup.ID, pgVersionEnum,
container.Host, container.Port,
container.Username, container.Password, newDBName,
user.Token,
)
restoreID := uuid.New()
restore := models.Restore{
ID: restoreID,
Backup: completedBackup,
Postgresql: &pgtypes.PostgresqlDatabase{
Version: pgVersionEnum,
Host: container.Host,
Port: container.Port,
Username: container.Username,
Password: container.Password,
Database: &newDBName,
IsHttps: false,
},
}
restore := waitForRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute)
assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status)
// Restore the backup
restoreBackupUC := usecases_postgresql_restore.GetRestorePostgresqlBackupUsecase()
err = restoreBackupUC.Execute(backupDb, backupConfig, restore, completedBackup, storage)
assert.NoError(t, err)
// Verify restored table exists
var tableExists bool
err = newDB.Get(
&tableExists,
@@ -359,17 +181,329 @@ func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {
assert.NoError(t, err)
assert.True(t, tableExists, "Table 'test_data' should exist in restored database")
// Verify data integrity
verifyDataIntegrity(t, container.DB, newDB)
// Clean up the backup file after the test
err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backupID.String()))
err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))
if err != nil {
t.Logf("Warning: Failed to delete backup file: %v", err)
}
test_utils.MakeDeleteRequest(
t,
router,
"/api/v1/databases/"+database.ID.String(),
"Bearer "+user.Token,
http.StatusNoContent,
)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}
func testBackupRestoreWithEncryptionForVersion(t *testing.T, pgVersion string, port string) {
container, err := connectToPostgresContainer(pgVersion, port)
assert.NoError(t, err)
defer func() {
if container.DB != nil {
container.DB.Close()
}
}()
_, err = container.DB.Exec(createAndFillTableQuery)
assert.NoError(t, err)
router := createTestRouter()
user := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", user, router)
storage := storages.CreateTestStorage(workspace.ID)
pgVersionEnum := tools.GetPostgresqlVersionEnum(pgVersion)
database := createDatabaseViaAPI(
t, router, "Test Database", workspace.ID,
pgVersionEnum, container.Host, container.Port,
container.Username, container.Password, container.Database,
user.Token,
)
enableBackupsViaAPI(
t, router, database.ID, storage.ID,
backups_config.BackupEncryptionEncrypted, user.Token,
)
createBackupViaAPI(t, router, database.ID, user.Token)
backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute)
assert.Equal(t, backups.BackupStatusCompleted, backup.Status)
assert.Equal(t, backups_config.BackupEncryptionEncrypted, backup.Encryption)
newDBName := "restoreddb_encrypted"
_, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName))
assert.NoError(t, err)
_, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName))
assert.NoError(t, err)
newDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
container.Host, container.Port, container.Username, container.Password, newDBName)
newDB, err := sqlx.Connect("postgres", newDSN)
assert.NoError(t, err)
defer newDB.Close()
createRestoreViaAPI(
t, router, backup.ID, pgVersionEnum,
container.Host, container.Port,
container.Username, container.Password, newDBName,
user.Token,
)
restore := waitForRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute)
assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status)
var tableExists bool
err = newDB.Get(
&tableExists,
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'test_data')",
)
assert.NoError(t, err)
assert.True(t, tableExists, "Table 'test_data' should exist in restored database")
verifyDataIntegrity(t, container.DB, newDB)
err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))
if err != nil {
t.Logf("Warning: Failed to delete backup file: %v", err)
}
test_utils.MakeDeleteRequest(
t,
router,
"/api/v1/databases/"+database.ID.String(),
"Bearer "+user.Token,
http.StatusNoContent,
)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}
func createTestRouter() *gin.Engine {
router := workspaces_testing.CreateTestRouter(
workspaces_controllers.GetWorkspaceController(),
workspaces_controllers.GetMembershipController(),
databases.GetDatabaseController(),
backups_config.GetBackupConfigController(),
backups.GetBackupController(),
restores.GetRestoreController(),
)
return router
}
func waitForBackupCompletion(
t *testing.T,
router *gin.Engine,
databaseID uuid.UUID,
token string,
timeout time.Duration,
) *backups.Backup {
startTime := time.Now()
pollInterval := 500 * time.Millisecond
for {
if time.Since(startTime) > timeout {
t.Fatalf("Timeout waiting for backup completion after %v", timeout)
}
var response backups.GetBackupsResponse
test_utils.MakeGetRequestAndUnmarshal(
t,
router,
fmt.Sprintf("/api/v1/backups?database_id=%s&limit=1", databaseID.String()),
"Bearer "+token,
http.StatusOK,
&response,
)
if len(response.Backups) > 0 {
backup := response.Backups[0]
if backup.Status == backups.BackupStatusCompleted {
return backup
}
if backup.Status == backups.BackupStatusFailed {
t.Fatalf("Backup failed: %v", backup.FailMessage)
}
}
time.Sleep(pollInterval)
}
}
func waitForRestoreCompletion(
t *testing.T,
router *gin.Engine,
backupID uuid.UUID,
token string,
timeout time.Duration,
) *restores_models.Restore {
startTime := time.Now()
pollInterval := 500 * time.Millisecond
for {
if time.Since(startTime) > timeout {
t.Fatalf("Timeout waiting for restore completion after %v", timeout)
}
var restores []*restores_models.Restore
test_utils.MakeGetRequestAndUnmarshal(
t,
router,
fmt.Sprintf("/api/v1/restores/%s", backupID.String()),
"Bearer "+token,
http.StatusOK,
&restores,
)
for _, restore := range restores {
if restore.Status == restores_enums.RestoreStatusCompleted {
return restore
}
if restore.Status == restores_enums.RestoreStatusFailed {
t.Fatalf("Restore failed: %v", restore.FailMessage)
}
}
time.Sleep(pollInterval)
}
}
func createDatabaseViaAPI(
t *testing.T,
router *gin.Engine,
name string,
workspaceID uuid.UUID,
pgVersion tools.PostgresqlVersion,
host string,
port int,
username string,
password string,
database string,
token string,
) *databases.Database {
request := databases.Database{
Name: name,
WorkspaceID: &workspaceID,
Type: databases.DatabaseTypePostgres,
Postgresql: &pgtypes.PostgresqlDatabase{
Version: pgVersion,
Host: host,
Port: port,
Username: username,
Password: password,
Database: &database,
},
}
w := workspaces_testing.MakeAPIRequest(
router,
"POST",
"/api/v1/databases/create",
"Bearer "+token,
request,
)
if w.Code != http.StatusCreated {
t.Fatalf("Failed to create database. Status: %d, Body: %s", w.Code, w.Body.String())
}
var createdDatabase databases.Database
if err := json.Unmarshal(w.Body.Bytes(), &createdDatabase); err != nil {
t.Fatalf("Failed to unmarshal database response: %v", err)
}
return &createdDatabase
}
func enableBackupsViaAPI(
t *testing.T,
router *gin.Engine,
databaseID uuid.UUID,
storageID uuid.UUID,
encryption backups_config.BackupEncryption,
token string,
) {
var backupConfig backups_config.BackupConfig
test_utils.MakeGetRequestAndUnmarshal(
t,
router,
fmt.Sprintf("/api/v1/backup-configs/database/%s", databaseID.String()),
"Bearer "+token,
http.StatusOK,
&backupConfig,
)
storage := &storages.Storage{ID: storageID}
backupConfig.IsBackupsEnabled = true
backupConfig.Storage = storage
backupConfig.Encryption = encryption
test_utils.MakePostRequest(
t,
router,
"/api/v1/backup-configs/save",
"Bearer "+token,
backupConfig,
http.StatusOK,
)
}
func createBackupViaAPI(
t *testing.T,
router *gin.Engine,
databaseID uuid.UUID,
token string,
) {
request := backups.MakeBackupRequest{DatabaseID: databaseID}
test_utils.MakePostRequest(
t,
router,
"/api/v1/backups",
"Bearer "+token,
request,
http.StatusOK,
)
}
func createRestoreViaAPI(
t *testing.T,
router *gin.Engine,
backupID uuid.UUID,
pgVersion tools.PostgresqlVersion,
host string,
port int,
username string,
password string,
database string,
token string,
) {
request := restores.RestoreBackupRequest{
PostgresqlDatabase: &pgtypes.PostgresqlDatabase{
Version: pgVersion,
Host: host,
Port: port,
Username: username,
Password: password,
Database: &database,
},
}
test_utils.MakePostRequest(
t,
router,
fmt.Sprintf("/api/v1/restores/%s/restore", backupID.String()),
"Bearer "+token,
request,
http.StatusOK,
)
}
// verifyDataIntegrity compares data in the original and restored databases
func verifyDataIntegrity(t *testing.T, originalDB *sqlx.DB, restoredDB *sqlx.DB) {
var originalData []TestDataItem
var restoredData []TestDataItem
@@ -382,7 +516,6 @@ func verifyDataIntegrity(t *testing.T, originalDB *sqlx.DB, restoredDB *sqlx.DB)
assert.Equal(t, len(originalData), len(restoredData), "Should have same number of rows")
// Only compare data if both slices have elements (to avoid panic)
if len(originalData) > 0 && len(restoredData) > 0 {
for i := range originalData {
assert.Equal(t, originalData[i].ID, restoredData[i].ID, "ID should match")

View File

@@ -1,13 +1,8 @@
package users_repositories
var secretKeyRepository = &SecretKeyRepository{}
var userRepository = &UserRepository{}
var usersSettingsRepository = &UsersSettingsRepository{}
func GetSecretKeyRepository() *SecretKeyRepository {
return secretKeyRepository
}
func GetUserRepository() *UserRepository {
return userRepository
}

View File

@@ -1,34 +0,0 @@
package users_repositories
import (
"errors"
user_models "postgresus-backend/internal/features/users/models"
"postgresus-backend/internal/storage"
"github.com/google/uuid"
"gorm.io/gorm"
)
type SecretKeyRepository struct{}
func (r *SecretKeyRepository) GetSecretKey() (string, error) {
var secretKey user_models.SecretKey
if err := storage.GetDb().First(&secretKey).Error; err != nil {
// create a new secret key if not found
if errors.Is(err, gorm.ErrRecordNotFound) {
newSecretKey := user_models.SecretKey{
Secret: uuid.New().String() + uuid.New().String(),
}
if err := storage.GetDb().Create(&newSecretKey).Error; err != nil {
return "", errors.New("failed to create new secret key")
}
return newSecretKey.Secret, nil
}
return "", err
}
return secretKey.Secret, nil
}

View File

@@ -1,10 +1,13 @@
package users_services
import users_repositories "postgresus-backend/internal/features/users/repositories"
import (
"postgresus-backend/internal/features/encryption/secrets"
users_repositories "postgresus-backend/internal/features/users/repositories"
)
var userService = &UserService{
users_repositories.GetUserRepository(),
users_repositories.GetSecretKeyRepository(),
secrets.GetSecretKeyService(),
settingsService,
nil,
}

View File

@@ -17,6 +17,7 @@ import (
"golang.org/x/oauth2/google"
"postgresus-backend/internal/config"
"postgresus-backend/internal/features/encryption/secrets"
users_dto "postgresus-backend/internal/features/users/dto"
users_enums "postgresus-backend/internal/features/users/enums"
users_interfaces "postgresus-backend/internal/features/users/interfaces"
@@ -25,10 +26,10 @@ import (
)
type UserService struct {
userRepository *users_repositories.UserRepository
secretKeyRepository *users_repositories.SecretKeyRepository
settingsService *SettingsService
auditLogWriter users_interfaces.AuditLogWriter
userRepository *users_repositories.UserRepository
secretKeyService *secrets.SecretKeyService
settingsService *SettingsService
auditLogWriter users_interfaces.AuditLogWriter
}
func (s *UserService) SetAuditLogWriter(writer users_interfaces.AuditLogWriter) {
@@ -162,7 +163,7 @@ func (s *UserService) SignIn(
}
func (s *UserService) GetUserFromToken(token string) (*users_models.User, error) {
secretKey, err := s.secretKeyRepository.GetSecretKey()
secretKey, err := s.secretKeyService.GetSecretKey()
if err != nil {
return nil, fmt.Errorf("failed to get secret key: %w", err)
}
@@ -221,7 +222,7 @@ func (s *UserService) GetUserFromToken(token string) (*users_models.User, error)
func (s *UserService) GenerateAccessToken(
user *users_models.User,
) (*users_dto.SignInResponseDTO, error) {
secretKey, err := s.secretKeyRepository.GetSecretKey()
secretKey, err := s.secretKeyService.GetSecretKey()
if err != nil {
return nil, fmt.Errorf("failed to get secret key: %w", err)
}

View File

@@ -1,9 +1,9 @@
package encryption
import users_repositories "postgresus-backend/internal/features/users/repositories"
import "postgresus-backend/internal/features/encryption/secrets"
var fieldEncryptor = &SecretKeyFieldEncryptor{
users_repositories.GetSecretKeyRepository(),
secrets.GetSecretKeyService(),
}
func GetFieldEncryptor() FieldEncryptor {

View File

@@ -8,17 +8,16 @@ import (
"encoding/base64"
"errors"
"fmt"
"postgresus-backend/internal/features/encryption/secrets"
"strings"
users_repositories "postgresus-backend/internal/features/users/repositories"
"github.com/google/uuid"
)
const encryptedPrefix = "enc:"
type SecretKeyFieldEncryptor struct {
secretKeyRepository *users_repositories.SecretKeyRepository
secretKeyService *secrets.SecretKeyService
}
func (e *SecretKeyFieldEncryptor) Encrypt(itemID uuid.UUID, plaintext string) (string, error) {
@@ -30,7 +29,7 @@ func (e *SecretKeyFieldEncryptor) Encrypt(itemID uuid.UUID, plaintext string) (s
return plaintext, nil
}
masterKey, err := e.secretKeyRepository.GetSecretKey()
masterKey, err := e.secretKeyService.GetSecretKey()
if err != nil {
return "", fmt.Errorf("failed to get master key: %w", err)
}
@@ -82,7 +81,7 @@ func (e *SecretKeyFieldEncryptor) Decrypt(itemID uuid.UUID, ciphertext string) (
return "", fmt.Errorf("failed to decode ciphertext: %w", err)
}
masterKey, err := e.secretKeyRepository.GetSecretKey()
masterKey, err := e.secretKeyService.GetSecretKey()
if err != nil {
return "", fmt.Errorf("failed to get master key: %w", err)
}

View File

@@ -140,6 +140,23 @@ export const MainScreenComponent = () => {
</div>
<div className="ml-auto flex items-center gap-5">
<a
className="!text-black hover:opacity-80"
href="https://postgresus.com/installation"
target="_blank"
rel="noreferrer"
>
Docs
</a>
<a
className="!text-black hover:opacity-80"
href="https://postgresus.com/contribute"
target="_blank"
rel="noreferrer"
>
Contribute
</a>
<a
className="!text-black hover:opacity-80"
href="https://t.me/postgresus_community"