Compare commits
57 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
41c72cf7b6 | ||
|
|
f04a8b7a82 | ||
|
|
552167e4ef | ||
|
|
be42cfab1f | ||
|
|
ea34ced676 | ||
|
|
09cb1488b3 | ||
|
|
b6518ef667 | ||
|
|
25c58e6209 | ||
|
|
97ee4b55c2 | ||
|
|
12eea72392 | ||
|
|
75c88bac50 | ||
|
|
ff1b6536bf | ||
|
|
06197f986d | ||
|
|
fe72e9e0a6 | ||
|
|
640cceadbd | ||
|
|
80e573fcb3 | ||
|
|
35498d83f1 | ||
|
|
77ae8d1ac7 | ||
|
|
2f20845b3d | ||
|
|
a3d3df4093 | ||
|
|
8db83d40d5 | ||
|
|
065ded37bd | ||
|
|
71e801debb | ||
|
|
ffd4e3a27b | ||
|
|
d2a9085591 | ||
|
|
6f0152b60c | ||
|
|
7007236f2f | ||
|
|
db55cad310 | ||
|
|
25bd096c81 | ||
|
|
7e98dd578c | ||
|
|
ba37b30e83 | ||
|
|
34b3f822e3 | ||
|
|
14700130b7 | ||
|
|
de11ab8d8a | ||
|
|
06282bb435 | ||
|
|
a3b263bbac | ||
|
|
a956dccf7c | ||
|
|
ce9fa18d58 | ||
|
|
281e185f21 | ||
|
|
bb5b0064ea | ||
|
|
da95bbb178 | ||
|
|
cfe5993831 | ||
|
|
fa0e3d1ce2 | ||
|
|
d07085c462 | ||
|
|
c89c1f9654 | ||
|
|
6cfc0ca79b | ||
|
|
5d27123bd7 | ||
|
|
79ca374bb6 | ||
|
|
b3f1a6f7e5 | ||
|
|
d521e2abc6 | ||
|
|
82eca7501b | ||
|
|
51866437fd | ||
|
|
244a56d1bb | ||
|
|
95c833b619 | ||
|
|
878fad5747 | ||
|
|
6ff3096695 | ||
|
|
b4b514c2d5 |
34
.github/workflows/ci-release.yml
vendored
@@ -465,3 +465,37 @@ jobs:
|
||||
body: ${{ steps.changelog.outputs.changelog }}
|
||||
draft: false
|
||||
prerelease: false
|
||||
|
||||
publish-helm-chart:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [determine-version, build-and-push]
|
||||
if: ${{ needs.determine-version.outputs.should_release == 'true' }}
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v4
|
||||
with:
|
||||
version: v3.14.0
|
||||
|
||||
- name: Log in to GHCR
|
||||
run: echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin
|
||||
|
||||
- name: Update Chart.yaml with release version
|
||||
run: |
|
||||
VERSION="${{ needs.determine-version.outputs.new_version }}"
|
||||
sed -i "s/^version: .*/version: ${VERSION}/" deploy/helm/Chart.yaml
|
||||
sed -i "s/^appVersion: .*/appVersion: \"v${VERSION}\"/" deploy/helm/Chart.yaml
|
||||
cat deploy/helm/Chart.yaml
|
||||
|
||||
- name: Package Helm chart
|
||||
run: helm package deploy/helm --destination .
|
||||
|
||||
- name: Push Helm chart to GHCR
|
||||
run: |
|
||||
VERSION="${{ needs.determine-version.outputs.new_version }}"
|
||||
helm push postgresus-${VERSION}.tgz oci://ghcr.io/rostislavdugin/charts
|
||||
|
||||
5
.gitignore
vendored
@@ -4,4 +4,7 @@ postgresus-data/
|
||||
pgdata/
|
||||
docker-compose.yml
|
||||
node_modules/
|
||||
.idea
|
||||
.idea
|
||||
/articles
|
||||
|
||||
.DS_Store
|
||||
59
README.md
@@ -25,6 +25,8 @@
|
||||
<a href="https://postgresus.com" target="_blank"><strong>🌐 Postgresus website</strong></a>
|
||||
</p>
|
||||
|
||||
<img src="assets/dashboard-dark.svg" alt="Postgresus Dark Dashboard" width="800" style="margin-bottom: 10px;"/>
|
||||
|
||||
<img src="assets/dashboard.svg" alt="Postgresus Dashboard" width="800"/>
|
||||
|
||||
|
||||
@@ -40,13 +42,13 @@
|
||||
- **Precise timing**: run backups at specific times (e.g., 4 AM during low traffic)
|
||||
- **Smart compression**: 4-8x space savings with balanced compression (~20% overhead)
|
||||
|
||||
### 🗄️ **Multiple Storage Destinations** <a href="https://postgresus.com/storages">(docs)</a>
|
||||
### 🗄️ **Multiple Storage Destinations** <a href="https://postgresus.com/storages">(view supported)</a>
|
||||
|
||||
- **Local storage**: Keep backups on your VPS/server
|
||||
- **Cloud storage**: S3, Cloudflare R2, Google Drive, NAS, Dropbox and more
|
||||
- **Secure**: All data stays under your control
|
||||
|
||||
### 📱 **Smart Notifications** <a href="https://postgresus.com/notifiers">(docs)</a>
|
||||
### 📱 **Smart Notifications** <a href="https://postgresus.com/notifiers">(view supported)</a>
|
||||
|
||||
- **Multiple channels**: Email, Telegram, Slack, Discord, webhooks
|
||||
- **Real-time updates**: Success and failure notifications
|
||||
@@ -58,6 +60,13 @@
|
||||
- **SSL support**: Secure connections available
|
||||
- **Easy restoration**: One-click restore from any backup
|
||||
|
||||
### 🔒 **Enterprise-grade security** <a href="https://postgresus.com/security">(docs)</a>
|
||||
|
||||
- **AES-256-GCM encryption**: Enterprise-grade protection for backup files
|
||||
- **Zero-trust storage**: Backups are encrypted and they are useless to attackers, so you can keep them in shared storages like S3, Azure Blob Storage, etc.
|
||||
- **Encryption for secrets**: Any sensitive data is encrypted and never exposed, even in logs or error messages
|
||||
- **Read-only user**: Postgresus uses by default a read-only user for backups and never stores anything that can change your data
|
||||
|
||||
### 👥 **Suitable for Teams** <a href="https://postgresus.com/access-management">(docs)</a>
|
||||
|
||||
- **Workspaces**: Group databases, notifiers and storages for different projects or teams
|
||||
@@ -65,6 +74,12 @@
|
||||
- **Audit logs**: Track all system activities and changes made by users
|
||||
- **User roles**: Assign viewer, member, admin or owner roles within workspaces
|
||||
|
||||
### 🎨 **UX-Friendly**
|
||||
|
||||
- **Designer-polished UI**: Clean, intuitive interface crafted with attention to detail
|
||||
- **Dark & light themes**: Choose the look that suits your workflow
|
||||
- **Mobile adaptive**: Check your backups from anywhere on any device
|
||||
|
||||
### 🐳 **Self-Hosted & Secure**
|
||||
|
||||
- **Docker-based**: Easy deployment and management
|
||||
@@ -142,6 +157,46 @@ Then run:
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
### Option 4: Kubernetes with Helm
|
||||
|
||||
For Kubernetes deployments, install directly from the OCI registry.
|
||||
|
||||
**With ClusterIP + port-forward (development/testing):**
|
||||
|
||||
```bash
|
||||
helm install postgresus oci://ghcr.io/rostislavdugin/charts/postgresus \
|
||||
-n postgresus --create-namespace
|
||||
```
|
||||
|
||||
```bash
|
||||
kubectl port-forward svc/postgresus-service 4005:4005 -n postgresus
|
||||
# Access at http://localhost:4005
|
||||
```
|
||||
|
||||
**With LoadBalancer (cloud environments):**
|
||||
|
||||
```bash
|
||||
helm install postgresus oci://ghcr.io/rostislavdugin/charts/postgresus \
|
||||
-n postgresus --create-namespace \
|
||||
--set service.type=LoadBalancer
|
||||
```
|
||||
|
||||
```bash
|
||||
kubectl get svc postgresus-service -n postgresus
|
||||
# Access at http://<EXTERNAL-IP>:4005
|
||||
```
|
||||
|
||||
**With Ingress (domain-based access):**
|
||||
|
||||
```bash
|
||||
helm install postgresus oci://ghcr.io/rostislavdugin/charts/postgresus \
|
||||
-n postgresus --create-namespace \
|
||||
--set ingress.enabled=true \
|
||||
--set ingress.hosts[0].host=backup.example.com
|
||||
```
|
||||
|
||||
For more options (NodePort, TLS, HTTPRoute for Gateway API), see the [Helm chart README](deploy/helm/README.md).
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Usage
|
||||
|
||||
764
assets/dashboard-dark.svg
Normal file
|
After Width: | Height: | Size: 766 KiB |
|
Before Width: | Height: | Size: 913 KiB After Width: | Height: | Size: 771 KiB |
|
Before Width: | Height: | Size: 34 KiB After Width: | Height: | Size: 13 KiB |
@@ -18,6 +18,7 @@ import (
|
||||
backups_config "postgresus-backend/internal/features/backups/config"
|
||||
"postgresus-backend/internal/features/databases"
|
||||
"postgresus-backend/internal/features/disk"
|
||||
"postgresus-backend/internal/features/encryption/secrets"
|
||||
healthcheck_attempt "postgresus-backend/internal/features/healthcheck/attempt"
|
||||
healthcheck_config "postgresus-backend/internal/features/healthcheck/config"
|
||||
"postgresus-backend/internal/features/notifiers"
|
||||
@@ -64,6 +65,12 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
err = secrets.GetSecretKeyService().MigrateKeyFromDbToFileIfExist()
|
||||
if err != nil {
|
||||
log.Error("Failed to migrate secret key from database to file", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
err = users_services.GetUserService().CreateInitialAdmin()
|
||||
if err != nil {
|
||||
log.Error("Failed to create initial admin", "error", err)
|
||||
|
||||
@@ -26,8 +26,9 @@ type EnvVariables struct {
|
||||
EnvMode env_utils.EnvMode `env:"ENV_MODE" required:"true"`
|
||||
PostgresesInstallDir string `env:"POSTGRES_INSTALL_DIR"`
|
||||
|
||||
DataFolder string
|
||||
TempFolder string
|
||||
DataFolder string
|
||||
TempFolder string
|
||||
SecretKeyPath string
|
||||
|
||||
TestGoogleDriveClientID string `env:"TEST_GOOGLE_DRIVE_CLIENT_ID"`
|
||||
TestGoogleDriveClientSecret string `env:"TEST_GOOGLE_DRIVE_CLIENT_SECRET"`
|
||||
@@ -146,6 +147,7 @@ func loadEnvVariables() {
|
||||
// (projectRoot/postgresus-data -> /postgresus-data)
|
||||
env.DataFolder = filepath.Join(filepath.Dir(backendRoot), "postgresus-data", "backups")
|
||||
env.TempFolder = filepath.Join(filepath.Dir(backendRoot), "postgresus-data", "temp")
|
||||
env.SecretKeyPath = filepath.Join(filepath.Dir(backendRoot), "postgresus-data", "secret.key")
|
||||
|
||||
if env.IsTesting {
|
||||
if env.TestPostgres12Port == "" {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"postgresus-backend/internal/config"
|
||||
backups_config "postgresus-backend/internal/features/backups/config"
|
||||
"postgresus-backend/internal/features/storages"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"postgresus-backend/internal/util/period"
|
||||
"time"
|
||||
)
|
||||
@@ -131,7 +132,8 @@ func (s *BackupBackgroundService) cleanOldBackups() error {
|
||||
continue
|
||||
}
|
||||
|
||||
err = storage.DeleteFile(backup.ID)
|
||||
encryptor := encryption.GetFieldEncryptor()
|
||||
err = storage.DeleteFile(encryptor, backup.ID)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to delete backup file", "backupId", backup.ID, "error", err)
|
||||
}
|
||||
|
||||
@@ -2,20 +2,21 @@ package backups
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type BackupContextManager struct {
|
||||
mu sync.RWMutex
|
||||
cancelFuncs map[uuid.UUID]context.CancelFunc
|
||||
mu sync.RWMutex
|
||||
cancelFuncs map[uuid.UUID]context.CancelFunc
|
||||
cancelledBackups map[uuid.UUID]bool
|
||||
}
|
||||
|
||||
func NewBackupContextManager() *BackupContextManager {
|
||||
return &BackupContextManager{
|
||||
cancelFuncs: make(map[uuid.UUID]context.CancelFunc),
|
||||
cancelFuncs: make(map[uuid.UUID]context.CancelFunc),
|
||||
cancelledBackups: make(map[uuid.UUID]bool),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,25 +24,37 @@ func (m *BackupContextManager) RegisterBackup(backupID uuid.UUID, cancelFunc con
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.cancelFuncs[backupID] = cancelFunc
|
||||
delete(m.cancelledBackups, backupID)
|
||||
}
|
||||
|
||||
func (m *BackupContextManager) CancelBackup(backupID uuid.UUID) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
cancelFunc, exists := m.cancelFuncs[backupID]
|
||||
if !exists {
|
||||
return errors.New("backup is not in progress or already completed")
|
||||
if m.cancelledBackups[backupID] {
|
||||
return nil
|
||||
}
|
||||
|
||||
cancelFunc()
|
||||
delete(m.cancelFuncs, backupID)
|
||||
cancelFunc, exists := m.cancelFuncs[backupID]
|
||||
if exists {
|
||||
cancelFunc()
|
||||
delete(m.cancelFuncs, backupID)
|
||||
}
|
||||
|
||||
m.cancelledBackups[backupID] = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *BackupContextManager) IsCancelled(backupID uuid.UUID) bool {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.cancelledBackups[backupID]
|
||||
}
|
||||
|
||||
func (m *BackupContextManager) UnregisterBackup(backupID uuid.UUID) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.cancelFuncs, backupID)
|
||||
delete(m.cancelledBackups, backupID)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package backups
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -26,6 +27,7 @@ import (
|
||||
users_testing "postgresus-backend/internal/features/users/testing"
|
||||
workspaces_models "postgresus-backend/internal/features/workspaces/models"
|
||||
workspaces_testing "postgresus-backend/internal/features/workspaces/testing"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
test_utils "postgresus-backend/internal/util/testing"
|
||||
"postgresus-backend/internal/util/tools"
|
||||
)
|
||||
@@ -524,7 +526,7 @@ func Test_CancelBackup_InProgressBackup_SuccessfullyCancelled(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Register a cancellable context for the backup
|
||||
GetBackupService().backupContextMgr.RegisterBackup(backup.ID, func() {})
|
||||
GetBackupService().backupContextManager.RegisterBackup(backup.ID, func() {})
|
||||
|
||||
resp := test_utils.MakePostRequest(
|
||||
t,
|
||||
@@ -700,7 +702,7 @@ func createTestBackup(
|
||||
dummyContent := []byte("dummy backup content for testing")
|
||||
reader := strings.NewReader(string(dummyContent))
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
if err := storages[0].SaveFile(logger, backup.ID, reader); err != nil {
|
||||
if err := storages[0].SaveFile(context.Background(), encryption.GetFieldEncryptor(), logger, backup.ID, reader); err != nil {
|
||||
panic(fmt.Sprintf("Failed to create test backup file: %v", err))
|
||||
}
|
||||
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
package backups
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
audit_logs "postgresus-backend/internal/features/audit_logs"
|
||||
"postgresus-backend/internal/features/backups/backups/usecases"
|
||||
backups_config "postgresus-backend/internal/features/backups/config"
|
||||
"postgresus-backend/internal/features/databases"
|
||||
encryption_secrets "postgresus-backend/internal/features/encryption/secrets"
|
||||
"postgresus-backend/internal/features/notifiers"
|
||||
"postgresus-backend/internal/features/storages"
|
||||
workspaces_services "postgresus-backend/internal/features/workspaces/services"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"postgresus-backend/internal/util/logger"
|
||||
"time"
|
||||
)
|
||||
|
||||
var backupRepository = &BackupRepository{}
|
||||
@@ -23,6 +26,8 @@ var backupService = &BackupService{
|
||||
notifiers.GetNotifierService(),
|
||||
notifiers.GetNotifierService(),
|
||||
backups_config.GetBackupConfigService(),
|
||||
encryption_secrets.GetSecretKeyService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
usecases.GetCreateBackupUsecase(),
|
||||
logger.GetLogger(),
|
||||
[]BackupRemoveListener{},
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
package backups
|
||||
|
||||
import (
|
||||
"io"
|
||||
"postgresus-backend/internal/features/backups/backups/encryption"
|
||||
)
|
||||
|
||||
type GetBackupsRequest struct {
|
||||
DatabaseID string `form:"database_id" binding:"required"`
|
||||
Limit int `form:"limit"`
|
||||
@@ -12,3 +17,12 @@ type GetBackupsResponse struct {
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
type decryptionReaderCloser struct {
|
||||
*encryption.DecryptionReader
|
||||
baseReader io.ReadCloser
|
||||
}
|
||||
|
||||
func (r *decryptionReaderCloser) Close() error {
|
||||
return r.baseReader.Close()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
package encryption
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type DecryptionReader struct {
|
||||
baseReader io.Reader
|
||||
cipher cipher.AEAD
|
||||
buffer []byte
|
||||
nonce []byte
|
||||
chunkIndex uint64
|
||||
headerRead bool
|
||||
eof bool
|
||||
}
|
||||
|
||||
func NewDecryptionReader(
|
||||
baseReader io.Reader,
|
||||
masterKey string,
|
||||
backupID uuid.UUID,
|
||||
salt []byte,
|
||||
nonce []byte,
|
||||
) (*DecryptionReader, error) {
|
||||
if len(salt) != SaltLen {
|
||||
return nil, fmt.Errorf("salt must be %d bytes, got %d", SaltLen, len(salt))
|
||||
}
|
||||
if len(nonce) != NonceLen {
|
||||
return nil, fmt.Errorf("nonce must be %d bytes, got %d", NonceLen, len(nonce))
|
||||
}
|
||||
|
||||
derivedKey, err := DeriveBackupKey(masterKey, backupID, salt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to derive backup key: %w", err)
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher(derivedKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create cipher: %w", err)
|
||||
}
|
||||
|
||||
aesgcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create GCM: %w", err)
|
||||
}
|
||||
|
||||
reader := &DecryptionReader{
|
||||
baseReader,
|
||||
aesgcm,
|
||||
make([]byte, 0),
|
||||
nonce,
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
}
|
||||
|
||||
if err := reader.readAndValidateHeader(salt, nonce); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func (r *DecryptionReader) Read(p []byte) (n int, err error) {
|
||||
for len(r.buffer) < len(p) && !r.eof {
|
||||
if err := r.readAndDecryptChunk(); err != nil {
|
||||
if err == io.EOF {
|
||||
r.eof = true
|
||||
break
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
if len(r.buffer) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
n = copy(p, r.buffer)
|
||||
r.buffer = r.buffer[n:]
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (r *DecryptionReader) readAndValidateHeader(expectedSalt, expectedNonce []byte) error {
|
||||
header := make([]byte, HeaderLen)
|
||||
|
||||
if _, err := io.ReadFull(r.baseReader, header); err != nil {
|
||||
return fmt.Errorf("failed to read header: %w", err)
|
||||
}
|
||||
|
||||
magic := string(header[0:MagicBytesLen])
|
||||
if magic != MagicBytes {
|
||||
return fmt.Errorf("invalid magic bytes: expected %s, got %s", MagicBytes, magic)
|
||||
}
|
||||
|
||||
salt := header[MagicBytesLen : MagicBytesLen+SaltLen]
|
||||
nonce := header[MagicBytesLen+SaltLen : MagicBytesLen+SaltLen+NonceLen]
|
||||
|
||||
if string(salt) != string(expectedSalt) {
|
||||
return fmt.Errorf("salt mismatch in file header")
|
||||
}
|
||||
|
||||
if string(nonce) != string(expectedNonce) {
|
||||
return fmt.Errorf("nonce mismatch in file header")
|
||||
}
|
||||
|
||||
r.headerRead = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *DecryptionReader) readAndDecryptChunk() error {
|
||||
lengthBuf := make([]byte, 4)
|
||||
if _, err := io.ReadFull(r.baseReader, lengthBuf); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunkLen := binary.BigEndian.Uint32(lengthBuf)
|
||||
if chunkLen == 0 || chunkLen > ChunkSize+16 {
|
||||
return fmt.Errorf("invalid chunk length: %d", chunkLen)
|
||||
}
|
||||
|
||||
encrypted := make([]byte, chunkLen)
|
||||
if _, err := io.ReadFull(r.baseReader, encrypted); err != nil {
|
||||
return fmt.Errorf("failed to read encrypted chunk: %w", err)
|
||||
}
|
||||
|
||||
chunkNonce := r.generateChunkNonce()
|
||||
|
||||
decrypted, err := r.cipher.Open(nil, chunkNonce, encrypted, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"failed to decrypt chunk (authentication failed - file may be corrupted or tampered): %w",
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
r.buffer = append(r.buffer, decrypted...)
|
||||
r.chunkIndex++
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *DecryptionReader) generateChunkNonce() []byte {
|
||||
chunkNonce := make([]byte, NonceLen)
|
||||
copy(chunkNonce, r.nonce)
|
||||
|
||||
binary.BigEndian.PutUint64(chunkNonce[4:], r.chunkIndex)
|
||||
|
||||
return chunkNonce
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package encryption
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type EncryptionWriter struct {
|
||||
baseWriter io.Writer
|
||||
cipher cipher.AEAD
|
||||
buffer []byte
|
||||
nonce []byte
|
||||
salt []byte
|
||||
chunkIndex uint64
|
||||
headerWritten bool
|
||||
}
|
||||
|
||||
func NewEncryptionWriter(
|
||||
baseWriter io.Writer,
|
||||
masterKey string,
|
||||
backupID uuid.UUID,
|
||||
salt []byte,
|
||||
nonce []byte,
|
||||
) (*EncryptionWriter, error) {
|
||||
if len(salt) != SaltLen {
|
||||
return nil, fmt.Errorf("salt must be %d bytes, got %d", SaltLen, len(salt))
|
||||
}
|
||||
if len(nonce) != NonceLen {
|
||||
return nil, fmt.Errorf("nonce must be %d bytes, got %d", NonceLen, len(nonce))
|
||||
}
|
||||
|
||||
derivedKey, err := DeriveBackupKey(masterKey, backupID, salt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to derive backup key: %w", err)
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher(derivedKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create cipher: %w", err)
|
||||
}
|
||||
|
||||
aesgcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create GCM: %w", err)
|
||||
}
|
||||
|
||||
writer := &EncryptionWriter{
|
||||
baseWriter: baseWriter,
|
||||
cipher: aesgcm,
|
||||
buffer: make([]byte, 0, ChunkSize),
|
||||
nonce: nonce,
|
||||
chunkIndex: 0,
|
||||
headerWritten: false,
|
||||
salt: salt, // Store salt for lazy header writing
|
||||
}
|
||||
|
||||
return writer, nil
|
||||
}
|
||||
|
||||
func (w *EncryptionWriter) Write(p []byte) (n int, err error) {
|
||||
// Write header on first write (lazy initialization)
|
||||
if !w.headerWritten {
|
||||
if err := w.writeHeader(w.salt, w.nonce); err != nil {
|
||||
return 0, fmt.Errorf("failed to write header: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
n = len(p)
|
||||
w.buffer = append(w.buffer, p...)
|
||||
|
||||
for len(w.buffer) >= ChunkSize {
|
||||
chunk := w.buffer[:ChunkSize]
|
||||
if err := w.encryptAndWriteChunk(chunk); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
w.buffer = w.buffer[ChunkSize:]
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (w *EncryptionWriter) Close() error {
|
||||
// Write header if it hasn't been written yet (in case Close is called without any writes)
|
||||
if !w.headerWritten {
|
||||
if err := w.writeHeader(w.salt, w.nonce); err != nil {
|
||||
return fmt.Errorf("failed to write header: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(w.buffer) > 0 {
|
||||
if err := w.encryptAndWriteChunk(w.buffer); err != nil {
|
||||
return err
|
||||
}
|
||||
w.buffer = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *EncryptionWriter) writeHeader(salt, nonce []byte) error {
|
||||
header := make([]byte, HeaderLen)
|
||||
|
||||
copy(header[0:MagicBytesLen], []byte(MagicBytes))
|
||||
copy(header[MagicBytesLen:MagicBytesLen+SaltLen], salt)
|
||||
copy(header[MagicBytesLen+SaltLen:MagicBytesLen+SaltLen+NonceLen], nonce)
|
||||
|
||||
_, err := w.baseWriter.Write(header)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write header: %w", err)
|
||||
}
|
||||
|
||||
w.headerWritten = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *EncryptionWriter) encryptAndWriteChunk(chunk []byte) error {
|
||||
chunkNonce := w.generateChunkNonce()
|
||||
|
||||
encrypted := w.cipher.Seal(nil, chunkNonce, chunk, nil)
|
||||
|
||||
lengthBuf := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(lengthBuf, uint32(len(encrypted)))
|
||||
|
||||
if _, err := w.baseWriter.Write(lengthBuf); err != nil {
|
||||
return fmt.Errorf("failed to write chunk length: %w", err)
|
||||
}
|
||||
|
||||
if _, err := w.baseWriter.Write(encrypted); err != nil {
|
||||
return fmt.Errorf("failed to write encrypted chunk: %w", err)
|
||||
}
|
||||
|
||||
w.chunkIndex++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *EncryptionWriter) generateChunkNonce() []byte {
|
||||
chunkNonce := make([]byte, NonceLen)
|
||||
copy(chunkNonce, w.nonce)
|
||||
|
||||
binary.BigEndian.PutUint64(chunkNonce[4:], w.chunkIndex)
|
||||
|
||||
return chunkNonce
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
package encryption
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_EncryptDecryptRoundTrip_ReturnsOriginalData(t *testing.T) {
|
||||
masterKey := uuid.New().String() + uuid.New().String()
|
||||
backupID := uuid.New()
|
||||
salt, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
nonce, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
originalData := []byte(
|
||||
"This is a test backup data that should be encrypted and then decrypted successfully.",
|
||||
)
|
||||
|
||||
var encrypted bytes.Buffer
|
||||
writer, err := NewEncryptionWriter(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
n, err := writer.Write(originalData)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, len(originalData), n)
|
||||
|
||||
err = writer.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
reader, err := NewDecryptionReader(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
decrypted := make([]byte, len(originalData))
|
||||
n, err = io.ReadFull(reader, decrypted)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, len(originalData), n)
|
||||
assert.Equal(t, originalData, decrypted)
|
||||
}
|
||||
|
||||
func Test_EncryptDecryptRoundTrip_LargeData_WorksCorrectly(t *testing.T) {
|
||||
masterKey := uuid.New().String() + uuid.New().String()
|
||||
backupID := uuid.New()
|
||||
salt, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
nonce, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
originalData := make([]byte, 100*1024)
|
||||
_, err = rand.Read(originalData)
|
||||
require.NoError(t, err)
|
||||
|
||||
var encrypted bytes.Buffer
|
||||
writer, err := NewEncryptionWriter(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
n, err := writer.Write(originalData)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, len(originalData), n)
|
||||
|
||||
err = writer.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
reader, err := NewDecryptionReader(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
decrypted, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, originalData, decrypted)
|
||||
}
|
||||
|
||||
func Test_EncryptionWriter_MultipleWrites_CombinesCorrectly(t *testing.T) {
|
||||
masterKey := uuid.New().String() + uuid.New().String()
|
||||
backupID := uuid.New()
|
||||
salt, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
nonce, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
part1 := []byte("First part of data. ")
|
||||
part2 := []byte("Second part of data. ")
|
||||
part3 := []byte("Third part of data.")
|
||||
expectedData := append(append(part1, part2...), part3...)
|
||||
|
||||
var encrypted bytes.Buffer
|
||||
writer, err := NewEncryptionWriter(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = writer.Write(part1)
|
||||
require.NoError(t, err)
|
||||
_, err = writer.Write(part2)
|
||||
require.NoError(t, err)
|
||||
_, err = writer.Write(part3)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = writer.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
reader, err := NewDecryptionReader(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
decrypted, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expectedData, decrypted)
|
||||
}
|
||||
|
||||
func Test_DecryptionReader_InvalidHeader_ReturnsError(t *testing.T) {
|
||||
masterKey := uuid.New().String() + uuid.New().String()
|
||||
backupID := uuid.New()
|
||||
salt, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
nonce, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
invalidHeader := make([]byte, HeaderLen)
|
||||
copy(invalidHeader, []byte("INVALID!"))
|
||||
|
||||
invalidData := bytes.NewBuffer(invalidHeader)
|
||||
|
||||
_, err = NewDecryptionReader(invalidData, masterKey, backupID, salt, nonce)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "invalid magic bytes")
|
||||
}
|
||||
|
||||
func Test_DecryptionReader_TamperedData_ReturnsError(t *testing.T) {
|
||||
masterKey := uuid.New().String() + uuid.New().String()
|
||||
backupID := uuid.New()
|
||||
salt, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
nonce, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
originalData := []byte("This data will be tampered with.")
|
||||
|
||||
var encrypted bytes.Buffer
|
||||
writer, err := NewEncryptionWriter(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = writer.Write(originalData)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = writer.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
encryptedBytes := encrypted.Bytes()
|
||||
if len(encryptedBytes) > HeaderLen+10 {
|
||||
encryptedBytes[HeaderLen+10] ^= 0xFF
|
||||
}
|
||||
|
||||
tamperedBuffer := bytes.NewBuffer(encryptedBytes)
|
||||
|
||||
reader, err := NewDecryptionReader(tamperedBuffer, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = io.ReadAll(reader)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "authentication failed")
|
||||
}
|
||||
|
||||
func Test_DeriveBackupKey_SameInputs_ReturnsSameKey(t *testing.T) {
|
||||
masterKey := uuid.New().String() + uuid.New().String()
|
||||
backupID := uuid.New()
|
||||
salt, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
|
||||
key1, err := DeriveBackupKey(masterKey, backupID, salt)
|
||||
require.NoError(t, err)
|
||||
|
||||
key2, err := DeriveBackupKey(masterKey, backupID, salt)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, key1, key2)
|
||||
}
|
||||
|
||||
func Test_DeriveBackupKey_DifferentInputs_ReturnsDifferentKeys(t *testing.T) {
|
||||
masterKey1 := uuid.New().String() + uuid.New().String()
|
||||
masterKey2 := uuid.New().String() + uuid.New().String()
|
||||
backupID1 := uuid.New()
|
||||
backupID2 := uuid.New()
|
||||
salt1, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
salt2, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
|
||||
key1, err := DeriveBackupKey(masterKey1, backupID1, salt1)
|
||||
require.NoError(t, err)
|
||||
|
||||
key2, err := DeriveBackupKey(masterKey2, backupID1, salt1)
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, key1, key2)
|
||||
|
||||
key3, err := DeriveBackupKey(masterKey1, backupID2, salt1)
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, key1, key3)
|
||||
|
||||
key4, err := DeriveBackupKey(masterKey1, backupID1, salt2)
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, key1, key4)
|
||||
}
|
||||
|
||||
func Test_EncryptionWriter_PartialChunk_HandledCorrectly(t *testing.T) {
|
||||
masterKey := uuid.New().String() + uuid.New().String()
|
||||
backupID := uuid.New()
|
||||
salt, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
nonce, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
smallData := []byte("Small data less than chunk size")
|
||||
|
||||
var encrypted bytes.Buffer
|
||||
writer, err := NewEncryptionWriter(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = writer.Write(smallData)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = writer.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
reader, err := NewDecryptionReader(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
decrypted, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, smallData, decrypted)
|
||||
}
|
||||
|
||||
func Test_GenerateSalt_ReturnsCorrectLength(t *testing.T) {
|
||||
salt, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, SaltLen, len(salt))
|
||||
}
|
||||
|
||||
func Test_GenerateSalt_GeneratesUniqueSalts(t *testing.T) {
|
||||
salt1, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
|
||||
salt2, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotEqual(t, salt1, salt2)
|
||||
}
|
||||
|
||||
func Test_GenerateNonce_ReturnsCorrectLength(t *testing.T) {
|
||||
nonce, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, NonceLen, len(nonce))
|
||||
}
|
||||
|
||||
func Test_GenerateNonce_GeneratesUniqueNonces(t *testing.T) {
|
||||
nonce1, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
nonce2, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotEqual(t, nonce1, nonce2)
|
||||
}
|
||||
|
||||
func Test_DecryptionReader_WrongMasterKey_ReturnsError(t *testing.T) {
|
||||
masterKey1 := uuid.New().String() + uuid.New().String()
|
||||
masterKey2 := uuid.New().String() + uuid.New().String()
|
||||
backupID := uuid.New()
|
||||
salt, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
nonce, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
originalData := []byte("Secret data")
|
||||
|
||||
var encrypted bytes.Buffer
|
||||
writer, err := NewEncryptionWriter(&encrypted, masterKey1, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = writer.Write(originalData)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = writer.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
reader, err := NewDecryptionReader(&encrypted, masterKey2, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = io.ReadAll(reader)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "authentication failed")
|
||||
}
|
||||
|
||||
func Test_EncryptionWriter_EmptyData_WorksCorrectly(t *testing.T) {
|
||||
masterKey := uuid.New().String() + uuid.New().String()
|
||||
backupID := uuid.New()
|
||||
salt, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
nonce, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
var encrypted bytes.Buffer
|
||||
writer, err := NewEncryptionWriter(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = writer.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
reader, err := NewDecryptionReader(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
decrypted, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, len(decrypted))
|
||||
}
|
||||
|
||||
func Test_EncryptionWriter_MultipleChunks_WorksCorrectly(t *testing.T) {
|
||||
masterKey := uuid.New().String() + uuid.New().String()
|
||||
backupID := uuid.New()
|
||||
salt, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
nonce, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
dataSize := ChunkSize*3 + 1000
|
||||
originalData := make([]byte, dataSize)
|
||||
_, err = rand.Read(originalData)
|
||||
require.NoError(t, err)
|
||||
|
||||
var encrypted bytes.Buffer
|
||||
writer, err := NewEncryptionWriter(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = writer.Write(originalData)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = writer.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
reader, err := NewDecryptionReader(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
decrypted, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, originalData, decrypted)
|
||||
}
|
||||
|
||||
func Test_DecryptionReader_SmallReads_WorksCorrectly(t *testing.T) {
|
||||
masterKey := uuid.New().String() + uuid.New().String()
|
||||
backupID := uuid.New()
|
||||
salt, err := GenerateSalt()
|
||||
require.NoError(t, err)
|
||||
nonce, err := GenerateNonce()
|
||||
require.NoError(t, err)
|
||||
|
||||
originalData := []byte("This is test data that will be read in small chunks.")
|
||||
|
||||
var encrypted bytes.Buffer
|
||||
writer, err := NewEncryptionWriter(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = writer.Write(originalData)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = writer.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
reader, err := NewDecryptionReader(&encrypted, masterKey, backupID, salt, nonce)
|
||||
require.NoError(t, err)
|
||||
|
||||
var decrypted []byte
|
||||
buf := make([]byte, 5)
|
||||
for {
|
||||
n, err := reader.Read(buf)
|
||||
if n > 0 {
|
||||
decrypted = append(decrypted, buf[:n]...)
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
assert.Equal(t, originalData, decrypted)
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package encryption
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/crypto/pbkdf2"
|
||||
)
|
||||
|
||||
const (
|
||||
MagicBytes = "PGRSUS01"
|
||||
MagicBytesLen = 8
|
||||
SaltLen = 32
|
||||
NonceLen = 12
|
||||
ReservedLen = 12
|
||||
HeaderLen = MagicBytesLen + SaltLen + NonceLen + ReservedLen
|
||||
ChunkSize = 1 * 1024 * 1024
|
||||
PBKDF2Iterations = 100000
|
||||
)
|
||||
|
||||
func DeriveBackupKey(masterKey string, backupID uuid.UUID, salt []byte) ([]byte, error) {
|
||||
if masterKey == "" {
|
||||
return nil, fmt.Errorf("master key cannot be empty")
|
||||
}
|
||||
if len(salt) != SaltLen {
|
||||
return nil, fmt.Errorf("salt must be %d bytes", SaltLen)
|
||||
}
|
||||
|
||||
keyMaterial := []byte(masterKey + backupID.String())
|
||||
|
||||
derivedKey := pbkdf2.Key(keyMaterial, salt, PBKDF2Iterations, 32, sha256.New)
|
||||
|
||||
return derivedKey, nil
|
||||
}
|
||||
|
||||
func GenerateSalt() ([]byte, error) {
|
||||
salt := make([]byte, SaltLen)
|
||||
if _, err := rand.Read(salt); err != nil {
|
||||
return nil, fmt.Errorf("failed to generate salt: %w", err)
|
||||
}
|
||||
return salt, nil
|
||||
}
|
||||
|
||||
func GenerateNonce() ([]byte, error) {
|
||||
nonce := make([]byte, NonceLen)
|
||||
if _, err := rand.Read(nonce); err != nil {
|
||||
return nil, fmt.Errorf("failed to generate nonce: %w", err)
|
||||
}
|
||||
return nonce, nil
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package backups
|
||||
import (
|
||||
"context"
|
||||
|
||||
usecases_postgresql "postgresus-backend/internal/features/backups/backups/usecases/postgresql"
|
||||
backups_config "postgresus-backend/internal/features/backups/config"
|
||||
"postgresus-backend/internal/features/databases"
|
||||
"postgresus-backend/internal/features/notifiers"
|
||||
@@ -29,7 +30,7 @@ type CreateBackupUsecase interface {
|
||||
backupProgressListener func(
|
||||
completedMBs float64,
|
||||
),
|
||||
) error
|
||||
) (*usecases_postgresql.BackupMetadata, error)
|
||||
}
|
||||
|
||||
type BackupRemoveListener interface {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package backups
|
||||
|
||||
import (
|
||||
backups_config "postgresus-backend/internal/features/backups/config"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -19,5 +20,9 @@ type Backup struct {
|
||||
|
||||
BackupDurationMs int64 `json:"backupDurationMs" gorm:"column:backup_duration_ms;default:0"`
|
||||
|
||||
EncryptionSalt *string `json:"-" gorm:"column:encryption_salt"`
|
||||
EncryptionIV *string `json:"-" gorm:"column:encryption_iv"`
|
||||
Encryption backups_config.BackupEncryption `json:"encryption" gorm:"column:encryption;type:text;not null;default:'NONE'"`
|
||||
|
||||
CreatedAt time.Time `json:"createdAt" gorm:"column:created_at"`
|
||||
}
|
||||
|
||||
@@ -2,20 +2,25 @@ package backups
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
audit_logs "postgresus-backend/internal/features/audit_logs"
|
||||
"postgresus-backend/internal/features/backups/backups/encryption"
|
||||
backups_config "postgresus-backend/internal/features/backups/config"
|
||||
"postgresus-backend/internal/features/databases"
|
||||
encryption_secrets "postgresus-backend/internal/features/encryption/secrets"
|
||||
"postgresus-backend/internal/features/notifiers"
|
||||
"postgresus-backend/internal/features/storages"
|
||||
users_models "postgresus-backend/internal/features/users/models"
|
||||
workspaces_services "postgresus-backend/internal/features/workspaces/services"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
util_encryption "postgresus-backend/internal/util/encryption"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -27,6 +32,8 @@ type BackupService struct {
|
||||
notifierService *notifiers.NotifierService
|
||||
notificationSender NotificationSender
|
||||
backupConfigService *backups_config.BackupConfigService
|
||||
secretKeyService *encryption_secrets.SecretKeyService
|
||||
fieldEncryptor util_encryption.FieldEncryptor
|
||||
|
||||
createBackupUseCase CreateBackupUsecase
|
||||
|
||||
@@ -34,9 +41,9 @@ type BackupService struct {
|
||||
|
||||
backupRemoveListeners []BackupRemoveListener
|
||||
|
||||
workspaceService *workspaces_services.WorkspaceService
|
||||
auditLogService *audit_logs.AuditLogService
|
||||
backupContextMgr *BackupContextManager
|
||||
workspaceService *workspaces_services.WorkspaceService
|
||||
auditLogService *audit_logs.AuditLogService
|
||||
backupContextManager *BackupContextManager
|
||||
}
|
||||
|
||||
func (s *BackupService) AddBackupRemoveListener(listener BackupRemoveListener) {
|
||||
@@ -253,10 +260,10 @@ func (s *BackupService) MakeBackup(databaseID uuid.UUID, isLastTry bool) {
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s.backupContextMgr.RegisterBackup(backup.ID, cancel)
|
||||
defer s.backupContextMgr.UnregisterBackup(backup.ID)
|
||||
s.backupContextManager.RegisterBackup(backup.ID, cancel)
|
||||
defer s.backupContextManager.UnregisterBackup(backup.ID)
|
||||
|
||||
err = s.createBackupUseCase.Execute(
|
||||
backupMetadata, err := s.createBackupUseCase.Execute(
|
||||
ctx,
|
||||
backup.ID,
|
||||
backupConfig,
|
||||
@@ -268,7 +275,12 @@ func (s *BackupService) MakeBackup(databaseID uuid.UUID, isLastTry bool) {
|
||||
errMsg := err.Error()
|
||||
|
||||
// Check if backup was cancelled (not due to shutdown)
|
||||
if strings.Contains(errMsg, "backup cancelled") && !strings.Contains(errMsg, "shutdown") {
|
||||
isCancelled := strings.Contains(errMsg, "backup cancelled") ||
|
||||
strings.Contains(errMsg, "context canceled") ||
|
||||
errors.Is(err, context.Canceled)
|
||||
isShutdown := strings.Contains(errMsg, "shutdown")
|
||||
|
||||
if isCancelled && !isShutdown {
|
||||
backup.Status = BackupStatusCanceled
|
||||
backup.BackupDurationMs = time.Since(start).Milliseconds()
|
||||
backup.BackupSizeMb = 0
|
||||
@@ -280,7 +292,7 @@ func (s *BackupService) MakeBackup(databaseID uuid.UUID, isLastTry bool) {
|
||||
// Delete partial backup from storage
|
||||
storage, storageErr := s.storageService.GetStorageByID(backup.StorageID)
|
||||
if storageErr == nil {
|
||||
if deleteErr := storage.DeleteFile(backup.ID); deleteErr != nil {
|
||||
if deleteErr := storage.DeleteFile(s.fieldEncryptor, backup.ID); deleteErr != nil {
|
||||
s.logger.Error(
|
||||
"Failed to delete partial backup file",
|
||||
"backupId",
|
||||
@@ -326,6 +338,13 @@ func (s *BackupService) MakeBackup(databaseID uuid.UUID, isLastTry bool) {
|
||||
backup.Status = BackupStatusCompleted
|
||||
backup.BackupDurationMs = time.Since(start).Milliseconds()
|
||||
|
||||
// Update backup with encryption metadata if provided
|
||||
if backupMetadata != nil {
|
||||
backup.EncryptionSalt = backupMetadata.EncryptionSalt
|
||||
backup.EncryptionIV = backupMetadata.EncryptionIV
|
||||
backup.Encryption = backupMetadata.Encryption
|
||||
}
|
||||
|
||||
if err := s.backupRepository.Save(backup); err != nil {
|
||||
s.logger.Error("Failed to save backup", "error", err)
|
||||
return
|
||||
@@ -463,7 +482,7 @@ func (s *BackupService) CancelBackup(
|
||||
return errors.New("backup is not in progress")
|
||||
}
|
||||
|
||||
if err := s.backupContextMgr.CancelBackup(backupID); err != nil {
|
||||
if err := s.backupContextManager.CancelBackup(backupID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -509,11 +528,6 @@ func (s *BackupService) GetBackupFile(
|
||||
return nil, errors.New("insufficient permissions to download backup for this database")
|
||||
}
|
||||
|
||||
storage, err := s.storageService.GetStorageByID(backup.StorageID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf(
|
||||
"Backup file downloaded for database: %s (ID: %s)",
|
||||
@@ -524,7 +538,7 @@ func (s *BackupService) GetBackupFile(
|
||||
database.WorkspaceID,
|
||||
)
|
||||
|
||||
return storage.GetFile(backup.ID)
|
||||
return s.getBackupReader(backupID)
|
||||
}
|
||||
|
||||
func (s *BackupService) deleteBackup(backup *Backup) error {
|
||||
@@ -539,7 +553,7 @@ func (s *BackupService) deleteBackup(backup *Backup) error {
|
||||
return err
|
||||
}
|
||||
|
||||
err = storage.DeleteFile(backup.ID)
|
||||
err = storage.DeleteFile(s.fieldEncryptor, backup.ID)
|
||||
if err != nil {
|
||||
// we do not return error here, because sometimes clean up performed
|
||||
// before unavailable storage removal or change - therefore we should
|
||||
@@ -579,3 +593,91 @@ func (s *BackupService) deleteDbBackups(databaseID uuid.UUID) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBackupReader returns a reader for the backup file
|
||||
// If encrypted, wraps with DecryptionReader
|
||||
func (s *BackupService) getBackupReader(backupID uuid.UUID) (io.ReadCloser, error) {
|
||||
backup, err := s.backupRepository.FindByID(backupID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find backup: %w", err)
|
||||
}
|
||||
|
||||
storage, err := s.storageService.GetStorageByID(backup.StorageID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get storage: %w", err)
|
||||
}
|
||||
|
||||
fileReader, err := storage.GetFile(s.fieldEncryptor, backup.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get backup file: %w", err)
|
||||
}
|
||||
|
||||
// If not encrypted, return raw reader
|
||||
if backup.Encryption == backups_config.BackupEncryptionNone {
|
||||
s.logger.Info("Returning non-encrypted backup", "backupId", backupID)
|
||||
return fileReader, nil
|
||||
}
|
||||
|
||||
// Decrypt on-the-fly for encrypted backups
|
||||
if backup.Encryption != backups_config.BackupEncryptionEncrypted {
|
||||
if err := fileReader.Close(); err != nil {
|
||||
s.logger.Error("Failed to close file reader", "error", err)
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported encryption type: %s", backup.Encryption)
|
||||
}
|
||||
|
||||
if backup.EncryptionSalt == nil || backup.EncryptionIV == nil {
|
||||
if err := fileReader.Close(); err != nil {
|
||||
s.logger.Error("Failed to close file reader", "error", err)
|
||||
}
|
||||
return nil, fmt.Errorf("backup marked as encrypted but missing encryption metadata")
|
||||
}
|
||||
|
||||
// Get master key
|
||||
masterKey, err := s.secretKeyService.GetSecretKey()
|
||||
if err != nil {
|
||||
if closeErr := fileReader.Close(); closeErr != nil {
|
||||
s.logger.Error("Failed to close file reader", "error", closeErr)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to get master key: %w", err)
|
||||
}
|
||||
|
||||
// Decode salt and IV
|
||||
salt, err := base64.StdEncoding.DecodeString(*backup.EncryptionSalt)
|
||||
if err != nil {
|
||||
if closeErr := fileReader.Close(); closeErr != nil {
|
||||
s.logger.Error("Failed to close file reader", "error", closeErr)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to decode salt: %w", err)
|
||||
}
|
||||
|
||||
iv, err := base64.StdEncoding.DecodeString(*backup.EncryptionIV)
|
||||
if err != nil {
|
||||
if closeErr := fileReader.Close(); closeErr != nil {
|
||||
s.logger.Error("Failed to close file reader", "error", closeErr)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to decode IV: %w", err)
|
||||
}
|
||||
|
||||
// Wrap with decrypting reader
|
||||
decryptionReader, err := encryption.NewDecryptionReader(
|
||||
fileReader,
|
||||
masterKey,
|
||||
backup.ID,
|
||||
salt,
|
||||
iv,
|
||||
)
|
||||
if err != nil {
|
||||
if closeErr := fileReader.Close(); closeErr != nil {
|
||||
s.logger.Error("Failed to close file reader", "error", closeErr)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to create decrypting reader: %w", err)
|
||||
}
|
||||
|
||||
s.logger.Info("Returning encrypted backup with decryption", "backupId", backupID)
|
||||
|
||||
return &decryptionReaderCloser{
|
||||
decryptionReader,
|
||||
fileReader,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -3,18 +3,22 @@ package backups
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
usecases_postgresql "postgresus-backend/internal/features/backups/backups/usecases/postgresql"
|
||||
backups_config "postgresus-backend/internal/features/backups/config"
|
||||
"postgresus-backend/internal/features/databases"
|
||||
encryption_secrets "postgresus-backend/internal/features/encryption/secrets"
|
||||
"postgresus-backend/internal/features/notifiers"
|
||||
"postgresus-backend/internal/features/storages"
|
||||
users_enums "postgresus-backend/internal/features/users/enums"
|
||||
users_testing "postgresus-backend/internal/features/users/testing"
|
||||
workspaces_services "postgresus-backend/internal/features/workspaces/services"
|
||||
workspaces_testing "postgresus-backend/internal/features/workspaces/testing"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"postgresus-backend/internal/util/logger"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -53,11 +57,13 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {
|
||||
notifiers.GetNotifierService(),
|
||||
mockNotificationSender,
|
||||
backups_config.GetBackupConfigService(),
|
||||
encryption_secrets.GetSecretKeyService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
&CreateFailedBackupUsecase{},
|
||||
logger.GetLogger(),
|
||||
[]BackupRemoveListener{},
|
||||
workspaces_services.GetWorkspaceService(),
|
||||
nil, // auditLogService
|
||||
nil,
|
||||
NewBackupContextManager(),
|
||||
}
|
||||
|
||||
@@ -99,11 +105,13 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {
|
||||
notifiers.GetNotifierService(),
|
||||
mockNotificationSender,
|
||||
backups_config.GetBackupConfigService(),
|
||||
encryption_secrets.GetSecretKeyService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
&CreateSuccessBackupUsecase{},
|
||||
logger.GetLogger(),
|
||||
[]BackupRemoveListener{},
|
||||
workspaces_services.GetWorkspaceService(),
|
||||
nil, // auditLogService
|
||||
nil,
|
||||
NewBackupContextManager(),
|
||||
}
|
||||
|
||||
@@ -122,11 +130,13 @@ func Test_BackupExecuted_NotificationSent(t *testing.T) {
|
||||
notifiers.GetNotifierService(),
|
||||
mockNotificationSender,
|
||||
backups_config.GetBackupConfigService(),
|
||||
encryption_secrets.GetSecretKeyService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
&CreateSuccessBackupUsecase{},
|
||||
logger.GetLogger(),
|
||||
[]BackupRemoveListener{},
|
||||
workspaces_services.GetWorkspaceService(),
|
||||
nil, // auditLogService
|
||||
nil,
|
||||
NewBackupContextManager(),
|
||||
}
|
||||
|
||||
@@ -171,9 +181,9 @@ func (uc *CreateFailedBackupUsecase) Execute(
|
||||
backupProgressListener func(
|
||||
completedMBs float64,
|
||||
),
|
||||
) error {
|
||||
) (*usecases_postgresql.BackupMetadata, error) {
|
||||
backupProgressListener(10) // Assume we completed 10MB
|
||||
return errors.New("backup failed")
|
||||
return nil, errors.New("backup failed")
|
||||
}
|
||||
|
||||
type CreateSuccessBackupUsecase struct {
|
||||
@@ -188,7 +198,11 @@ func (uc *CreateSuccessBackupUsecase) Execute(
|
||||
backupProgressListener func(
|
||||
completedMBs float64,
|
||||
),
|
||||
) error {
|
||||
) (*usecases_postgresql.BackupMetadata, error) {
|
||||
backupProgressListener(10) // Assume we completed 10MB
|
||||
return nil
|
||||
return &usecases_postgresql.BackupMetadata{
|
||||
EncryptionSalt: nil,
|
||||
EncryptionIV: nil,
|
||||
Encryption: backups_config.BackupEncryptionNone,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ type CreateBackupUsecase struct {
|
||||
CreatePostgresqlBackupUsecase *usecases_postgresql.CreatePostgresqlBackupUsecase
|
||||
}
|
||||
|
||||
// Execute creates a backup of the database and returns the backup size in MB
|
||||
// Execute creates a backup of the database and returns the backup metadata
|
||||
func (uc *CreateBackupUsecase) Execute(
|
||||
ctx context.Context,
|
||||
backupID uuid.UUID,
|
||||
@@ -25,7 +25,7 @@ func (uc *CreateBackupUsecase) Execute(
|
||||
backupProgressListener func(
|
||||
completedMBs float64,
|
||||
),
|
||||
) error {
|
||||
) (*usecases_postgresql.BackupMetadata, error) {
|
||||
if database.Type == databases.DatabaseTypePostgres {
|
||||
return uc.CreatePostgresqlBackupUsecase.Execute(
|
||||
ctx,
|
||||
@@ -37,5 +37,5 @@ func (uc *CreateBackupUsecase) Execute(
|
||||
)
|
||||
}
|
||||
|
||||
return errors.New("database type not supported")
|
||||
return nil, errors.New("database type not supported")
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package usecases_postgresql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -14,17 +15,39 @@ import (
|
||||
"time"
|
||||
|
||||
"postgresus-backend/internal/config"
|
||||
backup_encryption "postgresus-backend/internal/features/backups/backups/encryption"
|
||||
backups_config "postgresus-backend/internal/features/backups/config"
|
||||
"postgresus-backend/internal/features/databases"
|
||||
pgtypes "postgresus-backend/internal/features/databases/databases/postgresql"
|
||||
encryption_secrets "postgresus-backend/internal/features/encryption/secrets"
|
||||
"postgresus-backend/internal/features/storages"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"postgresus-backend/internal/util/tools"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
backupTimeout = 23 * time.Hour
|
||||
shutdownCheckInterval = 1 * time.Second
|
||||
copyBufferSize = 8 * 1024 * 1024
|
||||
progressReportIntervalMB = 1.0
|
||||
pgConnectTimeout = 30
|
||||
compressionLevel = 5
|
||||
exitCodeAccessViolation = -1073741819
|
||||
exitCodeGenericError = 1
|
||||
exitCodeConnectionError = 2
|
||||
)
|
||||
|
||||
type CreatePostgresqlBackupUsecase struct {
|
||||
logger *slog.Logger
|
||||
logger *slog.Logger
|
||||
secretKeyService *encryption_secrets.SecretKeyService
|
||||
fieldEncryptor encryption.FieldEncryptor
|
||||
}
|
||||
|
||||
type writeResult struct {
|
||||
bytesWritten int
|
||||
writeErr error
|
||||
}
|
||||
|
||||
// Execute creates a backup of the database
|
||||
@@ -37,7 +60,7 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(
|
||||
backupProgressListener func(
|
||||
completedMBs float64,
|
||||
),
|
||||
) error {
|
||||
) (*BackupMetadata, error) {
|
||||
uc.logger.Info(
|
||||
"Creating PostgreSQL backup via pg_dump custom format",
|
||||
"databaseId",
|
||||
@@ -47,38 +70,24 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(
|
||||
)
|
||||
|
||||
if !backupConfig.IsBackupsEnabled {
|
||||
return fmt.Errorf("backups are not enabled for this database: \"%s\"", db.Name)
|
||||
return nil, fmt.Errorf("backups are not enabled for this database: \"%s\"", db.Name)
|
||||
}
|
||||
|
||||
pg := db.Postgresql
|
||||
|
||||
if pg == nil {
|
||||
return fmt.Errorf("postgresql database configuration is required for pg_dump backups")
|
||||
return nil, fmt.Errorf("postgresql database configuration is required for pg_dump backups")
|
||||
}
|
||||
|
||||
if pg.Database == nil || *pg.Database == "" {
|
||||
return fmt.Errorf("database name is required for pg_dump backups")
|
||||
return nil, fmt.Errorf("database name is required for pg_dump backups")
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-Fc", // custom format with built-in compression
|
||||
"--no-password", // Use environment variable for password, prevent prompts
|
||||
"-h", pg.Host,
|
||||
"-p", strconv.Itoa(pg.Port),
|
||||
"-U", pg.Username,
|
||||
"-d", *pg.Database,
|
||||
"--verbose", // Add verbose output to help with debugging
|
||||
}
|
||||
args := uc.buildPgDumpArgs(pg)
|
||||
|
||||
// Use zstd compression level 5 for PostgreSQL 16+ (better compression and speed)
|
||||
// Fall back to gzip compression level 5 for older versions (12-15)
|
||||
if pg.Version == tools.PostgresqlVersion12 || pg.Version == tools.PostgresqlVersion13 ||
|
||||
pg.Version == tools.PostgresqlVersion14 || pg.Version == tools.PostgresqlVersion15 {
|
||||
args = append(args, "-Z", "5")
|
||||
uc.logger.Info("Using gzip compression level 5 (zstd not available)", "version", pg.Version)
|
||||
} else {
|
||||
args = append(args, "--compress=zstd:5")
|
||||
uc.logger.Info("Using zstd compression level 5", "version", pg.Version)
|
||||
decryptedPassword, err := uc.fieldEncryptor.Decrypt(db.ID, pg.Password)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decrypt database password: %w", err)
|
||||
}
|
||||
|
||||
return uc.streamToStorage(
|
||||
@@ -92,7 +101,7 @@ func (uc *CreatePostgresqlBackupUsecase) Execute(
|
||||
config.GetEnv().PostgresesInstallDir,
|
||||
),
|
||||
args,
|
||||
pg.Password,
|
||||
decryptedPassword,
|
||||
storage,
|
||||
db,
|
||||
backupProgressListener,
|
||||
@@ -110,36 +119,15 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
|
||||
storage *storages.Storage,
|
||||
db *databases.Database,
|
||||
backupProgressListener func(completedMBs float64),
|
||||
) error {
|
||||
) (*BackupMetadata, error) {
|
||||
uc.logger.Info("Streaming PostgreSQL backup to storage", "pgBin", pgBin, "args", args)
|
||||
|
||||
// if backup not fit into 23 hours, Postgresus
|
||||
// seems not to work for such database size
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 23*time.Hour)
|
||||
ctx, cancel := uc.createBackupContext(parentCtx)
|
||||
defer cancel()
|
||||
|
||||
// Monitor for shutdown and cancel context if needed
|
||||
go func() {
|
||||
ticker := time.NewTicker(1 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if config.IsShouldShutdown() {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Create temporary .pgpass file as a more reliable alternative to PGPASSWORD
|
||||
pgpassFile, err := uc.createTempPgpassFile(db.Postgresql, password)
|
||||
pgpassFile, err := uc.setupPgpassFile(db.Postgresql, password)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temporary .pgpass file: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if pgpassFile != "" {
|
||||
@@ -147,87 +135,21 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
|
||||
}
|
||||
}()
|
||||
|
||||
// Verify .pgpass file was created successfully
|
||||
if pgpassFile == "" {
|
||||
return fmt.Errorf("temporary .pgpass file was not created")
|
||||
}
|
||||
|
||||
// Verify .pgpass file was created correctly
|
||||
if info, err := os.Stat(pgpassFile); err == nil {
|
||||
uc.logger.Info("Temporary .pgpass file created successfully",
|
||||
"pgpassFile", pgpassFile,
|
||||
"size", info.Size(),
|
||||
"mode", info.Mode(),
|
||||
)
|
||||
} else {
|
||||
return fmt.Errorf("failed to verify .pgpass file: %w", err)
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, pgBin, args...)
|
||||
uc.logger.Info("Executing PostgreSQL backup command", "command", cmd.String())
|
||||
|
||||
// Start with system environment variables to preserve Windows PATH, SystemRoot, etc.
|
||||
cmd.Env = os.Environ()
|
||||
|
||||
// Use the .pgpass file for authentication
|
||||
cmd.Env = append(cmd.Env, "PGPASSFILE="+pgpassFile)
|
||||
uc.logger.Info("Using temporary .pgpass file for authentication", "pgpassFile", pgpassFile)
|
||||
|
||||
// Debug password setup (without exposing the actual password)
|
||||
uc.logger.Info("Setting up PostgreSQL environment",
|
||||
"passwordLength", len(password),
|
||||
"passwordEmpty", password == "",
|
||||
"pgBin", pgBin,
|
||||
"usingPgpassFile", true,
|
||||
"parallelJobs", backupConfig.CpuCount,
|
||||
)
|
||||
|
||||
// Add PostgreSQL-specific environment variables
|
||||
cmd.Env = append(cmd.Env, "PGCLIENTENCODING=UTF8")
|
||||
cmd.Env = append(cmd.Env, "PGCONNECT_TIMEOUT=30")
|
||||
|
||||
// Add encoding-related environment variables to handle character encoding issues
|
||||
cmd.Env = append(cmd.Env, "LC_ALL=C.UTF-8")
|
||||
cmd.Env = append(cmd.Env, "LANG=C.UTF-8")
|
||||
|
||||
// Add PostgreSQL-specific encoding settings
|
||||
cmd.Env = append(cmd.Env, "PGOPTIONS=--client-encoding=UTF8")
|
||||
|
||||
shouldRequireSSL := db.Postgresql.IsHttps
|
||||
|
||||
// Require SSL when explicitly configured
|
||||
if shouldRequireSSL {
|
||||
cmd.Env = append(cmd.Env, "PGSSLMODE=require")
|
||||
uc.logger.Info("Using required SSL mode", "configuredHttps", db.Postgresql.IsHttps)
|
||||
} else {
|
||||
// SSL not explicitly required, but prefer it if available
|
||||
cmd.Env = append(cmd.Env, "PGSSLMODE=prefer")
|
||||
uc.logger.Info("Using preferred SSL mode", "configuredHttps", db.Postgresql.IsHttps)
|
||||
}
|
||||
|
||||
// Set other SSL parameters to avoid certificate issues
|
||||
cmd.Env = append(cmd.Env, "PGSSLCERT=") // No client certificate
|
||||
cmd.Env = append(cmd.Env, "PGSSLKEY=") // No client key
|
||||
cmd.Env = append(cmd.Env, "PGSSLROOTCERT=") // No root certificate verification
|
||||
cmd.Env = append(cmd.Env, "PGSSLCRL=") // No certificate revocation list
|
||||
|
||||
// Verify executable exists and is accessible
|
||||
if _, err := exec.LookPath(pgBin); err != nil {
|
||||
return fmt.Errorf(
|
||||
"PostgreSQL executable not found or not accessible: %s - %w",
|
||||
pgBin,
|
||||
err,
|
||||
)
|
||||
if err := uc.setupPgEnvironment(cmd, pgpassFile, db.Postgresql.IsHttps, password, backupConfig.CpuCount, pgBin); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pgStdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("stdout pipe: %w", err)
|
||||
return nil, fmt.Errorf("stdout pipe: %w", err)
|
||||
}
|
||||
|
||||
pgStderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("stderr pipe: %w", err)
|
||||
return nil, fmt.Errorf("stderr pipe: %w", err)
|
||||
}
|
||||
|
||||
// Capture stderr in a separate goroutine to ensure we don't miss any error output
|
||||
@@ -237,23 +159,31 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
|
||||
stderrCh <- stderrOutput
|
||||
}()
|
||||
|
||||
// A pipe connecting pg_dump output → storage
|
||||
storageReader, storageWriter := io.Pipe()
|
||||
|
||||
// Create a counting writer to track bytes
|
||||
countingWriter := &CountingWriter{writer: storageWriter}
|
||||
finalWriter, encryptionWriter, backupMetadata, err := uc.setupBackupEncryption(
|
||||
backupID,
|
||||
backupConfig,
|
||||
storageWriter,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
countingWriter := &CountingWriter{writer: finalWriter}
|
||||
|
||||
// The backup ID becomes the object key / filename in storage
|
||||
|
||||
// Start streaming into storage in its own goroutine
|
||||
saveErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
saveErrCh <- storage.SaveFile(uc.logger, backupID, storageReader)
|
||||
saveErr := storage.SaveFile(ctx, uc.fieldEncryptor, uc.logger, backupID, storageReader)
|
||||
saveErrCh <- saveErr
|
||||
}()
|
||||
|
||||
// Start pg_dump
|
||||
if err = cmd.Start(); err != nil {
|
||||
return fmt.Errorf("start %s: %w", filepath.Base(pgBin), err)
|
||||
return nil, fmt.Errorf("start %s: %w", filepath.Base(pgBin), err)
|
||||
}
|
||||
|
||||
// Copy pg output directly to storage with shutdown checks
|
||||
@@ -270,37 +200,22 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
|
||||
copyResultCh <- err
|
||||
}()
|
||||
|
||||
// Wait for the copy to finish first, then the dump process
|
||||
copyErr := <-copyResultCh
|
||||
bytesWritten := <-bytesWrittenCh
|
||||
waitErr := cmd.Wait()
|
||||
|
||||
// Check for shutdown or cancellation before finalizing
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if pipeWriter, ok := countingWriter.writer.(*io.PipeWriter); ok {
|
||||
if err := pipeWriter.Close(); err != nil {
|
||||
uc.logger.Error("Failed to close counting writer", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
<-saveErrCh // Wait for storage to finish
|
||||
|
||||
if config.IsShouldShutdown() {
|
||||
return fmt.Errorf("backup cancelled due to shutdown")
|
||||
}
|
||||
return fmt.Errorf("backup cancelled")
|
||||
uc.cleanupOnCancellation(encryptionWriter, storageWriter, saveErrCh)
|
||||
return nil, uc.checkCancellationReason()
|
||||
default:
|
||||
}
|
||||
|
||||
// Close the pipe writer to signal end of data
|
||||
if pipeWriter, ok := countingWriter.writer.(*io.PipeWriter); ok {
|
||||
if err := pipeWriter.Close(); err != nil {
|
||||
uc.logger.Error("Failed to close counting writer", "error", err)
|
||||
}
|
||||
if err := uc.closeWriters(encryptionWriter, storageWriter); err != nil {
|
||||
<-saveErrCh
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Wait until storage ends reading
|
||||
saveErr := <-saveErrCh
|
||||
stderrOutput := <-stderrCh
|
||||
|
||||
@@ -312,149 +227,34 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage(
|
||||
|
||||
switch {
|
||||
case waitErr != nil:
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if config.IsShouldShutdown() {
|
||||
return fmt.Errorf("backup cancelled due to shutdown")
|
||||
}
|
||||
return fmt.Errorf("backup cancelled")
|
||||
default:
|
||||
if err := uc.checkCancellation(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Enhanced error handling for PostgreSQL connection and SSL issues
|
||||
stderrStr := string(stderrOutput)
|
||||
errorMsg := fmt.Sprintf(
|
||||
"%s failed: %v – stderr: %s",
|
||||
filepath.Base(pgBin),
|
||||
waitErr,
|
||||
stderrStr,
|
||||
)
|
||||
|
||||
// Check for specific PostgreSQL error patterns
|
||||
if exitErr, ok := waitErr.(*exec.ExitError); ok {
|
||||
exitCode := exitErr.ExitCode()
|
||||
|
||||
// Enhanced debugging for exit status 1 with empty stderr
|
||||
if exitCode == 1 && strings.TrimSpace(stderrStr) == "" {
|
||||
uc.logger.Error("pg_dump failed with exit status 1 but no stderr output",
|
||||
"pgBin", pgBin,
|
||||
"args", args,
|
||||
"env_vars", []string{
|
||||
"PGCLIENTENCODING=UTF8",
|
||||
"PGCONNECT_TIMEOUT=30",
|
||||
"LC_ALL=C.UTF-8",
|
||||
"LANG=C.UTF-8",
|
||||
"PGOPTIONS=--client-encoding=UTF8",
|
||||
},
|
||||
)
|
||||
|
||||
errorMsg = fmt.Sprintf(
|
||||
"%s failed with exit status 1 but provided no error details. "+
|
||||
"This often indicates: "+
|
||||
"1) Connection timeout or refused connection, "+
|
||||
"2) Authentication failure with incorrect credentials, "+
|
||||
"3) Database does not exist, "+
|
||||
"4) Network connectivity issues, "+
|
||||
"5) PostgreSQL server not running. "+
|
||||
"Command executed: %s %s",
|
||||
filepath.Base(pgBin),
|
||||
pgBin,
|
||||
strings.Join(args, " "),
|
||||
)
|
||||
} else if exitCode == -1073741819 { // 0xC0000005 in decimal
|
||||
uc.logger.Error("PostgreSQL tool crashed with access violation",
|
||||
"pgBin", pgBin,
|
||||
"args", args,
|
||||
"exitCode", fmt.Sprintf("0x%X", uint32(exitCode)),
|
||||
)
|
||||
|
||||
errorMsg = fmt.Sprintf(
|
||||
"%s crashed with access violation (0xC0000005). This may indicate incompatible PostgreSQL version, corrupted installation, or connection issues. stderr: %s",
|
||||
filepath.Base(pgBin),
|
||||
stderrStr,
|
||||
)
|
||||
} else if exitCode == 1 || exitCode == 2 {
|
||||
// Check for common connection and authentication issues
|
||||
if containsIgnoreCase(stderrStr, "pg_hba.conf") {
|
||||
errorMsg = fmt.Sprintf(
|
||||
"PostgreSQL connection rejected by server configuration (pg_hba.conf). The server may not allow connections from your IP address or may require different authentication settings. stderr: %s",
|
||||
stderrStr,
|
||||
)
|
||||
} else if containsIgnoreCase(stderrStr, "no password supplied") || containsIgnoreCase(stderrStr, "fe_sendauth") {
|
||||
errorMsg = fmt.Sprintf(
|
||||
"PostgreSQL authentication failed - no password supplied. "+
|
||||
"PGPASSWORD environment variable may not be working correctly on this system. "+
|
||||
"Password length: %d, Password empty: %v. "+
|
||||
"Consider using a .pgpass file as an alternative. stderr: %s",
|
||||
len(password),
|
||||
password == "",
|
||||
stderrStr,
|
||||
)
|
||||
} else if containsIgnoreCase(stderrStr, "ssl") && containsIgnoreCase(stderrStr, "connection") {
|
||||
errorMsg = fmt.Sprintf(
|
||||
"PostgreSQL SSL connection failed. The server may require SSL encryption or have SSL configuration issues. stderr: %s",
|
||||
stderrStr,
|
||||
)
|
||||
} else if containsIgnoreCase(stderrStr, "connection") && containsIgnoreCase(stderrStr, "refused") {
|
||||
errorMsg = fmt.Sprintf(
|
||||
"PostgreSQL connection refused. Check if the server is running and accessible from your network. stderr: %s",
|
||||
stderrStr,
|
||||
)
|
||||
} else if containsIgnoreCase(stderrStr, "authentication") || containsIgnoreCase(stderrStr, "password") {
|
||||
errorMsg = fmt.Sprintf(
|
||||
"PostgreSQL authentication failed. Check username and password. stderr: %s",
|
||||
stderrStr,
|
||||
)
|
||||
} else if containsIgnoreCase(stderrStr, "timeout") {
|
||||
errorMsg = fmt.Sprintf(
|
||||
"PostgreSQL connection timeout. The server may be unreachable or overloaded. stderr: %s",
|
||||
stderrStr,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return errors.New(errorMsg)
|
||||
return nil, uc.buildPgDumpErrorMessage(waitErr, stderrOutput, pgBin, args, password)
|
||||
case copyErr != nil:
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if config.IsShouldShutdown() {
|
||||
return fmt.Errorf("backup cancelled due to shutdown")
|
||||
}
|
||||
return fmt.Errorf("backup cancelled")
|
||||
default:
|
||||
if err := uc.checkCancellation(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return fmt.Errorf("copy to storage: %w", copyErr)
|
||||
return nil, fmt.Errorf("copy to storage: %w", copyErr)
|
||||
case saveErr != nil:
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if config.IsShouldShutdown() {
|
||||
return fmt.Errorf("backup cancelled due to shutdown")
|
||||
}
|
||||
return fmt.Errorf("backup cancelled")
|
||||
default:
|
||||
if err := uc.checkCancellation(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return fmt.Errorf("save to storage: %w", saveErr)
|
||||
return nil, fmt.Errorf("save to storage: %w", saveErr)
|
||||
}
|
||||
|
||||
return nil
|
||||
return &backupMetadata, nil
|
||||
}
|
||||
|
||||
// copyWithShutdownCheck copies data from src to dst while checking for shutdown
|
||||
func (uc *CreatePostgresqlBackupUsecase) copyWithShutdownCheck(
|
||||
ctx context.Context,
|
||||
dst io.Writer,
|
||||
src io.Reader,
|
||||
backupProgressListener func(completedMBs float64),
|
||||
) (int64, error) {
|
||||
buf := make([]byte, 32*1024) // 32KB buffer
|
||||
buf := make([]byte, copyBufferSize)
|
||||
var totalBytesWritten int64
|
||||
|
||||
// Progress reporting interval - report every 1MB of data
|
||||
var lastReportedMB float64
|
||||
const reportIntervalMB = 1.0
|
||||
|
||||
for {
|
||||
select {
|
||||
@@ -469,7 +269,23 @@ func (uc *CreatePostgresqlBackupUsecase) copyWithShutdownCheck(
|
||||
|
||||
bytesRead, readErr := src.Read(buf)
|
||||
if bytesRead > 0 {
|
||||
bytesWritten, writeErr := dst.Write(buf[0:bytesRead])
|
||||
writeResultCh := make(chan writeResult, 1)
|
||||
go func() {
|
||||
bytesWritten, writeErr := dst.Write(buf[0:bytesRead])
|
||||
writeResultCh <- writeResult{bytesWritten, writeErr}
|
||||
}()
|
||||
|
||||
var bytesWritten int
|
||||
var writeErr error
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return totalBytesWritten, fmt.Errorf("copy cancelled during write: %w", ctx.Err())
|
||||
case result := <-writeResultCh:
|
||||
bytesWritten = result.bytesWritten
|
||||
writeErr = result.writeErr
|
||||
}
|
||||
|
||||
if bytesWritten < 0 || bytesRead < bytesWritten {
|
||||
bytesWritten = 0
|
||||
if writeErr == nil {
|
||||
@@ -487,12 +303,9 @@ func (uc *CreatePostgresqlBackupUsecase) copyWithShutdownCheck(
|
||||
|
||||
totalBytesWritten += int64(bytesWritten)
|
||||
|
||||
// Report progress based on total size
|
||||
if backupProgressListener != nil {
|
||||
currentSizeMB := float64(totalBytesWritten) / (1024 * 1024)
|
||||
|
||||
// Only report if we've written at least 1MB more data than last report
|
||||
if currentSizeMB >= lastReportedMB+reportIntervalMB {
|
||||
if currentSizeMB >= lastReportedMB+progressReportIntervalMB {
|
||||
backupProgressListener(currentSizeMB)
|
||||
lastReportedMB = currentSizeMB
|
||||
}
|
||||
@@ -503,7 +316,6 @@ func (uc *CreatePostgresqlBackupUsecase) copyWithShutdownCheck(
|
||||
if readErr != io.EOF {
|
||||
return totalBytesWritten, readErr
|
||||
}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -511,12 +323,413 @@ func (uc *CreatePostgresqlBackupUsecase) copyWithShutdownCheck(
|
||||
return totalBytesWritten, nil
|
||||
}
|
||||
|
||||
// containsIgnoreCase checks if a string contains a substring, ignoring case
|
||||
func containsIgnoreCase(str, substr string) bool {
|
||||
return strings.Contains(strings.ToLower(str), strings.ToLower(substr))
|
||||
func (uc *CreatePostgresqlBackupUsecase) buildPgDumpArgs(pg *pgtypes.PostgresqlDatabase) []string {
|
||||
args := []string{
|
||||
"-Fc",
|
||||
"--no-password",
|
||||
"-h", pg.Host,
|
||||
"-p", strconv.Itoa(pg.Port),
|
||||
"-U", pg.Username,
|
||||
"-d", *pg.Database,
|
||||
"--verbose",
|
||||
}
|
||||
|
||||
compressionArgs := uc.getCompressionArgs(pg.Version)
|
||||
return append(args, compressionArgs...)
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) getCompressionArgs(
|
||||
version tools.PostgresqlVersion,
|
||||
) []string {
|
||||
if uc.isOlderPostgresVersion(version) {
|
||||
uc.logger.Info("Using gzip compression level 5 (zstd not available)", "version", version)
|
||||
return []string{"-Z", strconv.Itoa(compressionLevel)}
|
||||
}
|
||||
|
||||
uc.logger.Info("Using zstd compression level 5", "version", version)
|
||||
return []string{fmt.Sprintf("--compress=zstd:%d", compressionLevel)}
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) isOlderPostgresVersion(
|
||||
version tools.PostgresqlVersion,
|
||||
) bool {
|
||||
return version == tools.PostgresqlVersion12 ||
|
||||
version == tools.PostgresqlVersion13 ||
|
||||
version == tools.PostgresqlVersion14 ||
|
||||
version == tools.PostgresqlVersion15
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) createBackupContext(
|
||||
parentCtx context.Context,
|
||||
) (context.Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithTimeout(parentCtx, backupTimeout)
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(shutdownCheckInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-parentCtx.Done():
|
||||
cancel()
|
||||
return
|
||||
case <-ticker.C:
|
||||
if config.IsShouldShutdown() {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ctx, cancel
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) setupPgpassFile(
|
||||
pgConfig *pgtypes.PostgresqlDatabase,
|
||||
password string,
|
||||
) (string, error) {
|
||||
pgpassFile, err := uc.createTempPgpassFile(pgConfig, password)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temporary .pgpass file: %w", err)
|
||||
}
|
||||
|
||||
if pgpassFile == "" {
|
||||
return "", fmt.Errorf("temporary .pgpass file was not created")
|
||||
}
|
||||
|
||||
if info, err := os.Stat(pgpassFile); err == nil {
|
||||
uc.logger.Info("Temporary .pgpass file created successfully",
|
||||
"pgpassFile", pgpassFile,
|
||||
"size", info.Size(),
|
||||
"mode", info.Mode(),
|
||||
)
|
||||
} else {
|
||||
return "", fmt.Errorf("failed to verify .pgpass file: %w", err)
|
||||
}
|
||||
|
||||
return pgpassFile, nil
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) setupPgEnvironment(
|
||||
cmd *exec.Cmd,
|
||||
pgpassFile string,
|
||||
shouldRequireSSL bool,
|
||||
password string,
|
||||
cpuCount int,
|
||||
pgBin string,
|
||||
) error {
|
||||
cmd.Env = os.Environ()
|
||||
cmd.Env = append(cmd.Env, "PGPASSFILE="+pgpassFile)
|
||||
|
||||
uc.logger.Info("Using temporary .pgpass file for authentication", "pgpassFile", pgpassFile)
|
||||
uc.logger.Info("Setting up PostgreSQL environment",
|
||||
"passwordLength", len(password),
|
||||
"passwordEmpty", password == "",
|
||||
"pgBin", pgBin,
|
||||
"usingPgpassFile", true,
|
||||
"parallelJobs", cpuCount,
|
||||
)
|
||||
|
||||
cmd.Env = append(cmd.Env,
|
||||
"PGCLIENTENCODING=UTF8",
|
||||
"PGCONNECT_TIMEOUT="+strconv.Itoa(pgConnectTimeout),
|
||||
"LC_ALL=C.UTF-8",
|
||||
"LANG=C.UTF-8",
|
||||
)
|
||||
|
||||
if shouldRequireSSL {
|
||||
cmd.Env = append(cmd.Env, "PGSSLMODE=require")
|
||||
uc.logger.Info("Using required SSL mode", "configuredHttps", shouldRequireSSL)
|
||||
} else {
|
||||
cmd.Env = append(cmd.Env, "PGSSLMODE=prefer")
|
||||
uc.logger.Info("Using preferred SSL mode", "configuredHttps", shouldRequireSSL)
|
||||
}
|
||||
|
||||
cmd.Env = append(cmd.Env,
|
||||
"PGSSLCERT=",
|
||||
"PGSSLKEY=",
|
||||
"PGSSLROOTCERT=",
|
||||
"PGSSLCRL=",
|
||||
)
|
||||
|
||||
if _, err := exec.LookPath(pgBin); err != nil {
|
||||
return fmt.Errorf("PostgreSQL executable not found or not accessible: %s - %w", pgBin, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) setupBackupEncryption(
|
||||
backupID uuid.UUID,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
storageWriter io.WriteCloser,
|
||||
) (io.Writer, *backup_encryption.EncryptionWriter, BackupMetadata, error) {
|
||||
metadata := BackupMetadata{}
|
||||
|
||||
if backupConfig.Encryption != backups_config.BackupEncryptionEncrypted {
|
||||
metadata.Encryption = backups_config.BackupEncryptionNone
|
||||
uc.logger.Info("Encryption disabled for backup", "backupId", backupID)
|
||||
return storageWriter, nil, metadata, nil
|
||||
}
|
||||
|
||||
salt, err := backup_encryption.GenerateSalt()
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to generate salt: %w", err)
|
||||
}
|
||||
|
||||
nonce, err := backup_encryption.GenerateNonce()
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to generate nonce: %w", err)
|
||||
}
|
||||
|
||||
masterKey, err := uc.secretKeyService.GetSecretKey()
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to get master key: %w", err)
|
||||
}
|
||||
|
||||
encWriter, err := backup_encryption.NewEncryptionWriter(
|
||||
storageWriter,
|
||||
masterKey,
|
||||
backupID,
|
||||
salt,
|
||||
nonce,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, nil, metadata, fmt.Errorf("failed to create encrypting writer: %w", err)
|
||||
}
|
||||
|
||||
saltBase64 := base64.StdEncoding.EncodeToString(salt)
|
||||
nonceBase64 := base64.StdEncoding.EncodeToString(nonce)
|
||||
metadata.EncryptionSalt = &saltBase64
|
||||
metadata.EncryptionIV = &nonceBase64
|
||||
metadata.Encryption = backups_config.BackupEncryptionEncrypted
|
||||
|
||||
uc.logger.Info("Encryption enabled for backup", "backupId", backupID)
|
||||
return encWriter, encWriter, metadata, nil
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) cleanupOnCancellation(
|
||||
encryptionWriter *backup_encryption.EncryptionWriter,
|
||||
storageWriter io.WriteCloser,
|
||||
saveErrCh chan error,
|
||||
) {
|
||||
if encryptionWriter != nil {
|
||||
go func() {
|
||||
if closeErr := encryptionWriter.Close(); closeErr != nil {
|
||||
uc.logger.Error(
|
||||
"Failed to close encrypting writer during cancellation",
|
||||
"error",
|
||||
closeErr,
|
||||
)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if err := storageWriter.Close(); err != nil {
|
||||
uc.logger.Error("Failed to close pipe writer during cancellation", "error", err)
|
||||
}
|
||||
|
||||
<-saveErrCh
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) closeWriters(
|
||||
encryptionWriter *backup_encryption.EncryptionWriter,
|
||||
storageWriter io.WriteCloser,
|
||||
) error {
|
||||
encryptionCloseErrCh := make(chan error, 1)
|
||||
if encryptionWriter != nil {
|
||||
go func() {
|
||||
closeErr := encryptionWriter.Close()
|
||||
if closeErr != nil {
|
||||
uc.logger.Error("Failed to close encrypting writer", "error", closeErr)
|
||||
}
|
||||
encryptionCloseErrCh <- closeErr
|
||||
}()
|
||||
} else {
|
||||
encryptionCloseErrCh <- nil
|
||||
}
|
||||
|
||||
encryptionCloseErr := <-encryptionCloseErrCh
|
||||
if encryptionCloseErr != nil {
|
||||
if err := storageWriter.Close(); err != nil {
|
||||
uc.logger.Error("Failed to close pipe writer after encryption error", "error", err)
|
||||
}
|
||||
return fmt.Errorf("failed to close encryption writer: %w", encryptionCloseErr)
|
||||
}
|
||||
|
||||
if err := storageWriter.Close(); err != nil {
|
||||
uc.logger.Error("Failed to close pipe writer", "error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) checkCancellation(ctx context.Context) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if config.IsShouldShutdown() {
|
||||
return fmt.Errorf("backup cancelled due to shutdown")
|
||||
}
|
||||
return fmt.Errorf("backup cancelled")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) checkCancellationReason() error {
|
||||
if config.IsShouldShutdown() {
|
||||
return fmt.Errorf("backup cancelled due to shutdown")
|
||||
}
|
||||
return fmt.Errorf("backup cancelled")
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) buildPgDumpErrorMessage(
|
||||
waitErr error,
|
||||
stderrOutput []byte,
|
||||
pgBin string,
|
||||
args []string,
|
||||
password string,
|
||||
) error {
|
||||
stderrStr := string(stderrOutput)
|
||||
errorMsg := fmt.Sprintf("%s failed: %v – stderr: %s", filepath.Base(pgBin), waitErr, stderrStr)
|
||||
|
||||
exitErr, ok := waitErr.(*exec.ExitError)
|
||||
if !ok {
|
||||
return errors.New(errorMsg)
|
||||
}
|
||||
|
||||
exitCode := exitErr.ExitCode()
|
||||
|
||||
if exitCode == exitCodeGenericError && strings.TrimSpace(stderrStr) == "" {
|
||||
return uc.handleExitCode1NoStderr(pgBin, args)
|
||||
}
|
||||
|
||||
if exitCode == exitCodeAccessViolation {
|
||||
return uc.handleAccessViolation(pgBin, stderrStr)
|
||||
}
|
||||
|
||||
if exitCode == exitCodeGenericError || exitCode == exitCodeConnectionError {
|
||||
return uc.handleConnectionErrors(stderrStr, password)
|
||||
}
|
||||
|
||||
return errors.New(errorMsg)
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) handleExitCode1NoStderr(
|
||||
pgBin string,
|
||||
args []string,
|
||||
) error {
|
||||
uc.logger.Error("pg_dump failed with exit status 1 but no stderr output",
|
||||
"pgBin", pgBin,
|
||||
"args", args,
|
||||
"env_vars", []string{
|
||||
"PGCLIENTENCODING=UTF8",
|
||||
"PGCONNECT_TIMEOUT=" + strconv.Itoa(pgConnectTimeout),
|
||||
"LC_ALL=C.UTF-8",
|
||||
"LANG=C.UTF-8",
|
||||
},
|
||||
)
|
||||
|
||||
return fmt.Errorf(
|
||||
"%s failed with exit status 1 but provided no error details. "+
|
||||
"This often indicates: "+
|
||||
"1) Connection timeout or refused connection, "+
|
||||
"2) Authentication failure with incorrect credentials, "+
|
||||
"3) Database does not exist, "+
|
||||
"4) Network connectivity issues, "+
|
||||
"5) PostgreSQL server not running. "+
|
||||
"Command executed: %s %s",
|
||||
filepath.Base(pgBin),
|
||||
pgBin,
|
||||
strings.Join(args, " "),
|
||||
)
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) handleAccessViolation(
|
||||
pgBin string,
|
||||
stderrStr string,
|
||||
) error {
|
||||
uc.logger.Error("PostgreSQL tool crashed with access violation",
|
||||
"pgBin", pgBin,
|
||||
"exitCode", "0xC0000005",
|
||||
)
|
||||
|
||||
return fmt.Errorf(
|
||||
"%s crashed with access violation (0xC0000005). "+
|
||||
"This may indicate incompatible PostgreSQL version, corrupted installation, or connection issues. "+
|
||||
"stderr: %s",
|
||||
filepath.Base(pgBin),
|
||||
stderrStr,
|
||||
)
|
||||
}
|
||||
|
||||
func (uc *CreatePostgresqlBackupUsecase) handleConnectionErrors(
|
||||
stderrStr string,
|
||||
password string,
|
||||
) error {
|
||||
if containsIgnoreCase(stderrStr, "pg_hba.conf") {
|
||||
return fmt.Errorf(
|
||||
"PostgreSQL connection rejected by server configuration (pg_hba.conf). "+
|
||||
"The server may not allow connections from your IP address or may require different authentication settings. "+
|
||||
"stderr: %s",
|
||||
stderrStr,
|
||||
)
|
||||
}
|
||||
|
||||
if containsIgnoreCase(stderrStr, "no password supplied") ||
|
||||
containsIgnoreCase(stderrStr, "fe_sendauth") {
|
||||
return fmt.Errorf(
|
||||
"PostgreSQL authentication failed - no password supplied. "+
|
||||
"PGPASSWORD environment variable may not be working correctly on this system. "+
|
||||
"Password length: %d, Password empty: %v. "+
|
||||
"Consider using a .pgpass file as an alternative. "+
|
||||
"stderr: %s",
|
||||
len(password),
|
||||
password == "",
|
||||
stderrStr,
|
||||
)
|
||||
}
|
||||
|
||||
if containsIgnoreCase(stderrStr, "ssl") && containsIgnoreCase(stderrStr, "connection") {
|
||||
return fmt.Errorf(
|
||||
"PostgreSQL SSL connection failed. "+
|
||||
"The server may require SSL encryption or have SSL configuration issues. "+
|
||||
"stderr: %s",
|
||||
stderrStr,
|
||||
)
|
||||
}
|
||||
|
||||
if containsIgnoreCase(stderrStr, "connection") && containsIgnoreCase(stderrStr, "refused") {
|
||||
return fmt.Errorf(
|
||||
"PostgreSQL connection refused. "+
|
||||
"Check if the server is running and accessible from your network. "+
|
||||
"stderr: %s",
|
||||
stderrStr,
|
||||
)
|
||||
}
|
||||
|
||||
if containsIgnoreCase(stderrStr, "authentication") ||
|
||||
containsIgnoreCase(stderrStr, "password") {
|
||||
return fmt.Errorf(
|
||||
"PostgreSQL authentication failed. Check username and password. stderr: %s",
|
||||
stderrStr,
|
||||
)
|
||||
}
|
||||
|
||||
if containsIgnoreCase(stderrStr, "timeout") {
|
||||
return fmt.Errorf(
|
||||
"PostgreSQL connection timeout. The server may be unreachable or overloaded. stderr: %s",
|
||||
stderrStr,
|
||||
)
|
||||
}
|
||||
|
||||
return fmt.Errorf("PostgreSQL connection or authentication error. stderr: %s", stderrStr)
|
||||
}
|
||||
|
||||
// createTempPgpassFile creates a temporary .pgpass file with the given password
|
||||
func (uc *CreatePostgresqlBackupUsecase) createTempPgpassFile(
|
||||
pgConfig *pgtypes.PostgresqlDatabase,
|
||||
password string,
|
||||
@@ -525,14 +738,17 @@ func (uc *CreatePostgresqlBackupUsecase) createTempPgpassFile(
|
||||
return "", nil
|
||||
}
|
||||
|
||||
escapedHost := tools.EscapePgpassField(pgConfig.Host)
|
||||
escapedUsername := tools.EscapePgpassField(pgConfig.Username)
|
||||
escapedPassword := tools.EscapePgpassField(password)
|
||||
|
||||
pgpassContent := fmt.Sprintf("%s:%d:*:%s:%s",
|
||||
pgConfig.Host,
|
||||
escapedHost,
|
||||
pgConfig.Port,
|
||||
pgConfig.Username,
|
||||
password,
|
||||
escapedUsername,
|
||||
escapedPassword,
|
||||
)
|
||||
|
||||
// it always create unique directory like /tmp/pgpass-1234567890
|
||||
tempDir, err := os.MkdirTemp("", "pgpass")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temporary directory: %w", err)
|
||||
@@ -546,3 +762,7 @@ func (uc *CreatePostgresqlBackupUsecase) createTempPgpassFile(
|
||||
|
||||
return pgpassFile, nil
|
||||
}
|
||||
|
||||
func containsIgnoreCase(str, substr string) bool {
|
||||
return strings.Contains(strings.ToLower(str), strings.ToLower(substr))
|
||||
}
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
package usecases_postgresql
|
||||
|
||||
import (
|
||||
"postgresus-backend/internal/features/encryption/secrets"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"postgresus-backend/internal/util/logger"
|
||||
)
|
||||
|
||||
var createPostgresqlBackupUsecase = &CreatePostgresqlBackupUsecase{
|
||||
logger.GetLogger(),
|
||||
secrets.GetSecretKeyService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
}
|
||||
|
||||
func GetCreatePostgresqlBackupUsecase() *CreatePostgresqlBackupUsecase {
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package usecases_postgresql
|
||||
|
||||
import backups_config "postgresus-backend/internal/features/backups/config"
|
||||
|
||||
type EncryptionMetadata struct {
|
||||
Salt string
|
||||
IV string
|
||||
Encryption backups_config.BackupEncryption
|
||||
}
|
||||
|
||||
type BackupMetadata struct {
|
||||
EncryptionSalt *string
|
||||
EncryptionIV *string
|
||||
Encryption backups_config.BackupEncryption
|
||||
}
|
||||
@@ -20,15 +20,15 @@ func (c *BackupConfigController) RegisterRoutes(router *gin.RouterGroup) {
|
||||
|
||||
// SaveBackupConfig
|
||||
// @Summary Save backup configuration
|
||||
// @Description Save or update backup configuration for a database
|
||||
// @Description Save or update backup configuration for a database. Encryption can be set to NONE (no encryption) or ENCRYPTED (AES-256-GCM encryption).
|
||||
// @Tags backup-configs
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body BackupConfig true "Backup configuration data"
|
||||
// @Success 200 {object} BackupConfig
|
||||
// @Failure 400
|
||||
// @Failure 401
|
||||
// @Failure 500
|
||||
// @Param request body BackupConfig true "Backup configuration data (encryption field: NONE or ENCRYPTED)"
|
||||
// @Success 200 {object} BackupConfig "Returns the saved backup configuration including encryption settings"
|
||||
// @Failure 400 {object} map[string]string "Invalid encryption value or other validation errors"
|
||||
// @Failure 401 {object} map[string]string "User not authenticated"
|
||||
// @Failure 500 {object} map[string]string "Internal server error"
|
||||
// @Router /backup-configs/save [post]
|
||||
func (c *BackupConfigController) SaveBackupConfig(ctx *gin.Context) {
|
||||
user, ok := users_middleware.GetUserFromContext(ctx)
|
||||
@@ -57,14 +57,14 @@ func (c *BackupConfigController) SaveBackupConfig(ctx *gin.Context) {
|
||||
|
||||
// GetBackupConfigByDbID
|
||||
// @Summary Get backup configuration by database ID
|
||||
// @Description Get backup configuration for a specific database
|
||||
// @Description Get backup configuration for a specific database including encryption settings (NONE or ENCRYPTED)
|
||||
// @Tags backup-configs
|
||||
// @Produce json
|
||||
// @Param id path string true "Database ID"
|
||||
// @Success 200 {object} BackupConfig
|
||||
// @Failure 400
|
||||
// @Failure 401
|
||||
// @Failure 404
|
||||
// @Success 200 {object} BackupConfig "Returns backup configuration with encryption field"
|
||||
// @Failure 400 {object} map[string]string "Invalid database ID"
|
||||
// @Failure 401 {object} map[string]string "User not authenticated"
|
||||
// @Failure 404 {object} map[string]string "Backup configuration not found"
|
||||
// @Router /backup-configs/database/{id} [get]
|
||||
func (c *BackupConfigController) GetBackupConfigByDbID(ctx *gin.Context) {
|
||||
user, ok := users_middleware.GetUserFromContext(ctx)
|
||||
|
||||
@@ -368,6 +368,86 @@ func Test_IsStorageUsing_PermissionsEnforced(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func Test_SaveBackupConfig_WithEncryptionNone_ConfigSaved(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
|
||||
database := createTestDatabaseViaAPI("Test Database", workspace.ID, owner.Token, router)
|
||||
|
||||
timeOfDay := "04:00"
|
||||
request := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
},
|
||||
SendNotificationsOn: []BackupNotificationType{
|
||||
NotificationBackupFailed,
|
||||
},
|
||||
CpuCount: 2,
|
||||
IsRetryIfFailed: true,
|
||||
MaxFailedTriesCount: 3,
|
||||
Encryption: BackupEncryptionNone,
|
||||
}
|
||||
|
||||
var response BackupConfig
|
||||
test_utils.MakePostRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/backup-configs/save",
|
||||
"Bearer "+owner.Token,
|
||||
request,
|
||||
http.StatusOK,
|
||||
&response,
|
||||
)
|
||||
|
||||
assert.Equal(t, database.ID, response.DatabaseID)
|
||||
assert.Equal(t, BackupEncryptionNone, response.Encryption)
|
||||
}
|
||||
|
||||
func Test_SaveBackupConfig_WithEncryptionEncrypted_ConfigSaved(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
|
||||
database := createTestDatabaseViaAPI("Test Database", workspace.ID, owner.Token, router)
|
||||
|
||||
timeOfDay := "04:00"
|
||||
request := BackupConfig{
|
||||
DatabaseID: database.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodWeek,
|
||||
BackupInterval: &intervals.Interval{
|
||||
Interval: intervals.IntervalDaily,
|
||||
TimeOfDay: &timeOfDay,
|
||||
},
|
||||
SendNotificationsOn: []BackupNotificationType{
|
||||
NotificationBackupFailed,
|
||||
},
|
||||
CpuCount: 2,
|
||||
IsRetryIfFailed: true,
|
||||
MaxFailedTriesCount: 3,
|
||||
Encryption: BackupEncryptionEncrypted,
|
||||
}
|
||||
|
||||
var response BackupConfig
|
||||
test_utils.MakePostRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/backup-configs/save",
|
||||
"Bearer "+owner.Token,
|
||||
request,
|
||||
http.StatusOK,
|
||||
&response,
|
||||
)
|
||||
|
||||
assert.Equal(t, database.ID, response.DatabaseID)
|
||||
assert.Equal(t, BackupEncryptionEncrypted, response.Encryption)
|
||||
}
|
||||
|
||||
func createTestDatabaseViaAPI(
|
||||
name string,
|
||||
workspaceID uuid.UUID,
|
||||
|
||||
@@ -6,3 +6,10 @@ const (
|
||||
NotificationBackupFailed BackupNotificationType = "BACKUP_FAILED"
|
||||
NotificationBackupSuccess BackupNotificationType = "BACKUP_SUCCESS"
|
||||
)
|
||||
|
||||
type BackupEncryption string
|
||||
|
||||
const (
|
||||
BackupEncryptionNone BackupEncryption = "NONE"
|
||||
BackupEncryptionEncrypted BackupEncryption = "ENCRYPTED"
|
||||
)
|
||||
|
||||
@@ -31,6 +31,8 @@ type BackupConfig struct {
|
||||
MaxFailedTriesCount int `json:"maxFailedTriesCount" gorm:"column:max_failed_tries_count;type:int;not null"`
|
||||
|
||||
CpuCount int `json:"cpuCount" gorm:"type:int;not null"`
|
||||
|
||||
Encryption BackupEncryption `json:"encryption" gorm:"column:encryption;type:text;not null;default:'NONE'"`
|
||||
}
|
||||
|
||||
func (h *BackupConfig) TableName() string {
|
||||
@@ -88,6 +90,11 @@ func (b *BackupConfig) Validate() error {
|
||||
return errors.New("max failed tries count must be greater than 0")
|
||||
}
|
||||
|
||||
if b.Encryption != "" && b.Encryption != BackupEncryptionNone &&
|
||||
b.Encryption != BackupEncryptionEncrypted {
|
||||
return errors.New("encryption must be NONE or ENCRYPTED")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -103,5 +110,6 @@ func (b *BackupConfig) Copy(newDatabaseID uuid.UUID) *BackupConfig {
|
||||
IsRetryIfFailed: b.IsRetryIfFailed,
|
||||
MaxFailedTriesCount: b.MaxFailedTriesCount,
|
||||
CpuCount: b.CpuCount,
|
||||
Encryption: b.Encryption,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,6 +171,7 @@ func (s *BackupConfigService) initializeDefaultConfig(
|
||||
CpuCount: 1,
|
||||
IsRetryIfFailed: true,
|
||||
MaxFailedTriesCount: 3,
|
||||
Encryption: BackupEncryptionNone,
|
||||
})
|
||||
|
||||
return err
|
||||
|
||||
@@ -26,7 +26,8 @@ func (c *DatabaseController) RegisterRoutes(router *gin.RouterGroup) {
|
||||
router.POST("/databases/test-connection-direct", c.TestDatabaseConnectionDirect)
|
||||
router.POST("/databases/:id/copy", c.CopyDatabase)
|
||||
router.GET("/databases/notifier/:id/is-using", c.IsNotifierUsing)
|
||||
|
||||
router.POST("/databases/is-readonly", c.IsUserReadOnly)
|
||||
router.POST("/databases/create-readonly-user", c.CreateReadOnlyUser)
|
||||
}
|
||||
|
||||
// CreateDatabase
|
||||
@@ -330,3 +331,76 @@ func (c *DatabaseController) CopyDatabase(ctx *gin.Context) {
|
||||
|
||||
ctx.JSON(http.StatusCreated, copiedDatabase)
|
||||
}
|
||||
|
||||
// IsUserReadOnly
|
||||
// @Summary Check if database user is read-only
|
||||
// @Description Check if current database credentials have only read (SELECT) privileges
|
||||
// @Tags databases
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param request body Database true "Database configuration to check"
|
||||
// @Success 200 {object} IsReadOnlyResponse
|
||||
// @Failure 400 {object} map[string]string
|
||||
// @Failure 401 {object} map[string]string
|
||||
// @Failure 403 {object} map[string]string
|
||||
// @Router /databases/is-readonly [post]
|
||||
func (c *DatabaseController) IsUserReadOnly(ctx *gin.Context) {
|
||||
user, ok := users_middleware.GetUserFromContext(ctx)
|
||||
if !ok {
|
||||
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "User not authenticated"})
|
||||
return
|
||||
}
|
||||
|
||||
var request Database
|
||||
if err := ctx.ShouldBindJSON(&request); err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
isReadOnly, err := c.databaseService.IsUserReadOnly(user, &request)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
ctx.JSON(http.StatusOK, IsReadOnlyResponse{IsReadOnly: isReadOnly})
|
||||
}
|
||||
|
||||
// CreateReadOnlyUser
|
||||
// @Summary Create read-only database user
|
||||
// @Description Create a new PostgreSQL user with read-only privileges for backup operations
|
||||
// @Tags databases
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param request body Database true "Database configuration to create user for"
|
||||
// @Success 200 {object} CreateReadOnlyUserResponse
|
||||
// @Failure 400 {object} map[string]string
|
||||
// @Failure 401 {object} map[string]string
|
||||
// @Failure 403 {object} map[string]string
|
||||
// @Router /databases/create-readonly-user [post]
|
||||
func (c *DatabaseController) CreateReadOnlyUser(ctx *gin.Context) {
|
||||
user, ok := users_middleware.GetUserFromContext(ctx)
|
||||
if !ok {
|
||||
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "User not authenticated"})
|
||||
return
|
||||
}
|
||||
|
||||
var request Database
|
||||
if err := ctx.ShouldBindJSON(&request); err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
username, password, err := c.databaseService.CreateReadOnlyUser(user, &request)
|
||||
if err != nil {
|
||||
ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
ctx.JSON(http.StatusOK, CreateReadOnlyUserResponse{
|
||||
Username: username,
|
||||
Password: password,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
users_testing "postgresus-backend/internal/features/users/testing"
|
||||
workspaces_controllers "postgresus-backend/internal/features/workspaces/controllers"
|
||||
workspaces_testing "postgresus-backend/internal/features/workspaces/testing"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
test_utils "postgresus-backend/internal/util/testing"
|
||||
"postgresus-backend/internal/util/tools"
|
||||
)
|
||||
@@ -769,6 +770,71 @@ func createTestDatabaseViaAPI(
|
||||
return &database
|
||||
}
|
||||
|
||||
func Test_CreateDatabase_PasswordIsEncryptedInDB(t *testing.T) {
|
||||
router := createTestRouter()
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
|
||||
testDbName := "test_db"
|
||||
plainPassword := "my-super-secret-password-123"
|
||||
request := Database{
|
||||
Name: "Test Database",
|
||||
WorkspaceID: &workspace.ID,
|
||||
Type: DatabaseTypePostgres,
|
||||
Postgresql: &postgresql.PostgresqlDatabase{
|
||||
Version: tools.PostgresqlVersion16,
|
||||
Host: "localhost",
|
||||
Port: 5432,
|
||||
Username: "postgres",
|
||||
Password: plainPassword,
|
||||
Database: &testDbName,
|
||||
},
|
||||
}
|
||||
|
||||
var createdDatabase Database
|
||||
test_utils.MakePostRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/databases/create",
|
||||
"Bearer "+owner.Token,
|
||||
request,
|
||||
http.StatusCreated,
|
||||
&createdDatabase,
|
||||
)
|
||||
|
||||
repository := &DatabaseRepository{}
|
||||
databaseFromDB, err := repository.FindByID(createdDatabase.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, databaseFromDB)
|
||||
assert.NotNil(t, databaseFromDB.Postgresql)
|
||||
|
||||
assert.True(
|
||||
t,
|
||||
strings.HasPrefix(databaseFromDB.Postgresql.Password, "enc:"),
|
||||
"Password should be encrypted in database with 'enc:' prefix, got: %s",
|
||||
databaseFromDB.Postgresql.Password,
|
||||
)
|
||||
|
||||
encryptor := encryption.GetFieldEncryptor()
|
||||
decryptedPassword, err := encryptor.Decrypt(
|
||||
databaseFromDB.ID,
|
||||
databaseFromDB.Postgresql.Password,
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, plainPassword, decryptedPassword,
|
||||
"Decrypted password should match original plaintext password")
|
||||
|
||||
test_utils.MakeDeleteRequest(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/databases/"+createdDatabase.ID.String(),
|
||||
"Bearer "+owner.Token,
|
||||
http.StatusNoContent,
|
||||
)
|
||||
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}
|
||||
|
||||
func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
@@ -815,7 +881,15 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, database *Database) {
|
||||
assert.Equal(t, "original-password-secret", database.Postgresql.Password)
|
||||
// Verify password is encrypted
|
||||
assert.True(t, strings.HasPrefix(database.Postgresql.Password, "enc:"),
|
||||
"Password should be encrypted in database")
|
||||
|
||||
// Verify it can be decrypted back to original
|
||||
encryptor := encryption.GetFieldEncryptor()
|
||||
decrypted, err := encryptor.Decrypt(database.ID, database.Postgresql.Password)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "original-password-secret", decrypted)
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, database *Database) {
|
||||
assert.Equal(t, "", database.Postgresql.Password)
|
||||
|
||||
@@ -5,9 +5,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"postgresus-backend/internal/util/tools"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -18,7 +19,6 @@ type PostgresqlDatabase struct {
|
||||
ID uuid.UUID `json:"id" gorm:"primaryKey;type:uuid;default:gen_random_uuid()"`
|
||||
|
||||
DatabaseID *uuid.UUID `json:"databaseId" gorm:"type:uuid;column:database_id"`
|
||||
RestoreID *uuid.UUID `json:"restoreId" gorm:"type:uuid;column:restore_id"`
|
||||
|
||||
Version tools.PostgresqlVersion `json:"version" gorm:"type:text;not null"`
|
||||
|
||||
@@ -59,11 +59,15 @@ func (p *PostgresqlDatabase) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *PostgresqlDatabase) TestConnection(logger *slog.Logger) error {
|
||||
func (p *PostgresqlDatabase) TestConnection(
|
||||
logger *slog.Logger,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
databaseID uuid.UUID,
|
||||
) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
return testSingleDatabaseConnection(logger, ctx, p)
|
||||
return testSingleDatabaseConnection(logger, ctx, p, encryptor, databaseID)
|
||||
}
|
||||
|
||||
func (p *PostgresqlDatabase) HideSensitiveData() {
|
||||
@@ -87,19 +91,420 @@ func (p *PostgresqlDatabase) Update(incoming *PostgresqlDatabase) {
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PostgresqlDatabase) EncryptSensitiveFields(
|
||||
databaseID uuid.UUID,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
) error {
|
||||
if p.Password != "" {
|
||||
encrypted, err := encryptor.Encrypt(databaseID, p.Password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.Password = encrypted
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsUserReadOnly checks if the database user has read-only privileges.
|
||||
//
|
||||
// This method performs a comprehensive security check by examining:
|
||||
// - Role-level attributes (superuser, createrole, createdb)
|
||||
// - Database-level privileges (CREATE, TEMP)
|
||||
// - Table-level write permissions (INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER)
|
||||
//
|
||||
// A user is considered read-only only if they have ZERO write privileges
|
||||
// across all three levels. This ensures the database user follows the
|
||||
// principle of least privilege for backup operations.
|
||||
func (p *PostgresqlDatabase) IsUserReadOnly(
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
databaseID uuid.UUID,
|
||||
) (bool, error) {
|
||||
password, err := decryptPasswordIfNeeded(p.Password, encryptor, databaseID)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to decrypt password: %w", err)
|
||||
}
|
||||
|
||||
connStr := buildConnectionStringForDB(p, *p.Database, password)
|
||||
|
||||
conn, err := pgx.Connect(ctx, connStr)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to connect to database: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := conn.Close(ctx); closeErr != nil {
|
||||
logger.Error("Failed to close connection", "error", closeErr)
|
||||
}
|
||||
}()
|
||||
|
||||
// LEVEL 1: Check role-level attributes
|
||||
var isSuperuser, canCreateRole, canCreateDB bool
|
||||
err = conn.QueryRow(ctx, `
|
||||
SELECT
|
||||
rolsuper,
|
||||
rolcreaterole,
|
||||
rolcreatedb
|
||||
FROM pg_roles
|
||||
WHERE rolname = current_user
|
||||
`).Scan(&isSuperuser, &canCreateRole, &canCreateDB)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to check role attributes: %w", err)
|
||||
}
|
||||
|
||||
if isSuperuser || canCreateRole || canCreateDB {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// LEVEL 2: Check database-level privileges
|
||||
var canCreate, canTemp bool
|
||||
err = conn.QueryRow(ctx, `
|
||||
SELECT
|
||||
has_database_privilege(current_user, current_database(), 'CREATE') as can_create,
|
||||
has_database_privilege(current_user, current_database(), 'TEMP') as can_temp
|
||||
`).Scan(&canCreate, &canTemp)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to check database privileges: %w", err)
|
||||
}
|
||||
|
||||
if canCreate || canTemp {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// LEVEL 2.5: Check schema-level CREATE privileges
|
||||
schemaRows, err := conn.Query(ctx, `
|
||||
SELECT DISTINCT nspname
|
||||
FROM pg_namespace n
|
||||
WHERE has_schema_privilege(current_user, n.nspname, 'CREATE')
|
||||
AND nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
|
||||
`)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to check schema privileges: %w", err)
|
||||
}
|
||||
defer schemaRows.Close()
|
||||
|
||||
// If user has CREATE privilege on any schema, they're not read-only
|
||||
if schemaRows.Next() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if err := schemaRows.Err(); err != nil {
|
||||
return false, fmt.Errorf("error iterating schema privileges: %w", err)
|
||||
}
|
||||
|
||||
// LEVEL 3: Check table-level write permissions
|
||||
rows, err := conn.Query(ctx, `
|
||||
SELECT DISTINCT privilege_type
|
||||
FROM information_schema.role_table_grants
|
||||
WHERE grantee = current_user
|
||||
AND table_schema NOT IN ('pg_catalog', 'information_schema')
|
||||
`)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to check table privileges: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
writePrivileges := map[string]bool{
|
||||
"INSERT": true,
|
||||
"UPDATE": true,
|
||||
"DELETE": true,
|
||||
"TRUNCATE": true,
|
||||
"REFERENCES": true,
|
||||
"TRIGGER": true,
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
var privilege string
|
||||
if err := rows.Scan(&privilege); err != nil {
|
||||
return false, fmt.Errorf("failed to scan privilege: %w", err)
|
||||
}
|
||||
|
||||
if writePrivileges[privilege] {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return false, fmt.Errorf("error iterating privileges: %w", err)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// CreateReadOnlyUser creates a new PostgreSQL user with read-only privileges.
|
||||
//
|
||||
// This method performs the following operations atomically in a single transaction:
|
||||
// 1. Creates a PostgreSQL user with a UUID-based password
|
||||
// 2. Grants CONNECT privilege on the database
|
||||
// 3. Grants USAGE on all non-system schemas
|
||||
// 4. Grants SELECT on all existing tables and sequences
|
||||
// 5. Sets default privileges for future tables and sequences
|
||||
//
|
||||
// Security features:
|
||||
// - Username format: "postgresus-{8-char-uuid}" for uniqueness
|
||||
// - Password: Full UUID (36 characters) for strong entropy
|
||||
// - Transaction safety: All operations rollback on any failure
|
||||
// - Retry logic: Up to 3 attempts if username collision occurs
|
||||
// - Pre-validation: Checks CREATEROLE privilege before starting transaction
|
||||
func (p *PostgresqlDatabase) CreateReadOnlyUser(
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
databaseID uuid.UUID,
|
||||
) (string, string, error) {
|
||||
password, err := decryptPasswordIfNeeded(p.Password, encryptor, databaseID)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to decrypt password: %w", err)
|
||||
}
|
||||
|
||||
connStr := buildConnectionStringForDB(p, *p.Database, password)
|
||||
|
||||
conn, err := pgx.Connect(ctx, connStr)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to connect to database: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := conn.Close(ctx); closeErr != nil {
|
||||
logger.Error("Failed to close connection", "error", closeErr)
|
||||
}
|
||||
}()
|
||||
|
||||
// Pre-validate: Check if current user can create roles
|
||||
var canCreateRole, isSuperuser bool
|
||||
err = conn.QueryRow(ctx, `
|
||||
SELECT rolcreaterole, rolsuper
|
||||
FROM pg_roles
|
||||
WHERE rolname = current_user
|
||||
`).Scan(&canCreateRole, &isSuperuser)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to check permissions: %w", err)
|
||||
}
|
||||
if !canCreateRole && !isSuperuser {
|
||||
return "", "", errors.New("current database user lacks CREATEROLE privilege")
|
||||
}
|
||||
|
||||
// Retry logic for username collision
|
||||
maxRetries := 3
|
||||
for attempt := 0; attempt < maxRetries; attempt++ {
|
||||
username := fmt.Sprintf("postgresus-%s", uuid.New().String()[:8])
|
||||
newPassword := uuid.New().String()
|
||||
|
||||
tx, err := conn.Begin(ctx)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to begin transaction: %w", err)
|
||||
}
|
||||
|
||||
success := false
|
||||
defer func() {
|
||||
if !success {
|
||||
if rollbackErr := tx.Rollback(ctx); rollbackErr != nil {
|
||||
logger.Error("Failed to rollback transaction", "error", rollbackErr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Step 1: Create PostgreSQL user with LOGIN privilege
|
||||
_, err = tx.Exec(
|
||||
ctx,
|
||||
fmt.Sprintf(`CREATE USER "%s" WITH PASSWORD '%s' LOGIN`, username, newPassword),
|
||||
)
|
||||
if err != nil {
|
||||
if err.Error() != "" && attempt < maxRetries-1 {
|
||||
continue
|
||||
}
|
||||
return "", "", fmt.Errorf("failed to create user: %w", err)
|
||||
}
|
||||
|
||||
// Step 1.5: Revoke CREATE privilege from PUBLIC role on public schema
|
||||
// This is necessary because all PostgreSQL users inherit CREATE privilege on the
|
||||
// public schema through the PUBLIC role. This is a one-time operation that affects
|
||||
// the entire database, making it more secure by default.
|
||||
// Note: This only affects the public schema; other schemas are unaffected.
|
||||
_, err = tx.Exec(ctx, `REVOKE CREATE ON SCHEMA public FROM PUBLIC`)
|
||||
if err != nil {
|
||||
logger.Error("Failed to revoke CREATE on public from PUBLIC", "error", err)
|
||||
if !strings.Contains(err.Error(), "schema \"public\" does not exist") &&
|
||||
!strings.Contains(err.Error(), "permission denied") {
|
||||
return "", "", fmt.Errorf("failed to revoke CREATE from PUBLIC: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Now revoke from the specific user as well (belt and suspenders)
|
||||
_, err = tx.Exec(ctx, fmt.Sprintf(`REVOKE CREATE ON SCHEMA public FROM "%s"`, username))
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
"Failed to revoke CREATE on public schema from user",
|
||||
"error",
|
||||
err,
|
||||
"username",
|
||||
username,
|
||||
)
|
||||
}
|
||||
|
||||
// Step 2: Grant database connection privilege and revoke TEMP
|
||||
_, err = tx.Exec(
|
||||
ctx,
|
||||
fmt.Sprintf(`GRANT CONNECT ON DATABASE %s TO "%s"`, *p.Database, username),
|
||||
)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to grant connect privilege: %w", err)
|
||||
}
|
||||
|
||||
// Revoke TEMP privilege from PUBLIC role (like CREATE on public schema, TEMP is granted to PUBLIC by default)
|
||||
_, err = tx.Exec(ctx, fmt.Sprintf(`REVOKE TEMP ON DATABASE %s FROM PUBLIC`, *p.Database))
|
||||
if err != nil {
|
||||
logger.Warn("Failed to revoke TEMP from PUBLIC", "error", err)
|
||||
}
|
||||
|
||||
// Also revoke from the specific user (belt and suspenders)
|
||||
_, err = tx.Exec(
|
||||
ctx,
|
||||
fmt.Sprintf(`REVOKE TEMP ON DATABASE %s FROM "%s"`, *p.Database, username),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Warn("Failed to revoke TEMP privilege", "error", err, "username", username)
|
||||
}
|
||||
|
||||
// Step 3: Discover all user-created schemas
|
||||
rows, err := tx.Query(ctx, `
|
||||
SELECT schema_name
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name NOT IN ('pg_catalog', 'information_schema')
|
||||
`)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to get schemas: %w", err)
|
||||
}
|
||||
|
||||
var schemas []string
|
||||
for rows.Next() {
|
||||
var schema string
|
||||
if err := rows.Scan(&schema); err != nil {
|
||||
rows.Close()
|
||||
return "", "", fmt.Errorf("failed to scan schema: %w", err)
|
||||
}
|
||||
schemas = append(schemas, schema)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return "", "", fmt.Errorf("error iterating schemas: %w", err)
|
||||
}
|
||||
|
||||
// Step 4: Grant USAGE on each schema and explicitly prevent CREATE
|
||||
for _, schema := range schemas {
|
||||
// Revoke CREATE specifically (handles inheritance from PUBLIC role)
|
||||
_, err = tx.Exec(
|
||||
ctx,
|
||||
fmt.Sprintf(`REVOKE CREATE ON SCHEMA "%s" FROM "%s"`, schema, username),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Warn(
|
||||
"Failed to revoke CREATE on schema",
|
||||
"error",
|
||||
err,
|
||||
"schema",
|
||||
schema,
|
||||
"username",
|
||||
username,
|
||||
)
|
||||
}
|
||||
|
||||
// Grant only USAGE (not CREATE)
|
||||
_, err = tx.Exec(
|
||||
ctx,
|
||||
fmt.Sprintf(`GRANT USAGE ON SCHEMA "%s" TO "%s"`, schema, username),
|
||||
)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to grant usage on schema %s: %w", schema, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 5: Grant SELECT on ALL existing tables and sequences
|
||||
grantSelectSQL := fmt.Sprintf(`
|
||||
DO $$
|
||||
DECLARE
|
||||
schema_rec RECORD;
|
||||
BEGIN
|
||||
FOR schema_rec IN
|
||||
SELECT schema_name
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name NOT IN ('pg_catalog', 'information_schema')
|
||||
LOOP
|
||||
EXECUTE format('GRANT SELECT ON ALL TABLES IN SCHEMA %%I TO "%s"', schema_rec.schema_name);
|
||||
EXECUTE format('GRANT SELECT ON ALL SEQUENCES IN SCHEMA %%I TO "%s"', schema_rec.schema_name);
|
||||
END LOOP;
|
||||
END $$;
|
||||
`, username, username)
|
||||
|
||||
_, err = tx.Exec(ctx, grantSelectSQL)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to grant select on tables: %w", err)
|
||||
}
|
||||
|
||||
// Step 6: Set default privileges for FUTURE tables and sequences
|
||||
defaultPrivilegesSQL := fmt.Sprintf(`
|
||||
DO $$
|
||||
DECLARE
|
||||
schema_rec RECORD;
|
||||
BEGIN
|
||||
FOR schema_rec IN
|
||||
SELECT schema_name
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name NOT IN ('pg_catalog', 'information_schema')
|
||||
LOOP
|
||||
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %%I GRANT SELECT ON TABLES TO "%s"', schema_rec.schema_name);
|
||||
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %%I GRANT SELECT ON SEQUENCES TO "%s"', schema_rec.schema_name);
|
||||
END LOOP;
|
||||
END $$;
|
||||
`, username, username)
|
||||
|
||||
_, err = tx.Exec(ctx, defaultPrivilegesSQL)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to set default privileges: %w", err)
|
||||
}
|
||||
|
||||
// Step 7: Verify user creation before committing
|
||||
var verifyUsername string
|
||||
err = tx.QueryRow(ctx, fmt.Sprintf(`SELECT rolname FROM pg_roles WHERE rolname = '%s'`, username)).
|
||||
Scan(&verifyUsername)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to verify user creation: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return "", "", fmt.Errorf("failed to commit transaction: %w", err)
|
||||
}
|
||||
|
||||
success = true
|
||||
logger.Info("Read-only user created successfully", "username", username)
|
||||
return username, newPassword, nil
|
||||
}
|
||||
|
||||
return "", "", errors.New("failed to generate unique username after 3 attempts")
|
||||
}
|
||||
|
||||
// testSingleDatabaseConnection tests connection to a specific database for pg_dump
|
||||
func testSingleDatabaseConnection(
|
||||
logger *slog.Logger,
|
||||
ctx context.Context,
|
||||
postgresDb *PostgresqlDatabase,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
databaseID uuid.UUID,
|
||||
) error {
|
||||
// For single database backup, we need to connect to the specific database
|
||||
if postgresDb.Database == nil || *postgresDb.Database == "" {
|
||||
return errors.New("database name is required for single database backup (pg_dump)")
|
||||
}
|
||||
|
||||
// Decrypt password if needed
|
||||
password, err := decryptPasswordIfNeeded(postgresDb.Password, encryptor, databaseID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt password: %w", err)
|
||||
}
|
||||
|
||||
// Build connection string for the specific database
|
||||
connStr := buildConnectionStringForDB(postgresDb, *postgresDb.Database)
|
||||
connStr := buildConnectionStringForDB(postgresDb, *postgresDb.Database, password)
|
||||
|
||||
// Test connection
|
||||
conn, err := pgx.Connect(ctx, connStr)
|
||||
@@ -182,116 +587,30 @@ func testBasicOperations(ctx context.Context, conn *pgx.Conn, dbName string) err
|
||||
}
|
||||
|
||||
// buildConnectionStringForDB builds connection string for specific database
|
||||
func buildConnectionStringForDB(p *PostgresqlDatabase, dbName string) string {
|
||||
func buildConnectionStringForDB(p *PostgresqlDatabase, dbName string, password string) string {
|
||||
sslMode := "disable"
|
||||
if p.IsHttps {
|
||||
sslMode = "require"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s",
|
||||
return fmt.Sprintf(
|
||||
"host=%s port=%d user=%s password=%s dbname=%s sslmode=%s default_query_exec_mode=simple_protocol standard_conforming_strings=on",
|
||||
p.Host,
|
||||
p.Port,
|
||||
p.Username,
|
||||
p.Password,
|
||||
password,
|
||||
dbName,
|
||||
sslMode,
|
||||
)
|
||||
}
|
||||
|
||||
func (p *PostgresqlDatabase) InstallExtensions(extensions []tools.PostgresqlExtension) error {
|
||||
if len(extensions) == 0 {
|
||||
return nil
|
||||
func decryptPasswordIfNeeded(
|
||||
password string,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
databaseID uuid.UUID,
|
||||
) (string, error) {
|
||||
if encryptor == nil {
|
||||
return password, nil
|
||||
}
|
||||
|
||||
if p.Database == nil || *p.Database == "" {
|
||||
return errors.New("database name is required for installing extensions")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Build connection string for the specific database
|
||||
connStr := buildConnectionStringForDB(p, *p.Database)
|
||||
|
||||
// Connect to database
|
||||
conn, err := pgx.Connect(ctx, connStr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to database '%s': %w", *p.Database, err)
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := conn.Close(ctx); closeErr != nil {
|
||||
fmt.Println("failed to close connection: %w", closeErr)
|
||||
}
|
||||
}()
|
||||
|
||||
// Check which extensions are already installed
|
||||
installedExtensions, err := p.getInstalledExtensions(ctx, conn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check installed extensions: %w", err)
|
||||
}
|
||||
|
||||
// Install missing extensions
|
||||
for _, extension := range extensions {
|
||||
if contains(installedExtensions, string(extension)) {
|
||||
continue // Extension already installed
|
||||
}
|
||||
|
||||
if err := p.installExtension(ctx, conn, string(extension)); err != nil {
|
||||
return fmt.Errorf("failed to install extension '%s': %w", extension, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getInstalledExtensions queries the database for currently installed extensions
|
||||
func (p *PostgresqlDatabase) getInstalledExtensions(
|
||||
ctx context.Context,
|
||||
conn *pgx.Conn,
|
||||
) ([]string, error) {
|
||||
query := "SELECT extname FROM pg_extension"
|
||||
|
||||
rows, err := conn.Query(ctx, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query installed extensions: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var extensions []string
|
||||
for rows.Next() {
|
||||
var extname string
|
||||
|
||||
if err := rows.Scan(&extname); err != nil {
|
||||
return nil, fmt.Errorf("failed to scan extension name: %w", err)
|
||||
}
|
||||
|
||||
extensions = append(extensions, extname)
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("error iterating over extension rows: %w", err)
|
||||
}
|
||||
|
||||
return extensions, nil
|
||||
}
|
||||
|
||||
// installExtension installs a single PostgreSQL extension
|
||||
func (p *PostgresqlDatabase) installExtension(
|
||||
ctx context.Context,
|
||||
conn *pgx.Conn,
|
||||
extensionName string,
|
||||
) error {
|
||||
query := fmt.Sprintf("CREATE EXTENSION IF NOT EXISTS %s", extensionName)
|
||||
|
||||
_, err := conn.Exec(ctx, query)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to execute CREATE EXTENSION: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// contains checks if a string slice contains a specific string
|
||||
func contains(slice []string, item string) bool {
|
||||
return slices.Contains(slice, item)
|
||||
return encryptor.Decrypt(databaseID, password)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
package postgresql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/lib/pq"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"postgresus-backend/internal/config"
|
||||
"postgresus-backend/internal/util/tools"
|
||||
)
|
||||
|
||||
func Test_IsUserReadOnly_AdminUser_ReturnsFalse(t *testing.T) {
|
||||
env := config.GetEnv()
|
||||
cases := []struct {
|
||||
name string
|
||||
version string
|
||||
port string
|
||||
}{
|
||||
{"PostgreSQL 12", "12", env.TestPostgres12Port},
|
||||
{"PostgreSQL 13", "13", env.TestPostgres13Port},
|
||||
{"PostgreSQL 14", "14", env.TestPostgres14Port},
|
||||
{"PostgreSQL 15", "15", env.TestPostgres15Port},
|
||||
{"PostgreSQL 16", "16", env.TestPostgres16Port},
|
||||
{"PostgreSQL 17", "17", env.TestPostgres17Port},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
container := connectToPostgresContainer(t, tc.port)
|
||||
defer container.DB.Close()
|
||||
|
||||
pgModel := createPostgresModel(container)
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
|
||||
ctx := context.Background()
|
||||
|
||||
isReadOnly, err := pgModel.IsUserReadOnly(ctx, logger, nil, uuid.New())
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, isReadOnly, "Admin user should not be read-only")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_CreateReadOnlyUser_UserCanReadButNotWrite(t *testing.T) {
|
||||
env := config.GetEnv()
|
||||
cases := []struct {
|
||||
name string
|
||||
version string
|
||||
port string
|
||||
}{
|
||||
{"PostgreSQL 12", "12", env.TestPostgres12Port},
|
||||
{"PostgreSQL 13", "13", env.TestPostgres13Port},
|
||||
{"PostgreSQL 14", "14", env.TestPostgres14Port},
|
||||
{"PostgreSQL 15", "15", env.TestPostgres15Port},
|
||||
{"PostgreSQL 16", "16", env.TestPostgres16Port},
|
||||
{"PostgreSQL 17", "17", env.TestPostgres17Port},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
container := connectToPostgresContainer(t, tc.port)
|
||||
defer container.DB.Close()
|
||||
|
||||
_, err := container.DB.Exec(`
|
||||
DROP TABLE IF EXISTS readonly_test CASCADE;
|
||||
DROP TABLE IF EXISTS hack_table CASCADE;
|
||||
DROP TABLE IF EXISTS future_table CASCADE;
|
||||
CREATE TABLE readonly_test (
|
||||
id SERIAL PRIMARY KEY,
|
||||
data TEXT NOT NULL
|
||||
);
|
||||
INSERT INTO readonly_test (data) VALUES ('test1'), ('test2');
|
||||
`)
|
||||
assert.NoError(t, err)
|
||||
|
||||
pgModel := createPostgresModel(container)
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
|
||||
ctx := context.Background()
|
||||
|
||||
username, password, err := pgModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New())
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, username)
|
||||
assert.NotEmpty(t, password)
|
||||
assert.True(t, strings.HasPrefix(username, "postgresus-"))
|
||||
|
||||
readOnlyModel := &PostgresqlDatabase{
|
||||
Version: pgModel.Version,
|
||||
Host: pgModel.Host,
|
||||
Port: pgModel.Port,
|
||||
Username: username,
|
||||
Password: password,
|
||||
Database: pgModel.Database,
|
||||
IsHttps: false,
|
||||
}
|
||||
|
||||
isReadOnly, err := readOnlyModel.IsUserReadOnly(ctx, logger, nil, uuid.New())
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, isReadOnly, "Created user should be read-only")
|
||||
|
||||
readOnlyDSN := fmt.Sprintf(
|
||||
"host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||||
container.Host,
|
||||
container.Port,
|
||||
username,
|
||||
password,
|
||||
container.Database,
|
||||
)
|
||||
readOnlyConn, err := sqlx.Connect("postgres", readOnlyDSN)
|
||||
assert.NoError(t, err)
|
||||
defer readOnlyConn.Close()
|
||||
|
||||
var count int
|
||||
err = readOnlyConn.Get(&count, "SELECT COUNT(*) FROM readonly_test")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 2, count)
|
||||
|
||||
_, err = readOnlyConn.Exec("INSERT INTO readonly_test (data) VALUES ('should-fail')")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "permission denied")
|
||||
|
||||
_, err = readOnlyConn.Exec("UPDATE readonly_test SET data = 'hacked' WHERE id = 1")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "permission denied")
|
||||
|
||||
_, err = readOnlyConn.Exec("DELETE FROM readonly_test WHERE id = 1")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "permission denied")
|
||||
|
||||
_, err = readOnlyConn.Exec("CREATE TABLE hack_table (id INT)")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "permission denied")
|
||||
|
||||
// Clean up: Drop user with CASCADE to handle default privilege dependencies
|
||||
_, err = container.DB.Exec(fmt.Sprintf(`DROP OWNED BY "%s" CASCADE`, username))
|
||||
if err != nil {
|
||||
t.Logf("Warning: Failed to drop owned objects: %v", err)
|
||||
}
|
||||
|
||||
_, err = container.DB.Exec(fmt.Sprintf(`DROP USER IF EXISTS "%s"`, username))
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_ReadOnlyUser_FutureTables_HaveSelectPermission(t *testing.T) {
|
||||
env := config.GetEnv()
|
||||
container := connectToPostgresContainer(t, env.TestPostgres16Port)
|
||||
defer container.DB.Close()
|
||||
|
||||
pgModel := createPostgresModel(container)
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
|
||||
ctx := context.Background()
|
||||
|
||||
username, password, err := pgModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New())
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = container.DB.Exec(`
|
||||
CREATE TABLE future_table (
|
||||
id SERIAL PRIMARY KEY,
|
||||
data TEXT NOT NULL
|
||||
);
|
||||
INSERT INTO future_table (data) VALUES ('future_data');
|
||||
`)
|
||||
assert.NoError(t, err)
|
||||
|
||||
readOnlyDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||||
container.Host, container.Port, username, password, container.Database)
|
||||
readOnlyConn, err := sqlx.Connect("postgres", readOnlyDSN)
|
||||
assert.NoError(t, err)
|
||||
defer readOnlyConn.Close()
|
||||
|
||||
var data string
|
||||
err = readOnlyConn.Get(&data, "SELECT data FROM future_table LIMIT 1")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "future_data", data)
|
||||
|
||||
// Clean up: Drop user with CASCADE to handle default privilege dependencies
|
||||
_, err = container.DB.Exec(fmt.Sprintf(`DROP OWNED BY "%s" CASCADE`, username))
|
||||
if err != nil {
|
||||
t.Logf("Warning: Failed to drop owned objects: %v", err)
|
||||
}
|
||||
|
||||
_, err = container.DB.Exec(fmt.Sprintf(`DROP USER IF EXISTS "%s"`, username))
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func Test_ReadOnlyUser_MultipleSchemas_AllAccessible(t *testing.T) {
|
||||
env := config.GetEnv()
|
||||
container := connectToPostgresContainer(t, env.TestPostgres16Port)
|
||||
defer container.DB.Close()
|
||||
|
||||
_, err := container.DB.Exec(`
|
||||
CREATE SCHEMA IF NOT EXISTS schema_a;
|
||||
CREATE SCHEMA IF NOT EXISTS schema_b;
|
||||
CREATE TABLE schema_a.table_a (id INT, data TEXT);
|
||||
CREATE TABLE schema_b.table_b (id INT, data TEXT);
|
||||
INSERT INTO schema_a.table_a VALUES (1, 'data_a');
|
||||
INSERT INTO schema_b.table_b VALUES (2, 'data_b');
|
||||
`)
|
||||
assert.NoError(t, err)
|
||||
|
||||
pgModel := createPostgresModel(container)
|
||||
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
|
||||
ctx := context.Background()
|
||||
|
||||
username, password, err := pgModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New())
|
||||
assert.NoError(t, err)
|
||||
|
||||
readOnlyDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||||
container.Host, container.Port, username, password, container.Database)
|
||||
readOnlyConn, err := sqlx.Connect("postgres", readOnlyDSN)
|
||||
assert.NoError(t, err)
|
||||
defer readOnlyConn.Close()
|
||||
|
||||
var dataA string
|
||||
err = readOnlyConn.Get(&dataA, "SELECT data FROM schema_a.table_a LIMIT 1")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "data_a", dataA)
|
||||
|
||||
var dataB string
|
||||
err = readOnlyConn.Get(&dataB, "SELECT data FROM schema_b.table_b LIMIT 1")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "data_b", dataB)
|
||||
|
||||
// Clean up: Drop user with CASCADE to handle default privilege dependencies
|
||||
_, err = container.DB.Exec(fmt.Sprintf(`DROP OWNED BY "%s" CASCADE`, username))
|
||||
if err != nil {
|
||||
t.Logf("Warning: Failed to drop owned objects: %v", err)
|
||||
}
|
||||
|
||||
_, err = container.DB.Exec(fmt.Sprintf(`DROP USER IF EXISTS "%s"`, username))
|
||||
assert.NoError(t, err)
|
||||
_, err = container.DB.Exec(`DROP SCHEMA schema_a CASCADE; DROP SCHEMA schema_b CASCADE;`)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
type PostgresContainer struct {
|
||||
Host string
|
||||
Port int
|
||||
Username string
|
||||
Password string
|
||||
Database string
|
||||
DB *sqlx.DB
|
||||
}
|
||||
|
||||
func connectToPostgresContainer(t *testing.T, port string) *PostgresContainer {
|
||||
dbName := "testdb"
|
||||
password := "testpassword"
|
||||
username := "testuser"
|
||||
host := "localhost"
|
||||
|
||||
portInt, err := strconv.Atoi(port)
|
||||
assert.NoError(t, err)
|
||||
|
||||
dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||||
host, portInt, username, password, dbName)
|
||||
|
||||
db, err := sqlx.Connect("postgres", dsn)
|
||||
assert.NoError(t, err)
|
||||
|
||||
var versionStr string
|
||||
err = db.Get(&versionStr, "SELECT version()")
|
||||
assert.NoError(t, err)
|
||||
|
||||
return &PostgresContainer{
|
||||
Host: host,
|
||||
Port: portInt,
|
||||
Username: username,
|
||||
Password: password,
|
||||
Database: dbName,
|
||||
DB: db,
|
||||
}
|
||||
}
|
||||
|
||||
func createPostgresModel(container *PostgresContainer) *PostgresqlDatabase {
|
||||
var versionStr string
|
||||
err := container.DB.Get(&versionStr, "SELECT version()")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
version := extractPostgresVersion(versionStr)
|
||||
|
||||
return &PostgresqlDatabase{
|
||||
Version: version,
|
||||
Host: container.Host,
|
||||
Port: container.Port,
|
||||
Username: container.Username,
|
||||
Password: container.Password,
|
||||
Database: &container.Database,
|
||||
IsHttps: false,
|
||||
}
|
||||
}
|
||||
|
||||
func extractPostgresVersion(versionStr string) tools.PostgresqlVersion {
|
||||
if strings.Contains(versionStr, "PostgreSQL 12") {
|
||||
return tools.GetPostgresqlVersionEnum("12")
|
||||
} else if strings.Contains(versionStr, "PostgreSQL 13") {
|
||||
return tools.GetPostgresqlVersionEnum("13")
|
||||
} else if strings.Contains(versionStr, "PostgreSQL 14") {
|
||||
return tools.GetPostgresqlVersionEnum("14")
|
||||
} else if strings.Contains(versionStr, "PostgreSQL 15") {
|
||||
return tools.GetPostgresqlVersionEnum("15")
|
||||
} else if strings.Contains(versionStr, "PostgreSQL 16") {
|
||||
return tools.GetPostgresqlVersionEnum("16")
|
||||
} else if strings.Contains(versionStr, "PostgreSQL 17") {
|
||||
return tools.GetPostgresqlVersionEnum("17")
|
||||
}
|
||||
|
||||
return tools.GetPostgresqlVersionEnum("16")
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"postgresus-backend/internal/features/notifiers"
|
||||
users_services "postgresus-backend/internal/features/users/services"
|
||||
workspaces_services "postgresus-backend/internal/features/workspaces/services"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"postgresus-backend/internal/util/logger"
|
||||
)
|
||||
|
||||
@@ -19,6 +20,7 @@ var databaseService = &DatabaseService{
|
||||
[]DatabaseCopyListener{},
|
||||
workspaces_services.GetWorkspaceService(),
|
||||
audit_logs.GetAuditLogService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
}
|
||||
|
||||
var databaseController = &DatabaseController{
|
||||
|
||||
10
backend/internal/features/databases/dto.go
Normal file
@@ -0,0 +1,10 @@
|
||||
package databases
|
||||
|
||||
type CreateReadOnlyUserResponse struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
type IsReadOnlyResponse struct {
|
||||
IsReadOnly bool `json:"isReadOnly"`
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package databases
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -11,7 +12,11 @@ type DatabaseValidator interface {
|
||||
}
|
||||
|
||||
type DatabaseConnector interface {
|
||||
TestConnection(logger *slog.Logger) error
|
||||
TestConnection(
|
||||
logger *slog.Logger,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
databaseID uuid.UUID,
|
||||
) error
|
||||
|
||||
HideSensitiveData()
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"log/slog"
|
||||
"postgresus-backend/internal/features/databases/databases/postgresql"
|
||||
"postgresus-backend/internal/features/notifiers"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -56,14 +57,24 @@ func (d *Database) ValidateUpdate(old, new Database) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Database) TestConnection(logger *slog.Logger) error {
|
||||
return d.getSpecificDatabase().TestConnection(logger)
|
||||
func (d *Database) TestConnection(
|
||||
logger *slog.Logger,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
) error {
|
||||
return d.getSpecificDatabase().TestConnection(logger, encryptor, d.ID)
|
||||
}
|
||||
|
||||
func (d *Database) HideSensitiveData() {
|
||||
d.getSpecificDatabase().HideSensitiveData()
|
||||
}
|
||||
|
||||
func (d *Database) EncryptSensitiveFields(encryptor encryption.FieldEncryptor) error {
|
||||
if d.Postgresql != nil {
|
||||
return d.Postgresql.EncryptSensitiveFields(d.ID, encryptor)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Database) Update(incoming *Database) {
|
||||
d.Name = incoming.Name
|
||||
d.Type = incoming.Type
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package databases
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"postgresus-backend/internal/features/notifiers"
|
||||
users_models "postgresus-backend/internal/features/users/models"
|
||||
workspaces_services "postgresus-backend/internal/features/workspaces/services"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -26,6 +28,7 @@ type DatabaseService struct {
|
||||
|
||||
workspaceService *workspaces_services.WorkspaceService
|
||||
auditLogService *audit_logs.AuditLogService
|
||||
fieldEncryptor encryption.FieldEncryptor
|
||||
}
|
||||
|
||||
func (s *DatabaseService) AddDbCreationListener(
|
||||
@@ -65,6 +68,10 @@ func (s *DatabaseService) CreateDatabase(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := database.EncryptSensitiveFields(s.fieldEncryptor); err != nil {
|
||||
return nil, fmt.Errorf("failed to encrypt sensitive fields: %w", err)
|
||||
}
|
||||
|
||||
database, err = s.dbRepository.Save(database)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -118,6 +125,10 @@ func (s *DatabaseService) UpdateDatabase(
|
||||
return err
|
||||
}
|
||||
|
||||
if err := existingDatabase.EncryptSensitiveFields(s.fieldEncryptor); err != nil {
|
||||
return fmt.Errorf("failed to encrypt sensitive fields: %w", err)
|
||||
}
|
||||
|
||||
_, err = s.dbRepository.Save(existingDatabase)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -250,7 +261,7 @@ func (s *DatabaseService) TestDatabaseConnection(
|
||||
return errors.New("insufficient permissions to test connection for this database")
|
||||
}
|
||||
|
||||
err = database.TestConnection(s.logger)
|
||||
err = database.TestConnection(s.logger, s.fieldEncryptor)
|
||||
if err != nil {
|
||||
lastSaveError := err.Error()
|
||||
database.LastBackupErrorMessage = &lastSaveError
|
||||
@@ -294,7 +305,7 @@ func (s *DatabaseService) TestDatabaseConnectionDirect(
|
||||
usingDatabase = database
|
||||
}
|
||||
|
||||
return usingDatabase.TestConnection(s.logger)
|
||||
return usingDatabase.TestConnection(s.logger, s.fieldEncryptor)
|
||||
}
|
||||
|
||||
func (s *DatabaseService) GetDatabaseByID(
|
||||
@@ -446,3 +457,148 @@ func (s *DatabaseService) OnBeforeWorkspaceDeletion(workspaceID uuid.UUID) error
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DatabaseService) IsUserReadOnly(
|
||||
user *users_models.User,
|
||||
database *Database,
|
||||
) (bool, error) {
|
||||
var usingDatabase *Database
|
||||
|
||||
if database.ID != uuid.Nil {
|
||||
existingDatabase, err := s.dbRepository.FindByID(database.ID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if existingDatabase.WorkspaceID == nil {
|
||||
return false, errors.New("cannot check user for database without workspace")
|
||||
}
|
||||
|
||||
canAccess, _, err := s.workspaceService.CanUserAccessWorkspace(
|
||||
*existingDatabase.WorkspaceID,
|
||||
user,
|
||||
)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !canAccess {
|
||||
return false, errors.New("insufficient permissions to access this database")
|
||||
}
|
||||
|
||||
if database.WorkspaceID != nil && *existingDatabase.WorkspaceID != *database.WorkspaceID {
|
||||
return false, errors.New("database does not belong to this workspace")
|
||||
}
|
||||
|
||||
existingDatabase.Update(database)
|
||||
|
||||
if err := existingDatabase.Validate(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
usingDatabase = existingDatabase
|
||||
} else {
|
||||
if database.WorkspaceID != nil {
|
||||
canAccess, _, err := s.workspaceService.CanUserAccessWorkspace(*database.WorkspaceID, user)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !canAccess {
|
||||
return false, errors.New("insufficient permissions to access this workspace")
|
||||
}
|
||||
}
|
||||
|
||||
usingDatabase = database
|
||||
}
|
||||
|
||||
if usingDatabase.Type != DatabaseTypePostgres {
|
||||
return false, errors.New("read-only check only supported for PostgreSQL databases")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
return usingDatabase.Postgresql.IsUserReadOnly(
|
||||
ctx,
|
||||
s.logger,
|
||||
s.fieldEncryptor,
|
||||
usingDatabase.ID,
|
||||
)
|
||||
}
|
||||
|
||||
func (s *DatabaseService) CreateReadOnlyUser(
|
||||
user *users_models.User,
|
||||
database *Database,
|
||||
) (string, string, error) {
|
||||
var usingDatabase *Database
|
||||
|
||||
if database.ID != uuid.Nil {
|
||||
existingDatabase, err := s.dbRepository.FindByID(database.ID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
if existingDatabase.WorkspaceID == nil {
|
||||
return "", "", errors.New("cannot create user for database without workspace")
|
||||
}
|
||||
|
||||
canManage, err := s.workspaceService.CanUserManageDBs(*existingDatabase.WorkspaceID, user)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if !canManage {
|
||||
return "", "", errors.New("insufficient permissions to manage this database")
|
||||
}
|
||||
|
||||
if database.WorkspaceID != nil && *existingDatabase.WorkspaceID != *database.WorkspaceID {
|
||||
return "", "", errors.New("database does not belong to this workspace")
|
||||
}
|
||||
|
||||
existingDatabase.Update(database)
|
||||
|
||||
if err := existingDatabase.Validate(); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
usingDatabase = existingDatabase
|
||||
} else {
|
||||
if database.WorkspaceID != nil {
|
||||
canManage, err := s.workspaceService.CanUserManageDBs(*database.WorkspaceID, user)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if !canManage {
|
||||
return "", "", errors.New("insufficient permissions to manage this workspace")
|
||||
}
|
||||
}
|
||||
|
||||
usingDatabase = database
|
||||
}
|
||||
|
||||
if usingDatabase.Type != DatabaseTypePostgres {
|
||||
return "", "", errors.New("read-only user creation only supported for PostgreSQL")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
username, password, err := usingDatabase.Postgresql.CreateReadOnlyUser(
|
||||
ctx, s.logger, s.fieldEncryptor, usingDatabase.ID,
|
||||
)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
if usingDatabase.WorkspaceID != nil {
|
||||
s.auditLogService.WriteAuditLog(
|
||||
fmt.Sprintf(
|
||||
"Read-only user created for database: %s (username: %s)",
|
||||
usingDatabase.Name,
|
||||
username,
|
||||
),
|
||||
&user.ID,
|
||||
usingDatabase.WorkspaceID,
|
||||
)
|
||||
}
|
||||
|
||||
return username, password, nil
|
||||
}
|
||||
|
||||
9
backend/internal/features/encryption/secrets/di.go
Normal file
@@ -0,0 +1,9 @@
|
||||
package secrets
|
||||
|
||||
var secretKeyService = &SecretKeyService{
|
||||
nil,
|
||||
}
|
||||
|
||||
func GetSecretKeyService() *SecretKeyService {
|
||||
return secretKeyService
|
||||
}
|
||||
1
backend/internal/features/encryption/secrets/model.go
Normal file
@@ -0,0 +1 @@
|
||||
package secrets
|
||||
73
backend/internal/features/encryption/secrets/service.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package secrets
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"postgresus-backend/internal/config"
|
||||
user_models "postgresus-backend/internal/features/users/models"
|
||||
"postgresus-backend/internal/storage"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type SecretKeyService struct {
|
||||
cachedKey *string
|
||||
}
|
||||
|
||||
func (s *SecretKeyService) MigrateKeyFromDbToFileIfExist() error {
|
||||
var secretKey user_models.SecretKey
|
||||
|
||||
err := storage.GetDb().First(&secretKey).Error
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to check for secret key in database: %w", err)
|
||||
}
|
||||
|
||||
if secretKey.Secret == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
secretKeyPath := config.GetEnv().SecretKeyPath
|
||||
if err := os.WriteFile(secretKeyPath, []byte(secretKey.Secret), 0600); err != nil {
|
||||
return fmt.Errorf("failed to write secret key to file: %w", err)
|
||||
}
|
||||
|
||||
if err := storage.GetDb().Exec("DELETE FROM secret_keys").Error; err != nil {
|
||||
return fmt.Errorf("failed to delete secret key from database: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SecretKeyService) GetSecretKey() (string, error) {
|
||||
if s.cachedKey != nil {
|
||||
return *s.cachedKey, nil
|
||||
}
|
||||
|
||||
secretKeyPath := config.GetEnv().SecretKeyPath
|
||||
data, err := os.ReadFile(secretKeyPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
newKey := s.generateNewSecretKey()
|
||||
if err := os.WriteFile(secretKeyPath, []byte(newKey), 0600); err != nil {
|
||||
return "", fmt.Errorf("failed to write new secret key: %w", err)
|
||||
}
|
||||
s.cachedKey = &newKey
|
||||
return newKey, nil
|
||||
}
|
||||
return "", fmt.Errorf("failed to read secret key file: %w", err)
|
||||
}
|
||||
|
||||
key := string(data)
|
||||
s.cachedKey = &key
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func (s *SecretKeyService) generateNewSecretKey() string {
|
||||
return uuid.New().String() + uuid.New().String()
|
||||
}
|
||||
@@ -453,70 +453,6 @@ func Test_CrossWorkspaceSecurity_CannotAccessNotifierFromAnotherWorkspace(t *tes
|
||||
workspaces_testing.RemoveTestWorkspace(workspace2, router)
|
||||
}
|
||||
|
||||
func createRouter() *gin.Engine {
|
||||
gin.SetMode(gin.TestMode)
|
||||
router := gin.New()
|
||||
|
||||
v1 := router.Group("/api/v1")
|
||||
protected := v1.Group("").Use(users_middleware.AuthMiddleware(users_services.GetUserService()))
|
||||
|
||||
if routerGroup, ok := protected.(*gin.RouterGroup); ok {
|
||||
GetNotifierController().RegisterRoutes(routerGroup)
|
||||
workspaces_controllers.GetWorkspaceController().RegisterRoutes(routerGroup)
|
||||
workspaces_controllers.GetMembershipController().RegisterRoutes(routerGroup)
|
||||
}
|
||||
|
||||
audit_logs.SetupDependencies()
|
||||
|
||||
return router
|
||||
}
|
||||
|
||||
func createNewNotifier(workspaceID uuid.UUID) *Notifier {
|
||||
return &Notifier{
|
||||
WorkspaceID: workspaceID,
|
||||
Name: "Test Notifier " + uuid.New().String(),
|
||||
NotifierType: NotifierTypeWebhook,
|
||||
WebhookNotifier: &webhook_notifier.WebhookNotifier{
|
||||
WebhookURL: "https://webhook.site/test-" + uuid.New().String(),
|
||||
WebhookMethod: webhook_notifier.WebhookMethodPOST,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func createTelegramNotifier(workspaceID uuid.UUID) *Notifier {
|
||||
env := config.GetEnv()
|
||||
return &Notifier{
|
||||
WorkspaceID: workspaceID,
|
||||
Name: "Test Telegram Notifier " + uuid.New().String(),
|
||||
NotifierType: NotifierTypeTelegram,
|
||||
TelegramNotifier: &telegram_notifier.TelegramNotifier{
|
||||
BotToken: env.TestTelegramBotToken,
|
||||
TargetChatID: env.TestTelegramChatID,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func verifyNotifierData(t *testing.T, expected *Notifier, actual *Notifier) {
|
||||
assert.Equal(t, expected.Name, actual.Name)
|
||||
assert.Equal(t, expected.NotifierType, actual.NotifierType)
|
||||
assert.Equal(t, expected.WorkspaceID, actual.WorkspaceID)
|
||||
}
|
||||
|
||||
func deleteNotifier(
|
||||
t *testing.T,
|
||||
router *gin.Engine,
|
||||
notifierID, workspaceID uuid.UUID,
|
||||
token string,
|
||||
) {
|
||||
test_utils.MakeDeleteRequest(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/notifiers/%s", notifierID.String()),
|
||||
"Bearer "+token,
|
||||
http.StatusOK,
|
||||
)
|
||||
}
|
||||
|
||||
func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
@@ -553,7 +489,13 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, notifier *Notifier) {
|
||||
assert.Equal(t, "original-bot-token-12345", notifier.TelegramNotifier.BotToken)
|
||||
assert.True(
|
||||
t,
|
||||
isEncrypted(notifier.TelegramNotifier.BotToken),
|
||||
"BotToken should be encrypted in DB",
|
||||
)
|
||||
decrypted := decryptField(t, notifier.ID, notifier.TelegramNotifier.BotToken)
|
||||
assert.Equal(t, "original-bot-token-12345", decrypted)
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, notifier *Notifier) {
|
||||
assert.Equal(t, "", notifier.TelegramNotifier.BotToken)
|
||||
@@ -592,7 +534,13 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, notifier *Notifier) {
|
||||
assert.Equal(t, "original-password-secret", notifier.EmailNotifier.SMTPPassword)
|
||||
assert.True(
|
||||
t,
|
||||
isEncrypted(notifier.EmailNotifier.SMTPPassword),
|
||||
"SMTPPassword should be encrypted in DB",
|
||||
)
|
||||
decrypted := decryptField(t, notifier.ID, notifier.EmailNotifier.SMTPPassword)
|
||||
assert.Equal(t, "original-password-secret", decrypted)
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, notifier *Notifier) {
|
||||
assert.Equal(t, "", notifier.EmailNotifier.SMTPPassword)
|
||||
@@ -625,7 +573,13 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, notifier *Notifier) {
|
||||
assert.Equal(t, "xoxb-original-slack-token", notifier.SlackNotifier.BotToken)
|
||||
assert.True(
|
||||
t,
|
||||
isEncrypted(notifier.SlackNotifier.BotToken),
|
||||
"BotToken should be encrypted in DB",
|
||||
)
|
||||
decrypted := decryptField(t, notifier.ID, notifier.SlackNotifier.BotToken)
|
||||
assert.Equal(t, "xoxb-original-slack-token", decrypted)
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, notifier *Notifier) {
|
||||
assert.Equal(t, "", notifier.SlackNotifier.BotToken)
|
||||
@@ -656,11 +610,17 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, notifier *Notifier) {
|
||||
assert.Equal(
|
||||
assert.True(
|
||||
t,
|
||||
"https://discord.com/api/webhooks/123/original-token",
|
||||
isEncrypted(notifier.DiscordNotifier.ChannelWebhookURL),
|
||||
"WebhookURL should be encrypted in DB",
|
||||
)
|
||||
decrypted := decryptField(
|
||||
t,
|
||||
notifier.ID,
|
||||
notifier.DiscordNotifier.ChannelWebhookURL,
|
||||
)
|
||||
assert.Equal(t, "https://discord.com/api/webhooks/123/original-token", decrypted)
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, notifier *Notifier) {
|
||||
assert.Equal(t, "", notifier.DiscordNotifier.ChannelWebhookURL)
|
||||
@@ -691,10 +651,16 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, notifier *Notifier) {
|
||||
assert.True(
|
||||
t,
|
||||
isEncrypted(notifier.TeamsNotifier.WebhookURL),
|
||||
"WebhookURL should be encrypted in DB",
|
||||
)
|
||||
decrypted := decryptField(t, notifier.ID, notifier.TeamsNotifier.WebhookURL)
|
||||
assert.Equal(
|
||||
t,
|
||||
"https://outlook.office.com/webhook/original-token",
|
||||
notifier.TeamsNotifier.WebhookURL,
|
||||
decrypted,
|
||||
)
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, notifier *Notifier) {
|
||||
@@ -813,3 +779,263 @@ func Test_NotifierSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_CreateNotifier_AllSensitiveFieldsEncryptedInDB(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
createNotifier func(workspaceID uuid.UUID) *Notifier
|
||||
verifySensitiveEncryption func(t *testing.T, notifier *Notifier)
|
||||
}{
|
||||
{
|
||||
name: "Telegram Notifier - BotToken encrypted",
|
||||
createNotifier: func(workspaceID uuid.UUID) *Notifier {
|
||||
return &Notifier{
|
||||
WorkspaceID: workspaceID,
|
||||
Name: "Test Telegram",
|
||||
NotifierType: NotifierTypeTelegram,
|
||||
TelegramNotifier: &telegram_notifier.TelegramNotifier{
|
||||
BotToken: "plain-telegram-token-123",
|
||||
TargetChatID: "123456789",
|
||||
},
|
||||
}
|
||||
},
|
||||
verifySensitiveEncryption: func(t *testing.T, notifier *Notifier) {
|
||||
assert.True(
|
||||
t,
|
||||
isEncrypted(notifier.TelegramNotifier.BotToken),
|
||||
"BotToken should be encrypted",
|
||||
)
|
||||
decrypted := decryptField(t, notifier.ID, notifier.TelegramNotifier.BotToken)
|
||||
assert.Equal(t, "plain-telegram-token-123", decrypted)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Email Notifier - SMTPPassword encrypted",
|
||||
createNotifier: func(workspaceID uuid.UUID) *Notifier {
|
||||
return &Notifier{
|
||||
WorkspaceID: workspaceID,
|
||||
Name: "Test Email",
|
||||
NotifierType: NotifierTypeEmail,
|
||||
EmailNotifier: &email_notifier.EmailNotifier{
|
||||
TargetEmail: "test@example.com",
|
||||
SMTPHost: "smtp.example.com",
|
||||
SMTPPort: 587,
|
||||
SMTPUser: "user@example.com",
|
||||
SMTPPassword: "plain-smtp-password-456",
|
||||
From: "noreply@example.com",
|
||||
},
|
||||
}
|
||||
},
|
||||
verifySensitiveEncryption: func(t *testing.T, notifier *Notifier) {
|
||||
assert.True(
|
||||
t,
|
||||
isEncrypted(notifier.EmailNotifier.SMTPPassword),
|
||||
"SMTPPassword should be encrypted",
|
||||
)
|
||||
decrypted := decryptField(t, notifier.ID, notifier.EmailNotifier.SMTPPassword)
|
||||
assert.Equal(t, "plain-smtp-password-456", decrypted)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Slack Notifier - BotToken encrypted",
|
||||
createNotifier: func(workspaceID uuid.UUID) *Notifier {
|
||||
return &Notifier{
|
||||
WorkspaceID: workspaceID,
|
||||
Name: "Test Slack",
|
||||
NotifierType: NotifierTypeSlack,
|
||||
SlackNotifier: &slack_notifier.SlackNotifier{
|
||||
BotToken: "plain-slack-token-789",
|
||||
TargetChatID: "C0123456789",
|
||||
},
|
||||
}
|
||||
},
|
||||
verifySensitiveEncryption: func(t *testing.T, notifier *Notifier) {
|
||||
assert.True(
|
||||
t,
|
||||
isEncrypted(notifier.SlackNotifier.BotToken),
|
||||
"BotToken should be encrypted",
|
||||
)
|
||||
decrypted := decryptField(t, notifier.ID, notifier.SlackNotifier.BotToken)
|
||||
assert.Equal(t, "plain-slack-token-789", decrypted)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Discord Notifier - WebhookURL encrypted",
|
||||
createNotifier: func(workspaceID uuid.UUID) *Notifier {
|
||||
return &Notifier{
|
||||
WorkspaceID: workspaceID,
|
||||
Name: "Test Discord",
|
||||
NotifierType: NotifierTypeDiscord,
|
||||
DiscordNotifier: &discord_notifier.DiscordNotifier{
|
||||
ChannelWebhookURL: "https://discord.com/api/webhooks/123/abc",
|
||||
},
|
||||
}
|
||||
},
|
||||
verifySensitiveEncryption: func(t *testing.T, notifier *Notifier) {
|
||||
assert.True(
|
||||
t,
|
||||
isEncrypted(notifier.DiscordNotifier.ChannelWebhookURL),
|
||||
"WebhookURL should be encrypted",
|
||||
)
|
||||
decrypted := decryptField(
|
||||
t,
|
||||
notifier.ID,
|
||||
notifier.DiscordNotifier.ChannelWebhookURL,
|
||||
)
|
||||
assert.Equal(t, "https://discord.com/api/webhooks/123/abc", decrypted)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Teams Notifier - WebhookURL encrypted",
|
||||
createNotifier: func(workspaceID uuid.UUID) *Notifier {
|
||||
return &Notifier{
|
||||
WorkspaceID: workspaceID,
|
||||
Name: "Test Teams",
|
||||
NotifierType: NotifierTypeTeams,
|
||||
TeamsNotifier: &teams_notifier.TeamsNotifier{
|
||||
WebhookURL: "https://outlook.office.com/webhook/test123",
|
||||
},
|
||||
}
|
||||
},
|
||||
verifySensitiveEncryption: func(t *testing.T, notifier *Notifier) {
|
||||
assert.True(
|
||||
t,
|
||||
isEncrypted(notifier.TeamsNotifier.WebhookURL),
|
||||
"WebhookURL should be encrypted",
|
||||
)
|
||||
decrypted := decryptField(t, notifier.ID, notifier.TeamsNotifier.WebhookURL)
|
||||
assert.Equal(t, "https://outlook.office.com/webhook/test123", decrypted)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Webhook Notifier - WebhookURL encrypted",
|
||||
createNotifier: func(workspaceID uuid.UUID) *Notifier {
|
||||
return &Notifier{
|
||||
WorkspaceID: workspaceID,
|
||||
Name: "Test Webhook",
|
||||
NotifierType: NotifierTypeWebhook,
|
||||
WebhookNotifier: &webhook_notifier.WebhookNotifier{
|
||||
WebhookURL: "https://webhook.example.com/test456",
|
||||
WebhookMethod: webhook_notifier.WebhookMethodPOST,
|
||||
},
|
||||
}
|
||||
},
|
||||
verifySensitiveEncryption: func(t *testing.T, notifier *Notifier) {
|
||||
assert.True(
|
||||
t,
|
||||
isEncrypted(notifier.WebhookNotifier.WebhookURL),
|
||||
"WebhookURL should be encrypted",
|
||||
)
|
||||
decrypted := decryptField(t, notifier.ID, notifier.WebhookNotifier.WebhookURL)
|
||||
assert.Equal(t, "https://webhook.example.com/test456", decrypted)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
router := createRouter()
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
|
||||
// Create notifier via API (plaintext credentials)
|
||||
var createdNotifier Notifier
|
||||
test_utils.MakePostRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/notifiers",
|
||||
"Bearer "+owner.Token,
|
||||
tc.createNotifier(workspace.ID),
|
||||
http.StatusOK,
|
||||
&createdNotifier,
|
||||
)
|
||||
|
||||
// Read from DB directly (bypass service layer)
|
||||
repository := &NotifierRepository{}
|
||||
notifierFromDB, err := repository.FindByID(createdNotifier.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify encryption
|
||||
tc.verifySensitiveEncryption(t, notifierFromDB)
|
||||
|
||||
// Cleanup
|
||||
deleteNotifier(t, router, createdNotifier.ID, workspace.ID, owner.Token)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func createRouter() *gin.Engine {
|
||||
gin.SetMode(gin.TestMode)
|
||||
router := gin.New()
|
||||
|
||||
v1 := router.Group("/api/v1")
|
||||
protected := v1.Group("").Use(users_middleware.AuthMiddleware(users_services.GetUserService()))
|
||||
|
||||
if routerGroup, ok := protected.(*gin.RouterGroup); ok {
|
||||
GetNotifierController().RegisterRoutes(routerGroup)
|
||||
workspaces_controllers.GetWorkspaceController().RegisterRoutes(routerGroup)
|
||||
workspaces_controllers.GetMembershipController().RegisterRoutes(routerGroup)
|
||||
}
|
||||
|
||||
audit_logs.SetupDependencies()
|
||||
|
||||
return router
|
||||
}
|
||||
|
||||
func createNewNotifier(workspaceID uuid.UUID) *Notifier {
|
||||
return &Notifier{
|
||||
WorkspaceID: workspaceID,
|
||||
Name: "Test Notifier " + uuid.New().String(),
|
||||
NotifierType: NotifierTypeWebhook,
|
||||
WebhookNotifier: &webhook_notifier.WebhookNotifier{
|
||||
WebhookURL: "https://webhook.site/test-" + uuid.New().String(),
|
||||
WebhookMethod: webhook_notifier.WebhookMethodPOST,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func createTelegramNotifier(workspaceID uuid.UUID) *Notifier {
|
||||
env := config.GetEnv()
|
||||
return &Notifier{
|
||||
WorkspaceID: workspaceID,
|
||||
Name: "Test Telegram Notifier " + uuid.New().String(),
|
||||
NotifierType: NotifierTypeTelegram,
|
||||
TelegramNotifier: &telegram_notifier.TelegramNotifier{
|
||||
BotToken: env.TestTelegramBotToken,
|
||||
TargetChatID: env.TestTelegramChatID,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func verifyNotifierData(t *testing.T, expected *Notifier, actual *Notifier) {
|
||||
assert.Equal(t, expected.Name, actual.Name)
|
||||
assert.Equal(t, expected.NotifierType, actual.NotifierType)
|
||||
assert.Equal(t, expected.WorkspaceID, actual.WorkspaceID)
|
||||
}
|
||||
|
||||
func deleteNotifier(
|
||||
t *testing.T,
|
||||
router *gin.Engine,
|
||||
notifierID, workspaceID uuid.UUID,
|
||||
token string,
|
||||
) {
|
||||
test_utils.MakeDeleteRequest(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/notifiers/%s", notifierID.String()),
|
||||
"Bearer "+token,
|
||||
http.StatusOK,
|
||||
)
|
||||
}
|
||||
|
||||
func isEncrypted(value string) bool {
|
||||
return len(value) > 4 && value[:4] == "enc:"
|
||||
}
|
||||
|
||||
func decryptField(t *testing.T, notifierID uuid.UUID, encryptedValue string) string {
|
||||
encryptor := GetNotifierService().fieldEncryptor
|
||||
decrypted, err := encryptor.Decrypt(notifierID, encryptedValue)
|
||||
assert.NoError(t, err)
|
||||
return decrypted
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package notifiers
|
||||
import (
|
||||
audit_logs "postgresus-backend/internal/features/audit_logs"
|
||||
workspaces_services "postgresus-backend/internal/features/workspaces/services"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"postgresus-backend/internal/util/logger"
|
||||
)
|
||||
|
||||
@@ -12,6 +13,7 @@ var notifierService = &NotifierService{
|
||||
logger.GetLogger(),
|
||||
workspaces_services.GetWorkspaceService(),
|
||||
audit_logs.GetAuditLogService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
}
|
||||
var notifierController = &NotifierController{
|
||||
notifierService,
|
||||
|
||||
@@ -1,11 +1,21 @@
|
||||
package notifiers
|
||||
|
||||
import "log/slog"
|
||||
import (
|
||||
"log/slog"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
)
|
||||
|
||||
type NotificationSender interface {
|
||||
Send(logger *slog.Logger, heading string, message string) error
|
||||
Send(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
heading string,
|
||||
message string,
|
||||
) error
|
||||
|
||||
Validate() error
|
||||
Validate(encryptor encryption.FieldEncryptor) error
|
||||
|
||||
HideSensitiveData()
|
||||
|
||||
EncryptSensitiveData(encryptor encryption.FieldEncryptor) error
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
teams_notifier "postgresus-backend/internal/features/notifiers/models/teams"
|
||||
telegram_notifier "postgresus-backend/internal/features/notifiers/models/telegram"
|
||||
webhook_notifier "postgresus-backend/internal/features/notifiers/models/webhook"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -33,16 +34,21 @@ func (n *Notifier) TableName() string {
|
||||
return "notifiers"
|
||||
}
|
||||
|
||||
func (n *Notifier) Validate() error {
|
||||
func (n *Notifier) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
if n.Name == "" {
|
||||
return errors.New("name is required")
|
||||
}
|
||||
|
||||
return n.getSpecificNotifier().Validate()
|
||||
return n.getSpecificNotifier().Validate(encryptor)
|
||||
}
|
||||
|
||||
func (n *Notifier) Send(logger *slog.Logger, heading string, message string) error {
|
||||
err := n.getSpecificNotifier().Send(logger, heading, message)
|
||||
func (n *Notifier) Send(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
heading string,
|
||||
message string,
|
||||
) error {
|
||||
err := n.getSpecificNotifier().Send(encryptor, logger, heading, message)
|
||||
|
||||
if err != nil {
|
||||
lastSendError := err.Error()
|
||||
@@ -58,6 +64,10 @@ func (n *Notifier) HideSensitiveData() {
|
||||
n.getSpecificNotifier().HideSensitiveData()
|
||||
}
|
||||
|
||||
func (n *Notifier) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
return n.getSpecificNotifier().EncryptSensitiveData(encryptor)
|
||||
}
|
||||
|
||||
func (n *Notifier) Update(incoming *Notifier) {
|
||||
n.Name = incoming.Name
|
||||
n.NotifierType = incoming.NotifierType
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -21,7 +22,7 @@ func (d *DiscordNotifier) TableName() string {
|
||||
return "discord_notifiers"
|
||||
}
|
||||
|
||||
func (d *DiscordNotifier) Validate() error {
|
||||
func (d *DiscordNotifier) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
if d.ChannelWebhookURL == "" {
|
||||
return errors.New("webhook URL is required")
|
||||
}
|
||||
@@ -29,7 +30,17 @@ func (d *DiscordNotifier) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DiscordNotifier) Send(logger *slog.Logger, heading string, message string) error {
|
||||
func (d *DiscordNotifier) Send(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
heading string,
|
||||
message string,
|
||||
) error {
|
||||
webhookURL, err := encryptor.Decrypt(d.NotifierID, d.ChannelWebhookURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt webhook URL: %w", err)
|
||||
}
|
||||
|
||||
fullMessage := heading
|
||||
if message != "" {
|
||||
fullMessage = fmt.Sprintf("%s\n\n%s", heading, message)
|
||||
@@ -44,7 +55,7 @@ func (d *DiscordNotifier) Send(logger *slog.Logger, heading string, message stri
|
||||
return fmt.Errorf("failed to marshal Discord payload: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", d.ChannelWebhookURL, bytes.NewReader(jsonPayload))
|
||||
req, err := http.NewRequest("POST", webhookURL, bytes.NewReader(jsonPayload))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
@@ -81,3 +92,14 @@ func (d *DiscordNotifier) Update(incoming *DiscordNotifier) {
|
||||
d.ChannelWebhookURL = incoming.ChannelWebhookURL
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DiscordNotifier) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
if d.ChannelWebhookURL != "" {
|
||||
encrypted, err := encryptor.Encrypt(d.NotifierID, d.ChannelWebhookURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt webhook URL: %w", err)
|
||||
}
|
||||
d.ChannelWebhookURL = encrypted
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package email_notifier
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/smtp"
|
||||
)
|
||||
|
||||
type loginAuth struct {
|
||||
username, password string
|
||||
}
|
||||
|
||||
func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
|
||||
return "LOGIN", []byte{}, nil
|
||||
}
|
||||
|
||||
func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
|
||||
if more {
|
||||
switch string(fromServer) {
|
||||
case "Username:":
|
||||
return []byte(a.username), nil
|
||||
case "Password:":
|
||||
return []byte(a.password), nil
|
||||
default:
|
||||
return nil, errors.New("unknown LOGIN challenge: " + string(fromServer))
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/smtp"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -34,7 +35,7 @@ func (e *EmailNotifier) TableName() string {
|
||||
return "email_notifiers"
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) Validate() error {
|
||||
func (e *EmailNotifier) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
if e.TargetEmail == "" {
|
||||
return errors.New("target email is required")
|
||||
}
|
||||
@@ -55,8 +56,21 @@ func (e *EmailNotifier) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) Send(logger *slog.Logger, heading string, message string) error {
|
||||
// Compose email
|
||||
func (e *EmailNotifier) Send(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
_ *slog.Logger,
|
||||
heading string,
|
||||
message string,
|
||||
) error {
|
||||
var smtpPassword string
|
||||
if e.SMTPPassword != "" {
|
||||
decrypted, err := encryptor.Decrypt(e.NotifierID, e.SMTPPassword)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt SMTP password: %w", err)
|
||||
}
|
||||
smtpPassword = decrypted
|
||||
}
|
||||
|
||||
from := e.From
|
||||
if from == "" {
|
||||
from = e.SMTPUser
|
||||
@@ -65,153 +79,13 @@ func (e *EmailNotifier) Send(logger *slog.Logger, heading string, message string
|
||||
}
|
||||
}
|
||||
|
||||
to := []string{e.TargetEmail}
|
||||
emailContent := e.buildEmailContent(heading, message, from)
|
||||
isAuthRequired := e.SMTPUser != "" && smtpPassword != ""
|
||||
|
||||
// Format the email content
|
||||
subject := fmt.Sprintf("Subject: %s\r\n", heading)
|
||||
mime := fmt.Sprintf(
|
||||
"MIME-version: 1.0;\nContent-Type: %s; charset=\"%s\";\n\n",
|
||||
MIMETypeHTML,
|
||||
MIMECharsetUTF8,
|
||||
)
|
||||
body := message
|
||||
fromHeader := fmt.Sprintf("From: %s\r\n", from)
|
||||
toHeader := fmt.Sprintf("To: %s\r\n", e.TargetEmail)
|
||||
|
||||
// Combine all parts of the email
|
||||
emailContent := []byte(fromHeader + toHeader + subject + mime + body)
|
||||
|
||||
addr := net.JoinHostPort(e.SMTPHost, fmt.Sprintf("%d", e.SMTPPort))
|
||||
timeout := DefaultTimeout
|
||||
|
||||
// Determine if authentication is required
|
||||
isAuthRequired := e.SMTPUser != "" && e.SMTPPassword != ""
|
||||
|
||||
// Handle different port scenarios
|
||||
if e.SMTPPort == ImplicitTLSPort {
|
||||
// Implicit TLS (port 465)
|
||||
// Set up TLS config
|
||||
tlsConfig := &tls.Config{
|
||||
ServerName: e.SMTPHost,
|
||||
}
|
||||
|
||||
// Dial with timeout
|
||||
dialer := &net.Dialer{Timeout: timeout}
|
||||
conn, err := tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to SMTP server: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = conn.Close()
|
||||
}()
|
||||
|
||||
// Create SMTP client
|
||||
client, err := smtp.NewClient(conn, e.SMTPHost)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create SMTP client: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = client.Quit()
|
||||
}()
|
||||
|
||||
// Set up authentication only if credentials are provided
|
||||
if isAuthRequired {
|
||||
auth := smtp.PlainAuth("", e.SMTPUser, e.SMTPPassword, e.SMTPHost)
|
||||
if err := client.Auth(auth); err != nil {
|
||||
return fmt.Errorf("SMTP authentication failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Set sender and recipients
|
||||
if err := client.Mail(from); err != nil {
|
||||
return fmt.Errorf("failed to set sender: %w", err)
|
||||
}
|
||||
for _, recipient := range to {
|
||||
if err := client.Rcpt(recipient); err != nil {
|
||||
return fmt.Errorf("failed to set recipient: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Send the email body
|
||||
writer, err := client.Data()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get data writer: %w", err)
|
||||
}
|
||||
_, err = writer.Write(emailContent)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write email content: %w", err)
|
||||
}
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to close data writer: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
} else {
|
||||
// STARTTLS (port 587) or other ports
|
||||
// Create a custom dialer with timeout
|
||||
dialer := &net.Dialer{Timeout: timeout}
|
||||
conn, err := dialer.Dial("tcp", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to SMTP server: %w", err)
|
||||
}
|
||||
|
||||
// Create client from connection
|
||||
client, err := smtp.NewClient(conn, e.SMTPHost)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create SMTP client: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = client.Quit()
|
||||
}()
|
||||
|
||||
// Send email using the client
|
||||
if err := client.Hello(DefaultHelloName); err != nil {
|
||||
return fmt.Errorf("SMTP hello failed: %w", err)
|
||||
}
|
||||
|
||||
// Start TLS if available
|
||||
if ok, _ := client.Extension("STARTTLS"); ok {
|
||||
if err := client.StartTLS(&tls.Config{ServerName: e.SMTPHost}); err != nil {
|
||||
return fmt.Errorf("STARTTLS failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Authenticate only if credentials are provided
|
||||
if isAuthRequired {
|
||||
auth := smtp.PlainAuth("", e.SMTPUser, e.SMTPPassword, e.SMTPHost)
|
||||
if err := client.Auth(auth); err != nil {
|
||||
return fmt.Errorf("SMTP authentication failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := client.Mail(from); err != nil {
|
||||
return fmt.Errorf("failed to set sender: %w", err)
|
||||
}
|
||||
|
||||
for _, recipient := range to {
|
||||
if err := client.Rcpt(recipient); err != nil {
|
||||
return fmt.Errorf("failed to set recipient: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
writer, err := client.Data()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get data writer: %w", err)
|
||||
}
|
||||
|
||||
_, err = writer.Write(emailContent)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write email content: %w", err)
|
||||
}
|
||||
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to close data writer: %w", err)
|
||||
}
|
||||
|
||||
return client.Quit()
|
||||
return e.sendImplicitTLS(emailContent, from, smtpPassword, isAuthRequired)
|
||||
}
|
||||
return e.sendStartTLS(emailContent, from, smtpPassword, isAuthRequired)
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) HideSensitiveData() {
|
||||
@@ -229,3 +103,177 @@ func (e *EmailNotifier) Update(incoming *EmailNotifier) {
|
||||
e.SMTPPassword = incoming.SMTPPassword
|
||||
}
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
if e.SMTPPassword != "" {
|
||||
encrypted, err := encryptor.Encrypt(e.NotifierID, e.SMTPPassword)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt SMTP password: %w", err)
|
||||
}
|
||||
e.SMTPPassword = encrypted
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) buildEmailContent(heading, message, from string) []byte {
|
||||
subject := fmt.Sprintf("Subject: %s\r\n", heading)
|
||||
mime := fmt.Sprintf(
|
||||
"MIME-version: 1.0;\nContent-Type: %s; charset=\"%s\";\n\n",
|
||||
MIMETypeHTML,
|
||||
MIMECharsetUTF8,
|
||||
)
|
||||
fromHeader := fmt.Sprintf("From: %s\r\n", from)
|
||||
toHeader := fmt.Sprintf("To: %s\r\n", e.TargetEmail)
|
||||
return []byte(fromHeader + toHeader + subject + mime + message)
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) sendImplicitTLS(
|
||||
emailContent []byte,
|
||||
from string,
|
||||
password string,
|
||||
isAuthRequired bool,
|
||||
) error {
|
||||
createClient := func() (*smtp.Client, func(), error) {
|
||||
return e.createImplicitTLSClient()
|
||||
}
|
||||
|
||||
client, cleanup, err := e.authenticateWithRetry(createClient, password, isAuthRequired)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
return e.sendEmail(client, from, emailContent)
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) sendStartTLS(
|
||||
emailContent []byte,
|
||||
from string,
|
||||
password string,
|
||||
isAuthRequired bool,
|
||||
) error {
|
||||
createClient := func() (*smtp.Client, func(), error) {
|
||||
return e.createStartTLSClient()
|
||||
}
|
||||
|
||||
client, cleanup, err := e.authenticateWithRetry(createClient, password, isAuthRequired)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
return e.sendEmail(client, from, emailContent)
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) createImplicitTLSClient() (*smtp.Client, func(), error) {
|
||||
addr := net.JoinHostPort(e.SMTPHost, fmt.Sprintf("%d", e.SMTPPort))
|
||||
tlsConfig := &tls.Config{ServerName: e.SMTPHost}
|
||||
dialer := &net.Dialer{Timeout: DefaultTimeout}
|
||||
|
||||
conn, err := tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to connect to SMTP server: %w", err)
|
||||
}
|
||||
|
||||
client, err := smtp.NewClient(conn, e.SMTPHost)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, nil, fmt.Errorf("failed to create SMTP client: %w", err)
|
||||
}
|
||||
|
||||
return client, func() { _ = client.Quit() }, nil
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) createStartTLSClient() (*smtp.Client, func(), error) {
|
||||
addr := net.JoinHostPort(e.SMTPHost, fmt.Sprintf("%d", e.SMTPPort))
|
||||
dialer := &net.Dialer{Timeout: DefaultTimeout}
|
||||
|
||||
conn, err := dialer.Dial("tcp", addr)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to connect to SMTP server: %w", err)
|
||||
}
|
||||
|
||||
client, err := smtp.NewClient(conn, e.SMTPHost)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, nil, fmt.Errorf("failed to create SMTP client: %w", err)
|
||||
}
|
||||
|
||||
if err := client.Hello(DefaultHelloName); err != nil {
|
||||
_ = client.Quit()
|
||||
_ = conn.Close()
|
||||
return nil, nil, fmt.Errorf("SMTP hello failed: %w", err)
|
||||
}
|
||||
|
||||
if ok, _ := client.Extension("STARTTLS"); ok {
|
||||
if err := client.StartTLS(&tls.Config{ServerName: e.SMTPHost}); err != nil {
|
||||
_ = client.Quit()
|
||||
_ = conn.Close()
|
||||
return nil, nil, fmt.Errorf("STARTTLS failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return client, func() { _ = client.Quit() }, nil
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) authenticateWithRetry(
|
||||
createClient func() (*smtp.Client, func(), error),
|
||||
password string,
|
||||
isAuthRequired bool,
|
||||
) (*smtp.Client, func(), error) {
|
||||
client, cleanup, err := createClient()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if !isAuthRequired {
|
||||
return client, cleanup, nil
|
||||
}
|
||||
|
||||
// Try PLAIN auth first
|
||||
plainAuth := smtp.PlainAuth("", e.SMTPUser, password, e.SMTPHost)
|
||||
if err := client.Auth(plainAuth); err == nil {
|
||||
return client, cleanup, nil
|
||||
}
|
||||
|
||||
// PLAIN auth failed, connection may be closed - recreate and try LOGIN auth
|
||||
cleanup()
|
||||
|
||||
client, cleanup, err = createClient()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
loginAuth := &loginAuth{username: e.SMTPUser, password: password}
|
||||
if err := client.Auth(loginAuth); err != nil {
|
||||
cleanup()
|
||||
return nil, nil, fmt.Errorf("SMTP authentication failed: %w", err)
|
||||
}
|
||||
|
||||
return client, cleanup, nil
|
||||
}
|
||||
|
||||
func (e *EmailNotifier) sendEmail(client *smtp.Client, from string, content []byte) error {
|
||||
if err := client.Mail(from); err != nil {
|
||||
return fmt.Errorf("failed to set sender: %w", err)
|
||||
}
|
||||
|
||||
if err := client.Rcpt(e.TargetEmail); err != nil {
|
||||
return fmt.Errorf("failed to set recipient: %w", err)
|
||||
}
|
||||
|
||||
writer, err := client.Data()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get data writer: %w", err)
|
||||
}
|
||||
|
||||
if _, err = writer.Write(content); err != nil {
|
||||
return fmt.Errorf("failed to write email content: %w", err)
|
||||
}
|
||||
|
||||
if err = writer.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close data writer: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -23,7 +24,7 @@ type SlackNotifier struct {
|
||||
|
||||
func (s *SlackNotifier) TableName() string { return "slack_notifiers" }
|
||||
|
||||
func (s *SlackNotifier) Validate() error {
|
||||
func (s *SlackNotifier) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
if s.BotToken == "" {
|
||||
return errors.New("bot token is required")
|
||||
}
|
||||
@@ -43,7 +44,16 @@ func (s *SlackNotifier) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SlackNotifier) Send(logger *slog.Logger, heading, message string) error {
|
||||
func (s *SlackNotifier) Send(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
heading, message string,
|
||||
) error {
|
||||
botToken, err := encryptor.Decrypt(s.NotifierID, s.BotToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt bot token: %w", err)
|
||||
}
|
||||
|
||||
full := fmt.Sprintf("*%s*", heading)
|
||||
|
||||
if message != "" {
|
||||
@@ -60,6 +70,7 @@ func (s *SlackNotifier) Send(logger *slog.Logger, heading, message string) error
|
||||
maxAttempts = 5
|
||||
defaultBackoff = 2 * time.Second // when Retry-After header missing
|
||||
backoffMultiplier = 1.5 // use exponential growth
|
||||
requestTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -67,6 +78,10 @@ func (s *SlackNotifier) Send(logger *slog.Logger, heading, message string) error
|
||||
attempts = 0
|
||||
)
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: requestTimeout,
|
||||
}
|
||||
|
||||
for {
|
||||
attempts++
|
||||
|
||||
@@ -80,9 +95,9 @@ func (s *SlackNotifier) Send(logger *slog.Logger, heading, message string) error
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Set("Authorization", "Bearer "+s.BotToken)
|
||||
req.Header.Set("Authorization", "Bearer "+botToken)
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("send slack message: %w", err)
|
||||
}
|
||||
@@ -144,3 +159,14 @@ func (s *SlackNotifier) Update(incoming *SlackNotifier) {
|
||||
s.BotToken = incoming.BotToken
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SlackNotifier) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
if s.BotToken != "" {
|
||||
encrypted, err := encryptor.Encrypt(s.NotifierID, s.BotToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt bot token: %w", err)
|
||||
}
|
||||
s.BotToken = encrypted
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -21,11 +22,17 @@ func (TeamsNotifier) TableName() string {
|
||||
return "teams_notifiers"
|
||||
}
|
||||
|
||||
func (n *TeamsNotifier) Validate() error {
|
||||
func (n *TeamsNotifier) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
if n.WebhookURL == "" {
|
||||
return errors.New("webhook_url is required")
|
||||
}
|
||||
u, err := url.Parse(n.WebhookURL)
|
||||
|
||||
webhookURL, err := encryptor.Decrypt(n.NotifierID, n.WebhookURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt webhook URL: %w", err)
|
||||
}
|
||||
|
||||
u, err := url.Parse(webhookURL)
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
return errors.New("invalid webhook_url")
|
||||
}
|
||||
@@ -33,8 +40,8 @@ func (n *TeamsNotifier) Validate() error {
|
||||
}
|
||||
|
||||
type cardAttachment struct {
|
||||
ContentType string `json:"contentType"`
|
||||
Content interface{} `json:"content"`
|
||||
ContentType string `json:"contentType"`
|
||||
Content any `json:"content"`
|
||||
}
|
||||
|
||||
type payload struct {
|
||||
@@ -43,11 +50,20 @@ type payload struct {
|
||||
Attachments []cardAttachment `json:"attachments,omitempty"`
|
||||
}
|
||||
|
||||
func (n *TeamsNotifier) Send(logger *slog.Logger, heading, message string) error {
|
||||
if err := n.Validate(); err != nil {
|
||||
func (n *TeamsNotifier) Send(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
heading, message string,
|
||||
) error {
|
||||
if err := n.Validate(encryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
webhookURL, err := encryptor.Decrypt(n.NotifierID, n.WebhookURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt webhook URL: %w", err)
|
||||
}
|
||||
|
||||
card := map[string]any{
|
||||
"type": "AdaptiveCard",
|
||||
"version": "1.4",
|
||||
@@ -71,7 +87,7 @@ func (n *TeamsNotifier) Send(logger *slog.Logger, heading, message string) error
|
||||
}
|
||||
|
||||
body, _ := json.Marshal(p)
|
||||
req, err := http.NewRequest(http.MethodPost, n.WebhookURL, bytes.NewReader(body))
|
||||
req, err := http.NewRequest(http.MethodPost, webhookURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -104,3 +120,14 @@ func (n *TeamsNotifier) Update(incoming *TeamsNotifier) {
|
||||
n.WebhookURL = incoming.WebhookURL
|
||||
}
|
||||
}
|
||||
|
||||
func (n *TeamsNotifier) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
if n.WebhookURL != "" {
|
||||
encrypted, err := encryptor.Encrypt(n.NotifierID, n.WebhookURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt webhook URL: %w", err)
|
||||
}
|
||||
n.WebhookURL = encrypted
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -24,7 +25,7 @@ func (t *TelegramNotifier) TableName() string {
|
||||
return "telegram_notifiers"
|
||||
}
|
||||
|
||||
func (t *TelegramNotifier) Validate() error {
|
||||
func (t *TelegramNotifier) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
if t.BotToken == "" {
|
||||
return errors.New("bot token is required")
|
||||
}
|
||||
@@ -36,13 +37,23 @@ func (t *TelegramNotifier) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *TelegramNotifier) Send(logger *slog.Logger, heading string, message string) error {
|
||||
func (t *TelegramNotifier) Send(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
heading string,
|
||||
message string,
|
||||
) error {
|
||||
botToken, err := encryptor.Decrypt(t.NotifierID, t.BotToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt bot token: %w", err)
|
||||
}
|
||||
|
||||
fullMessage := heading
|
||||
if message != "" {
|
||||
fullMessage = fmt.Sprintf("%s\n\n%s", heading, message)
|
||||
}
|
||||
|
||||
apiURL := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", t.BotToken)
|
||||
apiURL := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", botToken)
|
||||
|
||||
data := url.Values{}
|
||||
data.Set("chat_id", t.TargetChatID)
|
||||
@@ -93,3 +104,14 @@ func (t *TelegramNotifier) Update(incoming *TelegramNotifier) {
|
||||
t.BotToken = incoming.BotToken
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TelegramNotifier) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
if t.BotToken != "" {
|
||||
encrypted, err := encryptor.Encrypt(t.NotifierID, t.BotToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt bot token: %w", err)
|
||||
}
|
||||
t.BotToken = encrypted
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -9,21 +9,59 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type WebhookHeader struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type WebhookNotifier struct {
|
||||
NotifierID uuid.UUID `json:"notifierId" gorm:"primaryKey;column:notifier_id"`
|
||||
WebhookURL string `json:"webhookUrl" gorm:"not null;column:webhook_url"`
|
||||
WebhookMethod WebhookMethod `json:"webhookMethod" gorm:"not null;column:webhook_method"`
|
||||
BodyTemplate *string `json:"bodyTemplate" gorm:"column:body_template;type:text"`
|
||||
HeadersJSON string `json:"-" gorm:"column:headers;type:text"`
|
||||
|
||||
Headers []WebhookHeader `json:"headers" gorm:"-"`
|
||||
}
|
||||
|
||||
func (t *WebhookNotifier) TableName() string {
|
||||
return "webhook_notifiers"
|
||||
}
|
||||
|
||||
func (t *WebhookNotifier) Validate() error {
|
||||
func (t *WebhookNotifier) BeforeSave(_ *gorm.DB) error {
|
||||
if len(t.Headers) > 0 {
|
||||
data, err := json.Marshal(t.Headers)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
t.HeadersJSON = string(data)
|
||||
} else {
|
||||
t.HeadersJSON = "[]"
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *WebhookNotifier) AfterFind(_ *gorm.DB) error {
|
||||
if t.HeadersJSON != "" {
|
||||
if err := json.Unmarshal([]byte(t.HeadersJSON), &t.Headers); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *WebhookNotifier) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
if t.WebhookURL == "" {
|
||||
return errors.New("webhook URL is required")
|
||||
}
|
||||
@@ -35,69 +73,22 @@ func (t *WebhookNotifier) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *WebhookNotifier) Send(logger *slog.Logger, heading string, message string) error {
|
||||
func (t *WebhookNotifier) Send(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
heading string,
|
||||
message string,
|
||||
) error {
|
||||
webhookURL, err := encryptor.Decrypt(t.NotifierID, t.WebhookURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt webhook URL: %w", err)
|
||||
}
|
||||
|
||||
switch t.WebhookMethod {
|
||||
case WebhookMethodGET:
|
||||
reqURL := fmt.Sprintf("%s?heading=%s&message=%s",
|
||||
t.WebhookURL,
|
||||
url.QueryEscape(heading),
|
||||
url.QueryEscape(message),
|
||||
)
|
||||
|
||||
resp, err := http.Get(reqURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send GET webhook: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if cerr := resp.Body.Close(); cerr != nil {
|
||||
logger.Error("failed to close response body", "error", cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf(
|
||||
"webhook GET returned status: %s, body: %s",
|
||||
resp.Status,
|
||||
string(body),
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
return t.sendGET(webhookURL, heading, message, logger)
|
||||
case WebhookMethodPOST:
|
||||
payload := map[string]string{
|
||||
"heading": heading,
|
||||
"message": message,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal webhook payload: %w", err)
|
||||
}
|
||||
|
||||
resp, err := http.Post(t.WebhookURL, "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send POST webhook: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if cerr := resp.Body.Close(); cerr != nil {
|
||||
logger.Error("failed to close response body", "error", cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf(
|
||||
"webhook POST returned status: %s, body: %s",
|
||||
resp.Status,
|
||||
string(body),
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
return t.sendPOST(webhookURL, heading, message, logger)
|
||||
default:
|
||||
return fmt.Errorf("unsupported webhook method: %s", t.WebhookMethod)
|
||||
}
|
||||
@@ -109,4 +100,144 @@ func (t *WebhookNotifier) HideSensitiveData() {
|
||||
func (t *WebhookNotifier) Update(incoming *WebhookNotifier) {
|
||||
t.WebhookURL = incoming.WebhookURL
|
||||
t.WebhookMethod = incoming.WebhookMethod
|
||||
t.BodyTemplate = incoming.BodyTemplate
|
||||
t.Headers = incoming.Headers
|
||||
}
|
||||
|
||||
func (t *WebhookNotifier) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
if t.WebhookURL != "" {
|
||||
encrypted, err := encryptor.Encrypt(t.NotifierID, t.WebhookURL)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt webhook URL: %w", err)
|
||||
}
|
||||
|
||||
t.WebhookURL = encrypted
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *WebhookNotifier) sendGET(webhookURL, heading, message string, logger *slog.Logger) error {
|
||||
reqURL := fmt.Sprintf("%s?heading=%s&message=%s",
|
||||
webhookURL,
|
||||
url.QueryEscape(heading),
|
||||
url.QueryEscape(message),
|
||||
)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, reqURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create GET request: %w", err)
|
||||
}
|
||||
|
||||
t.applyHeaders(req)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send GET webhook: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if cerr := resp.Body.Close(); cerr != nil {
|
||||
logger.Error("failed to close response body", "error", cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf(
|
||||
"webhook GET returned status: %s, body: %s",
|
||||
resp.Status,
|
||||
string(body),
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *WebhookNotifier) sendPOST(webhookURL, heading, message string, logger *slog.Logger) error {
|
||||
body := t.buildRequestBody(heading, message)
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, webhookURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create POST request: %w", err)
|
||||
}
|
||||
|
||||
hasContentType := false
|
||||
|
||||
for _, h := range t.Headers {
|
||||
if strings.EqualFold(h.Key, "Content-Type") {
|
||||
hasContentType = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !hasContentType {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
|
||||
t.applyHeaders(req)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send POST webhook: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if cerr := resp.Body.Close(); cerr != nil {
|
||||
logger.Error("failed to close response body", "error", cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf(
|
||||
"webhook POST returned status: %s, body: %s",
|
||||
resp.Status,
|
||||
string(respBody),
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *WebhookNotifier) buildRequestBody(heading, message string) []byte {
|
||||
if t.BodyTemplate != nil && *t.BodyTemplate != "" {
|
||||
result := *t.BodyTemplate
|
||||
result = strings.ReplaceAll(result, "{{heading}}", escapeJSONString(heading))
|
||||
result = strings.ReplaceAll(result, "{{message}}", escapeJSONString(message))
|
||||
return []byte(result)
|
||||
}
|
||||
|
||||
payload := map[string]string{
|
||||
"heading": heading,
|
||||
"message": message,
|
||||
}
|
||||
body, _ := json.Marshal(payload)
|
||||
|
||||
return body
|
||||
}
|
||||
|
||||
func (t *WebhookNotifier) applyHeaders(req *http.Request) {
|
||||
for _, h := range t.Headers {
|
||||
if h.Key != "" {
|
||||
req.Header.Set(h.Key, h.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func escapeJSONString(s string) string {
|
||||
b, err := json.Marshal(s)
|
||||
if err != nil || len(b) < 2 {
|
||||
escaped := strings.ReplaceAll(s, `\`, `\\`)
|
||||
escaped = strings.ReplaceAll(escaped, `"`, `\"`)
|
||||
escaped = strings.ReplaceAll(escaped, "\n", `\n`)
|
||||
escaped = strings.ReplaceAll(escaped, "\r", `\r`)
|
||||
escaped = strings.ReplaceAll(escaped, "\t", `\t`)
|
||||
return escaped
|
||||
}
|
||||
|
||||
return string(b[1 : len(b)-1])
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
audit_logs "postgresus-backend/internal/features/audit_logs"
|
||||
users_models "postgresus-backend/internal/features/users/models"
|
||||
workspaces_services "postgresus-backend/internal/features/workspaces/services"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -17,6 +18,7 @@ type NotifierService struct {
|
||||
logger *slog.Logger
|
||||
workspaceService *workspaces_services.WorkspaceService
|
||||
auditLogService *audit_logs.AuditLogService
|
||||
fieldEncryptor encryption.FieldEncryptor
|
||||
}
|
||||
|
||||
func (s *NotifierService) SaveNotifier(
|
||||
@@ -46,7 +48,11 @@ func (s *NotifierService) SaveNotifier(
|
||||
|
||||
existingNotifier.Update(notifier)
|
||||
|
||||
if err := existingNotifier.Validate(); err != nil {
|
||||
if err := existingNotifier.EncryptSensitiveData(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := existingNotifier.Validate(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -63,7 +69,11 @@ func (s *NotifierService) SaveNotifier(
|
||||
} else {
|
||||
notifier.WorkspaceID = workspaceID
|
||||
|
||||
if err := notifier.Validate(); err != nil {
|
||||
if err := notifier.EncryptSensitiveData(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := notifier.Validate(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -175,7 +185,7 @@ func (s *NotifierService) SendTestNotification(
|
||||
return errors.New("insufficient permissions to test notifier in this workspace")
|
||||
}
|
||||
|
||||
err = notifier.Send(s.logger, "Test message", "This is a test message")
|
||||
err = notifier.Send(s.fieldEncryptor, s.logger, "Test message", "This is a test message")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -205,16 +215,24 @@ func (s *NotifierService) SendTestNotificationToNotifier(
|
||||
|
||||
existingNotifier.Update(notifier)
|
||||
|
||||
if err := existingNotifier.Validate(); err != nil {
|
||||
if err := existingNotifier.EncryptSensitiveData(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := existingNotifier.Validate(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
usingNotifier = existingNotifier
|
||||
} else {
|
||||
if err := notifier.EncryptSensitiveData(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
usingNotifier = notifier
|
||||
}
|
||||
|
||||
return usingNotifier.Send(s.logger, "Test message", "This is a test message")
|
||||
return usingNotifier.Send(s.fieldEncryptor, s.logger, "Test message", "This is a test message")
|
||||
}
|
||||
|
||||
func (s *NotifierService) SendNotification(
|
||||
@@ -233,7 +251,7 @@ func (s *NotifierService) SendNotification(
|
||||
return
|
||||
}
|
||||
|
||||
err = notifiedFromDb.Send(s.logger, title, message)
|
||||
err = notifiedFromDb.Send(s.fieldEncryptor, s.logger, title, message)
|
||||
if err != nil {
|
||||
errMsg := err.Error()
|
||||
notifiedFromDb.LastSendError = &errMsg
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package restores
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -29,6 +30,7 @@ import (
|
||||
workspaces_controllers "postgresus-backend/internal/features/workspaces/controllers"
|
||||
workspaces_models "postgresus-backend/internal/features/workspaces/models"
|
||||
workspaces_testing "postgresus-backend/internal/features/workspaces/testing"
|
||||
util_encryption "postgresus-backend/internal/util/encryption"
|
||||
test_utils "postgresus-backend/internal/util/testing"
|
||||
"postgresus-backend/internal/util/tools"
|
||||
)
|
||||
@@ -309,6 +311,7 @@ func createTestBackup(
|
||||
database *databases.Database,
|
||||
owner *users_dto.SignInResponseDTO,
|
||||
) *backups.Backup {
|
||||
fieldEncryptor := util_encryption.GetFieldEncryptor()
|
||||
userService := users_services.GetUserService()
|
||||
user, err := userService.GetUserFromToken(owner.Token)
|
||||
if err != nil {
|
||||
@@ -338,7 +341,7 @@ func createTestBackup(
|
||||
dummyContent := []byte("dummy backup content for testing")
|
||||
reader := strings.NewReader(string(dummyContent))
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
if err := storages[0].SaveFile(logger, backup.ID, reader); err != nil {
|
||||
if err := storages[0].SaveFile(context.Background(), fieldEncryptor, logger, backup.ID, reader); err != nil {
|
||||
panic(fmt.Sprintf("Failed to create test backup file: %v", err))
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ package models
|
||||
|
||||
import (
|
||||
"postgresus-backend/internal/features/backups/backups"
|
||||
"postgresus-backend/internal/features/databases/databases/postgresql"
|
||||
"postgresus-backend/internal/features/restores/enums"
|
||||
"time"
|
||||
|
||||
@@ -16,8 +15,6 @@ type Restore struct {
|
||||
BackupID uuid.UUID `json:"backupId" gorm:"column:backup_id;type:uuid;not null"`
|
||||
Backup *backups.Backup
|
||||
|
||||
Postgresql *postgresql.PostgresqlDatabase `json:"postgresql,omitempty" gorm:"foreignKey:RestoreID"`
|
||||
|
||||
FailMessage *string `json:"failMessage" gorm:"column:fail_message"`
|
||||
|
||||
RestoreDurationMs int64 `json:"restoreDurationMs" gorm:"column:restore_duration_ms;default:0"`
|
||||
|
||||
@@ -32,7 +32,6 @@ func (r *RestoreRepository) FindByBackupID(backupID uuid.UUID) ([]*models.Restor
|
||||
if err := storage.
|
||||
GetDb().
|
||||
Preload("Backup").
|
||||
Preload("Postgresql").
|
||||
Where("backup_id = ?", backupID).
|
||||
Order("created_at DESC").
|
||||
Find(&restores).Error; err != nil {
|
||||
@@ -48,7 +47,6 @@ func (r *RestoreRepository) FindByID(id uuid.UUID) (*models.Restore, error) {
|
||||
if err := storage.
|
||||
GetDb().
|
||||
Preload("Backup").
|
||||
Preload("Postgresql").
|
||||
Where("id = ?", id).
|
||||
First(&restore).Error; err != nil {
|
||||
return nil, err
|
||||
@@ -63,7 +61,6 @@ func (r *RestoreRepository) FindByStatus(status enums.RestoreStatus) ([]*models.
|
||||
if err := storage.
|
||||
GetDb().
|
||||
Preload("Backup").
|
||||
Preload("Postgresql").
|
||||
Where("status = ?", status).
|
||||
Order("created_at DESC").
|
||||
Find(&restores).Error; err != nil {
|
||||
|
||||
@@ -191,15 +191,9 @@ func (s *RestoreService) RestoreBackup(
|
||||
return err
|
||||
}
|
||||
|
||||
// Set the RestoreID on the PostgreSQL database and save it
|
||||
if requestDTO.PostgresqlDatabase != nil {
|
||||
requestDTO.PostgresqlDatabase.RestoreID = &restore.ID
|
||||
restore.Postgresql = requestDTO.PostgresqlDatabase
|
||||
|
||||
// Save the restore again to include the postgresql database
|
||||
if err := s.restoreRepository.Save(&restore); err != nil {
|
||||
return err
|
||||
}
|
||||
// Save the restore again to include the postgresql database
|
||||
if err := s.restoreRepository.Save(&restore); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
storage, err := s.storageService.GetStorageByID(backup.StorageID)
|
||||
@@ -216,10 +210,15 @@ func (s *RestoreService) RestoreBackup(
|
||||
|
||||
start := time.Now().UTC()
|
||||
|
||||
restoringToDB := &databases.Database{
|
||||
Postgresql: requestDTO.PostgresqlDatabase,
|
||||
}
|
||||
|
||||
err = s.restoreBackupUsecase.Execute(
|
||||
backupConfig,
|
||||
restore,
|
||||
database,
|
||||
restoringToDB,
|
||||
backup,
|
||||
storage,
|
||||
)
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
package usecases_postgresql
|
||||
|
||||
import (
|
||||
"postgresus-backend/internal/features/encryption/secrets"
|
||||
"postgresus-backend/internal/util/logger"
|
||||
)
|
||||
|
||||
var restorePostgresqlBackupUsecase = &RestorePostgresqlBackupUsecase{
|
||||
logger.GetLogger(),
|
||||
secrets.GetSecretKeyService(),
|
||||
}
|
||||
|
||||
func GetRestorePostgresqlBackupUsecase() *RestorePostgresqlBackupUsecase {
|
||||
|
||||
@@ -2,6 +2,7 @@ package usecases_postgresql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -15,11 +16,14 @@ import (
|
||||
|
||||
"postgresus-backend/internal/config"
|
||||
"postgresus-backend/internal/features/backups/backups"
|
||||
"postgresus-backend/internal/features/backups/backups/encryption"
|
||||
backups_config "postgresus-backend/internal/features/backups/config"
|
||||
"postgresus-backend/internal/features/databases"
|
||||
pgtypes "postgresus-backend/internal/features/databases/databases/postgresql"
|
||||
encryption_secrets "postgresus-backend/internal/features/encryption/secrets"
|
||||
"postgresus-backend/internal/features/restores/models"
|
||||
"postgresus-backend/internal/features/storages"
|
||||
util_encryption "postgresus-backend/internal/util/encryption"
|
||||
files_utils "postgresus-backend/internal/util/files"
|
||||
"postgresus-backend/internal/util/tools"
|
||||
|
||||
@@ -27,17 +31,19 @@ import (
|
||||
)
|
||||
|
||||
type RestorePostgresqlBackupUsecase struct {
|
||||
logger *slog.Logger
|
||||
logger *slog.Logger
|
||||
secretKeyService *encryption_secrets.SecretKeyService
|
||||
}
|
||||
|
||||
func (uc *RestorePostgresqlBackupUsecase) Execute(
|
||||
database *databases.Database,
|
||||
originalDB *databases.Database,
|
||||
restoringToDB *databases.Database,
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
restore models.Restore,
|
||||
backup *backups.Backup,
|
||||
storage *storages.Storage,
|
||||
) error {
|
||||
if database.Type != databases.DatabaseTypePostgres {
|
||||
if originalDB.Type != databases.DatabaseTypePostgres {
|
||||
return errors.New("database type not supported")
|
||||
}
|
||||
|
||||
@@ -49,7 +55,7 @@ func (uc *RestorePostgresqlBackupUsecase) Execute(
|
||||
backup.ID,
|
||||
)
|
||||
|
||||
pg := restore.Postgresql
|
||||
pg := restoringToDB.Postgresql
|
||||
if pg == nil {
|
||||
return fmt.Errorf("postgresql configuration is required for restore")
|
||||
}
|
||||
@@ -73,11 +79,12 @@ func (uc *RestorePostgresqlBackupUsecase) Execute(
|
||||
"--verbose", // Add verbose output to help with debugging
|
||||
"--clean", // Clean (drop) database objects before recreating them
|
||||
"--if-exists", // Use IF EXISTS when dropping objects
|
||||
"--no-owner",
|
||||
"--no-owner", // Skip restoring ownership
|
||||
"--no-acl", // Skip restoring access privileges (GRANT/REVOKE commands)
|
||||
}
|
||||
|
||||
return uc.restoreFromStorage(
|
||||
database,
|
||||
originalDB,
|
||||
tools.GetPostgresqlExecutable(
|
||||
pg.Version,
|
||||
"pg_restore",
|
||||
@@ -202,18 +209,67 @@ func (uc *RestorePostgresqlBackupUsecase) downloadBackupToTempFile(
|
||||
backup.ID,
|
||||
"tempFile",
|
||||
tempBackupFile,
|
||||
"encrypted",
|
||||
backup.Encryption == backups_config.BackupEncryptionEncrypted,
|
||||
)
|
||||
backupReader, err := storage.GetFile(backup.ID)
|
||||
fieldEncryptor := util_encryption.GetFieldEncryptor()
|
||||
rawReader, err := storage.GetFile(fieldEncryptor, backup.ID)
|
||||
if err != nil {
|
||||
cleanupFunc()
|
||||
return "", nil, fmt.Errorf("failed to get backup file from storage: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := backupReader.Close(); err != nil {
|
||||
if err := rawReader.Close(); err != nil {
|
||||
uc.logger.Error("Failed to close backup reader", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Create a reader that handles decryption if needed
|
||||
var backupReader io.Reader = rawReader
|
||||
if backup.Encryption == backups_config.BackupEncryptionEncrypted {
|
||||
// Validate encryption metadata
|
||||
if backup.EncryptionSalt == nil || backup.EncryptionIV == nil {
|
||||
cleanupFunc()
|
||||
return "", nil, fmt.Errorf("backup is encrypted but missing encryption metadata")
|
||||
}
|
||||
|
||||
// Get master key
|
||||
masterKey, err := uc.secretKeyService.GetSecretKey()
|
||||
if err != nil {
|
||||
cleanupFunc()
|
||||
return "", nil, fmt.Errorf("failed to get master key for decryption: %w", err)
|
||||
}
|
||||
|
||||
// Decode salt and IV from base64
|
||||
salt, err := base64.StdEncoding.DecodeString(*backup.EncryptionSalt)
|
||||
if err != nil {
|
||||
cleanupFunc()
|
||||
return "", nil, fmt.Errorf("failed to decode encryption salt: %w", err)
|
||||
}
|
||||
|
||||
iv, err := base64.StdEncoding.DecodeString(*backup.EncryptionIV)
|
||||
if err != nil {
|
||||
cleanupFunc()
|
||||
return "", nil, fmt.Errorf("failed to decode encryption IV: %w", err)
|
||||
}
|
||||
|
||||
// Create decryption reader
|
||||
decryptReader, err := encryption.NewDecryptionReader(
|
||||
rawReader,
|
||||
masterKey,
|
||||
backup.ID,
|
||||
salt,
|
||||
iv,
|
||||
)
|
||||
if err != nil {
|
||||
cleanupFunc()
|
||||
return "", nil, fmt.Errorf("failed to create decryption reader: %w", err)
|
||||
}
|
||||
|
||||
backupReader = decryptReader
|
||||
uc.logger.Info("Using decryption for encrypted backup", "backupId", backup.ID)
|
||||
}
|
||||
|
||||
// Create temporary backup file
|
||||
tempFile, err := os.Create(tempBackupFile)
|
||||
if err != nil {
|
||||
@@ -322,7 +378,6 @@ func (uc *RestorePostgresqlBackupUsecase) setupPgRestoreEnvironment(
|
||||
// Add encoding-related environment variables
|
||||
cmd.Env = append(cmd.Env, "LC_ALL=C.UTF-8")
|
||||
cmd.Env = append(cmd.Env, "LANG=C.UTF-8")
|
||||
cmd.Env = append(cmd.Env, "PGOPTIONS=--client-encoding=UTF8")
|
||||
|
||||
shouldRequireSSL := pgConfig.IsHttps
|
||||
|
||||
@@ -447,7 +502,7 @@ func (uc *RestorePostgresqlBackupUsecase) copyWithShutdownCheck(
|
||||
dst io.Writer,
|
||||
src io.Reader,
|
||||
) (int64, error) {
|
||||
buf := make([]byte, 32*1024) // 32KB buffer
|
||||
buf := make([]byte, 16*1024*1024) // 16MB buffer
|
||||
var totalBytesWritten int64
|
||||
|
||||
for {
|
||||
@@ -508,11 +563,15 @@ func (uc *RestorePostgresqlBackupUsecase) createTempPgpassFile(
|
||||
return "", nil
|
||||
}
|
||||
|
||||
escapedHost := tools.EscapePgpassField(pgConfig.Host)
|
||||
escapedUsername := tools.EscapePgpassField(pgConfig.Username)
|
||||
escapedPassword := tools.EscapePgpassField(password)
|
||||
|
||||
pgpassContent := fmt.Sprintf("%s:%d:*:%s:%s",
|
||||
pgConfig.Host,
|
||||
escapedHost,
|
||||
pgConfig.Port,
|
||||
pgConfig.Username,
|
||||
password,
|
||||
escapedUsername,
|
||||
escapedPassword,
|
||||
)
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "pgpass")
|
||||
|
||||
@@ -17,13 +17,15 @@ type RestoreBackupUsecase struct {
|
||||
func (uc *RestoreBackupUsecase) Execute(
|
||||
backupConfig *backups_config.BackupConfig,
|
||||
restore models.Restore,
|
||||
database *databases.Database,
|
||||
originalDB *databases.Database,
|
||||
restoringToDB *databases.Database,
|
||||
backup *backups.Backup,
|
||||
storage *storages.Storage,
|
||||
) error {
|
||||
if database.Type == databases.DatabaseTypePostgres {
|
||||
if originalDB.Type == databases.DatabaseTypePostgres {
|
||||
return uc.restorePostgresqlBackupUsecase.Execute(
|
||||
database,
|
||||
originalDB,
|
||||
restoringToDB,
|
||||
backupConfig,
|
||||
restore,
|
||||
backup,
|
||||
|
||||
@@ -3,10 +3,14 @@ package storages
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
audit_logs "postgresus-backend/internal/features/audit_logs"
|
||||
azure_blob_storage "postgresus-backend/internal/features/storages/models/azure_blob"
|
||||
google_drive_storage "postgresus-backend/internal/features/storages/models/google_drive"
|
||||
local_storage "postgresus-backend/internal/features/storages/models/local"
|
||||
nas_storage "postgresus-backend/internal/features/storages/models/nas"
|
||||
s3_storage "postgresus-backend/internal/features/storages/models/s3"
|
||||
users_enums "postgresus-backend/internal/features/users/enums"
|
||||
users_middleware "postgresus-backend/internal/features/users/middleware"
|
||||
@@ -14,6 +18,7 @@ import (
|
||||
users_testing "postgresus-backend/internal/features/users/testing"
|
||||
workspaces_controllers "postgresus-backend/internal/features/workspaces/controllers"
|
||||
workspaces_testing "postgresus-backend/internal/features/workspaces/testing"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
test_utils "postgresus-backend/internal/util/testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
@@ -438,6 +443,386 @@ func Test_CrossWorkspaceSecurity_CannotAccessStorageFromAnotherWorkspace(t *test
|
||||
workspaces_testing.RemoveTestWorkspace(workspace2, router)
|
||||
}
|
||||
|
||||
func Test_StorageSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
storageType StorageType
|
||||
createStorage func(workspaceID uuid.UUID) *Storage
|
||||
updateStorage func(workspaceID uuid.UUID, storageID uuid.UUID) *Storage
|
||||
verifySensitiveData func(t *testing.T, storage *Storage)
|
||||
verifyHiddenData func(t *testing.T, storage *Storage)
|
||||
}{
|
||||
{
|
||||
name: "S3 Storage",
|
||||
storageType: StorageTypeS3,
|
||||
createStorage: func(workspaceID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeS3,
|
||||
Name: "Test S3 Storage",
|
||||
S3Storage: &s3_storage.S3Storage{
|
||||
S3Bucket: "test-bucket",
|
||||
S3Region: "us-east-1",
|
||||
S3AccessKey: "original-access-key",
|
||||
S3SecretKey: "original-secret-key",
|
||||
S3Endpoint: "https://s3.amazonaws.com",
|
||||
},
|
||||
}
|
||||
},
|
||||
updateStorage: func(workspaceID uuid.UUID, storageID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
ID: storageID,
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeS3,
|
||||
Name: "Updated S3 Storage",
|
||||
S3Storage: &s3_storage.S3Storage{
|
||||
S3Bucket: "updated-bucket",
|
||||
S3Region: "us-west-2",
|
||||
S3AccessKey: "",
|
||||
S3SecretKey: "",
|
||||
S3Endpoint: "https://s3.us-west-2.amazonaws.com",
|
||||
},
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, storage *Storage) {
|
||||
assert.True(t, strings.HasPrefix(storage.S3Storage.S3AccessKey, "enc:"),
|
||||
"S3AccessKey should be encrypted with 'enc:' prefix")
|
||||
assert.True(t, strings.HasPrefix(storage.S3Storage.S3SecretKey, "enc:"),
|
||||
"S3SecretKey should be encrypted with 'enc:' prefix")
|
||||
|
||||
encryptor := encryption.GetFieldEncryptor()
|
||||
accessKey, err := encryptor.Decrypt(storage.ID, storage.S3Storage.S3AccessKey)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "original-access-key", accessKey)
|
||||
|
||||
secretKey, err := encryptor.Decrypt(storage.ID, storage.S3Storage.S3SecretKey)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "original-secret-key", secretKey)
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, storage *Storage) {
|
||||
assert.Equal(t, "", storage.S3Storage.S3AccessKey)
|
||||
assert.Equal(t, "", storage.S3Storage.S3SecretKey)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Local Storage",
|
||||
storageType: StorageTypeLocal,
|
||||
createStorage: func(workspaceID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeLocal,
|
||||
Name: "Test Local Storage",
|
||||
LocalStorage: &local_storage.LocalStorage{},
|
||||
}
|
||||
},
|
||||
updateStorage: func(workspaceID uuid.UUID, storageID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
ID: storageID,
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeLocal,
|
||||
Name: "Updated Local Storage",
|
||||
LocalStorage: &local_storage.LocalStorage{},
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, storage *Storage) {
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, storage *Storage) {
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "NAS Storage",
|
||||
storageType: StorageTypeNAS,
|
||||
createStorage: func(workspaceID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeNAS,
|
||||
Name: "Test NAS Storage",
|
||||
NASStorage: &nas_storage.NASStorage{
|
||||
Host: "nas.example.com",
|
||||
Port: 445,
|
||||
Share: "backups",
|
||||
Username: "testuser",
|
||||
Password: "original-password",
|
||||
UseSSL: false,
|
||||
Domain: "WORKGROUP",
|
||||
Path: "/test",
|
||||
},
|
||||
}
|
||||
},
|
||||
updateStorage: func(workspaceID uuid.UUID, storageID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
ID: storageID,
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeNAS,
|
||||
Name: "Updated NAS Storage",
|
||||
NASStorage: &nas_storage.NASStorage{
|
||||
Host: "nas2.example.com",
|
||||
Port: 445,
|
||||
Share: "backups2",
|
||||
Username: "testuser2",
|
||||
Password: "",
|
||||
UseSSL: true,
|
||||
Domain: "WORKGROUP2",
|
||||
Path: "/test2",
|
||||
},
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, storage *Storage) {
|
||||
assert.True(t, strings.HasPrefix(storage.NASStorage.Password, "enc:"),
|
||||
"Password should be encrypted with 'enc:' prefix")
|
||||
|
||||
encryptor := encryption.GetFieldEncryptor()
|
||||
password, err := encryptor.Decrypt(storage.ID, storage.NASStorage.Password)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "original-password", password)
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, storage *Storage) {
|
||||
assert.Equal(t, "", storage.NASStorage.Password)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Azure Blob Storage (Connection String)",
|
||||
storageType: StorageTypeAzureBlob,
|
||||
createStorage: func(workspaceID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeAzureBlob,
|
||||
Name: "Test Azure Blob Storage",
|
||||
AzureBlobStorage: &azure_blob_storage.AzureBlobStorage{
|
||||
AuthMethod: azure_blob_storage.AuthMethodConnectionString,
|
||||
ConnectionString: "original-connection-string",
|
||||
ContainerName: "test-container",
|
||||
Endpoint: "",
|
||||
Prefix: "backups/",
|
||||
},
|
||||
}
|
||||
},
|
||||
updateStorage: func(workspaceID uuid.UUID, storageID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
ID: storageID,
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeAzureBlob,
|
||||
Name: "Updated Azure Blob Storage",
|
||||
AzureBlobStorage: &azure_blob_storage.AzureBlobStorage{
|
||||
AuthMethod: azure_blob_storage.AuthMethodConnectionString,
|
||||
ConnectionString: "",
|
||||
ContainerName: "updated-container",
|
||||
Endpoint: "https://custom.blob.core.windows.net",
|
||||
Prefix: "backups2/",
|
||||
},
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, storage *Storage) {
|
||||
assert.True(t, strings.HasPrefix(storage.AzureBlobStorage.ConnectionString, "enc:"),
|
||||
"ConnectionString should be encrypted with 'enc:' prefix")
|
||||
|
||||
encryptor := encryption.GetFieldEncryptor()
|
||||
connectionString, err := encryptor.Decrypt(
|
||||
storage.ID,
|
||||
storage.AzureBlobStorage.ConnectionString,
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "original-connection-string", connectionString)
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, storage *Storage) {
|
||||
assert.Equal(t, "", storage.AzureBlobStorage.ConnectionString)
|
||||
assert.Equal(t, "", storage.AzureBlobStorage.AccountKey)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Azure Blob Storage (Account Key)",
|
||||
storageType: StorageTypeAzureBlob,
|
||||
createStorage: func(workspaceID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeAzureBlob,
|
||||
Name: "Test Azure Blob with Account Key",
|
||||
AzureBlobStorage: &azure_blob_storage.AzureBlobStorage{
|
||||
AuthMethod: azure_blob_storage.AuthMethodAccountKey,
|
||||
AccountName: "testaccount",
|
||||
AccountKey: "original-account-key",
|
||||
ContainerName: "test-container",
|
||||
Endpoint: "",
|
||||
Prefix: "backups/",
|
||||
},
|
||||
}
|
||||
},
|
||||
updateStorage: func(workspaceID uuid.UUID, storageID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
ID: storageID,
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeAzureBlob,
|
||||
Name: "Updated Azure Blob with Account Key",
|
||||
AzureBlobStorage: &azure_blob_storage.AzureBlobStorage{
|
||||
AuthMethod: azure_blob_storage.AuthMethodAccountKey,
|
||||
AccountName: "updatedaccount",
|
||||
AccountKey: "",
|
||||
ContainerName: "updated-container",
|
||||
Endpoint: "https://custom.blob.core.windows.net",
|
||||
Prefix: "backups2/",
|
||||
},
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, storage *Storage) {
|
||||
assert.True(t, strings.HasPrefix(storage.AzureBlobStorage.AccountKey, "enc:"),
|
||||
"AccountKey should be encrypted with 'enc:' prefix")
|
||||
|
||||
encryptor := encryption.GetFieldEncryptor()
|
||||
accountKey, err := encryptor.Decrypt(
|
||||
storage.ID,
|
||||
storage.AzureBlobStorage.AccountKey,
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "original-account-key", accountKey)
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, storage *Storage) {
|
||||
assert.Equal(t, "", storage.AzureBlobStorage.ConnectionString)
|
||||
assert.Equal(t, "", storage.AzureBlobStorage.AccountKey)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Google Drive Storage",
|
||||
storageType: StorageTypeGoogleDrive,
|
||||
createStorage: func(workspaceID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeGoogleDrive,
|
||||
Name: "Test Google Drive Storage",
|
||||
GoogleDriveStorage: &google_drive_storage.GoogleDriveStorage{
|
||||
ClientID: "original-client-id",
|
||||
ClientSecret: "original-client-secret",
|
||||
TokenJSON: `{"access_token":"ya29.test-access-token","token_type":"Bearer","expiry":"2030-12-31T23:59:59Z","refresh_token":"1//test-refresh-token"}`,
|
||||
},
|
||||
}
|
||||
},
|
||||
updateStorage: func(workspaceID uuid.UUID, storageID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
ID: storageID,
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeGoogleDrive,
|
||||
Name: "Updated Google Drive Storage",
|
||||
GoogleDriveStorage: &google_drive_storage.GoogleDriveStorage{
|
||||
ClientID: "updated-client-id",
|
||||
ClientSecret: "",
|
||||
TokenJSON: "",
|
||||
},
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, storage *Storage) {
|
||||
assert.True(t, strings.HasPrefix(storage.GoogleDriveStorage.ClientSecret, "enc:"),
|
||||
"ClientSecret should be encrypted with 'enc:' prefix")
|
||||
assert.True(t, strings.HasPrefix(storage.GoogleDriveStorage.TokenJSON, "enc:"),
|
||||
"TokenJSON should be encrypted with 'enc:' prefix")
|
||||
|
||||
encryptor := encryption.GetFieldEncryptor()
|
||||
clientSecret, err := encryptor.Decrypt(
|
||||
storage.ID,
|
||||
storage.GoogleDriveStorage.ClientSecret,
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "original-client-secret", clientSecret)
|
||||
|
||||
tokenJSON, err := encryptor.Decrypt(
|
||||
storage.ID,
|
||||
storage.GoogleDriveStorage.TokenJSON,
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(
|
||||
t,
|
||||
`{"access_token":"ya29.test-access-token","token_type":"Bearer","expiry":"2030-12-31T23:59:59Z","refresh_token":"1//test-refresh-token"}`,
|
||||
tokenJSON,
|
||||
)
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, storage *Storage) {
|
||||
assert.Equal(t, "", storage.GoogleDriveStorage.ClientSecret)
|
||||
assert.Equal(t, "", storage.GoogleDriveStorage.TokenJSON)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
router := createRouter()
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
|
||||
// Phase 1: Create storage with sensitive data
|
||||
initialStorage := tc.createStorage(workspace.ID)
|
||||
var createdStorage Storage
|
||||
test_utils.MakePostRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/storages",
|
||||
"Bearer "+owner.Token,
|
||||
*initialStorage,
|
||||
http.StatusOK,
|
||||
&createdStorage,
|
||||
)
|
||||
|
||||
assert.NotEmpty(t, createdStorage.ID)
|
||||
assert.Equal(t, initialStorage.Name, createdStorage.Name)
|
||||
|
||||
// Phase 2: Verify sensitive data is encrypted in repository after creation
|
||||
repository := &StorageRepository{}
|
||||
storageFromDBAfterCreate, err := repository.FindByID(createdStorage.ID)
|
||||
assert.NoError(t, err)
|
||||
tc.verifySensitiveData(t, storageFromDBAfterCreate)
|
||||
|
||||
// Phase 3: Read via service - sensitive data should be hidden
|
||||
var retrievedStorage Storage
|
||||
test_utils.MakeGetRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/storages/%s", createdStorage.ID.String()),
|
||||
"Bearer "+owner.Token,
|
||||
http.StatusOK,
|
||||
&retrievedStorage,
|
||||
)
|
||||
|
||||
tc.verifyHiddenData(t, &retrievedStorage)
|
||||
assert.Equal(t, initialStorage.Name, retrievedStorage.Name)
|
||||
|
||||
// Phase 4: Update with non-sensitive changes only (sensitive fields empty)
|
||||
updatedStorage := tc.updateStorage(workspace.ID, createdStorage.ID)
|
||||
var updateResponse Storage
|
||||
test_utils.MakePostRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/storages",
|
||||
"Bearer "+owner.Token,
|
||||
*updatedStorage,
|
||||
http.StatusOK,
|
||||
&updateResponse,
|
||||
)
|
||||
|
||||
// Verify non-sensitive fields were updated
|
||||
assert.Equal(t, updatedStorage.Name, updateResponse.Name)
|
||||
|
||||
// Phase 5: Retrieve directly from repository to verify sensitive data preservation
|
||||
storageFromDB, err := repository.FindByID(createdStorage.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify original sensitive data is still present in DB
|
||||
tc.verifySensitiveData(t, storageFromDB)
|
||||
|
||||
// Verify non-sensitive fields were updated in DB
|
||||
assert.Equal(t, updatedStorage.Name, storageFromDB.Name)
|
||||
|
||||
// Additional verification: Check via GET that data is still hidden
|
||||
var finalRetrieved Storage
|
||||
test_utils.MakeGetRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/storages/%s", createdStorage.ID.String()),
|
||||
"Bearer "+owner.Token,
|
||||
http.StatusOK,
|
||||
&finalRetrieved,
|
||||
)
|
||||
tc.verifyHiddenData(t, &finalRetrieved)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func createRouter() *gin.Engine {
|
||||
gin.SetMode(gin.TestMode)
|
||||
router := gin.New()
|
||||
@@ -485,158 +870,3 @@ func deleteStorage(
|
||||
http.StatusOK,
|
||||
)
|
||||
}
|
||||
|
||||
func Test_StorageSensitiveDataLifecycle_AllTypes(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
storageType StorageType
|
||||
createStorage func(workspaceID uuid.UUID) *Storage
|
||||
updateStorage func(workspaceID uuid.UUID, storageID uuid.UUID) *Storage
|
||||
verifySensitiveData func(t *testing.T, storage *Storage)
|
||||
verifyHiddenData func(t *testing.T, storage *Storage)
|
||||
}{
|
||||
{
|
||||
name: "S3 Storage",
|
||||
storageType: StorageTypeS3,
|
||||
createStorage: func(workspaceID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeS3,
|
||||
Name: "Test S3 Storage",
|
||||
S3Storage: &s3_storage.S3Storage{
|
||||
S3Bucket: "test-bucket",
|
||||
S3Region: "us-east-1",
|
||||
S3AccessKey: "original-access-key",
|
||||
S3SecretKey: "original-secret-key",
|
||||
S3Endpoint: "https://s3.amazonaws.com",
|
||||
},
|
||||
}
|
||||
},
|
||||
updateStorage: func(workspaceID uuid.UUID, storageID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
ID: storageID,
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeS3,
|
||||
Name: "Updated S3 Storage",
|
||||
S3Storage: &s3_storage.S3Storage{
|
||||
S3Bucket: "updated-bucket",
|
||||
S3Region: "us-west-2",
|
||||
S3AccessKey: "",
|
||||
S3SecretKey: "",
|
||||
S3Endpoint: "https://s3.us-west-2.amazonaws.com",
|
||||
},
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, storage *Storage) {
|
||||
assert.Equal(t, "original-access-key", storage.S3Storage.S3AccessKey)
|
||||
assert.Equal(t, "original-secret-key", storage.S3Storage.S3SecretKey)
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, storage *Storage) {
|
||||
assert.Equal(t, "", storage.S3Storage.S3AccessKey)
|
||||
assert.Equal(t, "", storage.S3Storage.S3SecretKey)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Local Storage",
|
||||
storageType: StorageTypeLocal,
|
||||
createStorage: func(workspaceID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeLocal,
|
||||
Name: "Test Local Storage",
|
||||
LocalStorage: &local_storage.LocalStorage{},
|
||||
}
|
||||
},
|
||||
updateStorage: func(workspaceID uuid.UUID, storageID uuid.UUID) *Storage {
|
||||
return &Storage{
|
||||
ID: storageID,
|
||||
WorkspaceID: workspaceID,
|
||||
Type: StorageTypeLocal,
|
||||
Name: "Updated Local Storage",
|
||||
LocalStorage: &local_storage.LocalStorage{},
|
||||
}
|
||||
},
|
||||
verifySensitiveData: func(t *testing.T, storage *Storage) {
|
||||
},
|
||||
verifyHiddenData: func(t *testing.T, storage *Storage) {
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
owner := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
router := createRouter()
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router)
|
||||
|
||||
// Phase 1: Create storage with sensitive data
|
||||
initialStorage := tc.createStorage(workspace.ID)
|
||||
var createdStorage Storage
|
||||
test_utils.MakePostRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/storages",
|
||||
"Bearer "+owner.Token,
|
||||
*initialStorage,
|
||||
http.StatusOK,
|
||||
&createdStorage,
|
||||
)
|
||||
|
||||
assert.NotEmpty(t, createdStorage.ID)
|
||||
assert.Equal(t, initialStorage.Name, createdStorage.Name)
|
||||
|
||||
// Phase 2: Read via service - sensitive data should be hidden
|
||||
var retrievedStorage Storage
|
||||
test_utils.MakeGetRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/storages/%s", createdStorage.ID.String()),
|
||||
"Bearer "+owner.Token,
|
||||
http.StatusOK,
|
||||
&retrievedStorage,
|
||||
)
|
||||
|
||||
tc.verifyHiddenData(t, &retrievedStorage)
|
||||
assert.Equal(t, initialStorage.Name, retrievedStorage.Name)
|
||||
|
||||
// Phase 3: Update with non-sensitive changes only (sensitive fields empty)
|
||||
updatedStorage := tc.updateStorage(workspace.ID, createdStorage.ID)
|
||||
var updateResponse Storage
|
||||
test_utils.MakePostRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/storages",
|
||||
"Bearer "+owner.Token,
|
||||
*updatedStorage,
|
||||
http.StatusOK,
|
||||
&updateResponse,
|
||||
)
|
||||
|
||||
// Verify non-sensitive fields were updated
|
||||
assert.Equal(t, updatedStorage.Name, updateResponse.Name)
|
||||
|
||||
// Phase 4: Retrieve directly from repository to verify sensitive data preservation
|
||||
repository := &StorageRepository{}
|
||||
storageFromDB, err := repository.FindByID(createdStorage.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify original sensitive data is still present in DB
|
||||
tc.verifySensitiveData(t, storageFromDB)
|
||||
|
||||
// Verify non-sensitive fields were updated in DB
|
||||
assert.Equal(t, updatedStorage.Name, storageFromDB.Name)
|
||||
|
||||
// Additional verification: Check via GET that data is still hidden
|
||||
var finalRetrieved Storage
|
||||
test_utils.MakeGetRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/storages/%s", createdStorage.ID.String()),
|
||||
"Bearer "+owner.Token,
|
||||
http.StatusOK,
|
||||
&finalRetrieved,
|
||||
)
|
||||
tc.verifyHiddenData(t, &finalRetrieved)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package storages
|
||||
import (
|
||||
audit_logs "postgresus-backend/internal/features/audit_logs"
|
||||
workspaces_services "postgresus-backend/internal/features/workspaces/services"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
)
|
||||
|
||||
var storageRepository = &StorageRepository{}
|
||||
@@ -10,6 +11,7 @@ var storageService = &StorageService{
|
||||
storageRepository,
|
||||
workspaces_services.GetWorkspaceService(),
|
||||
audit_logs.GetAuditLogService(),
|
||||
encryption.GetFieldEncryptor(),
|
||||
}
|
||||
var storageController = &StorageController{
|
||||
storageService,
|
||||
|
||||
@@ -1,22 +1,32 @@
|
||||
package storages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type StorageFileSaver interface {
|
||||
SaveFile(logger *slog.Logger, fileID uuid.UUID, file io.Reader) error
|
||||
SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
file io.Reader,
|
||||
) error
|
||||
|
||||
GetFile(fileID uuid.UUID) (io.ReadCloser, error)
|
||||
GetFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) (io.ReadCloser, error)
|
||||
|
||||
DeleteFile(fileID uuid.UUID) error
|
||||
DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error
|
||||
|
||||
Validate() error
|
||||
Validate(encryptor encryption.FieldEncryptor) error
|
||||
|
||||
TestConnection() error
|
||||
TestConnection(encryptor encryption.FieldEncryptor) error
|
||||
|
||||
HideSensitiveData()
|
||||
|
||||
EncryptSensitiveData(encryptor encryption.FieldEncryptor) error
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package storages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
local_storage "postgresus-backend/internal/features/storages/models/local"
|
||||
nas_storage "postgresus-backend/internal/features/storages/models/nas"
|
||||
s3_storage "postgresus-backend/internal/features/storages/models/s3"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -28,8 +30,14 @@ type Storage struct {
|
||||
AzureBlobStorage *azure_blob_storage.AzureBlobStorage `json:"azureBlobStorage" gorm:"foreignKey:StorageID"`
|
||||
}
|
||||
|
||||
func (s *Storage) SaveFile(logger *slog.Logger, fileID uuid.UUID, file io.Reader) error {
|
||||
err := s.getSpecificStorage().SaveFile(logger, fileID, file)
|
||||
func (s *Storage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
file io.Reader,
|
||||
) error {
|
||||
err := s.getSpecificStorage().SaveFile(ctx, encryptor, logger, fileID, file)
|
||||
if err != nil {
|
||||
lastSaveError := err.Error()
|
||||
s.LastSaveError = &lastSaveError
|
||||
@@ -41,15 +49,18 @@ func (s *Storage) SaveFile(logger *slog.Logger, fileID uuid.UUID, file io.Reader
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Storage) GetFile(fileID uuid.UUID) (io.ReadCloser, error) {
|
||||
return s.getSpecificStorage().GetFile(fileID)
|
||||
func (s *Storage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
) (io.ReadCloser, error) {
|
||||
return s.getSpecificStorage().GetFile(encryptor, fileID)
|
||||
}
|
||||
|
||||
func (s *Storage) DeleteFile(fileID uuid.UUID) error {
|
||||
return s.getSpecificStorage().DeleteFile(fileID)
|
||||
func (s *Storage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
return s.getSpecificStorage().DeleteFile(encryptor, fileID)
|
||||
}
|
||||
|
||||
func (s *Storage) Validate() error {
|
||||
func (s *Storage) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
if s.Type == "" {
|
||||
return errors.New("storage type is required")
|
||||
}
|
||||
@@ -58,17 +69,21 @@ func (s *Storage) Validate() error {
|
||||
return errors.New("storage name is required")
|
||||
}
|
||||
|
||||
return s.getSpecificStorage().Validate()
|
||||
return s.getSpecificStorage().Validate(encryptor)
|
||||
}
|
||||
|
||||
func (s *Storage) TestConnection() error {
|
||||
return s.getSpecificStorage().TestConnection()
|
||||
func (s *Storage) TestConnection(encryptor encryption.FieldEncryptor) error {
|
||||
return s.getSpecificStorage().TestConnection(encryptor)
|
||||
}
|
||||
|
||||
func (s *Storage) HideSensitiveData() {
|
||||
s.getSpecificStorage().HideSensitiveData()
|
||||
}
|
||||
|
||||
func (s *Storage) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
return s.getSpecificStorage().EncryptSensitiveData(encryptor)
|
||||
}
|
||||
|
||||
func (s *Storage) Update(incoming *Storage) {
|
||||
s.Name = incoming.Name
|
||||
s.Type = incoming.Type
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
local_storage "postgresus-backend/internal/features/storages/models/local"
|
||||
nas_storage "postgresus-backend/internal/features/storages/models/nas"
|
||||
s3_storage "postgresus-backend/internal/features/storages/models/s3"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"postgresus-backend/internal/util/logger"
|
||||
"strconv"
|
||||
"testing"
|
||||
@@ -147,13 +148,15 @@ func Test_Storage_BasicOperations(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
encryptor := encryption.GetFieldEncryptor()
|
||||
|
||||
t.Run("Test_TestConnection_ConnectionSucceeds", func(t *testing.T) {
|
||||
err := tc.storage.TestConnection()
|
||||
err := tc.storage.TestConnection(encryptor)
|
||||
assert.NoError(t, err, "TestConnection should succeed")
|
||||
})
|
||||
|
||||
t.Run("Test_TestValidation_ValidationSucceeds", func(t *testing.T) {
|
||||
err := tc.storage.Validate()
|
||||
err := tc.storage.Validate(encryptor)
|
||||
assert.NoError(t, err, "Validate should succeed")
|
||||
})
|
||||
|
||||
@@ -163,10 +166,16 @@ func Test_Storage_BasicOperations(t *testing.T) {
|
||||
|
||||
fileID := uuid.New()
|
||||
|
||||
err = tc.storage.SaveFile(logger.GetLogger(), fileID, bytes.NewReader(fileData))
|
||||
err = tc.storage.SaveFile(
|
||||
context.Background(),
|
||||
encryptor,
|
||||
logger.GetLogger(),
|
||||
fileID,
|
||||
bytes.NewReader(fileData),
|
||||
)
|
||||
require.NoError(t, err, "SaveFile should succeed")
|
||||
|
||||
file, err := tc.storage.GetFile(fileID)
|
||||
file, err := tc.storage.GetFile(encryptor, fileID)
|
||||
assert.NoError(t, err, "GetFile should succeed")
|
||||
defer file.Close()
|
||||
|
||||
@@ -180,13 +189,19 @@ func Test_Storage_BasicOperations(t *testing.T) {
|
||||
require.NoError(t, err, "Should be able to read test file")
|
||||
|
||||
fileID := uuid.New()
|
||||
err = tc.storage.SaveFile(logger.GetLogger(), fileID, bytes.NewReader(fileData))
|
||||
err = tc.storage.SaveFile(
|
||||
context.Background(),
|
||||
encryptor,
|
||||
logger.GetLogger(),
|
||||
fileID,
|
||||
bytes.NewReader(fileData),
|
||||
)
|
||||
require.NoError(t, err, "SaveFile should succeed")
|
||||
|
||||
err = tc.storage.DeleteFile(fileID)
|
||||
err = tc.storage.DeleteFile(encryptor, fileID)
|
||||
assert.NoError(t, err, "DeleteFile should succeed")
|
||||
|
||||
file, err := tc.storage.GetFile(fileID)
|
||||
file, err := tc.storage.GetFile(encryptor, fileID)
|
||||
assert.Error(t, err, "GetFile should fail for non-existent file")
|
||||
if file != nil {
|
||||
file.Close()
|
||||
@@ -196,7 +211,7 @@ func Test_Storage_BasicOperations(t *testing.T) {
|
||||
t.Run("Test_TestDeleteNonExistentFile_DoesNotError", func(t *testing.T) {
|
||||
// Try to delete a non-existent file
|
||||
nonExistentID := uuid.New()
|
||||
err := tc.storage.DeleteFile(nonExistentID)
|
||||
err := tc.storage.DeleteFile(encryptor, nonExistentID)
|
||||
assert.NoError(t, err, "DeleteFile should not error for non-existent file")
|
||||
})
|
||||
})
|
||||
@@ -225,7 +240,7 @@ func setupS3Container(ctx context.Context) (*S3Container, error) {
|
||||
secretKey := "testpassword"
|
||||
bucketName := "test-bucket"
|
||||
region := "us-east-1"
|
||||
endpoint := fmt.Sprintf("localhost:%s", env.TestMinioPort)
|
||||
endpoint := fmt.Sprintf("127.0.0.1:%s", env.TestMinioPort)
|
||||
|
||||
// Create MinIO client and ensure bucket exists
|
||||
minioClient, err := minio.New(endpoint, &minio.Options{
|
||||
|
||||
@@ -3,18 +3,44 @@ package azure_blob_storage
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
azureConnectTimeout = 30 * time.Second
|
||||
azureResponseTimeout = 30 * time.Second
|
||||
azureIdleConnTimeout = 90 * time.Second
|
||||
azureTLSHandshakeTimeout = 30 * time.Second
|
||||
|
||||
// Chunk size for block blob uploads - 16MB provides good balance between
|
||||
// memory usage and upload efficiency. This creates backpressure to pg_dump
|
||||
// by only reading one chunk at a time and waiting for Azure to confirm receipt.
|
||||
azureChunkSize = 16 * 1024 * 1024
|
||||
)
|
||||
|
||||
type readSeekCloser struct {
|
||||
*bytes.Reader
|
||||
}
|
||||
|
||||
func (r *readSeekCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type AuthMethod string
|
||||
|
||||
const (
|
||||
@@ -37,30 +63,102 @@ func (s *AzureBlobStorage) TableName() string {
|
||||
return "azure_blob_storages"
|
||||
}
|
||||
|
||||
func (s *AzureBlobStorage) SaveFile(logger *slog.Logger, fileID uuid.UUID, file io.Reader) error {
|
||||
client, err := s.getClient()
|
||||
func (s *AzureBlobStorage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
file io.Reader,
|
||||
) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("upload cancelled before start: %w", ctx.Err())
|
||||
default:
|
||||
}
|
||||
|
||||
client, err := s.getClient(encryptor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
blobName := s.buildBlobName(fileID.String())
|
||||
blockBlobClient := client.ServiceClient().
|
||||
NewContainerClient(s.ContainerName).
|
||||
NewBlockBlobClient(blobName)
|
||||
|
||||
_, err = client.UploadStream(
|
||||
context.TODO(),
|
||||
s.ContainerName,
|
||||
blobName,
|
||||
file,
|
||||
nil,
|
||||
)
|
||||
var blockIDs []string
|
||||
blockNumber := 0
|
||||
buf := make([]byte, azureChunkSize)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("upload cancelled: %w", ctx.Err())
|
||||
default:
|
||||
}
|
||||
|
||||
n, readErr := io.ReadFull(file, buf)
|
||||
|
||||
if n == 0 && readErr == io.EOF {
|
||||
break
|
||||
}
|
||||
|
||||
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
|
||||
return fmt.Errorf("read error: %w", readErr)
|
||||
}
|
||||
|
||||
blockID := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%06d", blockNumber)))
|
||||
|
||||
_, err := blockBlobClient.StageBlock(
|
||||
ctx,
|
||||
blockID,
|
||||
&readSeekCloser{bytes.NewReader(buf[:n])},
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("upload cancelled: %w", ctx.Err())
|
||||
default:
|
||||
return fmt.Errorf("failed to stage block %d: %w", blockNumber, err)
|
||||
}
|
||||
}
|
||||
|
||||
blockIDs = append(blockIDs, blockID)
|
||||
blockNumber++
|
||||
|
||||
if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(blockIDs) == 0 {
|
||||
_, err = client.UploadStream(
|
||||
ctx,
|
||||
s.ContainerName,
|
||||
blobName,
|
||||
bytes.NewReader([]byte{}),
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload empty blob: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err = blockBlobClient.CommitBlockList(ctx, blockIDs, &blockblob.CommitBlockListOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload blob to Azure: %w", err)
|
||||
return fmt.Errorf("failed to commit block list: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *AzureBlobStorage) GetFile(fileID uuid.UUID) (io.ReadCloser, error) {
|
||||
client, err := s.getClient()
|
||||
func (s *AzureBlobStorage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
) (io.ReadCloser, error) {
|
||||
client, err := s.getClient(encryptor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -80,8 +178,8 @@ func (s *AzureBlobStorage) GetFile(fileID uuid.UUID) (io.ReadCloser, error) {
|
||||
return response.Body, nil
|
||||
}
|
||||
|
||||
func (s *AzureBlobStorage) DeleteFile(fileID uuid.UUID) error {
|
||||
client, err := s.getClient()
|
||||
func (s *AzureBlobStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
client, err := s.getClient(encryptor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -105,7 +203,7 @@ func (s *AzureBlobStorage) DeleteFile(fileID uuid.UUID) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *AzureBlobStorage) Validate() error {
|
||||
func (s *AzureBlobStorage) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
if s.ContainerName == "" {
|
||||
return errors.New("container name is required")
|
||||
}
|
||||
@@ -128,16 +226,11 @@ func (s *AzureBlobStorage) Validate() error {
|
||||
return fmt.Errorf("invalid auth method: %s", s.AuthMethod)
|
||||
}
|
||||
|
||||
_, err := s.getClient()
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid Azure Blob configuration: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *AzureBlobStorage) TestConnection() error {
|
||||
client, err := s.getClient()
|
||||
func (s *AzureBlobStorage) TestConnection(encryptor encryption.FieldEncryptor) error {
|
||||
client, err := s.getClient(encryptor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -192,6 +285,26 @@ func (s *AzureBlobStorage) HideSensitiveData() {
|
||||
s.AccountKey = ""
|
||||
}
|
||||
|
||||
func (s *AzureBlobStorage) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
var err error
|
||||
|
||||
if s.ConnectionString != "" {
|
||||
s.ConnectionString, err = encryptor.Encrypt(s.StorageID, s.ConnectionString)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt Azure connection string: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if s.AccountKey != "" {
|
||||
s.AccountKey, err = encryptor.Encrypt(s.StorageID, s.AccountKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt Azure account key: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *AzureBlobStorage) Update(incoming *AzureBlobStorage) {
|
||||
s.AuthMethod = incoming.AuthMethod
|
||||
s.ContainerName = incoming.ContainerName
|
||||
@@ -225,13 +338,20 @@ func (s *AzureBlobStorage) buildBlobName(fileName string) string {
|
||||
return prefix + fileName
|
||||
}
|
||||
|
||||
func (s *AzureBlobStorage) getClient() (*azblob.Client, error) {
|
||||
func (s *AzureBlobStorage) getClient(encryptor encryption.FieldEncryptor) (*azblob.Client, error) {
|
||||
var client *azblob.Client
|
||||
var err error
|
||||
|
||||
clientOptions := s.buildClientOptions()
|
||||
|
||||
switch s.AuthMethod {
|
||||
case AuthMethodConnectionString:
|
||||
client, err = azblob.NewClientFromConnectionString(s.ConnectionString, nil)
|
||||
connectionString, decryptErr := encryptor.Decrypt(s.StorageID, s.ConnectionString)
|
||||
if decryptErr != nil {
|
||||
return nil, fmt.Errorf("failed to decrypt Azure connection string: %w", decryptErr)
|
||||
}
|
||||
|
||||
client, err = azblob.NewClientFromConnectionString(connectionString, clientOptions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"failed to create Azure Blob client from connection string: %w",
|
||||
@@ -239,13 +359,18 @@ func (s *AzureBlobStorage) getClient() (*azblob.Client, error) {
|
||||
)
|
||||
}
|
||||
case AuthMethodAccountKey:
|
||||
accountKey, decryptErr := encryptor.Decrypt(s.StorageID, s.AccountKey)
|
||||
if decryptErr != nil {
|
||||
return nil, fmt.Errorf("failed to decrypt Azure account key: %w", decryptErr)
|
||||
}
|
||||
|
||||
accountURL := s.buildAccountURL()
|
||||
credential, credErr := azblob.NewSharedKeyCredential(s.AccountName, s.AccountKey)
|
||||
credential, credErr := azblob.NewSharedKeyCredential(s.AccountName, accountKey)
|
||||
if credErr != nil {
|
||||
return nil, fmt.Errorf("failed to create Azure shared key credential: %w", credErr)
|
||||
}
|
||||
|
||||
client, err = azblob.NewClientWithSharedKeyCredential(accountURL, credential, nil)
|
||||
client, err = azblob.NewClientWithSharedKeyCredential(accountURL, credential, clientOptions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Azure Blob client with shared key: %w", err)
|
||||
}
|
||||
@@ -256,6 +381,26 @@ func (s *AzureBlobStorage) getClient() (*azblob.Client, error) {
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (s *AzureBlobStorage) buildClientOptions() *azblob.ClientOptions {
|
||||
transport := &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: azureConnectTimeout,
|
||||
}).DialContext,
|
||||
TLSHandshakeTimeout: azureTLSHandshakeTimeout,
|
||||
ResponseHeaderTimeout: azureResponseTimeout,
|
||||
IdleConnTimeout: azureIdleConnTimeout,
|
||||
}
|
||||
|
||||
return &azblob.ClientOptions{
|
||||
ClientOptions: azcore.ClientOptions{
|
||||
Transport: &http.Client{Transport: transport},
|
||||
Retry: policy.RetryOptions{
|
||||
MaxRetries: 0,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *AzureBlobStorage) buildAccountURL() string {
|
||||
if s.Endpoint != "" {
|
||||
endpoint := s.Endpoint
|
||||
|
||||
@@ -7,6 +7,9 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -15,9 +18,22 @@ import (
|
||||
"golang.org/x/oauth2/google"
|
||||
|
||||
drive "google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
const (
|
||||
gdConnectTimeout = 30 * time.Second
|
||||
gdResponseTimeout = 30 * time.Second
|
||||
gdIdleConnTimeout = 90 * time.Second
|
||||
gdTLSHandshakeTimeout = 30 * time.Second
|
||||
|
||||
// Chunk size for Google Drive resumable uploads - 16MB provides good balance
|
||||
// between memory usage and upload efficiency. Google Drive requires chunks
|
||||
// to be multiples of 256KB for resumable uploads.
|
||||
gdChunkSize = 16 * 1024 * 1024
|
||||
)
|
||||
|
||||
type GoogleDriveStorage struct {
|
||||
StorageID uuid.UUID `json:"storageId" gorm:"primaryKey;type:uuid;column:storage_id"`
|
||||
ClientID string `json:"clientId" gorm:"not null;type:text;column:client_id"`
|
||||
@@ -30,30 +46,44 @@ func (s *GoogleDriveStorage) TableName() string {
|
||||
}
|
||||
|
||||
func (s *GoogleDriveStorage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
file io.Reader,
|
||||
) error {
|
||||
return s.withRetryOnAuth(func(driveService *drive.Service) error {
|
||||
ctx := context.Background()
|
||||
return s.withRetryOnAuth(ctx, encryptor, func(driveService *drive.Service) error {
|
||||
filename := fileID.String()
|
||||
|
||||
// Ensure the postgresus_backups folder exists
|
||||
folderID, err := s.ensureBackupsFolderExists(ctx, driveService)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create/find backups folder: %w", err)
|
||||
}
|
||||
|
||||
// Delete any previous copy so we keep at most one object per logical file.
|
||||
_ = s.deleteByName(ctx, driveService, filename, folderID) // ignore "not found"
|
||||
_ = s.deleteByName(ctx, driveService, filename, folderID)
|
||||
|
||||
fileMeta := &drive.File{
|
||||
Name: filename,
|
||||
Parents: []string{folderID},
|
||||
}
|
||||
|
||||
_, err = driveService.Files.Create(fileMeta).Media(file).Context(ctx).Do()
|
||||
backpressureReader := &backpressureReader{
|
||||
reader: file,
|
||||
ctx: ctx,
|
||||
chunkSize: gdChunkSize,
|
||||
buf: make([]byte, gdChunkSize),
|
||||
}
|
||||
|
||||
_, err = driveService.Files.Create(fileMeta).
|
||||
Media(backpressureReader, googleapi.ChunkSize(gdChunkSize)).
|
||||
Context(ctx).
|
||||
Do()
|
||||
if err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("upload cancelled: %w", ctx.Err())
|
||||
default:
|
||||
}
|
||||
return fmt.Errorf("failed to upload file to Google Drive: %w", err)
|
||||
}
|
||||
|
||||
@@ -68,34 +98,95 @@ func (s *GoogleDriveStorage) SaveFile(
|
||||
})
|
||||
}
|
||||
|
||||
func (s *GoogleDriveStorage) GetFile(fileID uuid.UUID) (io.ReadCloser, error) {
|
||||
type backpressureReader struct {
|
||||
reader io.Reader
|
||||
ctx context.Context
|
||||
chunkSize int
|
||||
buf []byte
|
||||
bufStart int
|
||||
bufEnd int
|
||||
totalBytes int64
|
||||
chunkCount int
|
||||
}
|
||||
|
||||
func (r *backpressureReader) Read(p []byte) (n int, err error) {
|
||||
select {
|
||||
case <-r.ctx.Done():
|
||||
return 0, r.ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
if r.bufStart >= r.bufEnd {
|
||||
r.chunkCount++
|
||||
|
||||
bytesRead, readErr := io.ReadFull(r.reader, r.buf)
|
||||
if bytesRead > 0 {
|
||||
r.bufStart = 0
|
||||
r.bufEnd = bytesRead
|
||||
}
|
||||
|
||||
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
|
||||
return 0, readErr
|
||||
}
|
||||
|
||||
if bytesRead == 0 && readErr == io.EOF {
|
||||
return 0, io.EOF
|
||||
}
|
||||
}
|
||||
|
||||
n = copy(p, r.buf[r.bufStart:r.bufEnd])
|
||||
r.bufStart += n
|
||||
r.totalBytes += int64(n)
|
||||
|
||||
if r.bufStart >= r.bufEnd {
|
||||
select {
|
||||
case <-r.ctx.Done():
|
||||
return n, r.ctx.Err()
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (s *GoogleDriveStorage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
) (io.ReadCloser, error) {
|
||||
var result io.ReadCloser
|
||||
err := s.withRetryOnAuth(func(driveService *drive.Service) error {
|
||||
folderID, err := s.findBackupsFolder(driveService)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find backups folder: %w", err)
|
||||
}
|
||||
err := s.withRetryOnAuth(
|
||||
context.Background(),
|
||||
encryptor,
|
||||
func(driveService *drive.Service) error {
|
||||
folderID, err := s.findBackupsFolder(driveService)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find backups folder: %w", err)
|
||||
}
|
||||
|
||||
fileIDGoogle, err := s.lookupFileID(driveService, fileID.String(), folderID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileIDGoogle, err := s.lookupFileID(driveService, fileID.String(), folderID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := driveService.Files.Get(fileIDGoogle).Download()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download file from Google Drive: %w", err)
|
||||
}
|
||||
resp, err := driveService.Files.Get(fileIDGoogle).Download()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download file from Google Drive: %w", err)
|
||||
}
|
||||
|
||||
result = resp.Body
|
||||
return nil
|
||||
})
|
||||
result = resp.Body
|
||||
return nil
|
||||
},
|
||||
)
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (s *GoogleDriveStorage) DeleteFile(fileID uuid.UUID) error {
|
||||
return s.withRetryOnAuth(func(driveService *drive.Service) error {
|
||||
ctx := context.Background()
|
||||
func (s *GoogleDriveStorage) DeleteFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
) error {
|
||||
ctx := context.Background()
|
||||
return s.withRetryOnAuth(ctx, encryptor, func(driveService *drive.Service) error {
|
||||
folderID, err := s.findBackupsFolder(driveService)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find backups folder: %w", err)
|
||||
@@ -105,7 +196,7 @@ func (s *GoogleDriveStorage) DeleteFile(fileID uuid.UUID) error {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *GoogleDriveStorage) Validate() error {
|
||||
func (s *GoogleDriveStorage) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
switch {
|
||||
case s.ClientID == "":
|
||||
return errors.New("client ID is required")
|
||||
@@ -115,7 +206,12 @@ func (s *GoogleDriveStorage) Validate() error {
|
||||
return errors.New("token JSON is required")
|
||||
}
|
||||
|
||||
// Also validate that the token JSON contains a refresh token
|
||||
// Skip JSON validation if token is already encrypted
|
||||
if strings.HasPrefix(s.TokenJSON, "enc:") {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate that the token JSON contains a refresh token
|
||||
var token oauth2.Token
|
||||
if err := json.Unmarshal([]byte(s.TokenJSON), &token); err != nil {
|
||||
return fmt.Errorf("invalid token JSON format: %w", err)
|
||||
@@ -128,9 +224,9 @@ func (s *GoogleDriveStorage) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *GoogleDriveStorage) TestConnection() error {
|
||||
return s.withRetryOnAuth(func(driveService *drive.Service) error {
|
||||
ctx := context.Background()
|
||||
func (s *GoogleDriveStorage) TestConnection(encryptor encryption.FieldEncryptor) error {
|
||||
ctx := context.Background()
|
||||
return s.withRetryOnAuth(ctx, encryptor, func(driveService *drive.Service) error {
|
||||
testFilename := "test-connection-" + uuid.New().String()
|
||||
testData := []byte("test")
|
||||
|
||||
@@ -196,6 +292,26 @@ func (s *GoogleDriveStorage) HideSensitiveData() {
|
||||
s.TokenJSON = ""
|
||||
}
|
||||
|
||||
func (s *GoogleDriveStorage) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
var err error
|
||||
|
||||
if s.ClientSecret != "" {
|
||||
s.ClientSecret, err = encryptor.Encrypt(s.StorageID, s.ClientSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt Google Drive client secret: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if s.TokenJSON != "" {
|
||||
s.TokenJSON, err = encryptor.Encrypt(s.StorageID, s.TokenJSON)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt Google Drive token JSON: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *GoogleDriveStorage) Update(incoming *GoogleDriveStorage) {
|
||||
s.ClientID = incoming.ClientID
|
||||
|
||||
@@ -209,18 +325,34 @@ func (s *GoogleDriveStorage) Update(incoming *GoogleDriveStorage) {
|
||||
}
|
||||
|
||||
// withRetryOnAuth executes the provided function with retry logic for authentication errors
|
||||
func (s *GoogleDriveStorage) withRetryOnAuth(fn func(*drive.Service) error) error {
|
||||
driveService, err := s.getDriveService()
|
||||
func (s *GoogleDriveStorage) withRetryOnAuth(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fn func(*drive.Service) error,
|
||||
) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
driveService, err := s.getDriveService(encryptor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = fn(driveService)
|
||||
if err != nil && s.isAuthError(err) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
// Try to refresh token and retry once
|
||||
fmt.Printf("Google Drive auth error detected, attempting token refresh: %v\n", err)
|
||||
|
||||
if refreshErr := s.refreshToken(); refreshErr != nil {
|
||||
if refreshErr := s.refreshToken(encryptor); refreshErr != nil {
|
||||
// If refresh fails, return a more helpful error message
|
||||
if strings.Contains(refreshErr.Error(), "invalid_grant") ||
|
||||
strings.Contains(refreshErr.Error(), "refresh token") {
|
||||
@@ -237,7 +369,7 @@ func (s *GoogleDriveStorage) withRetryOnAuth(fn func(*drive.Service) error) erro
|
||||
fmt.Printf("Token refresh successful, retrying operation\n")
|
||||
|
||||
// Get new service with refreshed token
|
||||
driveService, err = s.getDriveService()
|
||||
driveService, err = s.getDriveService(encryptor)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create service after token refresh: %w", err)
|
||||
}
|
||||
@@ -268,13 +400,24 @@ func (s *GoogleDriveStorage) isAuthError(err error) bool {
|
||||
}
|
||||
|
||||
// refreshToken refreshes the OAuth2 token and updates the TokenJSON field
|
||||
func (s *GoogleDriveStorage) refreshToken() error {
|
||||
if err := s.Validate(); err != nil {
|
||||
func (s *GoogleDriveStorage) refreshToken(encryptor encryption.FieldEncryptor) error {
|
||||
if err := s.Validate(encryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Decrypt credentials before use
|
||||
clientSecret, err := encryptor.Decrypt(s.StorageID, s.ClientSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt Google Drive client secret: %w", err)
|
||||
}
|
||||
|
||||
tokenJSON, err := encryptor.Decrypt(s.StorageID, s.TokenJSON)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt Google Drive token JSON: %w", err)
|
||||
}
|
||||
|
||||
var token oauth2.Token
|
||||
if err := json.Unmarshal([]byte(s.TokenJSON), &token); err != nil {
|
||||
if err := json.Unmarshal([]byte(tokenJSON), &token); err != nil {
|
||||
return fmt.Errorf("invalid token JSON: %w", err)
|
||||
}
|
||||
|
||||
@@ -289,12 +432,12 @@ func (s *GoogleDriveStorage) refreshToken() error {
|
||||
token.Expiry)
|
||||
|
||||
// Debug: Print the full token JSON structure (sensitive data masked)
|
||||
fmt.Printf("Original token JSON structure: %s\n", maskSensitiveData(s.TokenJSON))
|
||||
fmt.Printf("Original token JSON structure: %s\n", maskSensitiveData(tokenJSON))
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := &oauth2.Config{
|
||||
ClientID: s.ClientID,
|
||||
ClientSecret: s.ClientSecret,
|
||||
ClientSecret: clientSecret,
|
||||
Endpoint: google.Endpoint,
|
||||
Scopes: []string{"https://www.googleapis.com/auth/drive.file"},
|
||||
}
|
||||
@@ -330,7 +473,7 @@ func (s *GoogleDriveStorage) refreshToken() error {
|
||||
newToken.RefreshToken = token.RefreshToken
|
||||
}
|
||||
|
||||
// Update the stored token JSON
|
||||
// Update the stored token JSON (keep as plaintext in memory, encryption happens on save)
|
||||
newTokenJSON, err := json.Marshal(newToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal refreshed token: %w", err)
|
||||
@@ -368,13 +511,25 @@ func truncateString(s string, maxLen int) string {
|
||||
return s[:maxLen]
|
||||
}
|
||||
|
||||
func (s *GoogleDriveStorage) getDriveService() (*drive.Service, error) {
|
||||
if err := s.Validate(); err != nil {
|
||||
func (s *GoogleDriveStorage) getDriveService(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
) (*drive.Service, error) {
|
||||
if err := s.Validate(encryptor); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clientSecret, err := encryptor.Decrypt(s.StorageID, s.ClientSecret)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decrypt Google Drive client secret: %w", err)
|
||||
}
|
||||
|
||||
tokenJSON, err := encryptor.Decrypt(s.StorageID, s.TokenJSON)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decrypt Google Drive token JSON: %w", err)
|
||||
}
|
||||
|
||||
var token oauth2.Token
|
||||
if err := json.Unmarshal([]byte(s.TokenJSON), &token); err != nil {
|
||||
if err := json.Unmarshal([]byte(tokenJSON), &token); err != nil {
|
||||
return nil, fmt.Errorf("invalid token JSON: %w", err)
|
||||
}
|
||||
|
||||
@@ -382,23 +537,23 @@ func (s *GoogleDriveStorage) getDriveService() (*drive.Service, error) {
|
||||
|
||||
cfg := &oauth2.Config{
|
||||
ClientID: s.ClientID,
|
||||
ClientSecret: s.ClientSecret,
|
||||
ClientSecret: clientSecret,
|
||||
Endpoint: google.Endpoint,
|
||||
Scopes: []string{"https://www.googleapis.com/auth/drive.file"},
|
||||
}
|
||||
|
||||
tokenSource := cfg.TokenSource(ctx, &token)
|
||||
|
||||
// Force token validation to ensure we're using the current token
|
||||
currentToken, err := tokenSource.Token()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get current token: %w", err)
|
||||
}
|
||||
|
||||
// Create a new token source with the validated token
|
||||
validatedTokenSource := oauth2.StaticTokenSource(currentToken)
|
||||
|
||||
driveService, err := drive.NewService(ctx, option.WithTokenSource(validatedTokenSource))
|
||||
httpClient := s.buildHTTPClient(validatedTokenSource)
|
||||
|
||||
driveService, err := drive.NewService(ctx, option.WithHTTPClient(httpClient))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create Drive client: %w", err)
|
||||
}
|
||||
@@ -406,6 +561,24 @@ func (s *GoogleDriveStorage) getDriveService() (*drive.Service, error) {
|
||||
return driveService, nil
|
||||
}
|
||||
|
||||
func (s *GoogleDriveStorage) buildHTTPClient(tokenSource oauth2.TokenSource) *http.Client {
|
||||
transport := &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: gdConnectTimeout,
|
||||
}).DialContext,
|
||||
TLSHandshakeTimeout: gdTLSHandshakeTimeout,
|
||||
ResponseHeaderTimeout: gdResponseTimeout,
|
||||
IdleConnTimeout: gdIdleConnTimeout,
|
||||
}
|
||||
|
||||
return &http.Client{
|
||||
Transport: &oauth2.Transport{
|
||||
Source: tokenSource,
|
||||
Base: transport,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *GoogleDriveStorage) lookupFileID(
|
||||
driveService *drive.Service,
|
||||
name string,
|
||||
|
||||
@@ -1,17 +1,26 @@
|
||||
package local_storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"postgresus-backend/internal/config"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
files_utils "postgresus-backend/internal/util/files"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
// Chunk size for local storage writes - 8MB per buffer with double-buffering
|
||||
// allows overlapped I/O while keeping total memory under 32MB.
|
||||
// Two 8MB buffers = 16MB for local storage, plus 8MB for pg_dump buffer = ~25MB total.
|
||||
localChunkSize = 8 * 1024 * 1024
|
||||
)
|
||||
|
||||
// LocalStorage uses ./postgresus_local_backups folder as a
|
||||
// directory for backups and ./postgresus_local_temp folder as a
|
||||
// directory for temp files
|
||||
@@ -23,7 +32,19 @@ func (l *LocalStorage) TableName() string {
|
||||
return "local_storages"
|
||||
}
|
||||
|
||||
func (l *LocalStorage) SaveFile(logger *slog.Logger, fileID uuid.UUID, file io.Reader) error {
|
||||
func (l *LocalStorage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
file io.Reader,
|
||||
) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
logger.Info("Starting to save file to local storage", "fileId", fileID.String())
|
||||
|
||||
err := files_utils.EnsureDirectories([]string{
|
||||
@@ -54,7 +75,7 @@ func (l *LocalStorage) SaveFile(logger *slog.Logger, fileID uuid.UUID, file io.R
|
||||
}()
|
||||
|
||||
logger.Debug("Copying file data to temp file", "fileId", fileID.String())
|
||||
_, err = io.Copy(tempFile, file)
|
||||
_, err = copyWithContext(ctx, tempFile, file)
|
||||
if err != nil {
|
||||
logger.Error("Failed to write to temp file", "fileId", fileID.String(), "error", err)
|
||||
return fmt.Errorf("failed to write to temp file: %w", err)
|
||||
@@ -107,7 +128,10 @@ func (l *LocalStorage) SaveFile(logger *slog.Logger, fileID uuid.UUID, file io.R
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *LocalStorage) GetFile(fileID uuid.UUID) (io.ReadCloser, error) {
|
||||
func (l *LocalStorage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
) (io.ReadCloser, error) {
|
||||
filePath := filepath.Join(config.GetEnv().DataFolder, fileID.String())
|
||||
|
||||
if _, err := os.Stat(filePath); os.IsNotExist(err) {
|
||||
@@ -122,7 +146,7 @@ func (l *LocalStorage) GetFile(fileID uuid.UUID) (io.ReadCloser, error) {
|
||||
return file, nil
|
||||
}
|
||||
|
||||
func (l *LocalStorage) DeleteFile(fileID uuid.UUID) error {
|
||||
func (l *LocalStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
filePath := filepath.Join(config.GetEnv().DataFolder, fileID.String())
|
||||
|
||||
if _, err := os.Stat(filePath); os.IsNotExist(err) {
|
||||
@@ -136,11 +160,11 @@ func (l *LocalStorage) DeleteFile(fileID uuid.UUID) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *LocalStorage) Validate() error {
|
||||
func (l *LocalStorage) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *LocalStorage) TestConnection() error {
|
||||
func (l *LocalStorage) TestConnection(encryptor encryption.FieldEncryptor) error {
|
||||
testFile := filepath.Join(config.GetEnv().TempFolder, "test_connection")
|
||||
f, err := os.Create(testFile)
|
||||
if err != nil {
|
||||
@@ -160,5 +184,41 @@ func (l *LocalStorage) TestConnection() error {
|
||||
func (l *LocalStorage) HideSensitiveData() {
|
||||
}
|
||||
|
||||
func (l *LocalStorage) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *LocalStorage) Update(incoming *LocalStorage) {
|
||||
}
|
||||
|
||||
func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) {
|
||||
buf := make([]byte, localChunkSize)
|
||||
var written int64
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return written, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
nr, readErr := src.Read(buf)
|
||||
if nr > 0 {
|
||||
nw, writeErr := dst.Write(buf[:nr])
|
||||
written += int64(nw)
|
||||
if writeErr != nil {
|
||||
return written, writeErr
|
||||
}
|
||||
if nr != nw {
|
||||
return written, io.ErrShortWrite
|
||||
}
|
||||
}
|
||||
|
||||
if readErr == io.EOF {
|
||||
return written, nil
|
||||
}
|
||||
if readErr != nil {
|
||||
return written, readErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package nas_storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -8,6 +9,7 @@ import (
|
||||
"log/slog"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -15,6 +17,13 @@ import (
|
||||
"github.com/hirochachacha/go-smb2"
|
||||
)
|
||||
|
||||
const (
|
||||
// Chunk size for NAS uploads - 16MB provides good balance between
|
||||
// memory usage and upload efficiency. This creates backpressure to pg_dump
|
||||
// by only reading one chunk at a time and waiting for NAS to confirm receipt.
|
||||
nasChunkSize = 16 * 1024 * 1024
|
||||
)
|
||||
|
||||
type NASStorage struct {
|
||||
StorageID uuid.UUID `json:"storageId" gorm:"primaryKey;type:uuid;column:storage_id"`
|
||||
Host string `json:"host" gorm:"not null;type:text;column:host"`
|
||||
@@ -31,10 +40,22 @@ func (n *NASStorage) TableName() string {
|
||||
return "nas_storages"
|
||||
}
|
||||
|
||||
func (n *NASStorage) SaveFile(logger *slog.Logger, fileID uuid.UUID, file io.Reader) error {
|
||||
func (n *NASStorage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
file io.Reader,
|
||||
) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
logger.Info("Starting to save file to NAS storage", "fileId", fileID.String(), "host", n.Host)
|
||||
|
||||
session, err := n.createSession()
|
||||
session, err := n.createSessionWithContext(ctx, encryptor)
|
||||
if err != nil {
|
||||
logger.Error("Failed to create NAS session", "fileId", fileID.String(), "error", err)
|
||||
return fmt.Errorf("failed to create NAS session: %w", err)
|
||||
@@ -115,7 +136,7 @@ func (n *NASStorage) SaveFile(logger *slog.Logger, fileID uuid.UUID, file io.Rea
|
||||
}()
|
||||
|
||||
logger.Debug("Copying file data to NAS", "fileId", fileID.String())
|
||||
_, err = io.Copy(nasFile, file)
|
||||
_, err = copyWithContext(ctx, nasFile, file)
|
||||
if err != nil {
|
||||
logger.Error("Failed to write file to NAS", "fileId", fileID.String(), "error", err)
|
||||
return fmt.Errorf("failed to write file to NAS: %w", err)
|
||||
@@ -131,8 +152,11 @@ func (n *NASStorage) SaveFile(logger *slog.Logger, fileID uuid.UUID, file io.Rea
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NASStorage) GetFile(fileID uuid.UUID) (io.ReadCloser, error) {
|
||||
session, err := n.createSession()
|
||||
func (n *NASStorage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
) (io.ReadCloser, error) {
|
||||
session, err := n.createSession(encryptor)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create NAS session: %w", err)
|
||||
}
|
||||
@@ -168,8 +192,8 @@ func (n *NASStorage) GetFile(fileID uuid.UUID) (io.ReadCloser, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (n *NASStorage) DeleteFile(fileID uuid.UUID) error {
|
||||
session, err := n.createSession()
|
||||
func (n *NASStorage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
session, err := n.createSession(encryptor)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create NAS session: %w", err)
|
||||
}
|
||||
@@ -202,7 +226,7 @@ func (n *NASStorage) DeleteFile(fileID uuid.UUID) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NASStorage) Validate() error {
|
||||
func (n *NASStorage) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
if n.Host == "" {
|
||||
return errors.New("NAS host is required")
|
||||
}
|
||||
@@ -219,12 +243,11 @@ func (n *NASStorage) Validate() error {
|
||||
return errors.New("NAS port must be between 1 and 65535")
|
||||
}
|
||||
|
||||
// Test the configuration by creating a session
|
||||
return n.TestConnection()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NASStorage) TestConnection() error {
|
||||
session, err := n.createSession()
|
||||
func (n *NASStorage) TestConnection(encryptor encryption.FieldEncryptor) error {
|
||||
session, err := n.createSession(encryptor)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to NAS: %w", err)
|
||||
}
|
||||
@@ -255,6 +278,18 @@ func (n *NASStorage) HideSensitiveData() {
|
||||
n.Password = ""
|
||||
}
|
||||
|
||||
func (n *NASStorage) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
if n.Password != "" {
|
||||
encrypted, err := encryptor.Encrypt(n.StorageID, n.Password)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt NAS password: %w", err)
|
||||
}
|
||||
n.Password = encrypted
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *NASStorage) Update(incoming *NASStorage) {
|
||||
n.Host = incoming.Host
|
||||
n.Port = incoming.Port
|
||||
@@ -269,23 +304,33 @@ func (n *NASStorage) Update(incoming *NASStorage) {
|
||||
}
|
||||
}
|
||||
|
||||
func (n *NASStorage) createSession() (*smb2.Session, error) {
|
||||
// Create connection with timeout
|
||||
conn, err := n.createConnection()
|
||||
func (n *NASStorage) createSession(encryptor encryption.FieldEncryptor) (*smb2.Session, error) {
|
||||
return n.createSessionWithContext(context.Background(), encryptor)
|
||||
}
|
||||
|
||||
func (n *NASStorage) createSessionWithContext(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
) (*smb2.Session, error) {
|
||||
conn, err := n.createConnectionWithContext(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create SMB2 dialer
|
||||
password, err := encryptor.Decrypt(n.StorageID, n.Password)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("failed to decrypt NAS password: %w", err)
|
||||
}
|
||||
|
||||
d := &smb2.Dialer{
|
||||
Initiator: &smb2.NTLMInitiator{
|
||||
User: n.Username,
|
||||
Password: n.Password,
|
||||
Password: password,
|
||||
Domain: n.Domain,
|
||||
},
|
||||
}
|
||||
|
||||
// Create session
|
||||
session, err := d.Dial(conn)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
@@ -295,34 +340,30 @@ func (n *NASStorage) createSession() (*smb2.Session, error) {
|
||||
return session, nil
|
||||
}
|
||||
|
||||
func (n *NASStorage) createConnection() (net.Conn, error) {
|
||||
func (n *NASStorage) createConnectionWithContext(ctx context.Context) (net.Conn, error) {
|
||||
address := net.JoinHostPort(n.Host, fmt.Sprintf("%d", n.Port))
|
||||
|
||||
// Create connection with timeout
|
||||
dialer := &net.Dialer{
|
||||
Timeout: 10 * time.Second,
|
||||
Timeout: 30 * time.Second,
|
||||
}
|
||||
|
||||
if n.UseSSL {
|
||||
// Use TLS connection
|
||||
tlsConfig := &tls.Config{
|
||||
ServerName: n.Host,
|
||||
InsecureSkipVerify: false, // Change to true if you want to skip cert verification
|
||||
InsecureSkipVerify: false,
|
||||
}
|
||||
|
||||
conn, err := tls.DialWithDialer(dialer, "tcp", address, tlsConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create SSL connection to %s: %w", address, err)
|
||||
}
|
||||
return conn, nil
|
||||
} else {
|
||||
// Use regular TCP connection
|
||||
conn, err := dialer.Dial("tcp", address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create connection to %s: %w", address, err)
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
conn, err := dialer.DialContext(ctx, "tcp", address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create connection to %s: %w", address, err)
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func (n *NASStorage) ensureDirectory(fs *smb2.Share, path string) error {
|
||||
@@ -417,3 +458,71 @@ func (r *nasFileReader) Close() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type writeResult struct {
|
||||
bytesWritten int
|
||||
writeErr error
|
||||
}
|
||||
|
||||
func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) {
|
||||
buf := make([]byte, nasChunkSize)
|
||||
var written int64
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return written, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
nr, readErr := io.ReadFull(src, buf)
|
||||
|
||||
if nr == 0 && readErr == io.EOF {
|
||||
break
|
||||
}
|
||||
|
||||
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
|
||||
return written, readErr
|
||||
}
|
||||
|
||||
writeResultCh := make(chan writeResult, 1)
|
||||
go func() {
|
||||
nw, writeErr := dst.Write(buf[0:nr])
|
||||
writeResultCh <- writeResult{nw, writeErr}
|
||||
}()
|
||||
|
||||
var nw int
|
||||
var writeErr error
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return written, ctx.Err()
|
||||
case result := <-writeResultCh:
|
||||
nw = result.bytesWritten
|
||||
writeErr = result.writeErr
|
||||
}
|
||||
|
||||
if nw < 0 || nr < nw {
|
||||
nw = 0
|
||||
if writeErr == nil {
|
||||
writeErr = errors.New("invalid write result")
|
||||
}
|
||||
}
|
||||
|
||||
if writeErr != nil {
|
||||
return written, writeErr
|
||||
}
|
||||
|
||||
if nr != nw {
|
||||
return written, io.ErrShortWrite
|
||||
}
|
||||
|
||||
written += int64(nw)
|
||||
|
||||
if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return written, nil
|
||||
}
|
||||
|
||||
@@ -7,6 +7,9 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -15,6 +18,18 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
const (
|
||||
s3ConnectTimeout = 30 * time.Second
|
||||
s3ResponseTimeout = 30 * time.Second
|
||||
s3IdleConnTimeout = 90 * time.Second
|
||||
s3TLSHandshakeTimeout = 30 * time.Second
|
||||
|
||||
// Chunk size for multipart uploads - 16MB provides good balance between
|
||||
// memory usage and upload efficiency. This creates backpressure to pg_dump
|
||||
// by only reading one chunk at a time and waiting for S3 to confirm receipt.
|
||||
multipartChunkSize = 16 * 1024 * 1024
|
||||
)
|
||||
|
||||
type S3Storage struct {
|
||||
StorageID uuid.UUID `json:"storageId" gorm:"primaryKey;type:uuid;column:storage_id"`
|
||||
S3Bucket string `json:"s3Bucket" gorm:"not null;type:text;column:s3_bucket"`
|
||||
@@ -31,32 +46,134 @@ func (s *S3Storage) TableName() string {
|
||||
return "s3_storages"
|
||||
}
|
||||
|
||||
func (s *S3Storage) SaveFile(logger *slog.Logger, fileID uuid.UUID, file io.Reader) error {
|
||||
client, err := s.getClient()
|
||||
func (s *S3Storage) SaveFile(
|
||||
ctx context.Context,
|
||||
encryptor encryption.FieldEncryptor,
|
||||
logger *slog.Logger,
|
||||
fileID uuid.UUID,
|
||||
file io.Reader,
|
||||
) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("upload cancelled before start: %w", ctx.Err())
|
||||
default:
|
||||
}
|
||||
|
||||
coreClient, err := s.getCoreClient(encryptor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
objectKey := s.buildObjectKey(fileID.String())
|
||||
|
||||
// Upload the file using MinIO client with streaming (size = -1 for unknown size)
|
||||
_, err = client.PutObject(
|
||||
context.TODO(),
|
||||
uploadID, err := coreClient.NewMultipartUpload(
|
||||
ctx,
|
||||
s.S3Bucket,
|
||||
objectKey,
|
||||
file,
|
||||
-1,
|
||||
minio.PutObjectOptions{},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload file to S3: %w", err)
|
||||
return fmt.Errorf("failed to initiate multipart upload: %w", err)
|
||||
}
|
||||
|
||||
var parts []minio.CompletePart
|
||||
partNumber := 1
|
||||
buf := make([]byte, multipartChunkSize)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)
|
||||
return fmt.Errorf("upload cancelled: %w", ctx.Err())
|
||||
default:
|
||||
}
|
||||
|
||||
n, readErr := io.ReadFull(file, buf)
|
||||
|
||||
if n == 0 && readErr == io.EOF {
|
||||
break
|
||||
}
|
||||
|
||||
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
|
||||
_ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)
|
||||
return fmt.Errorf("read error: %w", readErr)
|
||||
}
|
||||
|
||||
part, err := coreClient.PutObjectPart(
|
||||
ctx,
|
||||
s.S3Bucket,
|
||||
objectKey,
|
||||
uploadID,
|
||||
partNumber,
|
||||
bytes.NewReader(buf[:n]),
|
||||
int64(n),
|
||||
minio.PutObjectPartOptions{},
|
||||
)
|
||||
if err != nil {
|
||||
_ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("upload cancelled: %w", ctx.Err())
|
||||
default:
|
||||
return fmt.Errorf("failed to upload part %d: %w", partNumber, err)
|
||||
}
|
||||
}
|
||||
|
||||
parts = append(parts, minio.CompletePart{
|
||||
PartNumber: partNumber,
|
||||
ETag: part.ETag,
|
||||
})
|
||||
|
||||
partNumber++
|
||||
|
||||
if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(parts) == 0 {
|
||||
_ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)
|
||||
|
||||
client, err := s.getClient(encryptor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = client.PutObject(
|
||||
ctx,
|
||||
s.S3Bucket,
|
||||
objectKey,
|
||||
bytes.NewReader([]byte{}),
|
||||
0,
|
||||
minio.PutObjectOptions{},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload empty file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err = coreClient.CompleteMultipartUpload(
|
||||
ctx,
|
||||
s.S3Bucket,
|
||||
objectKey,
|
||||
uploadID,
|
||||
parts,
|
||||
minio.PutObjectOptions{},
|
||||
)
|
||||
if err != nil {
|
||||
_ = coreClient.AbortMultipartUpload(ctx, s.S3Bucket, objectKey, uploadID)
|
||||
return fmt.Errorf("failed to complete multipart upload: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) GetFile(fileID uuid.UUID) (io.ReadCloser, error) {
|
||||
client, err := s.getClient()
|
||||
func (s *S3Storage) GetFile(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
fileID uuid.UUID,
|
||||
) (io.ReadCloser, error) {
|
||||
client, err := s.getClient(encryptor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -91,8 +208,8 @@ func (s *S3Storage) GetFile(fileID uuid.UUID) (io.ReadCloser, error) {
|
||||
return object, nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) DeleteFile(fileID uuid.UUID) error {
|
||||
client, err := s.getClient()
|
||||
func (s *S3Storage) DeleteFile(encryptor encryption.FieldEncryptor, fileID uuid.UUID) error {
|
||||
client, err := s.getClient(encryptor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -113,7 +230,7 @@ func (s *S3Storage) DeleteFile(fileID uuid.UUID) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) Validate() error {
|
||||
func (s *S3Storage) Validate(encryptor encryption.FieldEncryptor) error {
|
||||
if s.S3Bucket == "" {
|
||||
return errors.New("S3 bucket is required")
|
||||
}
|
||||
@@ -124,17 +241,11 @@ func (s *S3Storage) Validate() error {
|
||||
return errors.New("S3 secret key is required")
|
||||
}
|
||||
|
||||
// Try to create a client to validate the configuration
|
||||
_, err := s.getClient()
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid S3 configuration: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) TestConnection() error {
|
||||
client, err := s.getClient()
|
||||
func (s *S3Storage) TestConnection(encryptor encryption.FieldEncryptor) error {
|
||||
client, err := s.getClient(encryptor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -195,6 +306,26 @@ func (s *S3Storage) HideSensitiveData() {
|
||||
s.S3SecretKey = ""
|
||||
}
|
||||
|
||||
func (s *S3Storage) EncryptSensitiveData(encryptor encryption.FieldEncryptor) error {
|
||||
var err error
|
||||
|
||||
if s.S3AccessKey != "" {
|
||||
s.S3AccessKey, err = encryptor.Encrypt(s.StorageID, s.S3AccessKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt S3 access key: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if s.S3SecretKey != "" {
|
||||
s.S3SecretKey, err = encryptor.Encrypt(s.StorageID, s.S3SecretKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to encrypt S3 secret key: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) Update(incoming *S3Storage) {
|
||||
s.S3Bucket = incoming.S3Bucket
|
||||
s.S3Region = incoming.S3Region
|
||||
@@ -228,9 +359,55 @@ func (s *S3Storage) buildObjectKey(fileName string) string {
|
||||
return prefix + fileName
|
||||
}
|
||||
|
||||
func (s *S3Storage) getClient() (*minio.Client, error) {
|
||||
endpoint := s.S3Endpoint
|
||||
useSSL := true
|
||||
func (s *S3Storage) getClient(encryptor encryption.FieldEncryptor) (*minio.Client, error) {
|
||||
endpoint, useSSL, accessKey, secretKey, bucketLookup, transport, err := s.getClientParams(
|
||||
encryptor,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
minioClient, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: useSSL,
|
||||
Region: s.S3Region,
|
||||
BucketLookup: bucketLookup,
|
||||
Transport: transport,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize MinIO client: %w", err)
|
||||
}
|
||||
|
||||
return minioClient, nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) getCoreClient(encryptor encryption.FieldEncryptor) (*minio.Core, error) {
|
||||
endpoint, useSSL, accessKey, secretKey, bucketLookup, transport, err := s.getClientParams(
|
||||
encryptor,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
coreClient, err := minio.NewCore(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: useSSL,
|
||||
Region: s.S3Region,
|
||||
BucketLookup: bucketLookup,
|
||||
Transport: transport,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize MinIO Core client: %w", err)
|
||||
}
|
||||
|
||||
return coreClient, nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) getClientParams(
|
||||
encryptor encryption.FieldEncryptor,
|
||||
) (endpoint string, useSSL bool, accessKey string, secretKey string, bucketLookup minio.BucketLookupType, transport *http.Transport, err error) {
|
||||
endpoint = s.S3Endpoint
|
||||
useSSL = true
|
||||
|
||||
if strings.HasPrefix(endpoint, "http://") {
|
||||
useSSL = false
|
||||
@@ -239,27 +416,33 @@ func (s *S3Storage) getClient() (*minio.Client, error) {
|
||||
endpoint = strings.TrimPrefix(endpoint, "https://")
|
||||
}
|
||||
|
||||
// If no endpoint is provided, use the AWS S3 endpoint for the region
|
||||
if endpoint == "" {
|
||||
endpoint = fmt.Sprintf("s3.%s.amazonaws.com", s.S3Region)
|
||||
}
|
||||
|
||||
// Configure bucket lookup strategy
|
||||
bucketLookup := minio.BucketLookupAuto
|
||||
accessKey, err = encryptor.Decrypt(s.StorageID, s.S3AccessKey)
|
||||
if err != nil {
|
||||
return "", false, "", "", 0, nil, fmt.Errorf("failed to decrypt S3 access key: %w", err)
|
||||
}
|
||||
|
||||
secretKey, err = encryptor.Decrypt(s.StorageID, s.S3SecretKey)
|
||||
if err != nil {
|
||||
return "", false, "", "", 0, nil, fmt.Errorf("failed to decrypt S3 secret key: %w", err)
|
||||
}
|
||||
|
||||
bucketLookup = minio.BucketLookupAuto
|
||||
if s.S3UseVirtualHostedStyle {
|
||||
bucketLookup = minio.BucketLookupDNS
|
||||
}
|
||||
|
||||
// Initialize the MinIO client
|
||||
minioClient, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(s.S3AccessKey, s.S3SecretKey, ""),
|
||||
Secure: useSSL,
|
||||
Region: s.S3Region,
|
||||
BucketLookup: bucketLookup,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize MinIO client: %w", err)
|
||||
transport = &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: s3ConnectTimeout,
|
||||
}).DialContext,
|
||||
TLSHandshakeTimeout: s3TLSHandshakeTimeout,
|
||||
ResponseHeaderTimeout: s3ResponseTimeout,
|
||||
IdleConnTimeout: s3IdleConnTimeout,
|
||||
}
|
||||
|
||||
return minioClient, nil
|
||||
return endpoint, useSSL, accessKey, secretKey, bucketLookup, transport, nil
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
audit_logs "postgresus-backend/internal/features/audit_logs"
|
||||
users_models "postgresus-backend/internal/features/users/models"
|
||||
workspaces_services "postgresus-backend/internal/features/workspaces/services"
|
||||
"postgresus-backend/internal/util/encryption"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -15,6 +16,7 @@ type StorageService struct {
|
||||
storageRepository *StorageRepository
|
||||
workspaceService *workspaces_services.WorkspaceService
|
||||
auditLogService *audit_logs.AuditLogService
|
||||
fieldEncryptor encryption.FieldEncryptor
|
||||
}
|
||||
|
||||
func (s *StorageService) SaveStorage(
|
||||
@@ -44,7 +46,11 @@ func (s *StorageService) SaveStorage(
|
||||
|
||||
existingStorage.Update(storage)
|
||||
|
||||
if err := existingStorage.Validate(); err != nil {
|
||||
if err := existingStorage.EncryptSensitiveData(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := existingStorage.Validate(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -61,7 +67,11 @@ func (s *StorageService) SaveStorage(
|
||||
} else {
|
||||
storage.WorkspaceID = workspaceID
|
||||
|
||||
if err := storage.Validate(); err != nil {
|
||||
if err := storage.EncryptSensitiveData(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := storage.Validate(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -174,7 +184,7 @@ func (s *StorageService) TestStorageConnection(
|
||||
return errors.New("insufficient permissions to test storage in this workspace")
|
||||
}
|
||||
|
||||
err = storage.TestConnection()
|
||||
err = storage.TestConnection(s.fieldEncryptor)
|
||||
if err != nil {
|
||||
lastSaveError := err.Error()
|
||||
storage.LastSaveError = &lastSaveError
|
||||
@@ -207,7 +217,7 @@ func (s *StorageService) TestStorageConnectionDirect(
|
||||
|
||||
existingStorage.Update(storage)
|
||||
|
||||
if err := existingStorage.Validate(); err != nil {
|
||||
if err := existingStorage.Validate(s.fieldEncryptor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -216,7 +226,7 @@ func (s *StorageService) TestStorageConnectionDirect(
|
||||
usingStorage = storage
|
||||
}
|
||||
|
||||
return usingStorage.TestConnection()
|
||||
return usingStorage.TestConnection(s.fieldEncryptor)
|
||||
}
|
||||
|
||||
func (s *StorageService) GetStorageByID(
|
||||
|
||||
@@ -1,31 +1,36 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"postgresus-backend/internal/config"
|
||||
"postgresus-backend/internal/features/backups/backups"
|
||||
usecases_postgresql_backup "postgresus-backend/internal/features/backups/backups/usecases/postgresql"
|
||||
backups_config "postgresus-backend/internal/features/backups/config"
|
||||
"postgresus-backend/internal/features/databases"
|
||||
pgtypes "postgresus-backend/internal/features/databases/databases/postgresql"
|
||||
"postgresus-backend/internal/features/intervals"
|
||||
"postgresus-backend/internal/features/restores/models"
|
||||
usecases_postgresql_restore "postgresus-backend/internal/features/restores/usecases/postgresql"
|
||||
"postgresus-backend/internal/features/storages"
|
||||
local_storage "postgresus-backend/internal/features/storages/models/local"
|
||||
"postgresus-backend/internal/util/period"
|
||||
"postgresus-backend/internal/util/tools"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/lib/pq"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"postgresus-backend/internal/config"
|
||||
"postgresus-backend/internal/features/backups/backups"
|
||||
backups_config "postgresus-backend/internal/features/backups/config"
|
||||
"postgresus-backend/internal/features/databases"
|
||||
pgtypes "postgresus-backend/internal/features/databases/databases/postgresql"
|
||||
"postgresus-backend/internal/features/restores"
|
||||
restores_enums "postgresus-backend/internal/features/restores/enums"
|
||||
restores_models "postgresus-backend/internal/features/restores/models"
|
||||
"postgresus-backend/internal/features/storages"
|
||||
users_enums "postgresus-backend/internal/features/users/enums"
|
||||
users_testing "postgresus-backend/internal/features/users/testing"
|
||||
workspaces_controllers "postgresus-backend/internal/features/workspaces/controllers"
|
||||
workspaces_testing "postgresus-backend/internal/features/workspaces/testing"
|
||||
test_utils "postgresus-backend/internal/util/testing"
|
||||
"postgresus-backend/internal/util/tools"
|
||||
)
|
||||
|
||||
const createAndFillTableQuery = `
|
||||
@@ -61,7 +66,6 @@ type TestDataItem struct {
|
||||
CreatedAt time.Time `db:"created_at"`
|
||||
}
|
||||
|
||||
// Main test functions for each PostgreSQL version
|
||||
func Test_BackupAndRestorePostgresql_RestoreIsSuccesful(t *testing.T) {
|
||||
env := config.GetEnv()
|
||||
cases := []struct {
|
||||
@@ -79,17 +83,38 @@ func Test_BackupAndRestorePostgresql_RestoreIsSuccesful(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
tc := tc // capture loop variable
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel() // Enable parallel execution
|
||||
t.Parallel()
|
||||
testBackupRestoreForVersion(t, tc.version, tc.port)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Run a test for a specific PostgreSQL version
|
||||
func Test_BackupAndRestorePostgresqlWithEncryption_RestoreIsSuccessful(t *testing.T) {
|
||||
env := config.GetEnv()
|
||||
cases := []struct {
|
||||
name string
|
||||
version string
|
||||
port string
|
||||
}{
|
||||
{"PostgreSQL 12", "12", env.TestPostgres12Port},
|
||||
{"PostgreSQL 13", "13", env.TestPostgres13Port},
|
||||
{"PostgreSQL 14", "14", env.TestPostgres14Port},
|
||||
{"PostgreSQL 15", "15", env.TestPostgres15Port},
|
||||
{"PostgreSQL 16", "16", env.TestPostgres16Port},
|
||||
{"PostgreSQL 17", "17", env.TestPostgres17Port},
|
||||
{"PostgreSQL 18", "18", env.TestPostgres18Port},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
testBackupRestoreWithEncryptionForVersion(t, tc.version, tc.port)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {
|
||||
// Connect to pre-configured PostgreSQL container
|
||||
container, err := connectToPostgresContainer(pgVersion, port)
|
||||
assert.NoError(t, err)
|
||||
defer func() {
|
||||
@@ -101,55 +126,30 @@ func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {
|
||||
_, err = container.DB.Exec(createAndFillTableQuery)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Prepare data for backup
|
||||
backupID := uuid.New()
|
||||
router := createTestRouter()
|
||||
user := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", user, router)
|
||||
|
||||
storage := storages.CreateTestStorage(workspace.ID)
|
||||
|
||||
pgVersionEnum := tools.GetPostgresqlVersionEnum(pgVersion)
|
||||
|
||||
backupDb := &databases.Database{
|
||||
ID: uuid.New(),
|
||||
Type: databases.DatabaseTypePostgres,
|
||||
Name: "Test Database",
|
||||
Postgresql: &pgtypes.PostgresqlDatabase{
|
||||
Version: pgVersionEnum,
|
||||
Host: container.Host,
|
||||
Port: container.Port,
|
||||
Username: container.Username,
|
||||
Password: container.Password,
|
||||
Database: &container.Database,
|
||||
IsHttps: false,
|
||||
},
|
||||
}
|
||||
|
||||
storageID := uuid.New()
|
||||
backupConfig := &backups_config.BackupConfig{
|
||||
DatabaseID: backupDb.ID,
|
||||
IsBackupsEnabled: true,
|
||||
StorePeriod: period.PeriodDay,
|
||||
BackupInterval: &intervals.Interval{Interval: intervals.IntervalDaily},
|
||||
StorageID: &storageID,
|
||||
CpuCount: 1,
|
||||
}
|
||||
|
||||
storage := &storages.Storage{
|
||||
WorkspaceID: uuid.New(),
|
||||
Type: storages.StorageTypeLocal,
|
||||
Name: "Test Storage",
|
||||
LocalStorage: &local_storage.LocalStorage{},
|
||||
}
|
||||
|
||||
// Make backup
|
||||
progressTracker := func(completedMBs float64) {}
|
||||
err = usecases_postgresql_backup.GetCreatePostgresqlBackupUsecase().Execute(
|
||||
context.Background(),
|
||||
backupID,
|
||||
backupConfig,
|
||||
backupDb,
|
||||
storage,
|
||||
progressTracker,
|
||||
database := createDatabaseViaAPI(
|
||||
t, router, "Test Database", workspace.ID,
|
||||
pgVersionEnum, container.Host, container.Port,
|
||||
container.Username, container.Password, container.Database,
|
||||
user.Token,
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Create new database
|
||||
enableBackupsViaAPI(
|
||||
t, router, database.ID, storage.ID,
|
||||
backups_config.BackupEncryptionNone, user.Token,
|
||||
)
|
||||
|
||||
createBackupViaAPI(t, router, database.ID, user.Token)
|
||||
|
||||
backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute)
|
||||
assert.Equal(t, backups.BackupStatusCompleted, backup.Status)
|
||||
|
||||
newDBName := "restoreddb"
|
||||
_, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName))
|
||||
assert.NoError(t, err)
|
||||
@@ -157,43 +157,22 @@ func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {
|
||||
_, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName))
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Connect to the new database
|
||||
newDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||||
container.Host, container.Port, container.Username, container.Password, newDBName)
|
||||
newDB, err := sqlx.Connect("postgres", newDSN)
|
||||
assert.NoError(t, err)
|
||||
defer newDB.Close()
|
||||
|
||||
// Setup data for restore
|
||||
completedBackup := &backups.Backup{
|
||||
ID: backupID,
|
||||
DatabaseID: backupDb.ID,
|
||||
StorageID: storage.ID,
|
||||
Status: backups.BackupStatusCompleted,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
}
|
||||
createRestoreViaAPI(
|
||||
t, router, backup.ID, pgVersionEnum,
|
||||
container.Host, container.Port,
|
||||
container.Username, container.Password, newDBName,
|
||||
user.Token,
|
||||
)
|
||||
|
||||
restoreID := uuid.New()
|
||||
restore := models.Restore{
|
||||
ID: restoreID,
|
||||
Backup: completedBackup,
|
||||
Postgresql: &pgtypes.PostgresqlDatabase{
|
||||
Version: pgVersionEnum,
|
||||
Host: container.Host,
|
||||
Port: container.Port,
|
||||
Username: container.Username,
|
||||
Password: container.Password,
|
||||
Database: &newDBName,
|
||||
IsHttps: false,
|
||||
},
|
||||
}
|
||||
restore := waitForRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute)
|
||||
assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status)
|
||||
|
||||
// Restore the backup
|
||||
restoreBackupUC := usecases_postgresql_restore.GetRestorePostgresqlBackupUsecase()
|
||||
err = restoreBackupUC.Execute(backupDb, backupConfig, restore, completedBackup, storage)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify restored table exists
|
||||
var tableExists bool
|
||||
err = newDB.Get(
|
||||
&tableExists,
|
||||
@@ -202,17 +181,329 @@ func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, tableExists, "Table 'test_data' should exist in restored database")
|
||||
|
||||
// Verify data integrity
|
||||
verifyDataIntegrity(t, container.DB, newDB)
|
||||
|
||||
// Clean up the backup file after the test
|
||||
err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backupID.String()))
|
||||
err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))
|
||||
if err != nil {
|
||||
t.Logf("Warning: Failed to delete backup file: %v", err)
|
||||
}
|
||||
|
||||
test_utils.MakeDeleteRequest(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/databases/"+database.ID.String(),
|
||||
"Bearer "+user.Token,
|
||||
http.StatusNoContent,
|
||||
)
|
||||
storages.RemoveTestStorage(storage.ID)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}
|
||||
|
||||
func testBackupRestoreWithEncryptionForVersion(t *testing.T, pgVersion string, port string) {
|
||||
container, err := connectToPostgresContainer(pgVersion, port)
|
||||
assert.NoError(t, err)
|
||||
defer func() {
|
||||
if container.DB != nil {
|
||||
container.DB.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
_, err = container.DB.Exec(createAndFillTableQuery)
|
||||
assert.NoError(t, err)
|
||||
|
||||
router := createTestRouter()
|
||||
user := users_testing.CreateTestUser(users_enums.UserRoleMember)
|
||||
workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", user, router)
|
||||
|
||||
storage := storages.CreateTestStorage(workspace.ID)
|
||||
|
||||
pgVersionEnum := tools.GetPostgresqlVersionEnum(pgVersion)
|
||||
database := createDatabaseViaAPI(
|
||||
t, router, "Test Database", workspace.ID,
|
||||
pgVersionEnum, container.Host, container.Port,
|
||||
container.Username, container.Password, container.Database,
|
||||
user.Token,
|
||||
)
|
||||
|
||||
enableBackupsViaAPI(
|
||||
t, router, database.ID, storage.ID,
|
||||
backups_config.BackupEncryptionEncrypted, user.Token,
|
||||
)
|
||||
|
||||
createBackupViaAPI(t, router, database.ID, user.Token)
|
||||
|
||||
backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute)
|
||||
assert.Equal(t, backups.BackupStatusCompleted, backup.Status)
|
||||
assert.Equal(t, backups_config.BackupEncryptionEncrypted, backup.Encryption)
|
||||
|
||||
newDBName := "restoreddb_encrypted"
|
||||
_, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName))
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName))
|
||||
assert.NoError(t, err)
|
||||
|
||||
newDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||||
container.Host, container.Port, container.Username, container.Password, newDBName)
|
||||
newDB, err := sqlx.Connect("postgres", newDSN)
|
||||
assert.NoError(t, err)
|
||||
defer newDB.Close()
|
||||
|
||||
createRestoreViaAPI(
|
||||
t, router, backup.ID, pgVersionEnum,
|
||||
container.Host, container.Port,
|
||||
container.Username, container.Password, newDBName,
|
||||
user.Token,
|
||||
)
|
||||
|
||||
restore := waitForRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute)
|
||||
assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status)
|
||||
|
||||
var tableExists bool
|
||||
err = newDB.Get(
|
||||
&tableExists,
|
||||
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'test_data')",
|
||||
)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, tableExists, "Table 'test_data' should exist in restored database")
|
||||
|
||||
verifyDataIntegrity(t, container.DB, newDB)
|
||||
|
||||
err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))
|
||||
if err != nil {
|
||||
t.Logf("Warning: Failed to delete backup file: %v", err)
|
||||
}
|
||||
|
||||
test_utils.MakeDeleteRequest(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/databases/"+database.ID.String(),
|
||||
"Bearer "+user.Token,
|
||||
http.StatusNoContent,
|
||||
)
|
||||
storages.RemoveTestStorage(storage.ID)
|
||||
workspaces_testing.RemoveTestWorkspace(workspace, router)
|
||||
}
|
||||
|
||||
func createTestRouter() *gin.Engine {
|
||||
router := workspaces_testing.CreateTestRouter(
|
||||
workspaces_controllers.GetWorkspaceController(),
|
||||
workspaces_controllers.GetMembershipController(),
|
||||
databases.GetDatabaseController(),
|
||||
backups_config.GetBackupConfigController(),
|
||||
backups.GetBackupController(),
|
||||
restores.GetRestoreController(),
|
||||
)
|
||||
return router
|
||||
}
|
||||
|
||||
func waitForBackupCompletion(
|
||||
t *testing.T,
|
||||
router *gin.Engine,
|
||||
databaseID uuid.UUID,
|
||||
token string,
|
||||
timeout time.Duration,
|
||||
) *backups.Backup {
|
||||
startTime := time.Now()
|
||||
pollInterval := 500 * time.Millisecond
|
||||
|
||||
for {
|
||||
if time.Since(startTime) > timeout {
|
||||
t.Fatalf("Timeout waiting for backup completion after %v", timeout)
|
||||
}
|
||||
|
||||
var response backups.GetBackupsResponse
|
||||
test_utils.MakeGetRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/backups?database_id=%s&limit=1", databaseID.String()),
|
||||
"Bearer "+token,
|
||||
http.StatusOK,
|
||||
&response,
|
||||
)
|
||||
|
||||
if len(response.Backups) > 0 {
|
||||
backup := response.Backups[0]
|
||||
if backup.Status == backups.BackupStatusCompleted {
|
||||
return backup
|
||||
}
|
||||
if backup.Status == backups.BackupStatusFailed {
|
||||
t.Fatalf("Backup failed: %v", backup.FailMessage)
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(pollInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForRestoreCompletion(
|
||||
t *testing.T,
|
||||
router *gin.Engine,
|
||||
backupID uuid.UUID,
|
||||
token string,
|
||||
timeout time.Duration,
|
||||
) *restores_models.Restore {
|
||||
startTime := time.Now()
|
||||
pollInterval := 500 * time.Millisecond
|
||||
|
||||
for {
|
||||
if time.Since(startTime) > timeout {
|
||||
t.Fatalf("Timeout waiting for restore completion after %v", timeout)
|
||||
}
|
||||
|
||||
var restores []*restores_models.Restore
|
||||
test_utils.MakeGetRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/restores/%s", backupID.String()),
|
||||
"Bearer "+token,
|
||||
http.StatusOK,
|
||||
&restores,
|
||||
)
|
||||
|
||||
for _, restore := range restores {
|
||||
if restore.Status == restores_enums.RestoreStatusCompleted {
|
||||
return restore
|
||||
}
|
||||
if restore.Status == restores_enums.RestoreStatusFailed {
|
||||
t.Fatalf("Restore failed: %v", restore.FailMessage)
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(pollInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func createDatabaseViaAPI(
|
||||
t *testing.T,
|
||||
router *gin.Engine,
|
||||
name string,
|
||||
workspaceID uuid.UUID,
|
||||
pgVersion tools.PostgresqlVersion,
|
||||
host string,
|
||||
port int,
|
||||
username string,
|
||||
password string,
|
||||
database string,
|
||||
token string,
|
||||
) *databases.Database {
|
||||
request := databases.Database{
|
||||
Name: name,
|
||||
WorkspaceID: &workspaceID,
|
||||
Type: databases.DatabaseTypePostgres,
|
||||
Postgresql: &pgtypes.PostgresqlDatabase{
|
||||
Version: pgVersion,
|
||||
Host: host,
|
||||
Port: port,
|
||||
Username: username,
|
||||
Password: password,
|
||||
Database: &database,
|
||||
},
|
||||
}
|
||||
|
||||
w := workspaces_testing.MakeAPIRequest(
|
||||
router,
|
||||
"POST",
|
||||
"/api/v1/databases/create",
|
||||
"Bearer "+token,
|
||||
request,
|
||||
)
|
||||
|
||||
if w.Code != http.StatusCreated {
|
||||
t.Fatalf("Failed to create database. Status: %d, Body: %s", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
var createdDatabase databases.Database
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &createdDatabase); err != nil {
|
||||
t.Fatalf("Failed to unmarshal database response: %v", err)
|
||||
}
|
||||
|
||||
return &createdDatabase
|
||||
}
|
||||
|
||||
func enableBackupsViaAPI(
|
||||
t *testing.T,
|
||||
router *gin.Engine,
|
||||
databaseID uuid.UUID,
|
||||
storageID uuid.UUID,
|
||||
encryption backups_config.BackupEncryption,
|
||||
token string,
|
||||
) {
|
||||
var backupConfig backups_config.BackupConfig
|
||||
test_utils.MakeGetRequestAndUnmarshal(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/backup-configs/database/%s", databaseID.String()),
|
||||
"Bearer "+token,
|
||||
http.StatusOK,
|
||||
&backupConfig,
|
||||
)
|
||||
|
||||
storage := &storages.Storage{ID: storageID}
|
||||
backupConfig.IsBackupsEnabled = true
|
||||
backupConfig.Storage = storage
|
||||
backupConfig.Encryption = encryption
|
||||
|
||||
test_utils.MakePostRequest(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/backup-configs/save",
|
||||
"Bearer "+token,
|
||||
backupConfig,
|
||||
http.StatusOK,
|
||||
)
|
||||
}
|
||||
|
||||
func createBackupViaAPI(
|
||||
t *testing.T,
|
||||
router *gin.Engine,
|
||||
databaseID uuid.UUID,
|
||||
token string,
|
||||
) {
|
||||
request := backups.MakeBackupRequest{DatabaseID: databaseID}
|
||||
test_utils.MakePostRequest(
|
||||
t,
|
||||
router,
|
||||
"/api/v1/backups",
|
||||
"Bearer "+token,
|
||||
request,
|
||||
http.StatusOK,
|
||||
)
|
||||
}
|
||||
|
||||
func createRestoreViaAPI(
|
||||
t *testing.T,
|
||||
router *gin.Engine,
|
||||
backupID uuid.UUID,
|
||||
pgVersion tools.PostgresqlVersion,
|
||||
host string,
|
||||
port int,
|
||||
username string,
|
||||
password string,
|
||||
database string,
|
||||
token string,
|
||||
) {
|
||||
request := restores.RestoreBackupRequest{
|
||||
PostgresqlDatabase: &pgtypes.PostgresqlDatabase{
|
||||
Version: pgVersion,
|
||||
Host: host,
|
||||
Port: port,
|
||||
Username: username,
|
||||
Password: password,
|
||||
Database: &database,
|
||||
},
|
||||
}
|
||||
|
||||
test_utils.MakePostRequest(
|
||||
t,
|
||||
router,
|
||||
fmt.Sprintf("/api/v1/restores/%s/restore", backupID.String()),
|
||||
"Bearer "+token,
|
||||
request,
|
||||
http.StatusOK,
|
||||
)
|
||||
}
|
||||
|
||||
// verifyDataIntegrity compares data in the original and restored databases
|
||||
func verifyDataIntegrity(t *testing.T, originalDB *sqlx.DB, restoredDB *sqlx.DB) {
|
||||
var originalData []TestDataItem
|
||||
var restoredData []TestDataItem
|
||||
@@ -225,7 +516,6 @@ func verifyDataIntegrity(t *testing.T, originalDB *sqlx.DB, restoredDB *sqlx.DB)
|
||||
|
||||
assert.Equal(t, len(originalData), len(restoredData), "Should have same number of rows")
|
||||
|
||||
// Only compare data if both slices have elements (to avoid panic)
|
||||
if len(originalData) > 0 && len(restoredData) > 0 {
|
||||
for i := range originalData {
|
||||
assert.Equal(t, originalData[i].ID, restoredData[i].ID, "ID should match")
|
||||
|
||||
12
backend/internal/features/users/repositories/di.go
Normal file
@@ -0,0 +1,12 @@
|
||||
package users_repositories
|
||||
|
||||
var userRepository = &UserRepository{}
|
||||
var usersSettingsRepository = &UsersSettingsRepository{}
|
||||
|
||||
func GetUserRepository() *UserRepository {
|
||||
return userRepository
|
||||
}
|
||||
|
||||
func GetUsersSettingsRepository() *UsersSettingsRepository {
|
||||
return usersSettingsRepository
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
package users_repositories
|
||||
|
||||
import (
|
||||
"errors"
|
||||
user_models "postgresus-backend/internal/features/users/models"
|
||||
"postgresus-backend/internal/storage"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type SecretKeyRepository struct{}
|
||||
|
||||
func (r *SecretKeyRepository) GetSecretKey() (string, error) {
|
||||
var secretKey user_models.SecretKey
|
||||
|
||||
if err := storage.
|
||||
GetDb().
|
||||
First(&secretKey).Error; err != nil {
|
||||
// create a new secret key if not found
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
newSecretKey := user_models.SecretKey{
|
||||
Secret: uuid.New().String() + uuid.New().String(),
|
||||
}
|
||||
if err := storage.GetDb().Create(&newSecretKey).Error; err != nil {
|
||||
return "", errors.New("failed to create new secret key")
|
||||
}
|
||||
|
||||
return newSecretKey.Secret, nil
|
||||
}
|
||||
|
||||
return "", err
|
||||
}
|
||||
|
||||
return secretKey.Secret, nil
|
||||
}
|
||||
@@ -1,25 +1,22 @@
|
||||
package users_services
|
||||
|
||||
import (
|
||||
user_repositories "postgresus-backend/internal/features/users/repositories"
|
||||
"postgresus-backend/internal/features/encryption/secrets"
|
||||
users_repositories "postgresus-backend/internal/features/users/repositories"
|
||||
)
|
||||
|
||||
var secretKeyRepository = &user_repositories.SecretKeyRepository{}
|
||||
var userRepository = &user_repositories.UserRepository{}
|
||||
var usersSettingsRepository = &user_repositories.UsersSettingsRepository{}
|
||||
|
||||
var userService = &UserService{
|
||||
userRepository,
|
||||
secretKeyRepository,
|
||||
users_repositories.GetUserRepository(),
|
||||
secrets.GetSecretKeyService(),
|
||||
settingsService,
|
||||
nil,
|
||||
}
|
||||
var settingsService = &SettingsService{
|
||||
usersSettingsRepository,
|
||||
users_repositories.GetUsersSettingsRepository(),
|
||||
nil,
|
||||
}
|
||||
var managementService = &UserManagementService{
|
||||
userRepository,
|
||||
users_repositories.GetUserRepository(),
|
||||
nil,
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"golang.org/x/oauth2/google"
|
||||
|
||||
"postgresus-backend/internal/config"
|
||||
"postgresus-backend/internal/features/encryption/secrets"
|
||||
users_dto "postgresus-backend/internal/features/users/dto"
|
||||
users_enums "postgresus-backend/internal/features/users/enums"
|
||||
users_interfaces "postgresus-backend/internal/features/users/interfaces"
|
||||
@@ -25,10 +26,10 @@ import (
|
||||
)
|
||||
|
||||
type UserService struct {
|
||||
userRepository *users_repositories.UserRepository
|
||||
secretKeyRepository *users_repositories.SecretKeyRepository
|
||||
settingsService *SettingsService
|
||||
auditLogWriter users_interfaces.AuditLogWriter
|
||||
userRepository *users_repositories.UserRepository
|
||||
secretKeyService *secrets.SecretKeyService
|
||||
settingsService *SettingsService
|
||||
auditLogWriter users_interfaces.AuditLogWriter
|
||||
}
|
||||
|
||||
func (s *UserService) SetAuditLogWriter(writer users_interfaces.AuditLogWriter) {
|
||||
@@ -162,7 +163,7 @@ func (s *UserService) SignIn(
|
||||
}
|
||||
|
||||
func (s *UserService) GetUserFromToken(token string) (*users_models.User, error) {
|
||||
secretKey, err := s.secretKeyRepository.GetSecretKey()
|
||||
secretKey, err := s.secretKeyService.GetSecretKey()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get secret key: %w", err)
|
||||
}
|
||||
@@ -221,7 +222,7 @@ func (s *UserService) GetUserFromToken(token string) (*users_models.User, error)
|
||||
func (s *UserService) GenerateAccessToken(
|
||||
user *users_models.User,
|
||||
) (*users_dto.SignInResponseDTO, error) {
|
||||
secretKey, err := s.secretKeyRepository.GetSecretKey()
|
||||
secretKey, err := s.secretKeyService.GetSecretKey()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get secret key: %w", err)
|
||||
}
|
||||
@@ -309,15 +310,6 @@ func (s *UserService) ChangeUserPasswordByEmail(email string, newPassword string
|
||||
}
|
||||
|
||||
func (s *UserService) ChangeUserPassword(userID uuid.UUID, newPassword string) error {
|
||||
user, err := s.userRepository.GetUserByID(userID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get user: %w", err)
|
||||
}
|
||||
|
||||
if !user.HasPassword() {
|
||||
return errors.New("user has no password set")
|
||||
}
|
||||
|
||||
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(newPassword), bcrypt.DefaultCost)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to hash new password: %w", err)
|
||||
|
||||
11
backend/internal/util/encryption/di.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package encryption
|
||||
|
||||
import "postgresus-backend/internal/features/encryption/secrets"
|
||||
|
||||
var fieldEncryptor = &SecretKeyFieldEncryptor{
|
||||
secrets.GetSecretKeyService(),
|
||||
}
|
||||
|
||||
func GetFieldEncryptor() FieldEncryptor {
|
||||
return fieldEncryptor
|
||||
}
|
||||
15
backend/internal/util/encryption/field_encryptor.go
Normal file
@@ -0,0 +1,15 @@
|
||||
package encryption
|
||||
|
||||
import "github.com/google/uuid"
|
||||
|
||||
type FieldEncryptor interface {
|
||||
// Encrypt encrypts a plaintext string and returns an encrypted string.
|
||||
// If the string is already encrypted, returns it as-is.
|
||||
// Empty strings are returned unchanged.
|
||||
Encrypt(itemID uuid.UUID, plaintext string) (string, error)
|
||||
|
||||
// Decrypt decrypts an encrypted string and returns a plaintext string.
|
||||
// If the string is not encrypted, returns it as-is.
|
||||
// Empty strings are returned unchanged.
|
||||
Decrypt(itemID uuid.UUID, ciphertext string) (string, error)
|
||||
}
|
||||
120
backend/internal/util/encryption/secret_key_field_encryptor.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package encryption
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"postgresus-backend/internal/features/encryption/secrets"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const encryptedPrefix = "enc:"
|
||||
|
||||
type SecretKeyFieldEncryptor struct {
|
||||
secretKeyService *secrets.SecretKeyService
|
||||
}
|
||||
|
||||
func (e *SecretKeyFieldEncryptor) Encrypt(itemID uuid.UUID, plaintext string) (string, error) {
|
||||
if plaintext == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
if e.isEncrypted(plaintext) {
|
||||
return plaintext, nil
|
||||
}
|
||||
|
||||
masterKey, err := e.secretKeyService.GetSecretKey()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get master key: %w", err)
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher([]byte(masterKey)[:32])
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create cipher: %w", err)
|
||||
}
|
||||
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create GCM: %w", err)
|
||||
}
|
||||
|
||||
nonce := e.deriveNonce(itemID, masterKey, gcm.NonceSize())
|
||||
|
||||
ciphertext := gcm.Seal(nil, nonce, []byte(plaintext), nil)
|
||||
|
||||
nonceBase64 := base64.StdEncoding.EncodeToString(nonce)
|
||||
ciphertextBase64 := base64.StdEncoding.EncodeToString(ciphertext)
|
||||
|
||||
return fmt.Sprintf("%s%s:%s", encryptedPrefix, nonceBase64, ciphertextBase64), nil
|
||||
}
|
||||
|
||||
func (e *SecretKeyFieldEncryptor) Decrypt(itemID uuid.UUID, ciphertext string) (string, error) {
|
||||
if ciphertext == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
if !e.isEncrypted(ciphertext) {
|
||||
return ciphertext, nil
|
||||
}
|
||||
|
||||
parts := strings.SplitN(ciphertext, ":", 3)
|
||||
if len(parts) != 3 {
|
||||
return "", errors.New("invalid encrypted format")
|
||||
}
|
||||
|
||||
nonceBase64 := parts[1]
|
||||
ciphertextBase64 := parts[2]
|
||||
|
||||
nonce, err := base64.StdEncoding.DecodeString(nonceBase64)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to decode nonce: %w", err)
|
||||
}
|
||||
|
||||
encryptedData, err := base64.StdEncoding.DecodeString(ciphertextBase64)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to decode ciphertext: %w", err)
|
||||
}
|
||||
|
||||
masterKey, err := e.secretKeyService.GetSecretKey()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get master key: %w", err)
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher([]byte(masterKey)[:32])
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create cipher: %w", err)
|
||||
}
|
||||
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create GCM: %w", err)
|
||||
}
|
||||
|
||||
plaintext, err := gcm.Open(nil, nonce, encryptedData, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to decrypt: %w", err)
|
||||
}
|
||||
|
||||
return string(plaintext), nil
|
||||
}
|
||||
|
||||
func (e *SecretKeyFieldEncryptor) isEncrypted(value string) bool {
|
||||
return strings.HasPrefix(value, encryptedPrefix)
|
||||
}
|
||||
|
||||
func (e *SecretKeyFieldEncryptor) deriveNonce(
|
||||
itemID uuid.UUID,
|
||||
masterKey string,
|
||||
nonceSize int,
|
||||
) []byte {
|
||||
h := hmac.New(sha256.New, []byte(masterKey))
|
||||
h.Write(itemID[:])
|
||||
hash := h.Sum(nil)
|
||||
return hash[:nonceSize]
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package encryption
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func Test_Encrypt_Decrypt_RoundTrip(t *testing.T) {
|
||||
encryptor := GetFieldEncryptor()
|
||||
itemID := uuid.New()
|
||||
plaintext := "my-secret-password"
|
||||
|
||||
encrypted, err := encryptor.Encrypt(itemID, plaintext)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, encrypted)
|
||||
assert.NotEqual(t, plaintext, encrypted)
|
||||
assert.Contains(t, encrypted, "enc:")
|
||||
|
||||
decrypted, err := encryptor.Decrypt(itemID, encrypted)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, plaintext, decrypted)
|
||||
}
|
||||
|
||||
func Test_Encrypt_EmptyString_ReturnsEmpty(t *testing.T) {
|
||||
encryptor := GetFieldEncryptor()
|
||||
itemID := uuid.New()
|
||||
|
||||
encrypted, err := encryptor.Encrypt(itemID, "")
|
||||
assert.NoError(t, err)
|
||||
assert.Empty(t, encrypted)
|
||||
}
|
||||
|
||||
func Test_Decrypt_EmptyString_ReturnsEmpty(t *testing.T) {
|
||||
encryptor := GetFieldEncryptor()
|
||||
itemID := uuid.New()
|
||||
|
||||
decrypted, err := encryptor.Decrypt(itemID, "")
|
||||
assert.NoError(t, err)
|
||||
assert.Empty(t, decrypted)
|
||||
}
|
||||
|
||||
func Test_Decrypt_PlaintextValue_ReturnsAsIs(t *testing.T) {
|
||||
encryptor := GetFieldEncryptor()
|
||||
itemID := uuid.New()
|
||||
plaintext := "not-encrypted-password"
|
||||
|
||||
decrypted, err := encryptor.Decrypt(itemID, plaintext)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, plaintext, decrypted)
|
||||
}
|
||||
|
||||
func Test_Encrypt_DetectsAlreadyEncryptedFormat(t *testing.T) {
|
||||
encryptor := GetFieldEncryptor()
|
||||
itemID := uuid.New()
|
||||
alreadyEncrypted := "enc:nonce:ciphertext"
|
||||
|
||||
result, err := encryptor.Encrypt(itemID, alreadyEncrypted)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, alreadyEncrypted, result)
|
||||
}
|
||||
|
||||
func Test_Encrypt_SamePlaintext_DifferentItemIDs_ProducesDifferentCiphertext(t *testing.T) {
|
||||
encryptor := GetFieldEncryptor()
|
||||
plaintext := "shared-secret"
|
||||
itemID1 := uuid.New()
|
||||
itemID2 := uuid.New()
|
||||
|
||||
encrypted1, err := encryptor.Encrypt(itemID1, plaintext)
|
||||
assert.NoError(t, err)
|
||||
|
||||
encrypted2, err := encryptor.Encrypt(itemID2, plaintext)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.NotEqual(t, encrypted1, encrypted2)
|
||||
|
||||
decrypted1, err := encryptor.Decrypt(itemID1, encrypted1)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, plaintext, decrypted1)
|
||||
|
||||
decrypted2, err := encryptor.Decrypt(itemID2, encrypted2)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, plaintext, decrypted2)
|
||||
}
|
||||
|
||||
func Test_Encrypt_AlreadyEncrypted_ReturnsAsIs(t *testing.T) {
|
||||
encryptor := GetFieldEncryptor()
|
||||
itemID := uuid.New()
|
||||
plaintext := "my-password"
|
||||
|
||||
encrypted1, err := encryptor.Encrypt(itemID, plaintext)
|
||||
assert.NoError(t, err)
|
||||
|
||||
encrypted2, err := encryptor.Encrypt(itemID, encrypted1)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, encrypted1, encrypted2)
|
||||
}
|
||||
|
||||
func Test_Decrypt_MalformedData_ReturnsError(t *testing.T) {
|
||||
encryptor := GetFieldEncryptor()
|
||||
itemID := uuid.New()
|
||||
|
||||
_, err := encryptor.Decrypt(itemID, "enc:invalid")
|
||||
assert.Error(t, err)
|
||||
|
||||
_, err = encryptor.Decrypt(itemID, "enc:invalid:invalid-base64")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func Test_EncryptedFormat_ContainsPrefix(t *testing.T) {
|
||||
encryptor := GetFieldEncryptor()
|
||||
itemID := uuid.New()
|
||||
plaintext := "test-secret"
|
||||
|
||||
encrypted, err := encryptor.Encrypt(itemID, plaintext)
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, encrypted, "enc:")
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
env_utils "postgresus-backend/internal/util/env"
|
||||
)
|
||||
@@ -151,6 +152,24 @@ func VerifyPostgresesInstallation(
|
||||
logger.Info("All PostgreSQL version-specific client tools verification completed successfully!")
|
||||
}
|
||||
|
||||
// EscapePgpassField escapes special characters in a field value for .pgpass file format.
|
||||
// According to PostgreSQL documentation, the .pgpass file format requires:
|
||||
// - Backslash (\) must be escaped as \\
|
||||
// - Colon (:) must be escaped as \:
|
||||
// Additionally, newlines and carriage returns are removed to prevent format corruption.
|
||||
func EscapePgpassField(field string) string {
|
||||
// Remove newlines and carriage returns that would break .pgpass format
|
||||
field = strings.ReplaceAll(field, "\r", "")
|
||||
field = strings.ReplaceAll(field, "\n", "")
|
||||
|
||||
// Escape backslashes first (order matters!)
|
||||
// Then escape colons
|
||||
field = strings.ReplaceAll(field, "\\", "\\\\")
|
||||
field = strings.ReplaceAll(field, ":", "\\:")
|
||||
|
||||
return field
|
||||
}
|
||||
|
||||
func getPostgresqlBasePath(
|
||||
version PostgresqlVersion,
|
||||
envMode env_utils.EnvMode,
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
|
||||
ALTER TABLE backup_configs
|
||||
ADD COLUMN encryption TEXT NOT NULL DEFAULT 'NONE';
|
||||
|
||||
ALTER TABLE backups
|
||||
ADD COLUMN encryption_salt TEXT,
|
||||
ADD COLUMN encryption_iv TEXT,
|
||||
ADD COLUMN encryption TEXT NOT NULL DEFAULT 'NONE';
|
||||
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
|
||||
ALTER TABLE backups
|
||||
DROP COLUMN IF EXISTS encryption,
|
||||
DROP COLUMN IF EXISTS encryption_iv,
|
||||
DROP COLUMN IF EXISTS encryption_salt;
|
||||
|
||||
ALTER TABLE backup_configs
|
||||
DROP COLUMN IF EXISTS encryption;
|
||||
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,28 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
|
||||
ALTER TABLE postgresql_databases
|
||||
DROP CONSTRAINT IF EXISTS fk_postgresql_databases_restore_id;
|
||||
|
||||
DROP INDEX IF EXISTS idx_postgresql_databases_restore_id;
|
||||
|
||||
ALTER TABLE postgresql_databases
|
||||
DROP COLUMN IF EXISTS restore_id;
|
||||
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
|
||||
ALTER TABLE postgresql_databases
|
||||
ADD COLUMN restore_id UUID;
|
||||
|
||||
CREATE INDEX idx_postgresql_databases_restore_id ON postgresql_databases (restore_id);
|
||||
|
||||
ALTER TABLE postgresql_databases
|
||||
ADD CONSTRAINT fk_postgresql_databases_restore_id
|
||||
FOREIGN KEY (restore_id)
|
||||
REFERENCES restores (id)
|
||||
ON DELETE CASCADE;
|
||||
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,18 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
|
||||
ALTER TABLE webhook_notifiers
|
||||
ADD COLUMN body_template TEXT,
|
||||
ADD COLUMN headers TEXT DEFAULT '[]';
|
||||
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
|
||||
ALTER TABLE webhook_notifiers
|
||||
DROP COLUMN body_template,
|
||||
DROP COLUMN headers;
|
||||
|
||||
-- +goose StatementEnd
|
||||
|
||||
23
deploy/helm/.helmignore
Normal file
@@ -0,0 +1,23 @@
|
||||
# Patterns to ignore when building packages.
|
||||
# This supports shell glob matching, relative path matching, and
|
||||
# negation (prefixed with !). Only one pattern per line.
|
||||
.DS_Store
|
||||
# Common VCS dirs
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
# Common backup files
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*.orig
|
||||
*~
|
||||
# Various IDEs
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
||||
.vscode/
|
||||
22
deploy/helm/Chart.yaml
Normal file
@@ -0,0 +1,22 @@
|
||||
apiVersion: v2
|
||||
name: postgresus
|
||||
description: A Helm chart for Postgresus - PostgreSQL backup and management system
|
||||
type: application
|
||||
version: 0.0.0
|
||||
appVersion: "latest"
|
||||
keywords:
|
||||
- postgresql
|
||||
- backup
|
||||
- database
|
||||
- restore
|
||||
home: https://github.com/RostislavDugin/postgresus
|
||||
|
||||
sources:
|
||||
- https://github.com/RostislavDugin/postgresus
|
||||
- https://github.com/RostislavDugin/postgresus/tree/main/deploy/helm
|
||||
|
||||
maintainers:
|
||||
- name: Rostislav Dugin
|
||||
url: https://github.com/RostislavDugin
|
||||
|
||||
icon: https://raw.githubusercontent.com/RostislavDugin/postgresus/main/frontend/public/logo.svg
|
||||
222
deploy/helm/README.md
Normal file
@@ -0,0 +1,222 @@
|
||||
# Postgresus Helm Chart
|
||||
|
||||
## Installation
|
||||
|
||||
Install directly from the OCI registry (no need to clone the repository):
|
||||
|
||||
```bash
|
||||
helm install postgresus oci://ghcr.io/rostislavdugin/charts/postgresus \
|
||||
-n postgresus --create-namespace
|
||||
```
|
||||
|
||||
The `-n postgresus --create-namespace` flags control which namespace the chart is installed into. You can use any namespace name you prefer.
|
||||
|
||||
## Accessing Postgresus
|
||||
|
||||
By default, the chart creates a ClusterIP service. Use port-forward to access:
|
||||
|
||||
```bash
|
||||
kubectl port-forward svc/postgresus-service 4005:4005 -n postgresus
|
||||
```
|
||||
|
||||
Then open `http://localhost:4005` in your browser.
|
||||
|
||||
## Configuration
|
||||
|
||||
### Main Parameters
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
| ------------------ | ------------------ | --------------------------- |
|
||||
| `image.repository` | Docker image | `rostislavdugin/postgresus` |
|
||||
| `image.tag` | Image tag | `latest` |
|
||||
| `image.pullPolicy` | Image pull policy | `Always` |
|
||||
| `replicaCount` | Number of replicas | `1` |
|
||||
|
||||
### Service
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
| -------------------------- | ----------------------- | ------------- |
|
||||
| `service.type` | Service type | `ClusterIP` |
|
||||
| `service.port` | Service port | `4005` |
|
||||
| `service.targetPort` | Container port | `4005` |
|
||||
| `service.headless.enabled` | Enable headless service | `true` |
|
||||
|
||||
### Storage
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
| ------------------------------ | ------------------------- | ---------------------- |
|
||||
| `persistence.enabled` | Enable persistent storage | `true` |
|
||||
| `persistence.storageClassName` | Storage class | `""` (cluster default) |
|
||||
| `persistence.accessMode` | Access mode | `ReadWriteOnce` |
|
||||
| `persistence.size` | Storage size | `10Gi` |
|
||||
| `persistence.mountPath` | Mount path | `/postgresus-data` |
|
||||
|
||||
### Resources
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
| --------------------------- | -------------- | ------------- |
|
||||
| `resources.requests.memory` | Memory request | `1Gi` |
|
||||
| `resources.requests.cpu` | CPU request | `500m` |
|
||||
| `resources.limits.memory` | Memory limit | `1Gi` |
|
||||
| `resources.limits.cpu` | CPU limit | `500m` |
|
||||
|
||||
## External Access Options
|
||||
|
||||
### Option 1: Port Forward (Default)
|
||||
|
||||
Best for development or quick access:
|
||||
|
||||
```bash
|
||||
kubectl port-forward svc/postgresus-service 4005:4005 -n postgresus
|
||||
```
|
||||
|
||||
Access at `http://localhost:4005`
|
||||
|
||||
### Option 2: NodePort
|
||||
|
||||
For direct access via node IP:
|
||||
|
||||
```yaml
|
||||
# nodeport-values.yaml
|
||||
service:
|
||||
type: NodePort
|
||||
port: 4005
|
||||
targetPort: 4005
|
||||
nodePort: 30080
|
||||
```
|
||||
|
||||
```bash
|
||||
helm install postgresus oci://ghcr.io/rostislavdugin/charts/postgresus \
|
||||
-n postgresus --create-namespace \
|
||||
-f nodeport-values.yaml
|
||||
```
|
||||
|
||||
Access at `http://<NODE-IP>:30080`
|
||||
|
||||
### Option 3: LoadBalancer
|
||||
|
||||
For cloud environments with load balancer support:
|
||||
|
||||
```yaml
|
||||
# loadbalancer-values.yaml
|
||||
service:
|
||||
type: LoadBalancer
|
||||
port: 80
|
||||
targetPort: 4005
|
||||
```
|
||||
|
||||
```bash
|
||||
helm install postgresus oci://ghcr.io/rostislavdugin/charts/postgresus \
|
||||
-n postgresus --create-namespace \
|
||||
-f loadbalancer-values.yaml
|
||||
```
|
||||
|
||||
Get the external IP:
|
||||
|
||||
```bash
|
||||
kubectl get svc -n postgresus
|
||||
```
|
||||
|
||||
Access at `http://<EXTERNAL-IP>`
|
||||
|
||||
### Option 4: Ingress
|
||||
|
||||
For domain-based access with TLS:
|
||||
|
||||
```yaml
|
||||
# ingress-values.yaml
|
||||
ingress:
|
||||
enabled: true
|
||||
className: nginx
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/ssl-redirect: "true"
|
||||
cert-manager.io/cluster-issuer: "letsencrypt-prod"
|
||||
hosts:
|
||||
- host: backup.example.com
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
tls:
|
||||
- secretName: backup-example-com-tls
|
||||
hosts:
|
||||
- backup.example.com
|
||||
```
|
||||
|
||||
```bash
|
||||
helm install postgresus oci://ghcr.io/rostislavdugin/charts/postgresus \
|
||||
-n postgresus --create-namespace \
|
||||
-f ingress-values.yaml
|
||||
```
|
||||
|
||||
### Option 5: HTTPRoute (Gateway API)
|
||||
|
||||
For clusters using Istio, Envoy Gateway, Cilium, or other Gateway API implementations:
|
||||
|
||||
```yaml
|
||||
# httproute-values.yaml
|
||||
route:
|
||||
enabled: true
|
||||
hostnames:
|
||||
- backup.example.com
|
||||
parentRefs:
|
||||
- name: my-gateway
|
||||
namespace: istio-system
|
||||
```
|
||||
|
||||
```bash
|
||||
helm install postgresus oci://ghcr.io/rostislavdugin/charts/postgresus \
|
||||
-n postgresus --create-namespace \
|
||||
-f httproute-values.yaml
|
||||
```
|
||||
|
||||
## Ingress Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
| ----------------------- | ----------------- | ------------------------ |
|
||||
| `ingress.enabled` | Enable Ingress | `false` |
|
||||
| `ingress.className` | Ingress class | `nginx` |
|
||||
| `ingress.hosts[0].host` | Hostname | `postgresus.example.com` |
|
||||
| `ingress.tls` | TLS configuration | `[]` |
|
||||
|
||||
## HTTPRoute Configuration
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
| ------------------ | ----------------------- | ------------------------------ |
|
||||
| `route.enabled` | Enable HTTPRoute | `false` |
|
||||
| `route.apiVersion` | Gateway API version | `gateway.networking.k8s.io/v1` |
|
||||
| `route.hostnames` | Hostnames for the route | `["postgresus.example.com"]` |
|
||||
| `route.parentRefs` | Gateway references | `[]` |
|
||||
|
||||
## Health Checks
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
| ------------------------ | ---------------------- | ------------- |
|
||||
| `livenessProbe.enabled` | Enable liveness probe | `true` |
|
||||
| `readinessProbe.enabled` | Enable readiness probe | `true` |
|
||||
|
||||
## Custom Storage Size
|
||||
|
||||
```yaml
|
||||
# storage-values.yaml
|
||||
persistence:
|
||||
size: 50Gi
|
||||
storageClassName: "fast-ssd"
|
||||
```
|
||||
|
||||
```bash
|
||||
helm install postgresus oci://ghcr.io/rostislavdugin/charts/postgresus \
|
||||
-n postgresus --create-namespace \
|
||||
-f storage-values.yaml
|
||||
```
|
||||
|
||||
## Upgrade
|
||||
|
||||
```bash
|
||||
helm upgrade postgresus oci://ghcr.io/rostislavdugin/charts/postgresus -n postgresus
|
||||
```
|
||||
|
||||
## Uninstall
|
||||
|
||||
```bash
|
||||
helm uninstall postgresus -n postgresus
|
||||
```
|
||||
68
deploy/helm/templates/_helpers.tpl
Normal file
@@ -0,0 +1,68 @@
|
||||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "postgresus.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
*/}}
|
||||
{{- define "postgresus.fullname" -}}
|
||||
{{- if .Values.fullnameOverride }}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "postgresus.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "postgresus.labels" -}}
|
||||
helm.sh/chart: {{ include "postgresus.chart" . }}
|
||||
{{ include "postgresus.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels
|
||||
*/}}
|
||||
{{- define "postgresus.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "postgresus.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app: postgresus
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create the name of the service account to use
|
||||
*/}}
|
||||
{{- define "postgresus.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount.create }}
|
||||
{{- default (include "postgresus.fullname" .) .Values.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Namespace - uses the release namespace from helm install -n <namespace>
|
||||
*/}}
|
||||
{{- define "postgresus.namespace" -}}
|
||||
{{- .Release.Namespace }}
|
||||
{{- end }}
|
||||
35
deploy/helm/templates/httproute.yaml
Normal file
@@ -0,0 +1,35 @@
|
||||
{{- if .Values.route.enabled -}}
|
||||
apiVersion: {{ .Values.route.apiVersion}}
|
||||
kind: {{ .Values.route.kind}}
|
||||
metadata:
|
||||
name: {{ template "postgresus.fullname" . }}
|
||||
annotations: {{ toYaml .Values.route.annotations | nindent 4 }}
|
||||
labels:
|
||||
app.kubernetes.io/component: "app"
|
||||
{{- include "postgresus.labels" . | nindent 4 }}
|
||||
spec:
|
||||
{{- with .Values.route.parentRefs }}
|
||||
parentRefs:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.route.hostnames }}
|
||||
hostnames:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
- backendRefs:
|
||||
- name: {{ template "postgresus.fullname" . }}-service
|
||||
port: {{ .Values.service.port }}
|
||||
{{- with .Values.route.filters }}
|
||||
filters:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.route.matches }}
|
||||
matches:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.route.timeouts }}
|
||||
timeouts:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
42
deploy/helm/templates/ingress.yaml
Normal file
@@ -0,0 +1,42 @@
|
||||
{{- if .Values.ingress.enabled -}}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "postgresus.fullname" . }}-ingress
|
||||
namespace: {{ include "postgresus.namespace" . }}
|
||||
labels:
|
||||
{{- include "postgresus.labels" . | nindent 4 }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.ingress.className }}
|
||||
ingressClassName: {{ .Values.ingress.className }}
|
||||
{{- end }}
|
||||
{{- if .Values.ingress.tls }}
|
||||
tls:
|
||||
{{- range .Values.ingress.tls }}
|
||||
- hosts:
|
||||
{{- range .hosts }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
secretName: {{ .secretName }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.ingress.hosts }}
|
||||
- host: {{ .host | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- range .paths }}
|
||||
- path: {{ .path }}
|
||||
pathType: {{ .pathType }}
|
||||
backend:
|
||||
service:
|
||||
name: {{ include "postgresus.fullname" $ }}-service
|
||||
port:
|
||||
number: {{ $.Values.service.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
36
deploy/helm/templates/service.yaml
Normal file
@@ -0,0 +1,36 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "postgresus.fullname" . }}-service
|
||||
namespace: {{ include "postgresus.namespace" . }}
|
||||
labels:
|
||||
{{- include "postgresus.labels" . | nindent 4 }}
|
||||
spec:
|
||||
type: {{ .Values.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.service.port }}
|
||||
targetPort: {{ .Values.service.targetPort }}
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "postgresus.selectorLabels" . | nindent 4 }}
|
||||
---
|
||||
{{- if .Values.service.headless.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "postgresus.fullname" . }}-headless
|
||||
namespace: {{ include "postgresus.namespace" . }}
|
||||
labels:
|
||||
{{- include "postgresus.labels" . | nindent 4 }}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
clusterIP: None
|
||||
ports:
|
||||
- port: {{ .Values.service.port }}
|
||||
targetPort: {{ .Values.service.targetPort }}
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "postgresus.selectorLabels" . | nindent 4 }}
|
||||
{{- end }}
|
||||
84
deploy/helm/templates/statefulset.yaml
Normal file
@@ -0,0 +1,84 @@
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ include "postgresus.fullname" . }}
|
||||
namespace: {{ include "postgresus.namespace" . }}
|
||||
labels:
|
||||
{{- include "postgresus.labels" . | nindent 4 }}
|
||||
spec:
|
||||
serviceName: {{ include "postgresus.fullname" . }}-headless
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "postgresus.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "postgresus.selectorLabels" . | nindent 8 }}
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- with .Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.affinity }}
|
||||
affinity:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: {{ .Chart.Name }}
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: {{ .Values.service.targetPort }}
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: postgresus-storage
|
||||
mountPath: {{ .Values.persistence.mountPath }}
|
||||
resources:
|
||||
{{- toYaml .Values.resources | nindent 12 }}
|
||||
{{- if .Values.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
{{- toYaml .Values.livenessProbe.httpGet | nindent 14 }}
|
||||
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
|
||||
{{- end }}
|
||||
{{- if .Values.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
{{- toYaml .Values.readinessProbe.httpGet | nindent 14 }}
|
||||
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
|
||||
{{- end }}
|
||||
{{- if .Values.persistence.enabled }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: postgresus-storage
|
||||
spec:
|
||||
accessModes:
|
||||
- {{ .Values.persistence.accessMode }}
|
||||
{{- if .Values.persistence.storageClassName }}
|
||||
storageClassName: {{ .Values.persistence.storageClassName }}
|
||||
{{- end }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.persistence.size }}
|
||||
{{- end }}
|
||||
updateStrategy:
|
||||
{{- toYaml .Values.updateStrategy | nindent 4 }}
|
||||
101
deploy/helm/values.yaml
Normal file
@@ -0,0 +1,101 @@
|
||||
# Default values for postgresus
|
||||
|
||||
# Image configuration
|
||||
image:
|
||||
repository: rostislavdugin/postgresus
|
||||
tag: null
|
||||
pullPolicy: Always
|
||||
|
||||
# StatefulSet configuration
|
||||
replicaCount: 1
|
||||
|
||||
# Service configuration
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 4005 # Service port
|
||||
targetPort: 4005 # Internal container port
|
||||
# Headless service for StatefulSet
|
||||
headless:
|
||||
enabled: true
|
||||
|
||||
# Resource limits and requests
|
||||
resources:
|
||||
requests:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "500m"
|
||||
|
||||
# Persistent storage configuration
|
||||
persistence:
|
||||
enabled: true
|
||||
# Storage class name. Leave empty to use cluster default.
|
||||
# Examples: "longhorn", "standard", "gp2", etc.
|
||||
storageClassName: ""
|
||||
accessMode: ReadWriteOnce
|
||||
size: 10Gi
|
||||
# Mount path in container
|
||||
mountPath: /postgresus-data
|
||||
|
||||
# Ingress configuration (disabled by default - using LoadBalancer instead)
|
||||
ingress:
|
||||
enabled: false
|
||||
className: nginx
|
||||
annotations: {}
|
||||
hosts:
|
||||
- host: postgresus.example.com
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
tls: []
|
||||
|
||||
# HTTPRoute configuration for Gateway API
|
||||
route:
|
||||
enabled: false
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
annotations: {}
|
||||
hostnames:
|
||||
- postgresus.example.com
|
||||
parentRefs: []
|
||||
filters: []
|
||||
matches: []
|
||||
timeouts: {}
|
||||
|
||||
# Health checks configuration
|
||||
# Note: The application only has /api/v1/system/health endpoint
|
||||
livenessProbe:
|
||||
enabled: true
|
||||
httpGet:
|
||||
path: /api/v1/system/health
|
||||
port: 4005
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 3
|
||||
|
||||
readinessProbe:
|
||||
enabled: true
|
||||
httpGet:
|
||||
path: /api/v1/system/health
|
||||
port: 4005
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 3
|
||||
|
||||
# StatefulSet update strategy
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
partition: 0
|
||||
|
||||
# Pod labels and annotations
|
||||
podLabels: {}
|
||||
podAnnotations: {}
|
||||
|
||||
# Node selector, tolerations and affinity
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
@@ -3,7 +3,10 @@
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/logo.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta
|
||||
name="viewport"
|
||||
content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"
|
||||
/>
|
||||
<meta name="robots" content="noindex" />
|
||||
<title>Postgresus - PostgreSQL backups</title>
|
||||
|
||||
|
||||
|
Before Width: | Height: | Size: 32 KiB After Width: | Height: | Size: 32 KiB |
@@ -1,24 +1,12 @@
|
||||
<svg width="1000" height="1000" viewBox="0 0 1000 1000" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g clip-path="url(#clip0_221_160)">
|
||||
<path d="M213 149H388L725.5 233.5L856.5 602.5L811.5 685L733 650L679.5 631.5L581 591.5L492.5 570.5L430.5 545.5L371.5 494.5L166.5 386.5L213 149Z" fill="#155DFC"/>
|
||||
<path d="M66.5436 450.256C242.608 569.316 275.646 627.237 321.721 606.141C333.899 600.56 366.664 580.46 376.871 491.549C373.963 491.413 368.225 491.432 361.922 493.872C322.424 509.249 323.556 596.384 303.865 596.481C303.553 596.481 302.87 596.481 301.992 596.228C300.06 595.876 297.679 595.135 294.108 593.476C283.667 588.597 278.437 586.138 270.514 581.357C270.514 581.357 262.415 576.537 255.039 571.385C244.149 563.774 237.924 554.602 225.708 543.635C215.482 534.463 215.345 536.863 191.478 519.339C183.263 513.308 177.233 508.547 165.504 499.258C146.243 484.016 132.934 473.478 121.576 463.662C107.369 451.387 88.1466 433.258 66.8558 408.103C47.4969 371.708 24.4303 333.868 12.7213 337.674C5.24705 340.113 4.09568 358.535 5.53979 370.888C10.36 412.553 50.6584 439.542 66.5436 450.275V450.256Z" fill="#003A86"/>
|
||||
<path d="M86.9766 439.249C181.683 457.672 276.369 476.113 371.076 494.535C382.082 511.572 393.752 522.169 401.734 528.394C401.734 528.394 448.823 565.101 520.326 562.369C523.761 562.233 532.289 561.569 543.881 562.818C555.785 564.106 557.853 565.375 576.763 568.79C596.2 572.302 598.386 571.756 606.153 574.761C615.188 578.255 621.589 582.626 626.078 585.729C633.513 590.861 635.269 593.359 649.496 605.654C657.38 612.464 661.322 615.879 664.932 618.611C671.567 623.627 682.203 631.608 697.327 638.048C705.933 641.698 714.402 644.196 722.443 646.557C726.853 647.845 730.561 648.84 733.195 649.504C734.522 662.227 735.459 680.337 733.195 701.823C730.834 724.285 729.195 739.936 719.75 757.636C715.554 765.501 700.313 792.978 667.43 804.979C660.795 807.399 640.441 814.503 616.106 807.965C607.929 805.78 592.746 801.467 580.725 787.045C568.391 772.233 567.084 756.016 566.772 751.177C566.167 741.868 565.328 729.008 574.246 718.294C581.291 709.844 593.039 704.38 604.143 705.336C617.433 706.487 621.101 716.128 630.371 713.708C634.801 712.557 634.762 710.156 646.978 697.374C657.985 685.86 660.541 685.158 660.931 681.138C661.849 671.497 648.383 662.93 642.997 659.495C638.977 656.939 629.142 651.299 611.598 649.035C601.586 647.747 582.169 645.464 559.785 654.519C532.484 665.545 518.804 686.114 514.452 692.885C503.016 710.605 500.245 727.212 498.508 738.219C497.552 744.327 494.937 763.666 500.011 788.04C507.114 822.231 524.756 844.595 530.396 851.328C554.868 880.522 584.335 891.314 598.152 896.173C662.766 918.908 720.959 895.061 732.181 890.201C793.848 863.524 821.325 811.575 834.829 786.069C860.492 737.575 863.75 693.959 867.224 647.552C868.395 632.018 871.03 586.314 861.253 527.964C854.286 486.476 844.138 457.086 833.346 425.823C810.826 360.585 786.022 313.944 778.041 299.269C778.041 299.269 746.075 240.548 692.838 177.69C682.456 165.435 671.996 153.901 654.979 144.3C644.129 138.192 624.614 129.469 574.265 128.356C548.798 127.79 512.735 129.098 469.626 137.821C471.792 141.139 473.939 144.456 476.105 147.793C464.24 141.568 446.969 133.333 425.288 125.878C403.9 118.521 387.702 115.164 362.001 109.934C321.82 101.757 291.572 98.2838 284.273 97.4837C251.644 93.8539 227.758 93.1318 203.559 94.9858C188.259 96.1567 174.267 98.2447 159.709 106.441C150.459 111.652 144.019 117.623 136.291 124.883C121.342 138.914 111.878 151.657 107.389 157.765C75.5994 201.011 59.7142 222.653 44.6096 247.944C44.6096 247.944 33.6227 266.347 15.22 306.743C12.2538 313.241 7.12134 324.775 5.87238 340.64C5.67723 343.08 5.05274 351.003 6.26267 358.574C9.32652 377.816 23.5724 390.911 38.1501 403.42C70.3693 431.073 92.9677 446.763 92.9677 446.763C96.617 449.202 131.92 473.908 162.226 497.58C211.697 536.239 236.422 555.578 251.566 569.219C251.566 569.219 264.192 580.596 289.757 592.13C297.367 595.564 303.261 597.594 310.189 596.482C320.22 594.862 326.523 587.544 332.631 580.284C334.836 577.65 339.383 571.893 353.922 539.381C357.532 531.321 359.328 527.281 361.513 521.817C363.328 517.29 367.778 505.893 372.656 489.325C376.501 476.308 384.112 450.51 388.424 421.823C392.347 395.77 398.026 358.028 387.917 310.704C387 306.45 376.95 261.117 348.731 218.359C334.036 196.093 319.712 182.53 316.668 179.661C307.457 171.055 298.851 164.674 292.255 160.224C298.675 160.537 307.476 161.395 317.663 163.718C325.176 165.415 345.374 170.626 367.485 185.145C389.107 199.332 402.065 215.549 407.842 223.511C428.254 251.632 433.66 279.539 436.997 296.751C440.9 316.91 441.915 337.752 442.735 354.047C443.691 373.64 443.008 379.592 445.232 393.916C446.755 403.752 449.409 420.164 458.015 439.249C470.953 467.936 489.531 485.402 496.537 491.569C509.007 502.517 520.131 508.157 542.378 519.475C560.995 528.94 573.134 532.98 586.716 534.424C592.668 535.048 597.683 535.048 601.157 534.931C597.722 526.794 595.576 519.515 594.19 513.758C593 508.84 591.165 501.034 590.697 490.593C590.326 482.358 590.951 475.372 591.692 470.161C593.156 476.504 595.342 484.719 598.659 494.086C600.259 498.594 606.036 514.441 618.096 534.951C627.053 550.212 637.904 563.618 655.955 582.294C673.167 600.072 698.517 623.959 733.176 649.562C623.326 589.359 513.496 529.155 403.646 468.951" fill="#155DFC"/>
|
||||
<path d="M562.652 325.145C573.171 318.588 582.07 313.885 588.568 310.704C594.501 307.777 597.565 306.645 601.194 305.728C603.712 305.084 611.811 303.249 622.29 304.674C626.642 305.279 632.77 306.177 639.736 310.041C648.069 314.666 652.577 320.774 653.69 322.335C655.017 324.209 657.905 328.326 659.661 334.298C660.754 337.947 660.52 339.313 661.984 342.611C663.525 346.065 665.223 347.9 667.955 351.256C670.16 353.949 673.166 357.852 676.6 362.887C674.785 361.872 665.594 356.896 656.012 357.072C651.68 357.15 647.367 358.243 647.367 358.243C643.893 359.121 641.571 360.214 640.732 360.565C632.984 363.785 615.284 359.511 608.181 347.939C603.692 340.64 606.834 335.918 601.507 331.156C599.536 329.4 596.94 328.092 585.27 326.687C579.748 326.023 572.039 325.321 562.691 325.184L562.652 325.145Z" fill="#003C8D"/>
|
||||
<path d="M820.797 391.692C811.059 390.95 795.467 391.009 777.572 396.726C763.833 401.117 753.373 407.343 746.348 412.338C752.319 411.031 760.086 409.509 769.258 408.182C771.444 407.87 778.489 406.874 788.363 406.016C797.906 405.196 810.552 404.435 825.715 404.513C824.076 400.239 822.437 395.965 820.797 391.672V391.692Z" fill="#0052C9"/>
|
||||
<path d="M841.795 450.627C830.925 450.256 815.625 450.997 798.218 456.032C786.177 459.506 776.283 464.15 768.711 468.405C774.019 467.624 780.693 466.765 788.401 466.043C790.977 465.809 798.549 465.107 808.834 464.677C818.455 464.267 831.12 464.033 846.244 464.599C844.761 459.955 843.278 455.291 841.814 450.646L841.795 450.627Z" fill="#0052C9"/>
|
||||
<path d="M855.848 500.39C845.895 499.824 829.952 500.331 812.583 507.65C805.694 510.557 799.938 513.953 795.273 517.231C798.415 516.607 802.748 515.846 807.919 515.143C818.808 513.699 827.141 513.445 835.494 513.231C841.524 513.075 849.369 513.016 858.6 513.328L855.848 500.37V500.39Z" fill="#0052C9"/>
|
||||
<path d="M570.282 597.028C599.399 605.985 630.096 616.016 657.475 624.934C674.434 630.457 688.289 635.043 697.344 638.048C665.281 603.663 633.238 569.297 601.174 534.912C596.471 534.853 589.095 534.502 580.254 532.824C573.971 531.633 559.627 528.394 529.769 512.157C516.031 504.683 509.161 500.956 502.526 496.038C469.975 471.917 457.427 437.142 453.836 426.799C450.656 417.607 448.47 397.116 444.235 356.525C438.732 303.893 439.552 300.537 435.766 285.276C432.819 273.392 426.203 245.329 407.859 219.511C401.81 210.983 378.099 178.607 333.858 164.927C315.378 159.209 299.141 158.565 288.271 158.956C297.034 165.298 309.016 174.86 321.662 188.111C327.926 194.668 354.408 223.16 373.493 269.859C379.465 284.457 389.866 310.489 392.423 346.085C394.316 372.371 391.154 392.94 390.432 397.409C385.671 426.877 376.264 444.87 371.854 486.436C371.483 489.91 371.23 492.759 371.093 494.555C377.318 504.059 383.368 511.142 387.973 516.06C392.442 520.822 396.092 523.964 397.224 525.076C397.224 525.076 403.605 530.716 410.923 535.614C444.43 558.095 570.282 597.028 570.282 597.028Z" fill="#0051C8"/>
|
||||
<path d="M591.456 468.893C590.109 479.079 589.289 494.594 593.427 512.626C595.495 521.602 598.013 527.886 600.881 534.912C607.555 551.304 615.908 571.619 634.018 591.915C647.424 606.941 661.007 616.035 671.916 623.393C676.56 626.515 682.786 630.691 691.47 635.16C708.175 643.786 723.24 648.001 733.759 650.226C732.139 643.434 730.5 636.663 728.88 629.872C712.097 621.968 691.86 610.435 671.155 593.476C663.622 587.29 658.177 582.157 655.64 579.718C646.019 570.448 627.792 552.729 612.512 524.881C600.53 503.044 594.597 483.021 591.456 468.854V468.893Z" fill="#00398B"/>
|
||||
<path d="M680.074 799.203C689.988 795.768 704.312 789.367 718.519 777.307C731.438 766.34 738.912 755.236 741.937 750.396C750.816 736.248 754.251 723.836 756.631 715.015C759.305 705.121 762.154 690.485 761.861 672.414C765.569 674.502 770.819 677.215 777.298 679.888C791.524 685.743 803.175 687.85 810.434 689.099C824.914 691.617 836.272 691.773 845.054 691.831C852.469 691.89 858.695 691.617 863.222 691.324C863.749 686.64 864.257 681.898 864.725 677.117C865.72 667.047 866.54 657.153 867.223 647.474C861.134 647.396 854.733 647.142 848.039 646.674C842.927 646.303 837.989 645.815 833.267 645.269C826.846 646.147 818.24 647.006 808.053 647.064C778.586 647.201 757.178 640.507 751.87 638.77C741.917 635.511 733.955 631.862 728.452 629.052C729.428 632.682 730.696 637.853 731.808 644.117C733.135 651.553 736.238 671.458 733.174 701.784C731.125 722.06 730.091 732.423 725.368 744.854C721.309 755.567 710.322 778.888 680.035 799.144L680.074 799.203Z" fill="#0050C8"/>
|
||||
<path d="M493.94 571.483C489.179 567.034 474.133 552.846 461.233 537.625C458.15 533.995 453.017 527.867 446.89 519.183C444.86 516.295 442.733 513.153 440.547 509.718C435.395 501.62 428.682 490.925 422.789 475.626C419.139 466.141 416.934 457.847 415.568 451.7C411.88 469.225 412.465 483.783 413.578 493.306C414.846 504.274 417.246 512.255 419.315 518.968C421.54 526.208 424.857 535.497 429.931 545.918C436.937 549.49 442.967 552.28 447.553 554.31C460.453 560.028 470.756 563.677 477.45 566.019C484.144 568.38 489.823 570.195 493.96 571.483H493.94Z" fill="#003C8D"/>
|
||||
<path d="M668.582 578.488C670.768 569.18 674.593 562.798 677.13 559.227C684.135 549.333 692.507 545.586 701.855 541.293C709.407 537.819 715.437 535.165 720.999 537.78C722.014 538.248 722.482 538.658 728.337 542.425C730.366 543.771 733.333 545.703 736.982 547.908C746.525 553.685 784.618 575.854 840.177 582.294C877.822 586.665 906.762 581.864 923.467 578.957C957.599 573.024 985.271 562.545 1005 553.392C994.15 582.001 979.26 598.999 970.186 607.8C935.371 641.58 892.731 644.976 865.839 647.591C859.79 648.176 838.518 649.757 809.344 645.269C745.822 635.511 697.503 602.824 668.582 578.508V578.488Z" fill="#8BC7FE"/>
|
||||
<path d="M664.369 591.27C671.004 599.311 678.791 607.839 687.826 616.425C700.764 628.72 713.391 638.38 724.631 645.912C724.475 642.595 723.89 638.341 722.172 633.794C721.665 632.467 721.119 631.257 720.572 630.144C726.017 633.15 732.008 636.018 738.487 638.536C769.477 650.537 798.183 649.386 817.874 646.225C791.353 642.653 756.792 634.262 720.143 614.318C707.361 607.351 695.964 599.877 685.953 592.441C685.484 592.09 681.874 589.202 676.8 585.221C673.404 582.567 670.614 580.381 668.779 578.957C667.296 583.074 665.813 587.172 664.33 591.29L664.369 591.27Z" fill="#00398B"/>
|
||||
<path d="M992.766 578.937C975.417 607.683 953.073 622.065 944.135 627.237C936.231 631.803 924.366 637.56 908.91 641.092C908.715 641.151 888.81 646.4 853.956 648.235C848.863 648.508 843.887 648.567 843.887 648.567C843.887 648.567 837.759 648.645 831.436 648.391C776.54 646.283 723.362 623.88 723.362 623.88C715.146 620.641 701.642 614.864 687.747 606.434C675.511 599.018 668.74 593.066 665.149 589.709C663.432 588.09 662.046 586.704 661.109 585.728C668.447 572.77 675.785 559.832 683.142 546.874C690.011 553.743 700.198 563.228 713.468 573.024C732.007 586.704 785.107 625.949 857.059 623.587C937.5 620.972 991.069 568.028 1005 553.333C1002.6 559.851 998.777 568.926 992.747 578.898L992.766 578.937Z" fill="#0087F7"/>
|
||||
<path d="M649.906 574.117C651.233 571.268 653.146 567.423 655.761 563.111C656.971 561.12 661.323 553.997 666.533 548.416C670.67 543.986 682.047 531.789 698.733 531.282C708.119 530.989 715.847 534.502 721.019 537.761C717.604 538.073 712.94 538.736 707.534 540.239C701.796 541.859 695.279 543.323 689.502 547.909C681.462 554.29 677.013 562.896 668.582 580.791C666.884 584.421 665.577 587.426 664.757 589.358C661.596 586.294 658.415 583.094 655.195 579.737C653.38 577.845 651.624 575.971 649.906 574.098V574.117Z" fill="#0051CB"/>
|
||||
<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g clip-path="url(#clip0_286_152)">
|
||||
<path d="M12.5378 28.7972C12.5378 30.2972 14.2889 30.2982 14.7498 29.4999C15.0384 28.9999 15.2498 28.4999 15.2498 26.9999C15.2498 25.4999 14.5258 24.2384 13.8296 22.8794C13.6504 22.5296 13.4578 22.1537 13.253 21.7309C12.9016 21.0056 12.4714 20.3367 11.8469 19.8035C11.7511 19.7217 11.7231 19.5023 11.7404 19.3555C11.8085 18.7759 11.8827 18.1969 11.9568 17.6179C12.0794 16.6609 12.202 15.7039 12.2975 14.7442C12.3645 14.0718 12.3633 13.378 12.2786 12.7082C12.0956 11.2586 11.2821 10.1946 10.0335 9.47786C9.6702 9.26926 9.56307 9.03637 9.60844 8.64632C9.73481 7.56183 10.1635 6.58265 10.7083 5.66605C11.2087 4.82425 11.7757 4.02144 12.3417 3.2201C12.4142 3.11747 12.4866 3.01486 12.5589 2.9122C12.6515 2.78077 12.8805 2.68113 13.049 2.67687C15.103 2.62543 17.1573 2.59121 19.2117 2.56589C19.3528 2.56407 19.5394 2.62989 19.6303 2.73115C20.9118 4.1585 22.0569 5.678 22.7442 7.49459C22.7912 7.61894 22.8303 7.74673 22.8661 7.87492C23.1241 8.79206 23.1199 8.79508 22.3848 9.31247L22.3515 9.33589C20.9673 10.3108 20.4612 11.705 20.5228 13.3373C20.5933 15.2138 21.0153 17.0328 21.4965 18.8385C21.5943 19.2054 21.6577 19.4768 21.3049 19.8152C20.5985 20.4924 20.3047 21.4218 20.1641 22.3672C20.0179 23.352 19.8995 24.3409 19.7811 25.3299C19.7154 25.8788 19.7122 26.3421 19.5792 26.9759C19.0702 29.4029 17.7498 31.9999 15.7498 31.9999H12.5378C11.2498 31.9999 10.2498 30.7972 10.2498 28.7972H12.5378Z" fill="#155DFC"/>
|
||||
<path d="M11.5607 1.64008C10.8347 2.775 10.091 3.85078 9.44034 4.98022C8.85324 5.9991 8.46218 7.10566 8.27283 8.28027C8.02494 9.8182 8.4235 11.1797 9.51913 12.2444C10.4211 13.1209 10.6538 14.1118 10.538 15.2919C10.2857 17.8647 9.77936 20.385 9.05131 22.863C9.04422 22.8869 9.02214 22.9064 8.97435 22.9744C8.53792 22.8381 8.07901 22.7111 7.63165 22.5524C1.38253 20.3344 -3.18062 16.2382 -6.04606 10.2575C-6.29374 9.74063 -6.51975 9.21246 -6.73158 8.68004C-6.77189 8.57878 -6.74374 8.38234 -6.66961 8.31328C-4.28698 6.09307 -1.91002 3.86617 0.50117 1.67735C2.16142 0.170403 4.14529 -0.303895 6.31143 0.184986C7.98444 0.562481 9.62828 1.06959 11.2841 1.52262C11.3961 1.5532 11.5 1.61376 11.5607 1.64008Z" fill="#155DFC"/>
|
||||
<path d="M24.2397 22.3314C23.8972 21.1864 23.5199 20.1003 23.2528 18.9878C22.9016 17.5244 22.6171 16.0438 22.3435 14.5632C22.1888 13.7264 22.35 12.9544 22.9725 12.3027C24.5667 10.6339 24.5983 8.69929 23.8328 6.67592C23.1868 4.96849 22.1482 3.49759 20.9256 2.15226C20.7825 1.9947 20.6423 1.8341 20.4795 1.65123C20.5718 1.6006 20.6427 1.54693 20.7227 1.52C22.532 0.910212 24.3624 0.387106 26.2602 0.122211C28.1953 -0.147949 29.8448 0.454544 31.2612 1.74398C32.5042 2.87545 33.7011 4.05816 34.9529 5.17971C35.9556 6.07788 37.0032 6.92664 38.0476 7.77681C38.2361 7.93012 38.2841 8.05386 38.2282 8.27602C37.2646 12.1148 35.328 15.3721 32.3643 17.9969C30.0283 20.0657 27.3376 21.4946 24.2397 22.3316V22.3314Z" fill="#155DFC"/>
|
||||
</g>
|
||||
<defs>
|
||||
<clipPath id="clip0_221_160">
|
||||
<rect width="1000" height="1000" fill="white"/>
|
||||
<clipPath id="clip0_286_152">
|
||||
<rect width="32" height="32" rx="6" fill="white"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 3.0 KiB |
@@ -1,24 +1,12 @@
|
||||
<svg width="1000" height="1000" viewBox="0 0 1000 1000" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g clip-path="url(#clip0_221_160)">
|
||||
<path d="M213 149H388L725.5 233.5L856.5 602.5L811.5 685L733 650L679.5 631.5L581 591.5L492.5 570.5L430.5 545.5L371.5 494.5L166.5 386.5L213 149Z" fill="#155DFC"/>
|
||||
<path d="M66.5436 450.256C242.608 569.316 275.646 627.237 321.721 606.141C333.899 600.56 366.664 580.46 376.871 491.549C373.963 491.413 368.225 491.432 361.922 493.872C322.424 509.249 323.556 596.384 303.865 596.481C303.553 596.481 302.87 596.481 301.992 596.228C300.06 595.876 297.679 595.135 294.108 593.476C283.667 588.597 278.437 586.138 270.514 581.357C270.514 581.357 262.415 576.537 255.039 571.385C244.149 563.774 237.924 554.602 225.708 543.635C215.482 534.463 215.345 536.863 191.478 519.339C183.263 513.308 177.233 508.547 165.504 499.258C146.243 484.016 132.934 473.478 121.576 463.662C107.369 451.387 88.1466 433.258 66.8558 408.103C47.4969 371.708 24.4303 333.868 12.7213 337.674C5.24705 340.113 4.09568 358.535 5.53979 370.888C10.36 412.553 50.6584 439.542 66.5436 450.275V450.256Z" fill="#003A86"/>
|
||||
<path d="M86.9766 439.249C181.683 457.672 276.369 476.113 371.076 494.535C382.082 511.572 393.752 522.169 401.734 528.394C401.734 528.394 448.823 565.101 520.326 562.369C523.761 562.233 532.289 561.569 543.881 562.818C555.785 564.106 557.853 565.375 576.763 568.79C596.2 572.302 598.386 571.756 606.153 574.761C615.188 578.255 621.589 582.626 626.078 585.729C633.513 590.861 635.269 593.359 649.496 605.654C657.38 612.464 661.322 615.879 664.932 618.611C671.567 623.627 682.203 631.608 697.327 638.048C705.933 641.698 714.402 644.196 722.443 646.557C726.853 647.845 730.561 648.84 733.195 649.504C734.522 662.227 735.459 680.337 733.195 701.823C730.834 724.285 729.195 739.936 719.75 757.636C715.554 765.501 700.313 792.978 667.43 804.979C660.795 807.399 640.441 814.503 616.106 807.965C607.929 805.78 592.746 801.467 580.725 787.045C568.391 772.233 567.084 756.016 566.772 751.177C566.167 741.868 565.328 729.008 574.246 718.294C581.291 709.844 593.039 704.38 604.143 705.336C617.433 706.487 621.101 716.128 630.371 713.708C634.801 712.557 634.762 710.156 646.978 697.374C657.985 685.86 660.541 685.158 660.931 681.138C661.849 671.497 648.383 662.93 642.997 659.495C638.977 656.939 629.142 651.299 611.598 649.035C601.586 647.747 582.169 645.464 559.785 654.519C532.484 665.545 518.804 686.114 514.452 692.885C503.016 710.605 500.245 727.212 498.508 738.219C497.552 744.327 494.937 763.666 500.011 788.04C507.114 822.231 524.756 844.595 530.396 851.328C554.868 880.522 584.335 891.314 598.152 896.173C662.766 918.908 720.959 895.061 732.181 890.201C793.848 863.524 821.325 811.575 834.829 786.069C860.492 737.575 863.75 693.959 867.224 647.552C868.395 632.018 871.03 586.314 861.253 527.964C854.286 486.476 844.138 457.086 833.346 425.823C810.826 360.585 786.022 313.944 778.041 299.269C778.041 299.269 746.075 240.548 692.838 177.69C682.456 165.435 671.996 153.901 654.979 144.3C644.129 138.192 624.614 129.469 574.265 128.356C548.798 127.79 512.735 129.098 469.626 137.821C471.792 141.139 473.939 144.456 476.105 147.793C464.24 141.568 446.969 133.333 425.288 125.878C403.9 118.521 387.702 115.164 362.001 109.934C321.82 101.757 291.572 98.2838 284.273 97.4837C251.644 93.8539 227.758 93.1318 203.559 94.9858C188.259 96.1567 174.267 98.2447 159.709 106.441C150.459 111.652 144.019 117.623 136.291 124.883C121.342 138.914 111.878 151.657 107.389 157.765C75.5994 201.011 59.7142 222.653 44.6096 247.944C44.6096 247.944 33.6227 266.347 15.22 306.743C12.2538 313.241 7.12134 324.775 5.87238 340.64C5.67723 343.08 5.05274 351.003 6.26267 358.574C9.32652 377.816 23.5724 390.911 38.1501 403.42C70.3693 431.073 92.9677 446.763 92.9677 446.763C96.617 449.202 131.92 473.908 162.226 497.58C211.697 536.239 236.422 555.578 251.566 569.219C251.566 569.219 264.192 580.596 289.757 592.13C297.367 595.564 303.261 597.594 310.189 596.482C320.22 594.862 326.523 587.544 332.631 580.284C334.836 577.65 339.383 571.893 353.922 539.381C357.532 531.321 359.328 527.281 361.513 521.817C363.328 517.29 367.778 505.893 372.656 489.325C376.501 476.308 384.112 450.51 388.424 421.823C392.347 395.77 398.026 358.028 387.917 310.704C387 306.45 376.95 261.117 348.731 218.359C334.036 196.093 319.712 182.53 316.668 179.661C307.457 171.055 298.851 164.674 292.255 160.224C298.675 160.537 307.476 161.395 317.663 163.718C325.176 165.415 345.374 170.626 367.485 185.145C389.107 199.332 402.065 215.549 407.842 223.511C428.254 251.632 433.66 279.539 436.997 296.751C440.9 316.91 441.915 337.752 442.735 354.047C443.691 373.64 443.008 379.592 445.232 393.916C446.755 403.752 449.409 420.164 458.015 439.249C470.953 467.936 489.531 485.402 496.537 491.569C509.007 502.517 520.131 508.157 542.378 519.475C560.995 528.94 573.134 532.98 586.716 534.424C592.668 535.048 597.683 535.048 601.157 534.931C597.722 526.794 595.576 519.515 594.19 513.758C593 508.84 591.165 501.034 590.697 490.593C590.326 482.358 590.951 475.372 591.692 470.161C593.156 476.504 595.342 484.719 598.659 494.086C600.259 498.594 606.036 514.441 618.096 534.951C627.053 550.212 637.904 563.618 655.955 582.294C673.167 600.072 698.517 623.959 733.176 649.562C623.326 589.359 513.496 529.155 403.646 468.951" fill="#155DFC"/>
|
||||
<path d="M562.652 325.145C573.171 318.588 582.07 313.885 588.568 310.704C594.501 307.777 597.565 306.645 601.194 305.728C603.712 305.084 611.811 303.249 622.29 304.674C626.642 305.279 632.77 306.177 639.736 310.041C648.069 314.666 652.577 320.774 653.69 322.335C655.017 324.209 657.905 328.326 659.661 334.298C660.754 337.947 660.52 339.313 661.984 342.611C663.525 346.065 665.223 347.9 667.955 351.256C670.16 353.949 673.166 357.852 676.6 362.887C674.785 361.872 665.594 356.896 656.012 357.072C651.68 357.15 647.367 358.243 647.367 358.243C643.893 359.121 641.571 360.214 640.732 360.565C632.984 363.785 615.284 359.511 608.181 347.939C603.692 340.64 606.834 335.918 601.507 331.156C599.536 329.4 596.94 328.092 585.27 326.687C579.748 326.023 572.039 325.321 562.691 325.184L562.652 325.145Z" fill="#003C8D"/>
|
||||
<path d="M820.797 391.692C811.059 390.95 795.467 391.009 777.572 396.726C763.833 401.117 753.373 407.343 746.348 412.338C752.319 411.031 760.086 409.509 769.258 408.182C771.444 407.87 778.489 406.874 788.363 406.016C797.906 405.196 810.552 404.435 825.715 404.513C824.076 400.239 822.437 395.965 820.797 391.672V391.692Z" fill="#0052C9"/>
|
||||
<path d="M841.795 450.627C830.925 450.256 815.625 450.997 798.218 456.032C786.177 459.506 776.283 464.15 768.711 468.405C774.019 467.624 780.693 466.765 788.401 466.043C790.977 465.809 798.549 465.107 808.834 464.677C818.455 464.267 831.12 464.033 846.244 464.599C844.761 459.955 843.278 455.291 841.814 450.646L841.795 450.627Z" fill="#0052C9"/>
|
||||
<path d="M855.848 500.39C845.895 499.824 829.952 500.331 812.583 507.65C805.694 510.557 799.938 513.953 795.273 517.231C798.415 516.607 802.748 515.846 807.919 515.143C818.808 513.699 827.141 513.445 835.494 513.231C841.524 513.075 849.369 513.016 858.6 513.328L855.848 500.37V500.39Z" fill="#0052C9"/>
|
||||
<path d="M570.282 597.028C599.399 605.985 630.096 616.016 657.475 624.934C674.434 630.457 688.289 635.043 697.344 638.048C665.281 603.663 633.238 569.297 601.174 534.912C596.471 534.853 589.095 534.502 580.254 532.824C573.971 531.633 559.627 528.394 529.769 512.157C516.031 504.683 509.161 500.956 502.526 496.038C469.975 471.917 457.427 437.142 453.836 426.799C450.656 417.607 448.47 397.116 444.235 356.525C438.732 303.893 439.552 300.537 435.766 285.276C432.819 273.392 426.203 245.329 407.859 219.511C401.81 210.983 378.099 178.607 333.858 164.927C315.378 159.209 299.141 158.565 288.271 158.956C297.034 165.298 309.016 174.86 321.662 188.111C327.926 194.668 354.408 223.16 373.493 269.859C379.465 284.457 389.866 310.489 392.423 346.085C394.316 372.371 391.154 392.94 390.432 397.409C385.671 426.877 376.264 444.87 371.854 486.436C371.483 489.91 371.23 492.759 371.093 494.555C377.318 504.059 383.368 511.142 387.973 516.06C392.442 520.822 396.092 523.964 397.224 525.076C397.224 525.076 403.605 530.716 410.923 535.614C444.43 558.095 570.282 597.028 570.282 597.028Z" fill="#0051C8"/>
|
||||
<path d="M591.456 468.893C590.109 479.079 589.289 494.594 593.427 512.626C595.495 521.602 598.013 527.886 600.881 534.912C607.555 551.304 615.908 571.619 634.018 591.915C647.424 606.941 661.007 616.035 671.916 623.393C676.56 626.515 682.786 630.691 691.47 635.16C708.175 643.786 723.24 648.001 733.759 650.226C732.139 643.434 730.5 636.663 728.88 629.872C712.097 621.968 691.86 610.435 671.155 593.476C663.622 587.29 658.177 582.157 655.64 579.718C646.019 570.448 627.792 552.729 612.512 524.881C600.53 503.044 594.597 483.021 591.456 468.854V468.893Z" fill="#00398B"/>
|
||||
<path d="M680.074 799.203C689.988 795.768 704.312 789.367 718.519 777.307C731.438 766.34 738.912 755.236 741.937 750.396C750.816 736.248 754.251 723.836 756.631 715.015C759.305 705.121 762.154 690.485 761.861 672.414C765.569 674.502 770.819 677.215 777.298 679.888C791.524 685.743 803.175 687.85 810.434 689.099C824.914 691.617 836.272 691.773 845.054 691.831C852.469 691.89 858.695 691.617 863.222 691.324C863.749 686.64 864.257 681.898 864.725 677.117C865.72 667.047 866.54 657.153 867.223 647.474C861.134 647.396 854.733 647.142 848.039 646.674C842.927 646.303 837.989 645.815 833.267 645.269C826.846 646.147 818.24 647.006 808.053 647.064C778.586 647.201 757.178 640.507 751.87 638.77C741.917 635.511 733.955 631.862 728.452 629.052C729.428 632.682 730.696 637.853 731.808 644.117C733.135 651.553 736.238 671.458 733.174 701.784C731.125 722.06 730.091 732.423 725.368 744.854C721.309 755.567 710.322 778.888 680.035 799.144L680.074 799.203Z" fill="#0050C8"/>
|
||||
<path d="M493.94 571.483C489.179 567.034 474.133 552.846 461.233 537.625C458.15 533.995 453.017 527.867 446.89 519.183C444.86 516.295 442.733 513.153 440.547 509.718C435.395 501.62 428.682 490.925 422.789 475.626C419.139 466.141 416.934 457.847 415.568 451.7C411.88 469.225 412.465 483.783 413.578 493.306C414.846 504.274 417.246 512.255 419.315 518.968C421.54 526.208 424.857 535.497 429.931 545.918C436.937 549.49 442.967 552.28 447.553 554.31C460.453 560.028 470.756 563.677 477.45 566.019C484.144 568.38 489.823 570.195 493.96 571.483H493.94Z" fill="#003C8D"/>
|
||||
<path d="M668.582 578.488C670.768 569.18 674.593 562.798 677.13 559.227C684.135 549.333 692.507 545.586 701.855 541.293C709.407 537.819 715.437 535.165 720.999 537.78C722.014 538.248 722.482 538.658 728.337 542.425C730.366 543.771 733.333 545.703 736.982 547.908C746.525 553.685 784.618 575.854 840.177 582.294C877.822 586.665 906.762 581.864 923.467 578.957C957.599 573.024 985.271 562.545 1005 553.392C994.15 582.001 979.26 598.999 970.186 607.8C935.371 641.58 892.731 644.976 865.839 647.591C859.79 648.176 838.518 649.757 809.344 645.269C745.822 635.511 697.503 602.824 668.582 578.508V578.488Z" fill="#8BC7FE"/>
|
||||
<path d="M664.369 591.27C671.004 599.311 678.791 607.839 687.826 616.425C700.764 628.72 713.391 638.38 724.631 645.912C724.475 642.595 723.89 638.341 722.172 633.794C721.665 632.467 721.119 631.257 720.572 630.144C726.017 633.15 732.008 636.018 738.487 638.536C769.477 650.537 798.183 649.386 817.874 646.225C791.353 642.653 756.792 634.262 720.143 614.318C707.361 607.351 695.964 599.877 685.953 592.441C685.484 592.09 681.874 589.202 676.8 585.221C673.404 582.567 670.614 580.381 668.779 578.957C667.296 583.074 665.813 587.172 664.33 591.29L664.369 591.27Z" fill="#00398B"/>
|
||||
<path d="M992.766 578.937C975.417 607.683 953.073 622.065 944.135 627.237C936.231 631.803 924.366 637.56 908.91 641.092C908.715 641.151 888.81 646.4 853.956 648.235C848.863 648.508 843.887 648.567 843.887 648.567C843.887 648.567 837.759 648.645 831.436 648.391C776.54 646.283 723.362 623.88 723.362 623.88C715.146 620.641 701.642 614.864 687.747 606.434C675.511 599.018 668.74 593.066 665.149 589.709C663.432 588.09 662.046 586.704 661.109 585.728C668.447 572.77 675.785 559.832 683.142 546.874C690.011 553.743 700.198 563.228 713.468 573.024C732.007 586.704 785.107 625.949 857.059 623.587C937.5 620.972 991.069 568.028 1005 553.333C1002.6 559.851 998.777 568.926 992.747 578.898L992.766 578.937Z" fill="#0087F7"/>
|
||||
<path d="M649.906 574.117C651.233 571.268 653.146 567.423 655.761 563.111C656.971 561.12 661.323 553.997 666.533 548.416C670.67 543.986 682.047 531.789 698.733 531.282C708.119 530.989 715.847 534.502 721.019 537.761C717.604 538.073 712.94 538.736 707.534 540.239C701.796 541.859 695.279 543.323 689.502 547.909C681.462 554.29 677.013 562.896 668.582 580.791C666.884 584.421 665.577 587.426 664.757 589.358C661.596 586.294 658.415 583.094 655.195 579.737C653.38 577.845 651.624 575.971 649.906 574.098V574.117Z" fill="#0051CB"/>
|
||||
<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g clip-path="url(#clip0_286_152)">
|
||||
<path d="M12.5378 28.7972C12.5378 30.2972 14.2889 30.2982 14.7498 29.4999C15.0384 28.9999 15.2498 28.4999 15.2498 26.9999C15.2498 25.4999 14.5258 24.2384 13.8296 22.8794C13.6504 22.5296 13.4578 22.1537 13.253 21.7309C12.9016 21.0056 12.4714 20.3367 11.8469 19.8035C11.7511 19.7217 11.7231 19.5023 11.7404 19.3555C11.8085 18.7759 11.8827 18.1969 11.9568 17.6179C12.0794 16.6609 12.202 15.7039 12.2975 14.7442C12.3645 14.0718 12.3633 13.378 12.2786 12.7082C12.0956 11.2586 11.2821 10.1946 10.0335 9.47786C9.6702 9.26926 9.56307 9.03637 9.60844 8.64632C9.73481 7.56183 10.1635 6.58265 10.7083 5.66605C11.2087 4.82425 11.7757 4.02144 12.3417 3.2201C12.4142 3.11747 12.4866 3.01486 12.5589 2.9122C12.6515 2.78077 12.8805 2.68113 13.049 2.67687C15.103 2.62543 17.1573 2.59121 19.2117 2.56589C19.3528 2.56407 19.5394 2.62989 19.6303 2.73115C20.9118 4.1585 22.0569 5.678 22.7442 7.49459C22.7912 7.61894 22.8303 7.74673 22.8661 7.87492C23.1241 8.79206 23.1199 8.79508 22.3848 9.31247L22.3515 9.33589C20.9673 10.3108 20.4612 11.705 20.5228 13.3373C20.5933 15.2138 21.0153 17.0328 21.4965 18.8385C21.5943 19.2054 21.6577 19.4768 21.3049 19.8152C20.5985 20.4924 20.3047 21.4218 20.1641 22.3672C20.0179 23.352 19.8995 24.3409 19.7811 25.3299C19.7154 25.8788 19.7122 26.3421 19.5792 26.9759C19.0702 29.4029 17.7498 31.9999 15.7498 31.9999H12.5378C11.2498 31.9999 10.2498 30.7972 10.2498 28.7972H12.5378Z" fill="#155DFC"/>
|
||||
<path d="M11.5607 1.64008C10.8347 2.775 10.091 3.85078 9.44034 4.98022C8.85324 5.9991 8.46218 7.10566 8.27283 8.28027C8.02494 9.8182 8.4235 11.1797 9.51913 12.2444C10.4211 13.1209 10.6538 14.1118 10.538 15.2919C10.2857 17.8647 9.77936 20.385 9.05131 22.863C9.04422 22.8869 9.02214 22.9064 8.97435 22.9744C8.53792 22.8381 8.07901 22.7111 7.63165 22.5524C1.38253 20.3344 -3.18062 16.2382 -6.04606 10.2575C-6.29374 9.74063 -6.51975 9.21246 -6.73158 8.68004C-6.77189 8.57878 -6.74374 8.38234 -6.66961 8.31328C-4.28698 6.09307 -1.91002 3.86617 0.50117 1.67735C2.16142 0.170403 4.14529 -0.303895 6.31143 0.184986C7.98444 0.562481 9.62828 1.06959 11.2841 1.52262C11.3961 1.5532 11.5 1.61376 11.5607 1.64008Z" fill="#155DFC"/>
|
||||
<path d="M24.2397 22.3314C23.8972 21.1864 23.5199 20.1003 23.2528 18.9878C22.9016 17.5244 22.6171 16.0438 22.3435 14.5632C22.1888 13.7264 22.35 12.9544 22.9725 12.3027C24.5667 10.6339 24.5983 8.69929 23.8328 6.67592C23.1868 4.96849 22.1482 3.49759 20.9256 2.15226C20.7825 1.9947 20.6423 1.8341 20.4795 1.65123C20.5718 1.6006 20.6427 1.54693 20.7227 1.52C22.532 0.910212 24.3624 0.387106 26.2602 0.122211C28.1953 -0.147949 29.8448 0.454544 31.2612 1.74398C32.5042 2.87545 33.7011 4.05816 34.9529 5.17971C35.9556 6.07788 37.0032 6.92664 38.0476 7.77681C38.2361 7.93012 38.2841 8.05386 38.2282 8.27602C37.2646 12.1148 35.328 15.3721 32.3643 17.9969C30.0283 20.0657 27.3376 21.4946 24.2397 22.3316V22.3314Z" fill="#155DFC"/>
|
||||
</g>
|
||||
<defs>
|
||||
<clipPath id="clip0_221_160">
|
||||
<rect width="1000" height="1000" fill="white"/>
|
||||
<clipPath id="clip0_286_152">
|
||||
<rect width="32" height="32" rx="6" fill="white"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 3.0 KiB |
@@ -1,4 +1,4 @@
|
||||
import { App as AntdApp, ConfigProvider } from 'antd';
|
||||
import { App as AntdApp, ConfigProvider, theme } from 'antd';
|
||||
import { useEffect, useState } from 'react';
|
||||
import { BrowserRouter, Route } from 'react-router';
|
||||
import { Routes } from 'react-router';
|
||||
@@ -7,10 +7,12 @@ import { userApi } from './entity/users';
|
||||
import { AuthPageComponent } from './pages/AuthPageComponent';
|
||||
import { OAuthCallbackPage } from './pages/OAuthCallbackPage';
|
||||
import { OauthStorageComponent } from './pages/OauthStorageComponent';
|
||||
import { ThemeProvider, useTheme } from './shared/theme';
|
||||
import { MainScreenComponent } from './widgets/main/MainScreenComponent';
|
||||
|
||||
function App() {
|
||||
function AppContent() {
|
||||
const [isAuthorized, setIsAuthorized] = useState(false);
|
||||
const { resolvedTheme } = useTheme();
|
||||
|
||||
useEffect(() => {
|
||||
const isAuthorized = userApi.isAuthorized();
|
||||
@@ -24,6 +26,7 @@ function App() {
|
||||
return (
|
||||
<ConfigProvider
|
||||
theme={{
|
||||
algorithm: resolvedTheme === 'dark' ? theme.darkAlgorithm : theme.defaultAlgorithm,
|
||||
token: {
|
||||
colorPrimary: '#155dfc', // Tailwind blue-600
|
||||
},
|
||||
@@ -45,4 +48,12 @@ function App() {
|
||||
);
|
||||
}
|
||||
|
||||
function App() {
|
||||
return (
|
||||
<ThemeProvider>
|
||||
<AppContent />
|
||||
</ThemeProvider>
|
||||
);
|
||||
}
|
||||
|
||||
export default App;
|
||||
|
||||
@@ -4,3 +4,4 @@ export { BackupStatus } from './model/BackupStatus';
|
||||
export type { Backup } from './model/Backup';
|
||||
export type { BackupConfig } from './model/BackupConfig';
|
||||
export { BackupNotificationType } from './model/BackupNotificationType';
|
||||
export { BackupEncryption } from './model/BackupEncryption';
|
||||
|
||||