diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bff3ae3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,67 @@ +# Git and GitHub +.git +.gitignore +.github + +# Node modules everywhere +node_modules +**/node_modules + +# Backend - exclude everything except what's needed for build +backend/tools +backend/mysqldata +backend/pgdata +backend/temp +backend/images +backend/bin +backend/*.exe + +# Scripts and data directories +scripts +postgresus-data + +# IDE and editor files +.idea +.vscode +.cursor +**/*.swp +**/*.swo + +# Documentation and articles (not needed for build) +articles +docs +pages + +# Notifiers not needed in container +notifiers + +# Dist (will be built fresh) +frontend/dist + +# Environment files (handled separately) +.env.local +.env.development + +# Logs and temp files +**/*.log +tmp +temp + +# OS files +.DS_Store +Thumbs.db + +# Helm charts and deployment configs +deploy + +# License and other root files +LICENSE +CITATION.cff +*.md +assets + +# Python cache +**/__pycache__ + +# Pre-commit config +.pre-commit-config.yaml diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index b662a90..035ce74 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -169,6 +169,10 @@ jobs: TEST_FTP_PORT=7007 # testing SFTP TEST_SFTP_PORT=7008 + # testing MySQL + TEST_MYSQL_57_PORT=33057 + TEST_MYSQL_80_PORT=33080 + TEST_MYSQL_84_PORT=33084 # testing Telegram TEST_TELEGRAM_BOT_TOKEN=${{ secrets.TEST_TELEGRAM_BOT_TOKEN }} TEST_TELEGRAM_CHAT_ID=${{ secrets.TEST_TELEGRAM_CHAT_ID }} @@ -210,6 +214,14 @@ jobs: # Wait for SFTP timeout 60 bash -c 'until nc -z localhost 7008; do sleep 2; done' + # Wait for MySQL containers + echo "Waiting for MySQL 5.7..." + timeout 120 bash -c 'until docker exec test-mysql-57 mysqladmin ping -h localhost -u root -prootpassword --silent 2>/dev/null; do sleep 2; done' + echo "Waiting for MySQL 8.0..." + timeout 120 bash -c 'until docker exec test-mysql-80 mysqladmin ping -h localhost -u root -prootpassword --silent 2>/dev/null; do sleep 2; done' + echo "Waiting for MySQL 8.4..." + timeout 120 bash -c 'until docker exec test-mysql-84 mysqladmin ping -h localhost -u root -prootpassword --silent 2>/dev/null; do sleep 2; done' + - name: Create data and temp directories run: | # Create directories that are used for backups and restore @@ -217,7 +229,7 @@ jobs: mkdir -p postgresus-data/backups mkdir -p postgresus-data/temp - - name: Install PostgreSQL client tools + - name: Install PostgreSQL and MySQL client tools run: | chmod +x backend/tools/download_linux.sh cd backend/tools diff --git a/Dockerfile b/Dockerfile index a4c819a..6b3ff0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,16 +77,57 @@ ENV APP_VERSION=$APP_VERSION # Set production mode for Docker containers ENV ENV_MODE=production -# Install PostgreSQL server and client tools (versions 12-18) and rclone +# Install PostgreSQL server and client tools (versions 12-18), MySQL client tools (5.7, 8.0, 8.4), and rclone +# Note: MySQL 5.7 is only available for x86_64, MySQL 8.0+ supports both x86_64 and ARM64 +# Note: MySQL binaries require libncurses5 for terminal handling +ARG TARGETARCH RUN apt-get update && apt-get install -y --no-install-recommends \ - wget ca-certificates gnupg lsb-release sudo gosu curl unzip && \ + wget ca-certificates gnupg lsb-release sudo gosu curl unzip xz-utils libncurses5 && \ + # Add PostgreSQL repository wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ > /etc/apt/sources.list.d/pgdg.list && \ apt-get update && \ + # Install PostgreSQL apt-get install -y --no-install-recommends \ postgresql-17 postgresql-18 postgresql-client-12 postgresql-client-13 postgresql-client-14 postgresql-client-15 \ postgresql-client-16 postgresql-client-17 postgresql-client-18 rclone && \ + # Create MySQL directories + mkdir -p /usr/local/mysql-5.7/bin /usr/local/mysql-8.0/bin /usr/local/mysql-8.4/bin && \ + # Download and install MySQL client tools (architecture-aware) + # MySQL 5.7: Only available for x86_64 + if [ "$TARGETARCH" = "amd64" ]; then \ + wget -q https://dev.mysql.com/get/Downloads/MySQL-5.7/mysql-5.7.44-linux-glibc2.12-x86_64.tar.gz -O /tmp/mysql57.tar.gz && \ + tar -xzf /tmp/mysql57.tar.gz -C /tmp && \ + cp /tmp/mysql-5.7.*/bin/mysql /usr/local/mysql-5.7/bin/ && \ + cp /tmp/mysql-5.7.*/bin/mysqldump /usr/local/mysql-5.7/bin/ && \ + rm -rf /tmp/mysql-5.7.* /tmp/mysql57.tar.gz; \ + else \ + echo "MySQL 5.7 not available for $TARGETARCH, skipping..."; \ + fi && \ + # MySQL 8.0: Available for both x86_64 and ARM64 + if [ "$TARGETARCH" = "amd64" ]; then \ + wget -q https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-8.0.40-linux-glibc2.17-x86_64-minimal.tar.xz -O /tmp/mysql80.tar.xz; \ + elif [ "$TARGETARCH" = "arm64" ]; then \ + wget -q https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-8.0.40-linux-glibc2.17-aarch64-minimal.tar.xz -O /tmp/mysql80.tar.xz; \ + fi && \ + tar -xJf /tmp/mysql80.tar.xz -C /tmp && \ + cp /tmp/mysql-8.0.*/bin/mysql /usr/local/mysql-8.0/bin/ && \ + cp /tmp/mysql-8.0.*/bin/mysqldump /usr/local/mysql-8.0/bin/ && \ + rm -rf /tmp/mysql-8.0.* /tmp/mysql80.tar.xz && \ + # MySQL 8.4: Available for both x86_64 and ARM64 + if [ "$TARGETARCH" = "amd64" ]; then \ + wget -q https://dev.mysql.com/get/Downloads/MySQL-8.4/mysql-8.4.3-linux-glibc2.17-x86_64-minimal.tar.xz -O /tmp/mysql84.tar.xz; \ + elif [ "$TARGETARCH" = "arm64" ]; then \ + wget -q https://dev.mysql.com/get/Downloads/MySQL-8.4/mysql-8.4.3-linux-glibc2.17-aarch64-minimal.tar.xz -O /tmp/mysql84.tar.xz; \ + fi && \ + tar -xJf /tmp/mysql84.tar.xz -C /tmp && \ + cp /tmp/mysql-8.4.*/bin/mysql /usr/local/mysql-8.4/bin/ && \ + cp /tmp/mysql-8.4.*/bin/mysqldump /usr/local/mysql-8.4/bin/ && \ + rm -rf /tmp/mysql-8.4.* /tmp/mysql84.tar.xz && \ + # Make MySQL binaries executable (ignore errors for empty dirs on ARM64) + chmod +x /usr/local/mysql-*/bin/* 2>/dev/null || true && \ + # Cleanup rm -rf /var/lib/apt/lists/* # Create postgres user and set up directories diff --git a/README.md b/README.md index 3811811..a319832 100644 --- a/README.md +++ b/README.md @@ -36,25 +36,25 @@ ## ✨ Features -### 🔄 **Scheduled Backups** +### 🔄 **Scheduled backups** - **Flexible scheduling**: hourly, daily, weekly, monthly or cron - **Precise timing**: run backups at specific times (e.g., 4 AM during low traffic) - **Smart compression**: 4-8x space savings with balanced compression (~20% overhead) -### 🗄️ **Multiple Storage Destinations** (view supported) +### 🗄️ **Multiple storage destinations** (view supported) - **Local storage**: Keep backups on your VPS/server - **Cloud storage**: S3, Cloudflare R2, Google Drive, NAS, Dropbox, SFTP, Rclone and more - **Secure**: All data stays under your control -### 📱 **Smart Notifications** (view supported) +### 📱 **Smart notifications** (view supported) - **Multiple channels**: Email, Telegram, Slack, Discord, webhooks - **Real-time updates**: Success and failure notifications - **Team integration**: Perfect for DevOps workflows -### 🐘 **PostgreSQL Support** +### 🐘 **PostgreSQL support** - **Multiple versions**: PostgreSQL 12, 13, 14, 15, 16, 17 and 18 - **SSL support**: Secure connections available @@ -67,7 +67,7 @@ - **Encryption for secrets**: Any sensitive data is encrypted and never exposed, even in logs or error messages - **Read-only user**: Postgresus uses by default a read-only user for backups and never stores anything that can change your data -### 👥 **Suitable for Teams** (docs) +### 👥 **Suitable for teams** (docs) - **Workspaces**: Group databases, notifiers and storages for different projects or teams - **Access management**: Control who can view or manage specific databases with role-based permissions @@ -80,7 +80,7 @@ - **Dark & light themes**: Choose the look that suits your workflow - **Mobile adaptive**: Check your backups from anywhere on any device -### ☁️ **Works with Self-Hosted & Cloud Databases** +### ☁️ **Works with self-hosted & cloud databases** Postgresus works seamlessly with both self-hosted PostgreSQL and cloud-managed databases: @@ -89,7 +89,7 @@ Postgresus works seamlessly with both self-hosted PostgreSQL and cloud-managed d - **Why no PITR?**: Cloud providers already offer native PITR, and external PITR backups cannot be restored to managed cloud databases — making them impractical for cloud-hosted PostgreSQL - **Practical granularity**: Hourly and daily backups are sufficient for 99% of projects without the operational complexity of WAL archiving -### 🐳 **Self-Hosted & Secure** +### 🐳 **Self-hosted & secure** - **Docker-based**: Easy deployment and management - **Privacy-first**: All your data stays on your infrastructure @@ -111,7 +111,7 @@ You have several ways to install Postgresus: You have three ways to install Postgresus: automated script (recommended), simple Docker run, or Docker Compose setup. -### Option 1: Automated Installation Script (Recommended, Linux only) +### Option 1: Automated installation script (recommended, Linux only) The installation script will: @@ -125,7 +125,7 @@ sudo curl -sSL https://raw.githubusercontent.com/RostislavDugin/postgresus/refs/ | sudo bash ``` -### Option 2: Simple Docker Run +### Option 2: Simple Docker run The easiest way to run Postgresus with embedded PostgreSQL: @@ -144,7 +144,7 @@ This single command will: - ✅ Store all data in `./postgresus-data` directory - ✅ Automatically restart on system reboot -### Option 3: Docker Compose Setup +### Option 3: Docker Compose setup Create a `docker-compose.yml` file with the following configuration: @@ -218,7 +218,7 @@ For more options (NodePort, TLS, HTTPRoute for Gateway API), see the [Helm chart 6. **Add notifications** (optional): Configure email, Telegram, Slack, or webhook notifications 7. **Save and start**: Postgresus will validate settings and begin the backup schedule -### 🔑 Resetting Password (docs) +### 🔑 Resetting password (docs) If you need to reset the password, you can use the built-in password reset command: diff --git a/backend/.gitignore b/backend/.gitignore index 0f645f6..35a6c2f 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -3,6 +3,7 @@ main docker-compose.yml pgdata pgdata_test/ +mysqldata/ main.exe swagger/ swagger/* diff --git a/backend/docker-compose.yml.example b/backend/docker-compose.yml.example index 7e01fc7..2b1d777 100644 --- a/backend/docker-compose.yml.example +++ b/backend/docker-compose.yml.example @@ -31,14 +31,6 @@ services: container_name: test-minio command: server /data --console-address ":9001" - # Test Azurite container - test-azurite: - image: mcr.microsoft.com/azure-storage/azurite - ports: - - "${TEST_AZURITE_BLOB_PORT:-10000}:10000" - container_name: test-azurite - command: azurite-blob --blobHost 0.0.0.0 - # Test PostgreSQL containers test-postgres-12: image: postgres:12 @@ -117,6 +109,14 @@ services: container_name: test-postgres-18 shm_size: 1gb + # Test Azurite container + test-azurite: + image: mcr.microsoft.com/azure-storage/azurite + ports: + - "${TEST_AZURITE_BLOB_PORT:-10000}:10000" + container_name: test-azurite + command: azurite-blob --blobHost 0.0.0.0 + # Test NAS server (Samba) test-nas: image: dperson/samba:latest @@ -154,3 +154,61 @@ services: - "${TEST_SFTP_PORT:-7008}:22" command: testuser:testpassword:1001::upload container_name: test-sftp + + # Test MySQL containers + test-mysql-57: + image: mysql:5.7 + ports: + - "${TEST_MYSQL_57_PORT:-33057}:3306" + environment: + - MYSQL_ROOT_PASSWORD=rootpassword + - MYSQL_DATABASE=testdb + - MYSQL_USER=testuser + - MYSQL_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - ./mysqldata/mysql-57:/var/lib/mysql + container_name: test-mysql-57 + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-prootpassword"] + interval: 5s + timeout: 5s + retries: 10 + + test-mysql-80: + image: mysql:8.0 + ports: + - "${TEST_MYSQL_80_PORT:-33080}:3306" + environment: + - MYSQL_ROOT_PASSWORD=rootpassword + - MYSQL_DATABASE=testdb + - MYSQL_USER=testuser + - MYSQL_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --default-authentication-plugin=mysql_native_password + volumes: + - ./mysqldata/mysql-80:/var/lib/mysql + container_name: test-mysql-80 + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-prootpassword"] + interval: 5s + timeout: 5s + retries: 10 + + test-mysql-84: + image: mysql:8.4 + ports: + - "${TEST_MYSQL_84_PORT:-33084}:3306" + environment: + - MYSQL_ROOT_PASSWORD=rootpassword + - MYSQL_DATABASE=testdb + - MYSQL_USER=testuser + - MYSQL_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - ./mysqldata/mysql-84:/var/lib/mysql + container_name: test-mysql-84 + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-prootpassword"] + interval: 5s + timeout: 5s + retries: 10 diff --git a/backend/go.mod b/backend/go.mod index e8aa924..d2ad648 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -32,6 +32,7 @@ require ( ) require ( + filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect @@ -229,7 +230,7 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.28.0 // indirect - github.com/go-sql-driver/mysql v1.9.2 // indirect + github.com/go-sql-driver/mysql v1.9.2 github.com/goccy/go-json v0.10.5 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -238,7 +239,7 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.1 // indirect + github.com/klauspost/compress v1.18.1 github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.9.1 // indirect diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 789bd49..20e3ff3 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -25,6 +25,7 @@ type EnvVariables struct { DatabaseDsn string `env:"DATABASE_DSN" required:"true"` EnvMode env_utils.EnvMode `env:"ENV_MODE" required:"true"` PostgresesInstallDir string `env:"POSTGRES_INSTALL_DIR"` + MysqlInstallDir string `env:"MYSQL_INSTALL_DIR"` DataFolder string TempFolder string @@ -51,6 +52,10 @@ type EnvVariables struct { TestFTPPort string `env:"TEST_FTP_PORT"` TestSFTPPort string `env:"TEST_SFTP_PORT"` + TestMysql57Port string `env:"TEST_MYSQL_57_PORT"` + TestMysql80Port string `env:"TEST_MYSQL_80_PORT"` + TestMysql84Port string `env:"TEST_MYSQL_84_PORT"` + // oauth GitHubClientID string `env:"GITHUB_CLIENT_ID"` GitHubClientSecret string `env:"GITHUB_CLIENT_SECRET"` @@ -152,6 +157,9 @@ func loadEnvVariables() { env.PostgresesInstallDir = filepath.Join(backendRoot, "tools", "postgresql") tools.VerifyPostgresesInstallation(log, env.EnvMode, env.PostgresesInstallDir) + env.MysqlInstallDir = filepath.Join(backendRoot, "tools", "mysql") + tools.VerifyMysqlInstallation(log, env.EnvMode, env.MysqlInstallDir) + // Store the data and temp folders one level below the root // (projectRoot/postgresus-data -> /postgresus-data) env.DataFolder = filepath.Join(filepath.Dir(backendRoot), "postgresus-data", "backups") diff --git a/backend/internal/features/backups/backups/controller.go b/backend/internal/features/backups/backups/controller.go index c528581..bc812b9 100644 --- a/backend/internal/features/backups/backups/controller.go +++ b/backend/internal/features/backups/backups/controller.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "net/http" + "postgresus-backend/internal/features/databases" users_middleware "postgresus-backend/internal/features/users/middleware" "github.com/gin-gonic/gin" @@ -181,7 +182,7 @@ func (c *BackupController) GetFile(ctx *gin.Context) { return } - fileReader, err := c.backupService.GetBackupFile(user, id) + fileReader, dbType, err := c.backupService.GetBackupFile(user, id) if err != nil { ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return @@ -192,10 +193,15 @@ func (c *BackupController) GetFile(ctx *gin.Context) { } }() + extension := ".dump.zst" + if dbType == databases.DatabaseTypeMysql { + extension = ".sql.zst" + } + ctx.Header("Content-Type", "application/octet-stream") ctx.Header( "Content-Disposition", - fmt.Sprintf("attachment; filename=\"backup_%s.dump\"", id.String()), + fmt.Sprintf("attachment; filename=\"backup_%s%s\"", id.String(), extension), ) _, err = io.Copy(ctx.Writer, fileReader) diff --git a/backend/internal/features/backups/backups/interfaces.go b/backend/internal/features/backups/backups/interfaces.go index 93b92c5..bb2721c 100644 --- a/backend/internal/features/backups/backups/interfaces.go +++ b/backend/internal/features/backups/backups/interfaces.go @@ -3,7 +3,7 @@ package backups import ( "context" - usecases_postgresql "postgresus-backend/internal/features/backups/backups/usecases/postgresql" + usecases_common "postgresus-backend/internal/features/backups/backups/usecases/common" backups_config "postgresus-backend/internal/features/backups/config" "postgresus-backend/internal/features/databases" "postgresus-backend/internal/features/notifiers" @@ -27,10 +27,8 @@ type CreateBackupUsecase interface { backupConfig *backups_config.BackupConfig, database *databases.Database, storage *storages.Storage, - backupProgressListener func( - completedMBs float64, - ), - ) (*usecases_postgresql.BackupMetadata, error) + backupProgressListener func(completedMBs float64), + ) (*usecases_common.BackupMetadata, error) } type BackupRemoveListener interface { diff --git a/backend/internal/features/backups/backups/service.go b/backend/internal/features/backups/backups/service.go index d01ef83..a244f89 100644 --- a/backend/internal/features/backups/backups/service.go +++ b/backend/internal/features/backups/backups/service.go @@ -502,19 +502,19 @@ func (s *BackupService) CancelBackup( func (s *BackupService) GetBackupFile( user *users_models.User, backupID uuid.UUID, -) (io.ReadCloser, error) { +) (io.ReadCloser, databases.DatabaseType, error) { backup, err := s.backupRepository.FindByID(backupID) if err != nil { - return nil, err + return nil, "", err } database, err := s.databaseService.GetDatabaseByID(backup.DatabaseID) if err != nil { - return nil, err + return nil, "", err } if database.WorkspaceID == nil { - return nil, errors.New("cannot download backup for database without workspace") + return nil, "", errors.New("cannot download backup for database without workspace") } canAccess, _, err := s.workspaceService.CanUserAccessWorkspace( @@ -522,10 +522,10 @@ func (s *BackupService) GetBackupFile( user, ) if err != nil { - return nil, err + return nil, "", err } if !canAccess { - return nil, errors.New("insufficient permissions to download backup for this database") + return nil, "", errors.New("insufficient permissions to download backup for this database") } s.auditLogService.WriteAuditLog( @@ -538,7 +538,12 @@ func (s *BackupService) GetBackupFile( database.WorkspaceID, ) - return s.getBackupReader(backupID) + reader, err := s.getBackupReader(backupID) + if err != nil { + return nil, "", err + } + + return reader, database.Type, nil } func (s *BackupService) deleteBackup(backup *Backup) error { diff --git a/backend/internal/features/backups/backups/service_test.go b/backend/internal/features/backups/backups/service_test.go index 56117c4..b3d5f5c 100644 --- a/backend/internal/features/backups/backups/service_test.go +++ b/backend/internal/features/backups/backups/service_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - usecases_postgresql "postgresus-backend/internal/features/backups/backups/usecases/postgresql" + "postgresus-backend/internal/features/backups/backups/usecases/common" backups_config "postgresus-backend/internal/features/backups/config" "postgresus-backend/internal/features/databases" encryption_secrets "postgresus-backend/internal/features/encryption/secrets" @@ -178,16 +178,13 @@ func (uc *CreateFailedBackupUsecase) Execute( backupConfig *backups_config.BackupConfig, database *databases.Database, storage *storages.Storage, - backupProgressListener func( - completedMBs float64, - ), -) (*usecases_postgresql.BackupMetadata, error) { - backupProgressListener(10) // Assume we completed 10MB + backupProgressListener func(completedMBs float64), +) (*common.BackupMetadata, error) { + backupProgressListener(10) return nil, errors.New("backup failed") } -type CreateSuccessBackupUsecase struct { -} +type CreateSuccessBackupUsecase struct{} func (uc *CreateSuccessBackupUsecase) Execute( ctx context.Context, @@ -195,12 +192,10 @@ func (uc *CreateSuccessBackupUsecase) Execute( backupConfig *backups_config.BackupConfig, database *databases.Database, storage *storages.Storage, - backupProgressListener func( - completedMBs float64, - ), -) (*usecases_postgresql.BackupMetadata, error) { - backupProgressListener(10) // Assume we completed 10MB - return &usecases_postgresql.BackupMetadata{ + backupProgressListener func(completedMBs float64), +) (*common.BackupMetadata, error) { + backupProgressListener(10) + return &common.BackupMetadata{ EncryptionSalt: nil, EncryptionIV: nil, Encryption: backups_config.BackupEncryptionNone, diff --git a/backend/internal/features/backups/backups/usecases/postgresql/dto.go b/backend/internal/features/backups/backups/usecases/common/dto.go similarity index 58% rename from backend/internal/features/backups/backups/usecases/postgresql/dto.go rename to backend/internal/features/backups/backups/usecases/common/dto.go index 594613c..f46a4d3 100644 --- a/backend/internal/features/backups/backups/usecases/postgresql/dto.go +++ b/backend/internal/features/backups/backups/usecases/common/dto.go @@ -1,13 +1,7 @@ -package usecases_postgresql +package common import backups_config "postgresus-backend/internal/features/backups/config" -type EncryptionMetadata struct { - Salt string - IV string - Encryption backups_config.BackupEncryption -} - type BackupMetadata struct { EncryptionSalt *string EncryptionIV *string diff --git a/backend/internal/features/backups/backups/usecases/common/interfaces.go b/backend/internal/features/backups/backups/usecases/common/interfaces.go new file mode 100644 index 0000000..c9a0852 --- /dev/null +++ b/backend/internal/features/backups/backups/usecases/common/interfaces.go @@ -0,0 +1,22 @@ +package common + +import "io" + +type CountingWriter struct { + Writer io.Writer + BytesWritten int64 +} + +func (cw *CountingWriter) Write(p []byte) (n int, err error) { + n, err = cw.Writer.Write(p) + cw.BytesWritten += int64(n) + return n, err +} + +func (cw *CountingWriter) GetBytesWritten() int64 { + return cw.BytesWritten +} + +func NewCountingWriter(writer io.Writer) *CountingWriter { + return &CountingWriter{Writer: writer} +} diff --git a/backend/internal/features/backups/backups/usecases/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/create_backup_uc.go index 0010165..33c575e 100644 --- a/backend/internal/features/backups/backups/usecases/create_backup_uc.go +++ b/backend/internal/features/backups/backups/usecases/create_backup_uc.go @@ -3,6 +3,9 @@ package usecases import ( "context" "errors" + + usecases_common "postgresus-backend/internal/features/backups/backups/usecases/common" + usecases_mysql "postgresus-backend/internal/features/backups/backups/usecases/mysql" usecases_postgresql "postgresus-backend/internal/features/backups/backups/usecases/postgresql" backups_config "postgresus-backend/internal/features/backups/config" "postgresus-backend/internal/features/databases" @@ -13,20 +16,19 @@ import ( type CreateBackupUsecase struct { CreatePostgresqlBackupUsecase *usecases_postgresql.CreatePostgresqlBackupUsecase + CreateMysqlBackupUsecase *usecases_mysql.CreateMysqlBackupUsecase } -// Execute creates a backup of the database and returns the backup metadata func (uc *CreateBackupUsecase) Execute( ctx context.Context, backupID uuid.UUID, backupConfig *backups_config.BackupConfig, database *databases.Database, storage *storages.Storage, - backupProgressListener func( - completedMBs float64, - ), -) (*usecases_postgresql.BackupMetadata, error) { - if database.Type == databases.DatabaseTypePostgres { + backupProgressListener func(completedMBs float64), +) (*usecases_common.BackupMetadata, error) { + switch database.Type { + case databases.DatabaseTypePostgres: return uc.CreatePostgresqlBackupUsecase.Execute( ctx, backupID, @@ -35,7 +37,18 @@ func (uc *CreateBackupUsecase) Execute( storage, backupProgressListener, ) - } - return nil, errors.New("database type not supported") + case databases.DatabaseTypeMysql: + return uc.CreateMysqlBackupUsecase.Execute( + ctx, + backupID, + backupConfig, + database, + storage, + backupProgressListener, + ) + + default: + return nil, errors.New("database type not supported") + } } diff --git a/backend/internal/features/backups/backups/usecases/di.go b/backend/internal/features/backups/backups/usecases/di.go index 0bda57c..edbcaf8 100644 --- a/backend/internal/features/backups/backups/usecases/di.go +++ b/backend/internal/features/backups/backups/usecases/di.go @@ -1,11 +1,13 @@ package usecases import ( + usecases_mysql "postgresus-backend/internal/features/backups/backups/usecases/mysql" usecases_postgresql "postgresus-backend/internal/features/backups/backups/usecases/postgresql" ) var createBackupUsecase = &CreateBackupUsecase{ usecases_postgresql.GetCreatePostgresqlBackupUsecase(), + usecases_mysql.GetCreateMysqlBackupUsecase(), } func GetCreateBackupUsecase() *CreateBackupUsecase { diff --git a/backend/internal/features/backups/backups/usecases/mysql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/mysql/create_backup_uc.go new file mode 100644 index 0000000..9836415 --- /dev/null +++ b/backend/internal/features/backups/backups/usecases/mysql/create_backup_uc.go @@ -0,0 +1,608 @@ +package usecases_mysql + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/klauspost/compress/zstd" + + "postgresus-backend/internal/config" + backup_encryption "postgresus-backend/internal/features/backups/backups/encryption" + usecases_common "postgresus-backend/internal/features/backups/backups/usecases/common" + backups_config "postgresus-backend/internal/features/backups/config" + "postgresus-backend/internal/features/databases" + mysqltypes "postgresus-backend/internal/features/databases/databases/mysql" + encryption_secrets "postgresus-backend/internal/features/encryption/secrets" + "postgresus-backend/internal/features/storages" + "postgresus-backend/internal/util/encryption" + "postgresus-backend/internal/util/tools" +) + +const ( + backupTimeout = 23 * time.Hour + shutdownCheckInterval = 1 * time.Second + copyBufferSize = 8 * 1024 * 1024 + progressReportIntervalMB = 1.0 + zstdStorageCompressionLevel = 3 + exitCodeGenericError = 1 + exitCodeConnectionError = 2 +) + +type CreateMysqlBackupUsecase struct { + logger *slog.Logger + secretKeyService *encryption_secrets.SecretKeyService + fieldEncryptor encryption.FieldEncryptor +} + +type writeResult struct { + bytesWritten int + writeErr error +} + +func (uc *CreateMysqlBackupUsecase) Execute( + ctx context.Context, + backupID uuid.UUID, + backupConfig *backups_config.BackupConfig, + db *databases.Database, + storage *storages.Storage, + backupProgressListener func(completedMBs float64), +) (*usecases_common.BackupMetadata, error) { + uc.logger.Info( + "Creating MySQL backup via mysqldump", + "databaseId", db.ID, + "storageId", storage.ID, + ) + + if !backupConfig.IsBackupsEnabled { + return nil, fmt.Errorf("backups are not enabled for this database: \"%s\"", db.Name) + } + + my := db.Mysql + if my == nil { + return nil, fmt.Errorf("mysql database configuration is required") + } + + if my.Database == nil || *my.Database == "" { + return nil, fmt.Errorf("database name is required for mysqldump backups") + } + + decryptedPassword, err := uc.fieldEncryptor.Decrypt(db.ID, my.Password) + if err != nil { + return nil, fmt.Errorf("failed to decrypt database password: %w", err) + } + + args := uc.buildMysqldumpArgs(my) + + return uc.streamToStorage( + ctx, + backupID, + backupConfig, + tools.GetMysqlExecutable( + my.Version, + tools.MysqlExecutableMysqldump, + config.GetEnv().EnvMode, + config.GetEnv().MysqlInstallDir, + ), + args, + decryptedPassword, + storage, + backupProgressListener, + my, + ) +} + +func (uc *CreateMysqlBackupUsecase) buildMysqldumpArgs(my *mysqltypes.MysqlDatabase) []string { + args := []string{ + "--host=" + my.Host, + "--port=" + strconv.Itoa(my.Port), + "--user=" + my.Username, + "--single-transaction", + "--routines", + "--triggers", + "--events", + "--set-gtid-purged=OFF", + "--quick", + "--verbose", + } + + args = append(args, uc.getNetworkCompressionArgs(my.Version)...) + + if my.IsHttps { + args = append(args, "--ssl-mode=REQUIRED") + } + + if my.Database != nil && *my.Database != "" { + args = append(args, *my.Database) + } + + return args +} + +func (uc *CreateMysqlBackupUsecase) getNetworkCompressionArgs(version tools.MysqlVersion) []string { + const zstdCompressionLevel = 3 + + switch version { + case tools.MysqlVersion80, tools.MysqlVersion84: + return []string{ + "--compression-algorithms=zstd", + fmt.Sprintf("--zstd-compression-level=%d", zstdCompressionLevel), + } + case tools.MysqlVersion57: + return []string{"--compress"} + default: + return []string{"--compress"} + } +} + +func (uc *CreateMysqlBackupUsecase) streamToStorage( + parentCtx context.Context, + backupID uuid.UUID, + backupConfig *backups_config.BackupConfig, + mysqlBin string, + args []string, + password string, + storage *storages.Storage, + backupProgressListener func(completedMBs float64), + myConfig *mysqltypes.MysqlDatabase, +) (*usecases_common.BackupMetadata, error) { + uc.logger.Info("Streaming MySQL backup to storage", "mysqlBin", mysqlBin) + + ctx, cancel := uc.createBackupContext(parentCtx) + defer cancel() + + myCnfFile, err := uc.createTempMyCnfFile(myConfig, password) + if err != nil { + return nil, fmt.Errorf("failed to create .my.cnf: %w", err) + } + defer func() { _ = os.RemoveAll(filepath.Dir(myCnfFile)) }() + + fullArgs := append([]string{"--defaults-file=" + myCnfFile}, args...) + + cmd := exec.CommandContext(ctx, mysqlBin, fullArgs...) + uc.logger.Info("Executing MySQL backup command", "command", cmd.String()) + + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, + "MYSQL_PWD=", + "LC_ALL=C.UTF-8", + "LANG=C.UTF-8", + ) + + pgStdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("stdout pipe: %w", err) + } + + pgStderr, err := cmd.StderrPipe() + if err != nil { + return nil, fmt.Errorf("stderr pipe: %w", err) + } + + stderrCh := make(chan []byte, 1) + go func() { + stderrOutput, _ := io.ReadAll(pgStderr) + stderrCh <- stderrOutput + }() + + storageReader, storageWriter := io.Pipe() + + finalWriter, encryptionWriter, backupMetadata, err := uc.setupBackupEncryption( + backupID, + backupConfig, + storageWriter, + ) + if err != nil { + return nil, err + } + + zstdWriter, err := zstd.NewWriter(finalWriter, + zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(zstdStorageCompressionLevel))) + if err != nil { + return nil, fmt.Errorf("failed to create zstd writer: %w", err) + } + countingWriter := usecases_common.NewCountingWriter(zstdWriter) + + saveErrCh := make(chan error, 1) + go func() { + saveErr := storage.SaveFile(ctx, uc.fieldEncryptor, uc.logger, backupID, storageReader) + saveErrCh <- saveErr + }() + + if err = cmd.Start(); err != nil { + return nil, fmt.Errorf("start %s: %w", filepath.Base(mysqlBin), err) + } + + copyResultCh := make(chan error, 1) + bytesWrittenCh := make(chan int64, 1) + go func() { + bytesWritten, err := uc.copyWithShutdownCheck( + ctx, + countingWriter, + pgStdout, + backupProgressListener, + ) + bytesWrittenCh <- bytesWritten + copyResultCh <- err + }() + + copyErr := <-copyResultCh + bytesWritten := <-bytesWrittenCh + waitErr := cmd.Wait() + + select { + case <-ctx.Done(): + uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, saveErrCh) + return nil, uc.checkCancellationReason() + default: + } + + if err := zstdWriter.Close(); err != nil { + uc.logger.Error("Failed to close zstd writer", "error", err) + } + if err := uc.closeWriters(encryptionWriter, storageWriter); err != nil { + <-saveErrCh + return nil, err + } + + saveErr := <-saveErrCh + stderrOutput := <-stderrCh + + if waitErr == nil && copyErr == nil && saveErr == nil && backupProgressListener != nil { + sizeMB := float64(bytesWritten) / (1024 * 1024) + backupProgressListener(sizeMB) + } + + switch { + case waitErr != nil: + return nil, uc.buildMysqldumpErrorMessage(waitErr, stderrOutput, mysqlBin) + case copyErr != nil: + return nil, fmt.Errorf("copy to storage: %w", copyErr) + case saveErr != nil: + return nil, fmt.Errorf("save to storage: %w", saveErr) + } + + return &backupMetadata, nil +} + +func (uc *CreateMysqlBackupUsecase) createTempMyCnfFile( + myConfig *mysqltypes.MysqlDatabase, + password string, +) (string, error) { + tempDir, err := os.MkdirTemp("", "mycnf") + if err != nil { + return "", fmt.Errorf("failed to create temp directory: %w", err) + } + + myCnfFile := filepath.Join(tempDir, ".my.cnf") + + content := fmt.Sprintf(`[client] +user=%s +password="%s" +host=%s +port=%d +`, myConfig.Username, tools.EscapeMysqlPassword(password), myConfig.Host, myConfig.Port) + + if myConfig.IsHttps { + content += "ssl-mode=REQUIRED\n" + } + + err = os.WriteFile(myCnfFile, []byte(content), 0600) + if err != nil { + return "", fmt.Errorf("failed to write .my.cnf: %w", err) + } + + return myCnfFile, nil +} + +func (uc *CreateMysqlBackupUsecase) copyWithShutdownCheck( + ctx context.Context, + dst io.Writer, + src io.Reader, + backupProgressListener func(completedMBs float64), +) (int64, error) { + buf := make([]byte, copyBufferSize) + var totalBytesWritten int64 + var lastReportedMB float64 + + for { + select { + case <-ctx.Done(): + return totalBytesWritten, fmt.Errorf("copy cancelled: %w", ctx.Err()) + default: + } + + if config.IsShouldShutdown() { + return totalBytesWritten, fmt.Errorf("copy cancelled due to shutdown") + } + + bytesRead, readErr := src.Read(buf) + if bytesRead > 0 { + writeResultCh := make(chan writeResult, 1) + go func() { + bytesWritten, writeErr := dst.Write(buf[0:bytesRead]) + writeResultCh <- writeResult{bytesWritten, writeErr} + }() + + var bytesWritten int + var writeErr error + + select { + case <-ctx.Done(): + return totalBytesWritten, fmt.Errorf("copy cancelled during write: %w", ctx.Err()) + case result := <-writeResultCh: + bytesWritten = result.bytesWritten + writeErr = result.writeErr + } + + if bytesWritten < 0 || bytesRead < bytesWritten { + bytesWritten = 0 + if writeErr == nil { + writeErr = fmt.Errorf("invalid write result") + } + } + + if writeErr != nil { + return totalBytesWritten, writeErr + } + + if bytesRead != bytesWritten { + return totalBytesWritten, io.ErrShortWrite + } + + totalBytesWritten += int64(bytesWritten) + + if backupProgressListener != nil { + currentSizeMB := float64(totalBytesWritten) / (1024 * 1024) + if currentSizeMB >= lastReportedMB+progressReportIntervalMB { + backupProgressListener(currentSizeMB) + lastReportedMB = currentSizeMB + } + } + } + + if readErr != nil { + if readErr != io.EOF { + return totalBytesWritten, readErr + } + break + } + } + + return totalBytesWritten, nil +} + +func (uc *CreateMysqlBackupUsecase) createBackupContext( + parentCtx context.Context, +) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithTimeout(parentCtx, backupTimeout) + + go func() { + ticker := time.NewTicker(shutdownCheckInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-parentCtx.Done(): + cancel() + return + case <-ticker.C: + if config.IsShouldShutdown() { + cancel() + return + } + } + } + }() + + return ctx, cancel +} + +func (uc *CreateMysqlBackupUsecase) setupBackupEncryption( + backupID uuid.UUID, + backupConfig *backups_config.BackupConfig, + storageWriter io.WriteCloser, +) (io.Writer, *backup_encryption.EncryptionWriter, usecases_common.BackupMetadata, error) { + metadata := usecases_common.BackupMetadata{} + + if backupConfig.Encryption != backups_config.BackupEncryptionEncrypted { + metadata.Encryption = backups_config.BackupEncryptionNone + uc.logger.Info("Encryption disabled for backup", "backupId", backupID) + return storageWriter, nil, metadata, nil + } + + salt, err := backup_encryption.GenerateSalt() + if err != nil { + return nil, nil, metadata, fmt.Errorf("failed to generate salt: %w", err) + } + + nonce, err := backup_encryption.GenerateNonce() + if err != nil { + return nil, nil, metadata, fmt.Errorf("failed to generate nonce: %w", err) + } + + masterKey, err := uc.secretKeyService.GetSecretKey() + if err != nil { + return nil, nil, metadata, fmt.Errorf("failed to get master key: %w", err) + } + + encWriter, err := backup_encryption.NewEncryptionWriter( + storageWriter, + masterKey, + backupID, + salt, + nonce, + ) + if err != nil { + return nil, nil, metadata, fmt.Errorf("failed to create encrypting writer: %w", err) + } + + saltBase64 := base64.StdEncoding.EncodeToString(salt) + nonceBase64 := base64.StdEncoding.EncodeToString(nonce) + metadata.EncryptionSalt = &saltBase64 + metadata.EncryptionIV = &nonceBase64 + metadata.Encryption = backups_config.BackupEncryptionEncrypted + + uc.logger.Info("Encryption enabled for backup", "backupId", backupID) + return encWriter, encWriter, metadata, nil +} + +func (uc *CreateMysqlBackupUsecase) cleanupOnCancellation( + zstdWriter *zstd.Encoder, + encryptionWriter *backup_encryption.EncryptionWriter, + storageWriter io.WriteCloser, + saveErrCh chan error, +) { + if zstdWriter != nil { + go func() { + if closeErr := zstdWriter.Close(); closeErr != nil { + uc.logger.Error( + "Failed to close zstd writer during cancellation", + "error", + closeErr, + ) + } + }() + } + + if encryptionWriter != nil { + go func() { + if closeErr := encryptionWriter.Close(); closeErr != nil { + uc.logger.Error( + "Failed to close encrypting writer during cancellation", + "error", + closeErr, + ) + } + }() + } + + if err := storageWriter.Close(); err != nil { + uc.logger.Error("Failed to close pipe writer during cancellation", "error", err) + } + + <-saveErrCh +} + +func (uc *CreateMysqlBackupUsecase) closeWriters( + encryptionWriter *backup_encryption.EncryptionWriter, + storageWriter io.WriteCloser, +) error { + encryptionCloseErrCh := make(chan error, 1) + if encryptionWriter != nil { + go func() { + closeErr := encryptionWriter.Close() + if closeErr != nil { + uc.logger.Error("Failed to close encrypting writer", "error", closeErr) + } + encryptionCloseErrCh <- closeErr + }() + } else { + encryptionCloseErrCh <- nil + } + + encryptionCloseErr := <-encryptionCloseErrCh + if encryptionCloseErr != nil { + if err := storageWriter.Close(); err != nil { + uc.logger.Error("Failed to close pipe writer after encryption error", "error", err) + } + return fmt.Errorf("failed to close encryption writer: %w", encryptionCloseErr) + } + + if err := storageWriter.Close(); err != nil { + uc.logger.Error("Failed to close pipe writer", "error", err) + return err + } + + return nil +} + +func (uc *CreateMysqlBackupUsecase) checkCancellationReason() error { + if config.IsShouldShutdown() { + return fmt.Errorf("backup cancelled due to shutdown") + } + return fmt.Errorf("backup cancelled") +} + +func (uc *CreateMysqlBackupUsecase) buildMysqldumpErrorMessage( + waitErr error, + stderrOutput []byte, + mysqlBin string, +) error { + stderrStr := string(stderrOutput) + errorMsg := fmt.Sprintf( + "%s failed: %v – stderr: %s", + filepath.Base(mysqlBin), + waitErr, + stderrStr, + ) + + exitErr, ok := waitErr.(*exec.ExitError) + if !ok { + return errors.New(errorMsg) + } + + exitCode := exitErr.ExitCode() + + if exitCode == exitCodeGenericError || exitCode == exitCodeConnectionError { + return uc.handleConnectionErrors(stderrStr) + } + + return errors.New(errorMsg) +} + +func (uc *CreateMysqlBackupUsecase) handleConnectionErrors(stderrStr string) error { + if containsIgnoreCase(stderrStr, "access denied") { + return fmt.Errorf( + "MySQL access denied. Check username and password. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "can't connect") || + containsIgnoreCase(stderrStr, "connection refused") { + return fmt.Errorf( + "MySQL connection refused. Check if the server is running and accessible. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "unknown database") { + return fmt.Errorf( + "MySQL database does not exist. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "ssl") { + return fmt.Errorf( + "MySQL SSL connection failed. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "timeout") { + return fmt.Errorf( + "MySQL connection timeout. stderr: %s", + stderrStr, + ) + } + + return fmt.Errorf("MySQL connection or authentication error. stderr: %s", stderrStr) +} + +func containsIgnoreCase(str, substr string) bool { + return strings.Contains(strings.ToLower(str), strings.ToLower(substr)) +} diff --git a/backend/internal/features/backups/backups/usecases/mysql/di.go b/backend/internal/features/backups/backups/usecases/mysql/di.go new file mode 100644 index 0000000..b6c15c8 --- /dev/null +++ b/backend/internal/features/backups/backups/usecases/mysql/di.go @@ -0,0 +1,17 @@ +package usecases_mysql + +import ( + "postgresus-backend/internal/features/encryption/secrets" + "postgresus-backend/internal/util/encryption" + "postgresus-backend/internal/util/logger" +) + +var createMysqlBackupUsecase = &CreateMysqlBackupUsecase{ + logger.GetLogger(), + secrets.GetSecretKeyService(), + encryption.GetFieldEncryptor(), +} + +func GetCreateMysqlBackupUsecase() *CreateMysqlBackupUsecase { + return createMysqlBackupUsecase +} diff --git a/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go index 79e3db9..66826ef 100644 --- a/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go +++ b/backend/internal/features/backups/backups/usecases/postgresql/create_backup_uc.go @@ -16,6 +16,7 @@ import ( "postgresus-backend/internal/config" backup_encryption "postgresus-backend/internal/features/backups/backups/encryption" + usecases_common "postgresus-backend/internal/features/backups/backups/usecases/common" backups_config "postgresus-backend/internal/features/backups/config" "postgresus-backend/internal/features/databases" pgtypes "postgresus-backend/internal/features/databases/databases/postgresql" @@ -50,7 +51,6 @@ type writeResult struct { writeErr error } -// Execute creates a backup of the database func (uc *CreatePostgresqlBackupUsecase) Execute( ctx context.Context, backupID uuid.UUID, @@ -60,7 +60,7 @@ func (uc *CreatePostgresqlBackupUsecase) Execute( backupProgressListener func( completedMBs float64, ), -) (*BackupMetadata, error) { +) (*usecases_common.BackupMetadata, error) { uc.logger.Info( "Creating PostgreSQL backup via pg_dump custom format", "databaseId", @@ -119,7 +119,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage( storage *storages.Storage, db *databases.Database, backupProgressListener func(completedMBs float64), -) (*BackupMetadata, error) { +) (*usecases_common.BackupMetadata, error) { uc.logger.Info("Streaming PostgreSQL backup to storage", "pgBin", pgBin, "args", args) ctx, cancel := uc.createBackupContext(parentCtx) @@ -171,7 +171,7 @@ func (uc *CreatePostgresqlBackupUsecase) streamToStorage( return nil, err } - countingWriter := &CountingWriter{writer: finalWriter} + countingWriter := usecases_common.NewCountingWriter(finalWriter) // The backup ID becomes the object key / filename in storage @@ -471,8 +471,8 @@ func (uc *CreatePostgresqlBackupUsecase) setupBackupEncryption( backupID uuid.UUID, backupConfig *backups_config.BackupConfig, storageWriter io.WriteCloser, -) (io.Writer, *backup_encryption.EncryptionWriter, BackupMetadata, error) { - metadata := BackupMetadata{} +) (io.Writer, *backup_encryption.EncryptionWriter, usecases_common.BackupMetadata, error) { + metadata := usecases_common.BackupMetadata{} if backupConfig.Encryption != backups_config.BackupEncryptionEncrypted { metadata.Encryption = backups_config.BackupEncryptionNone diff --git a/backend/internal/features/backups/backups/usecases/postgresql/interfaces.go b/backend/internal/features/backups/backups/usecases/postgresql/interfaces.go deleted file mode 100644 index 0e9ab18..0000000 --- a/backend/internal/features/backups/backups/usecases/postgresql/interfaces.go +++ /dev/null @@ -1,20 +0,0 @@ -package usecases_postgresql - -import "io" - -// CountingWriter wraps an io.Writer and counts the bytes written to it -type CountingWriter struct { - writer io.Writer - bytesWritten int64 -} - -func (cw *CountingWriter) Write(p []byte) (n int, err error) { - n, err = cw.writer.Write(p) - cw.bytesWritten += int64(n) - return n, err -} - -// GetBytesWritten returns the total number of bytes written -func (cw *CountingWriter) GetBytesWritten() int64 { - return cw.bytesWritten -} diff --git a/backend/internal/features/databases/databases/mysql/model.go b/backend/internal/features/databases/databases/mysql/model.go new file mode 100644 index 0000000..ee1f32d --- /dev/null +++ b/backend/internal/features/databases/databases/mysql/model.go @@ -0,0 +1,375 @@ +package mysql + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "regexp" + "time" + + "postgresus-backend/internal/util/encryption" + "postgresus-backend/internal/util/tools" + + _ "github.com/go-sql-driver/mysql" + "github.com/google/uuid" +) + +type MysqlDatabase struct { + ID uuid.UUID `json:"id" gorm:"primaryKey;type:uuid;default:gen_random_uuid()"` + DatabaseID *uuid.UUID `json:"databaseId" gorm:"type:uuid;column:database_id"` + + Version tools.MysqlVersion `json:"version" gorm:"type:text;not null"` + + Host string `json:"host" gorm:"type:text;not null"` + Port int `json:"port" gorm:"type:int;not null"` + Username string `json:"username" gorm:"type:text;not null"` + Password string `json:"password" gorm:"type:text;not null"` + Database *string `json:"database" gorm:"type:text"` + IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"` +} + +func (m *MysqlDatabase) TableName() string { + return "mysql_databases" +} + +func (m *MysqlDatabase) Validate() error { + if m.Host == "" { + return errors.New("host is required") + } + if m.Port == 0 { + return errors.New("port is required") + } + if m.Username == "" { + return errors.New("username is required") + } + if m.Password == "" { + return errors.New("password is required") + } + return nil +} + +func (m *MysqlDatabase) TestConnection( + logger *slog.Logger, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) error { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + if m.Database == nil || *m.Database == "" { + return errors.New("database name is required for MySQL backup") + } + + password, err := decryptPasswordIfNeeded(m.Password, encryptor, databaseID) + if err != nil { + return fmt.Errorf("failed to decrypt password: %w", err) + } + + dsn := m.buildDSN(password, *m.Database) + + db, err := sql.Open("mysql", dsn) + if err != nil { + return fmt.Errorf("failed to connect to MySQL database '%s': %w", *m.Database, err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + logger.Error("Failed to close MySQL connection", "error", closeErr) + } + }() + + db.SetConnMaxLifetime(15 * time.Second) + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + + if err := db.PingContext(ctx); err != nil { + return fmt.Errorf("failed to ping MySQL database '%s': %w", *m.Database, err) + } + + detectedVersion, err := detectMysqlVersion(ctx, db) + if err != nil { + return err + } + m.Version = detectedVersion + + return nil +} + +func (m *MysqlDatabase) HideSensitiveData() { + if m == nil { + return + } + m.Password = "" +} + +func (m *MysqlDatabase) Update(incoming *MysqlDatabase) { + m.Version = incoming.Version + m.Host = incoming.Host + m.Port = incoming.Port + m.Username = incoming.Username + m.Database = incoming.Database + m.IsHttps = incoming.IsHttps + + if incoming.Password != "" { + m.Password = incoming.Password + } +} + +func (m *MysqlDatabase) EncryptSensitiveFields( + databaseID uuid.UUID, + encryptor encryption.FieldEncryptor, +) error { + if m.Password != "" { + encrypted, err := encryptor.Encrypt(databaseID, m.Password) + if err != nil { + return err + } + m.Password = encrypted + } + return nil +} + +func (m *MysqlDatabase) PopulateVersionIfEmpty( + logger *slog.Logger, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) error { + if m.Version != "" { + return nil + } + + if m.Database == nil || *m.Database == "" { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + password, err := decryptPasswordIfNeeded(m.Password, encryptor, databaseID) + if err != nil { + return fmt.Errorf("failed to decrypt password: %w", err) + } + + dsn := m.buildDSN(password, *m.Database) + + db, err := sql.Open("mysql", dsn) + if err != nil { + return fmt.Errorf("failed to connect to database: %w", err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + logger.Error("Failed to close connection", "error", closeErr) + } + }() + + detectedVersion, err := detectMysqlVersion(ctx, db) + if err != nil { + return err + } + + m.Version = detectedVersion + return nil +} + +func (m *MysqlDatabase) IsUserReadOnly( + ctx context.Context, + logger *slog.Logger, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) (bool, error) { + password, err := decryptPasswordIfNeeded(m.Password, encryptor, databaseID) + if err != nil { + return false, fmt.Errorf("failed to decrypt password: %w", err) + } + + dsn := m.buildDSN(password, *m.Database) + + db, err := sql.Open("mysql", dsn) + if err != nil { + return false, fmt.Errorf("failed to connect to database: %w", err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + logger.Error("Failed to close connection", "error", closeErr) + } + }() + + rows, err := db.QueryContext(ctx, "SHOW GRANTS FOR CURRENT_USER()") + if err != nil { + return false, fmt.Errorf("failed to check grants: %w", err) + } + defer func() { _ = rows.Close() }() + + writePrivileges := []string{ + "INSERT", "UPDATE", "DELETE", "CREATE", "DROP", "ALTER", + "INDEX", "GRANT OPTION", "ALL PRIVILEGES", "SUPER", + } + + for rows.Next() { + var grant string + if err := rows.Scan(&grant); err != nil { + return false, fmt.Errorf("failed to scan grant: %w", err) + } + + for _, priv := range writePrivileges { + if regexp.MustCompile(`(?i)\b` + priv + `\b`).MatchString(grant) { + return false, nil + } + } + } + + if err := rows.Err(); err != nil { + return false, fmt.Errorf("error iterating grants: %w", err) + } + + return true, nil +} + +func (m *MysqlDatabase) CreateReadOnlyUser( + ctx context.Context, + logger *slog.Logger, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) (string, string, error) { + password, err := decryptPasswordIfNeeded(m.Password, encryptor, databaseID) + if err != nil { + return "", "", fmt.Errorf("failed to decrypt password: %w", err) + } + + dsn := m.buildDSN(password, *m.Database) + + db, err := sql.Open("mysql", dsn) + if err != nil { + return "", "", fmt.Errorf("failed to connect to database: %w", err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + logger.Error("Failed to close connection", "error", closeErr) + } + }() + + maxRetries := 3 + for attempt := range maxRetries { + newUsername := fmt.Sprintf("postgresus-%s", uuid.New().String()[:8]) + newPassword := uuid.New().String() + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return "", "", fmt.Errorf("failed to begin transaction: %w", err) + } + + success := false + defer func() { + if !success { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + logger.Error("Failed to rollback transaction", "error", rollbackErr) + } + } + }() + + _, err = tx.ExecContext(ctx, fmt.Sprintf( + "CREATE USER '%s'@'%%' IDENTIFIED BY '%s'", + newUsername, + newPassword, + )) + if err != nil { + if attempt < maxRetries-1 { + continue + } + return "", "", fmt.Errorf("failed to create user: %w", err) + } + + _, err = tx.ExecContext(ctx, fmt.Sprintf( + "GRANT SELECT, SHOW VIEW, LOCK TABLES, TRIGGER, EVENT ON `%s`.* TO '%s'@'%%'", + *m.Database, + newUsername, + )) + if err != nil { + return "", "", fmt.Errorf("failed to grant database privileges: %w", err) + } + + _, err = tx.ExecContext(ctx, fmt.Sprintf( + "GRANT PROCESS ON *.* TO '%s'@'%%'", + newUsername, + )) + if err != nil { + return "", "", fmt.Errorf("failed to grant PROCESS privilege: %w", err) + } + + _, err = tx.ExecContext(ctx, "FLUSH PRIVILEGES") + if err != nil { + return "", "", fmt.Errorf("failed to flush privileges: %w", err) + } + + if err := tx.Commit(); err != nil { + return "", "", fmt.Errorf("failed to commit transaction: %w", err) + } + + success = true + logger.Info( + "Read-only MySQL user created successfully", + "username", + newUsername, + ) + return newUsername, newPassword, nil + } + + return "", "", errors.New("failed to generate unique username after 3 attempts") +} + +func (m *MysqlDatabase) buildDSN(password string, database string) string { + tlsConfig := "false" + if m.IsHttps { + tlsConfig = "true" + } + + return fmt.Sprintf( + "%s:%s@tcp(%s:%d)/%s?parseTime=true&timeout=15s&tls=%s&charset=utf8mb4", + m.Username, + password, + m.Host, + m.Port, + database, + tlsConfig, + ) +} + +func detectMysqlVersion(ctx context.Context, db *sql.DB) (tools.MysqlVersion, error) { + var versionStr string + err := db.QueryRowContext(ctx, "SELECT VERSION()").Scan(&versionStr) + if err != nil { + return "", fmt.Errorf("failed to query MySQL version: %w", err) + } + + re := regexp.MustCompile(`^(\d+)\.(\d+)`) + matches := re.FindStringSubmatch(versionStr) + if len(matches) < 3 { + return "", fmt.Errorf("could not parse MySQL version: %s", versionStr) + } + + major := matches[1] + minor := matches[2] + + switch { + case major == "5" && minor == "7": + return tools.MysqlVersion57, nil + case major == "8" && minor == "0": + return tools.MysqlVersion80, nil + case major == "8" && minor == "4": + return tools.MysqlVersion84, nil + default: + return "", fmt.Errorf("unsupported MySQL version: %s.%s", major, minor) + } +} + +func decryptPasswordIfNeeded( + password string, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) (string, error) { + if encryptor == nil { + return password, nil + } + return encryptor.Decrypt(databaseID, password) +} diff --git a/backend/internal/features/databases/databases/mysql/readonly_user_test.go b/backend/internal/features/databases/databases/mysql/readonly_user_test.go new file mode 100644 index 0000000..8ec4539 --- /dev/null +++ b/backend/internal/features/databases/databases/mysql/readonly_user_test.go @@ -0,0 +1,366 @@ +package mysql + +import ( + "context" + "fmt" + "log/slog" + "os" + "strconv" + "strings" + "testing" + + _ "github.com/go-sql-driver/mysql" + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/assert" + + "postgresus-backend/internal/config" + "postgresus-backend/internal/util/tools" +) + +func Test_IsUserReadOnly_AdminUser_ReturnsFalse(t *testing.T) { + env := config.GetEnv() + cases := []struct { + name string + version tools.MysqlVersion + port string + }{ + {"MySQL 5.7", tools.MysqlVersion57, env.TestMysql57Port}, + {"MySQL 8.0", tools.MysqlVersion80, env.TestMysql80Port}, + {"MySQL 8.4", tools.MysqlVersion84, env.TestMysql84Port}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + container := connectToMysqlContainer(t, tc.port, tc.version) + defer container.DB.Close() + + mysqlModel := createMysqlModel(container) + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + ctx := context.Background() + + isReadOnly, err := mysqlModel.IsUserReadOnly(ctx, logger, nil, uuid.New()) + assert.NoError(t, err) + assert.False(t, isReadOnly, "Admin user should not be read-only") + }) + } +} + +func Test_CreateReadOnlyUser_UserCanReadButNotWrite(t *testing.T) { + env := config.GetEnv() + cases := []struct { + name string + version tools.MysqlVersion + port string + }{ + {"MySQL 5.7", tools.MysqlVersion57, env.TestMysql57Port}, + {"MySQL 8.0", tools.MysqlVersion80, env.TestMysql80Port}, + {"MySQL 8.4", tools.MysqlVersion84, env.TestMysql84Port}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + container := connectToMysqlContainer(t, tc.port, tc.version) + defer container.DB.Close() + + _, err := container.DB.Exec(`DROP TABLE IF EXISTS readonly_test`) + assert.NoError(t, err) + _, err = container.DB.Exec(`DROP TABLE IF EXISTS hack_table`) + assert.NoError(t, err) + _, err = container.DB.Exec(`DROP TABLE IF EXISTS future_table`) + assert.NoError(t, err) + + _, err = container.DB.Exec(` + CREATE TABLE readonly_test ( + id INT AUTO_INCREMENT PRIMARY KEY, + data VARCHAR(255) NOT NULL + ) + `) + assert.NoError(t, err) + + _, err = container.DB.Exec( + `INSERT INTO readonly_test (data) VALUES ('test1'), ('test2')`, + ) + assert.NoError(t, err) + + mysqlModel := createMysqlModel(container) + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + ctx := context.Background() + + username, password, err := mysqlModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New()) + assert.NoError(t, err) + assert.NotEmpty(t, username) + assert.NotEmpty(t, password) + assert.True(t, strings.HasPrefix(username, "postgresus-")) + + readOnlyModel := &MysqlDatabase{ + Version: mysqlModel.Version, + Host: mysqlModel.Host, + Port: mysqlModel.Port, + Username: username, + Password: password, + Database: mysqlModel.Database, + IsHttps: false, + } + + isReadOnly, err := readOnlyModel.IsUserReadOnly(ctx, logger, nil, uuid.New()) + assert.NoError(t, err) + assert.True(t, isReadOnly, "Created user should be read-only") + + readOnlyDSN := fmt.Sprintf( + "%s:%s@tcp(%s:%d)/%s?parseTime=true", + username, + password, + container.Host, + container.Port, + container.Database, + ) + readOnlyConn, err := sqlx.Connect("mysql", readOnlyDSN) + assert.NoError(t, err) + defer readOnlyConn.Close() + + var count int + err = readOnlyConn.Get(&count, "SELECT COUNT(*) FROM readonly_test") + assert.NoError(t, err) + assert.Equal(t, 2, count) + + _, err = readOnlyConn.Exec("INSERT INTO readonly_test (data) VALUES ('should-fail')") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = readOnlyConn.Exec("UPDATE readonly_test SET data = 'hacked' WHERE id = 1") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = readOnlyConn.Exec("DELETE FROM readonly_test WHERE id = 1") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = readOnlyConn.Exec("CREATE TABLE hack_table (id INT)") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = container.DB.Exec(fmt.Sprintf("DROP USER IF EXISTS '%s'@'%%'", username)) + assert.NoError(t, err) + }) + } +} + +func Test_ReadOnlyUser_FutureTables_NoSelectPermission(t *testing.T) { + env := config.GetEnv() + container := connectToMysqlContainer(t, env.TestMysql80Port, tools.MysqlVersion80) + defer container.DB.Close() + + mysqlModel := createMysqlModel(container) + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + ctx := context.Background() + + username, password, err := mysqlModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New()) + assert.NoError(t, err) + + _, err = container.DB.Exec(`DROP TABLE IF EXISTS future_table`) + assert.NoError(t, err) + _, err = container.DB.Exec(` + CREATE TABLE future_table ( + id INT AUTO_INCREMENT PRIMARY KEY, + data VARCHAR(255) NOT NULL + ) + `) + assert.NoError(t, err) + _, err = container.DB.Exec(`INSERT INTO future_table (data) VALUES ('future_data')`) + assert.NoError(t, err) + + readOnlyDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + username, password, container.Host, container.Port, container.Database) + readOnlyConn, err := sqlx.Connect("mysql", readOnlyDSN) + assert.NoError(t, err) + defer readOnlyConn.Close() + + var data string + err = readOnlyConn.Get(&data, "SELECT data FROM future_table LIMIT 1") + assert.NoError(t, err) + assert.Equal(t, "future_data", data) + + _, err = container.DB.Exec(fmt.Sprintf("DROP USER IF EXISTS '%s'@'%%'", username)) + assert.NoError(t, err) +} + +func Test_CreateReadOnlyUser_DatabaseNameWithDash_Success(t *testing.T) { + env := config.GetEnv() + container := connectToMysqlContainer(t, env.TestMysql80Port, tools.MysqlVersion80) + defer container.DB.Close() + + dashDbName := "test-db-with-dash" + + _, err := container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", dashDbName)) + assert.NoError(t, err) + + _, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE `%s`", dashDbName)) + assert.NoError(t, err) + + defer func() { + _, _ = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", dashDbName)) + }() + + dashDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + container.Username, container.Password, container.Host, container.Port, dashDbName) + dashDB, err := sqlx.Connect("mysql", dashDSN) + assert.NoError(t, err) + defer dashDB.Close() + + _, err = dashDB.Exec(` + CREATE TABLE dash_test ( + id INT AUTO_INCREMENT PRIMARY KEY, + data VARCHAR(255) NOT NULL + ) + `) + assert.NoError(t, err) + + _, err = dashDB.Exec(`INSERT INTO dash_test (data) VALUES ('test1'), ('test2')`) + assert.NoError(t, err) + + mysqlModel := &MysqlDatabase{ + Version: tools.MysqlVersion80, + Host: container.Host, + Port: container.Port, + Username: container.Username, + Password: container.Password, + Database: &dashDbName, + IsHttps: false, + } + + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + ctx := context.Background() + + username, password, err := mysqlModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New()) + assert.NoError(t, err) + assert.NotEmpty(t, username) + assert.NotEmpty(t, password) + assert.True(t, strings.HasPrefix(username, "postgresus-")) + + readOnlyDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + username, password, container.Host, container.Port, dashDbName) + readOnlyConn, err := sqlx.Connect("mysql", readOnlyDSN) + assert.NoError(t, err) + defer readOnlyConn.Close() + + var count int + err = readOnlyConn.Get(&count, "SELECT COUNT(*) FROM dash_test") + assert.NoError(t, err) + assert.Equal(t, 2, count) + + _, err = readOnlyConn.Exec("INSERT INTO dash_test (data) VALUES ('should-fail')") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = dashDB.Exec(fmt.Sprintf("DROP USER IF EXISTS '%s'@'%%'", username)) + assert.NoError(t, err) +} + +func Test_ReadOnlyUser_CannotDropOrAlterTables(t *testing.T) { + env := config.GetEnv() + container := connectToMysqlContainer(t, env.TestMysql80Port, tools.MysqlVersion80) + defer container.DB.Close() + + _, err := container.DB.Exec(`DROP TABLE IF EXISTS drop_test`) + assert.NoError(t, err) + _, err = container.DB.Exec(` + CREATE TABLE drop_test ( + id INT AUTO_INCREMENT PRIMARY KEY, + data VARCHAR(255) NOT NULL + ) + `) + assert.NoError(t, err) + _, err = container.DB.Exec(`INSERT INTO drop_test (data) VALUES ('test1')`) + assert.NoError(t, err) + + mysqlModel := createMysqlModel(container) + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + ctx := context.Background() + + username, password, err := mysqlModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New()) + assert.NoError(t, err) + + readOnlyDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + username, password, container.Host, container.Port, container.Database) + readOnlyConn, err := sqlx.Connect("mysql", readOnlyDSN) + assert.NoError(t, err) + defer readOnlyConn.Close() + + _, err = readOnlyConn.Exec("DROP TABLE drop_test") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = readOnlyConn.Exec("ALTER TABLE drop_test ADD COLUMN new_col VARCHAR(100)") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = readOnlyConn.Exec("TRUNCATE TABLE drop_test") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = container.DB.Exec(fmt.Sprintf("DROP USER IF EXISTS '%s'@'%%'", username)) + assert.NoError(t, err) +} + +type MysqlContainer struct { + Host string + Port int + Username string + Password string + Database string + Version tools.MysqlVersion + DB *sqlx.DB +} + +func connectToMysqlContainer( + t *testing.T, + port string, + version tools.MysqlVersion, +) *MysqlContainer { + if port == "" { + t.Skipf("MySQL port not configured for version %s", version) + } + + dbName := "testdb" + password := "testpassword" + username := "testuser" + host := "localhost" + + portInt, err := strconv.Atoi(port) + assert.NoError(t, err) + + dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + username, password, host, portInt, dbName) + + db, err := sqlx.Connect("mysql", dsn) + if err != nil { + t.Skipf("Failed to connect to MySQL %s: %v", version, err) + } + + return &MysqlContainer{ + Host: host, + Port: portInt, + Username: username, + Password: password, + Database: dbName, + Version: version, + DB: db, + } +} + +func createMysqlModel(container *MysqlContainer) *MysqlDatabase { + return &MysqlDatabase{ + Version: container.Version, + Host: container.Host, + Port: container.Port, + Username: container.Username, + Password: container.Password, + Database: &container.Database, + IsHttps: false, + } +} diff --git a/backend/internal/features/databases/enums.go b/backend/internal/features/databases/enums.go index 722115d..8a65530 100644 --- a/backend/internal/features/databases/enums.go +++ b/backend/internal/features/databases/enums.go @@ -4,6 +4,7 @@ type DatabaseType string const ( DatabaseTypePostgres DatabaseType = "POSTGRES" + DatabaseTypeMysql DatabaseType = "MYSQL" ) type HealthStatus string diff --git a/backend/internal/features/databases/model.go b/backend/internal/features/databases/model.go index f25cc09..fc6b167 100644 --- a/backend/internal/features/databases/model.go +++ b/backend/internal/features/databases/model.go @@ -3,6 +3,7 @@ package databases import ( "errors" "log/slog" + "postgresus-backend/internal/features/databases/databases/mysql" "postgresus-backend/internal/features/databases/databases/postgresql" "postgresus-backend/internal/features/notifiers" "postgresus-backend/internal/util/encryption" @@ -21,6 +22,7 @@ type Database struct { Type DatabaseType `json:"type" gorm:"column:type;type:text;not null"` Postgresql *postgresql.PostgresqlDatabase `json:"postgresql,omitempty" gorm:"foreignKey:DatabaseID"` + Mysql *mysql.MysqlDatabase `json:"mysql,omitempty" gorm:"foreignKey:DatabaseID"` Notifiers []notifiers.Notifier `json:"notifiers" gorm:"many2many:database_notifiers;"` @@ -42,8 +44,12 @@ func (d *Database) Validate() error { if d.Postgresql == nil { return errors.New("postgresql database is required") } - return d.Postgresql.Validate() + case DatabaseTypeMysql: + if d.Mysql == nil { + return errors.New("mysql database is required") + } + return d.Mysql.Validate() default: return errors.New("invalid database type: " + string(d.Type)) } @@ -72,6 +78,9 @@ func (d *Database) EncryptSensitiveFields(encryptor encryption.FieldEncryptor) e if d.Postgresql != nil { return d.Postgresql.EncryptSensitiveFields(d.ID, encryptor) } + if d.Mysql != nil { + return d.Mysql.EncryptSensitiveFields(d.ID, encryptor) + } return nil } @@ -82,6 +91,9 @@ func (d *Database) PopulateVersionIfEmpty( if d.Postgresql != nil { return d.Postgresql.PopulateVersionIfEmpty(logger, encryptor, d.ID) } + if d.Mysql != nil { + return d.Mysql.PopulateVersionIfEmpty(logger, encryptor, d.ID) + } return nil } @@ -95,6 +107,10 @@ func (d *Database) Update(incoming *Database) { if d.Postgresql != nil && incoming.Postgresql != nil { d.Postgresql.Update(incoming.Postgresql) } + case DatabaseTypeMysql: + if d.Mysql != nil && incoming.Mysql != nil { + d.Mysql.Update(incoming.Mysql) + } } } @@ -102,6 +118,8 @@ func (d *Database) getSpecificDatabase() DatabaseConnector { switch d.Type { case DatabaseTypePostgres: return d.Postgresql + case DatabaseTypeMysql: + return d.Mysql } panic("invalid database type: " + string(d.Type)) diff --git a/backend/internal/features/databases/repository.go b/backend/internal/features/databases/repository.go index 020902f..42621ab 100644 --- a/backend/internal/features/databases/repository.go +++ b/backend/internal/features/databases/repository.go @@ -2,6 +2,7 @@ package databases import ( "errors" + "postgresus-backend/internal/features/databases/databases/mysql" "postgresus-backend/internal/features/databases/databases/postgresql" "postgresus-backend/internal/storage" @@ -25,26 +26,28 @@ func (r *DatabaseRepository) Save(database *Database) (*Database, error) { if database.Postgresql == nil { return errors.New("postgresql configuration is required for PostgreSQL database") } - - // Ensure DatabaseID is always set and never nil database.Postgresql.DatabaseID = &database.ID + case DatabaseTypeMysql: + if database.Mysql == nil { + return errors.New("mysql configuration is required for MySQL database") + } + database.Mysql.DatabaseID = &database.ID } if isNew { if err := tx.Create(database). - Omit("Postgresql", "Notifiers"). + Omit("Postgresql", "Mysql", "Notifiers"). Error; err != nil { return err } } else { if err := tx.Save(database). - Omit("Postgresql", "Notifiers"). + Omit("Postgresql", "Mysql", "Notifiers"). Error; err != nil { return err } } - // Save the specific database type switch database.Type { case DatabaseTypePostgres: database.Postgresql.DatabaseID = &database.ID @@ -58,6 +61,18 @@ func (r *DatabaseRepository) Save(database *Database) (*Database, error) { return err } } + case DatabaseTypeMysql: + database.Mysql.DatabaseID = &database.ID + if database.Mysql.ID == uuid.Nil { + database.Mysql.ID = uuid.New() + if err := tx.Create(database.Mysql).Error; err != nil { + return err + } + } else { + if err := tx.Save(database.Mysql).Error; err != nil { + return err + } + } } if err := tx. @@ -83,6 +98,7 @@ func (r *DatabaseRepository) FindByID(id uuid.UUID) (*Database, error) { if err := storage. GetDb(). Preload("Postgresql"). + Preload("Mysql"). Preload("Notifiers"). Where("id = ?", id). First(&database).Error; err != nil { @@ -98,6 +114,7 @@ func (r *DatabaseRepository) FindByWorkspaceID(workspaceID uuid.UUID) ([]*Databa if err := storage. GetDb(). Preload("Postgresql"). + Preload("Mysql"). Preload("Notifiers"). Where("workspace_id = ?", workspaceID). Order("CASE WHEN health_status = 'UNAVAILABLE' THEN 1 WHEN health_status = 'AVAILABLE' THEN 2 WHEN health_status IS NULL THEN 3 ELSE 4 END, name ASC"). @@ -128,6 +145,12 @@ func (r *DatabaseRepository) Delete(id uuid.UUID) error { Delete(&postgresql.PostgresqlDatabase{}).Error; err != nil { return err } + case DatabaseTypeMysql: + if err := tx. + Where("database_id = ?", id). + Delete(&mysql.MysqlDatabase{}).Error; err != nil { + return err + } } if err := tx.Delete(&Database{}, id).Error; err != nil { @@ -158,6 +181,7 @@ func (r *DatabaseRepository) GetAllDatabases() ([]*Database, error) { if err := storage. GetDb(). Preload("Postgresql"). + Preload("Mysql"). Preload("Notifiers"). Find(&databases).Error; err != nil { return nil, err diff --git a/backend/internal/features/databases/service.go b/backend/internal/features/databases/service.go index bf81e3f..51b7a60 100644 --- a/backend/internal/features/databases/service.go +++ b/backend/internal/features/databases/service.go @@ -8,6 +8,7 @@ import ( "time" audit_logs "postgresus-backend/internal/features/audit_logs" + "postgresus-backend/internal/features/databases/databases/mysql" "postgresus-backend/internal/features/databases/databases/postgresql" "postgresus-backend/internal/features/notifiers" users_models "postgresus-backend/internal/features/users/models" @@ -404,6 +405,20 @@ func (s *DatabaseService) CopyDatabase( IsHttps: existingDatabase.Postgresql.IsHttps, } } + case DatabaseTypeMysql: + if existingDatabase.Mysql != nil { + newDatabase.Mysql = &mysql.MysqlDatabase{ + ID: uuid.Nil, + DatabaseID: nil, + Version: existingDatabase.Mysql.Version, + Host: existingDatabase.Mysql.Host, + Port: existingDatabase.Mysql.Port, + Username: existingDatabase.Mysql.Username, + Password: existingDatabase.Mysql.Password, + Database: existingDatabase.Mysql.Database, + IsHttps: existingDatabase.Mysql.IsHttps, + } + } } if err := newDatabase.Validate(); err != nil { @@ -518,19 +533,27 @@ func (s *DatabaseService) IsUserReadOnly( usingDatabase = database } - if usingDatabase.Type != DatabaseTypePostgres { - return false, errors.New("read-only check only supported for PostgreSQL databases") - } - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() - return usingDatabase.Postgresql.IsUserReadOnly( - ctx, - s.logger, - s.fieldEncryptor, - usingDatabase.ID, - ) + switch usingDatabase.Type { + case DatabaseTypePostgres: + return usingDatabase.Postgresql.IsUserReadOnly( + ctx, + s.logger, + s.fieldEncryptor, + usingDatabase.ID, + ) + case DatabaseTypeMysql: + return usingDatabase.Mysql.IsUserReadOnly( + ctx, + s.logger, + s.fieldEncryptor, + usingDatabase.ID, + ) + default: + return false, errors.New("read-only check not supported for this database type") + } } func (s *DatabaseService) CreateReadOnlyUser( @@ -582,16 +605,25 @@ func (s *DatabaseService) CreateReadOnlyUser( usingDatabase = database } - if usingDatabase.Type != DatabaseTypePostgres { - return "", "", errors.New("read-only user creation only supported for PostgreSQL") - } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - username, password, err := usingDatabase.Postgresql.CreateReadOnlyUser( - ctx, s.logger, s.fieldEncryptor, usingDatabase.ID, - ) + var username, password string + var err error + + switch usingDatabase.Type { + case DatabaseTypePostgres: + username, password, err = usingDatabase.Postgresql.CreateReadOnlyUser( + ctx, s.logger, s.fieldEncryptor, usingDatabase.ID, + ) + case DatabaseTypeMysql: + username, password, err = usingDatabase.Mysql.CreateReadOnlyUser( + ctx, s.logger, s.fieldEncryptor, usingDatabase.ID, + ) + default: + return "", "", errors.New("read-only user creation not supported for this database type") + } + if err != nil { return "", "", err } diff --git a/backend/internal/features/restores/dto.go b/backend/internal/features/restores/dto.go index fd62531..acd8e4a 100644 --- a/backend/internal/features/restores/dto.go +++ b/backend/internal/features/restores/dto.go @@ -1,9 +1,11 @@ package restores import ( + "postgresus-backend/internal/features/databases/databases/mysql" "postgresus-backend/internal/features/databases/databases/postgresql" ) type RestoreBackupRequest struct { PostgresqlDatabase *postgresql.PostgresqlDatabase `json:"postgresqlDatabase"` + MysqlDatabase *mysql.MysqlDatabase `json:"mysqlDatabase"` } diff --git a/backend/internal/features/restores/service.go b/backend/internal/features/restores/service.go index a30e987..7011f59 100644 --- a/backend/internal/features/restores/service.go +++ b/backend/internal/features/restores/service.go @@ -122,13 +122,8 @@ func (s *RestoreService) RestoreBackupWithAuth( return err } - if tools.IsBackupDbVersionHigherThanRestoreDbVersion( - backupDatabase.Postgresql.Version, - requestDTO.PostgresqlDatabase.Version, - ) { - return errors.New(`backup database version is higher than restore database version. ` + - `Should be restored to the same version as the backup database or higher. ` + - `For example, you can restore PG 15 backup to PG 15, 16 or higher. But cannot restore to 14 and lower`) + if err := s.validateVersionCompatibility(backupDatabase, requestDTO); err != nil { + return err } go func() { @@ -163,10 +158,15 @@ func (s *RestoreService) RestoreBackup( return err } - if database.Type == databases.DatabaseTypePostgres { + switch database.Type { + case databases.DatabaseTypePostgres: if requestDTO.PostgresqlDatabase == nil { return errors.New("postgresql database is required") } + case databases.DatabaseTypeMysql: + if requestDTO.MysqlDatabase == nil { + return errors.New("mysql database is required") + } } restore := models.Restore{ @@ -207,7 +207,9 @@ func (s *RestoreService) RestoreBackup( start := time.Now().UTC() restoringToDB := &databases.Database{ + Type: database.Type, Postgresql: requestDTO.PostgresqlDatabase, + Mysql: requestDTO.MysqlDatabase, } if err := restoringToDB.PopulateVersionIfEmpty(s.logger, s.fieldEncryptor); err != nil { @@ -250,3 +252,36 @@ func (s *RestoreService) RestoreBackup( return nil } + +func (s *RestoreService) validateVersionCompatibility( + backupDatabase *databases.Database, + requestDTO RestoreBackupRequest, +) error { + switch backupDatabase.Type { + case databases.DatabaseTypePostgres: + if requestDTO.PostgresqlDatabase == nil { + return errors.New("postgresql database configuration is required for restore") + } + if tools.IsBackupDbVersionHigherThanRestoreDbVersion( + backupDatabase.Postgresql.Version, + requestDTO.PostgresqlDatabase.Version, + ) { + return errors.New(`backup database version is higher than restore database version. ` + + `Should be restored to the same version as the backup database or higher. ` + + `For example, you can restore PG 15 backup to PG 15, 16 or higher. But cannot restore to 14 and lower`) + } + case databases.DatabaseTypeMysql: + if requestDTO.MysqlDatabase == nil { + return errors.New("mysql database configuration is required for restore") + } + if tools.IsMysqlBackupVersionHigherThanRestoreVersion( + backupDatabase.Mysql.Version, + requestDTO.MysqlDatabase.Version, + ) { + return errors.New(`backup database version is higher than restore database version. ` + + `Should be restored to the same version as the backup database or higher. ` + + `For example, you can restore MySQL 8.0 backup to MySQL 8.0, 8.4 or higher. But cannot restore to 5.7`) + } + } + return nil +} diff --git a/backend/internal/features/restores/usecases/di.go b/backend/internal/features/restores/usecases/di.go index a4ec802..764755d 100644 --- a/backend/internal/features/restores/usecases/di.go +++ b/backend/internal/features/restores/usecases/di.go @@ -1,11 +1,13 @@ package usecases import ( + usecases_mysql "postgresus-backend/internal/features/restores/usecases/mysql" usecases_postgresql "postgresus-backend/internal/features/restores/usecases/postgresql" ) var restoreBackupUsecase = &RestoreBackupUsecase{ usecases_postgresql.GetRestorePostgresqlBackupUsecase(), + usecases_mysql.GetRestoreMysqlBackupUsecase(), } func GetRestoreBackupUsecase() *RestoreBackupUsecase { diff --git a/backend/internal/features/restores/usecases/mysql/di.go b/backend/internal/features/restores/usecases/mysql/di.go new file mode 100644 index 0000000..d3bc6c9 --- /dev/null +++ b/backend/internal/features/restores/usecases/mysql/di.go @@ -0,0 +1,15 @@ +package usecases_mysql + +import ( + "postgresus-backend/internal/features/encryption/secrets" + "postgresus-backend/internal/util/logger" +) + +var restoreMysqlBackupUsecase = &RestoreMysqlBackupUsecase{ + logger.GetLogger(), + secrets.GetSecretKeyService(), +} + +func GetRestoreMysqlBackupUsecase() *RestoreMysqlBackupUsecase { + return restoreMysqlBackupUsecase +} diff --git a/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go new file mode 100644 index 0000000..988b0a1 --- /dev/null +++ b/backend/internal/features/restores/usecases/mysql/restore_backup_uc.go @@ -0,0 +1,463 @@ +package usecases_mysql + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/klauspost/compress/zstd" + + "postgresus-backend/internal/config" + "postgresus-backend/internal/features/backups/backups" + "postgresus-backend/internal/features/backups/backups/encryption" + backups_config "postgresus-backend/internal/features/backups/config" + "postgresus-backend/internal/features/databases" + mysqltypes "postgresus-backend/internal/features/databases/databases/mysql" + encryption_secrets "postgresus-backend/internal/features/encryption/secrets" + "postgresus-backend/internal/features/restores/models" + "postgresus-backend/internal/features/storages" + util_encryption "postgresus-backend/internal/util/encryption" + files_utils "postgresus-backend/internal/util/files" + "postgresus-backend/internal/util/tools" +) + +type RestoreMysqlBackupUsecase struct { + logger *slog.Logger + secretKeyService *encryption_secrets.SecretKeyService +} + +func (uc *RestoreMysqlBackupUsecase) Execute( + originalDB *databases.Database, + restoringToDB *databases.Database, + backupConfig *backups_config.BackupConfig, + restore models.Restore, + backup *backups.Backup, + storage *storages.Storage, +) error { + if originalDB.Type != databases.DatabaseTypeMysql { + return errors.New("database type not supported") + } + + uc.logger.Info( + "Restoring MySQL backup via mysql client", + "restoreId", restore.ID, + "backupId", backup.ID, + ) + + my := restoringToDB.Mysql + if my == nil { + return fmt.Errorf("mysql configuration is required for restore") + } + + if my.Database == nil || *my.Database == "" { + return fmt.Errorf("target database name is required for mysql restore") + } + + args := []string{ + "--host=" + my.Host, + "--port=" + strconv.Itoa(my.Port), + "--user=" + my.Username, + "--verbose", + } + + if my.IsHttps { + args = append(args, "--ssl-mode=REQUIRED") + } + + if my.Database != nil && *my.Database != "" { + args = append(args, *my.Database) + } + + return uc.restoreFromStorage( + originalDB, + tools.GetMysqlExecutable( + my.Version, + tools.MysqlExecutableMysql, + config.GetEnv().EnvMode, + config.GetEnv().MysqlInstallDir, + ), + args, + my.Password, + backup, + storage, + my, + ) +} + +func (uc *RestoreMysqlBackupUsecase) restoreFromStorage( + database *databases.Database, + mysqlBin string, + args []string, + password string, + backup *backups.Backup, + storage *storages.Storage, + myConfig *mysqltypes.MysqlDatabase, +) error { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + go func() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if config.IsShouldShutdown() { + cancel() + return + } + } + } + }() + + fieldEncryptor := util_encryption.GetFieldEncryptor() + decryptedPassword, err := fieldEncryptor.Decrypt(database.ID, password) + if err != nil { + return fmt.Errorf("failed to decrypt password: %w", err) + } + + myCnfFile, err := uc.createTempMyCnfFile(myConfig, decryptedPassword) + if err != nil { + return fmt.Errorf("failed to create .my.cnf: %w", err) + } + defer func() { _ = os.RemoveAll(filepath.Dir(myCnfFile)) }() + + tempBackupFile, cleanupFunc, err := uc.downloadBackupToTempFile(ctx, backup, storage) + if err != nil { + return fmt.Errorf("failed to download backup: %w", err) + } + defer cleanupFunc() + + return uc.executeMysqlRestore(ctx, database, mysqlBin, args, myCnfFile, tempBackupFile, backup) +} + +func (uc *RestoreMysqlBackupUsecase) executeMysqlRestore( + ctx context.Context, + database *databases.Database, + mysqlBin string, + args []string, + myCnfFile string, + backupFile string, + backup *backups.Backup, +) error { + fullArgs := append([]string{"--defaults-file=" + myCnfFile}, args...) + + cmd := exec.CommandContext(ctx, mysqlBin, fullArgs...) + uc.logger.Info("Executing MySQL restore command", "command", cmd.String()) + + backupFileHandle, err := os.Open(backupFile) + if err != nil { + return fmt.Errorf("failed to open backup file: %w", err) + } + defer func() { _ = backupFileHandle.Close() }() + + var inputReader io.Reader = backupFileHandle + + if backup.Encryption == backups_config.BackupEncryptionEncrypted { + decryptReader, err := uc.setupDecryption(backupFileHandle, backup) + if err != nil { + return fmt.Errorf("failed to setup decryption: %w", err) + } + inputReader = decryptReader + } + + zstdReader, err := zstd.NewReader(inputReader) + if err != nil { + return fmt.Errorf("failed to create zstd reader: %w", err) + } + defer zstdReader.Close() + + cmd.Stdin = zstdReader + + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, + "MYSQL_PWD=", + "LC_ALL=C.UTF-8", + "LANG=C.UTF-8", + ) + + stderrPipe, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("stderr pipe: %w", err) + } + + stderrCh := make(chan []byte, 1) + go func() { + output, _ := io.ReadAll(stderrPipe) + stderrCh <- output + }() + + if err = cmd.Start(); err != nil { + return fmt.Errorf("start mysql: %w", err) + } + + waitErr := cmd.Wait() + stderrOutput := <-stderrCh + + if config.IsShouldShutdown() { + return fmt.Errorf("restore cancelled due to shutdown") + } + + if waitErr != nil { + return uc.handleMysqlRestoreError(database, waitErr, stderrOutput, mysqlBin) + } + + return nil +} + +func (uc *RestoreMysqlBackupUsecase) downloadBackupToTempFile( + ctx context.Context, + backup *backups.Backup, + storage *storages.Storage, +) (string, func(), error) { + err := files_utils.EnsureDirectories([]string{ + config.GetEnv().TempFolder, + }) + if err != nil { + return "", nil, fmt.Errorf("failed to ensure directories: %w", err) + } + + tempDir, err := os.MkdirTemp(config.GetEnv().TempFolder, "restore_"+uuid.New().String()) + if err != nil { + return "", nil, fmt.Errorf("failed to create temporary directory: %w", err) + } + + cleanupFunc := func() { + _ = os.RemoveAll(tempDir) + } + + tempBackupFile := filepath.Join(tempDir, "backup.sql.zst") + + uc.logger.Info( + "Downloading backup file from storage to temporary file", + "backupId", backup.ID, + "tempFile", tempBackupFile, + "encrypted", backup.Encryption == backups_config.BackupEncryptionEncrypted, + ) + + fieldEncryptor := util_encryption.GetFieldEncryptor() + rawReader, err := storage.GetFile(fieldEncryptor, backup.ID) + if err != nil { + cleanupFunc() + return "", nil, fmt.Errorf("failed to get backup file from storage: %w", err) + } + defer func() { + if err := rawReader.Close(); err != nil { + uc.logger.Error("Failed to close backup reader", "error", err) + } + }() + + tempFile, err := os.Create(tempBackupFile) + if err != nil { + cleanupFunc() + return "", nil, fmt.Errorf("failed to create temporary backup file: %w", err) + } + defer func() { + if err := tempFile.Close(); err != nil { + uc.logger.Error("Failed to close temporary file", "error", err) + } + }() + + _, err = uc.copyWithShutdownCheck(ctx, tempFile, rawReader) + if err != nil { + cleanupFunc() + return "", nil, fmt.Errorf("failed to write backup to temporary file: %w", err) + } + + uc.logger.Info("Backup file written to temporary location", "tempFile", tempBackupFile) + return tempBackupFile, cleanupFunc, nil +} + +func (uc *RestoreMysqlBackupUsecase) setupDecryption( + reader io.Reader, + backup *backups.Backup, +) (io.Reader, error) { + if backup.EncryptionSalt == nil || backup.EncryptionIV == nil { + return nil, fmt.Errorf("backup is encrypted but missing encryption metadata") + } + + masterKey, err := uc.secretKeyService.GetSecretKey() + if err != nil { + return nil, fmt.Errorf("failed to get master key for decryption: %w", err) + } + + salt, err := base64.StdEncoding.DecodeString(*backup.EncryptionSalt) + if err != nil { + return nil, fmt.Errorf("failed to decode encryption salt: %w", err) + } + + iv, err := base64.StdEncoding.DecodeString(*backup.EncryptionIV) + if err != nil { + return nil, fmt.Errorf("failed to decode encryption IV: %w", err) + } + + decryptReader, err := encryption.NewDecryptionReader( + reader, + masterKey, + backup.ID, + salt, + iv, + ) + if err != nil { + return nil, fmt.Errorf("failed to create decryption reader: %w", err) + } + + uc.logger.Info("Using decryption for encrypted backup", "backupId", backup.ID) + return decryptReader, nil +} + +func (uc *RestoreMysqlBackupUsecase) createTempMyCnfFile( + myConfig *mysqltypes.MysqlDatabase, + password string, +) (string, error) { + tempDir, err := os.MkdirTemp("", "mycnf") + if err != nil { + return "", fmt.Errorf("failed to create temp directory: %w", err) + } + + myCnfFile := filepath.Join(tempDir, ".my.cnf") + + content := fmt.Sprintf(`[client] +user=%s +password="%s" +host=%s +port=%d +`, myConfig.Username, tools.EscapeMysqlPassword(password), myConfig.Host, myConfig.Port) + + if myConfig.IsHttps { + content += "ssl-mode=REQUIRED\n" + } + + err = os.WriteFile(myCnfFile, []byte(content), 0600) + if err != nil { + return "", fmt.Errorf("failed to write .my.cnf: %w", err) + } + + return myCnfFile, nil +} + +func (uc *RestoreMysqlBackupUsecase) copyWithShutdownCheck( + ctx context.Context, + dst io.Writer, + src io.Reader, +) (int64, error) { + buf := make([]byte, 16*1024*1024) + var totalBytesWritten int64 + + for { + select { + case <-ctx.Done(): + return totalBytesWritten, fmt.Errorf("copy cancelled: %w", ctx.Err()) + default: + } + + if config.IsShouldShutdown() { + return totalBytesWritten, fmt.Errorf("copy cancelled due to shutdown") + } + + bytesRead, readErr := src.Read(buf) + if bytesRead > 0 { + bytesWritten, writeErr := dst.Write(buf[0:bytesRead]) + if bytesWritten < 0 || bytesRead < bytesWritten { + bytesWritten = 0 + if writeErr == nil { + writeErr = fmt.Errorf("invalid write result") + } + } + + if writeErr != nil { + return totalBytesWritten, writeErr + } + + if bytesRead != bytesWritten { + return totalBytesWritten, io.ErrShortWrite + } + + totalBytesWritten += int64(bytesWritten) + } + + if readErr != nil { + if readErr != io.EOF { + return totalBytesWritten, readErr + } + break + } + } + + return totalBytesWritten, nil +} + +func (uc *RestoreMysqlBackupUsecase) handleMysqlRestoreError( + database *databases.Database, + waitErr error, + stderrOutput []byte, + mysqlBin string, +) error { + stderrStr := string(stderrOutput) + errorMsg := fmt.Sprintf( + "%s failed: %v – stderr: %s", + filepath.Base(mysqlBin), + waitErr, + stderrStr, + ) + + if containsIgnoreCase(stderrStr, "access denied") { + return fmt.Errorf( + "MySQL access denied. Check username and password. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "can't connect") || + containsIgnoreCase(stderrStr, "connection refused") { + return fmt.Errorf( + "MySQL connection refused. Check if the server is running and accessible. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "unknown database") { + backupDbName := "unknown" + if database.Mysql != nil && database.Mysql.Database != nil { + backupDbName = *database.Mysql.Database + } + + return fmt.Errorf( + "target database does not exist (backup db %s). Create the database before restoring. stderr: %s", + backupDbName, + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "ssl") { + return fmt.Errorf( + "MySQL SSL connection failed. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "timeout") { + return fmt.Errorf( + "MySQL connection timeout. stderr: %s", + stderrStr, + ) + } + + return errors.New(errorMsg) +} + +func containsIgnoreCase(str, substr string) bool { + return strings.Contains(strings.ToLower(str), strings.ToLower(substr)) +} diff --git a/backend/internal/features/restores/usecases/restore_backup_uc.go b/backend/internal/features/restores/usecases/restore_backup_uc.go index 2879e99..9be2fcc 100644 --- a/backend/internal/features/restores/usecases/restore_backup_uc.go +++ b/backend/internal/features/restores/usecases/restore_backup_uc.go @@ -2,16 +2,19 @@ package usecases import ( "errors" + "postgresus-backend/internal/features/backups/backups" backups_config "postgresus-backend/internal/features/backups/config" "postgresus-backend/internal/features/databases" "postgresus-backend/internal/features/restores/models" + usecases_mysql "postgresus-backend/internal/features/restores/usecases/mysql" usecases_postgresql "postgresus-backend/internal/features/restores/usecases/postgresql" "postgresus-backend/internal/features/storages" ) type RestoreBackupUsecase struct { restorePostgresqlBackupUsecase *usecases_postgresql.RestorePostgresqlBackupUsecase + restoreMysqlBackupUsecase *usecases_mysql.RestoreMysqlBackupUsecase } func (uc *RestoreBackupUsecase) Execute( @@ -23,7 +26,8 @@ func (uc *RestoreBackupUsecase) Execute( storage *storages.Storage, isExcludeExtensions bool, ) error { - if originalDB.Type == databases.DatabaseTypePostgres { + switch originalDB.Type { + case databases.DatabaseTypePostgres: return uc.restorePostgresqlBackupUsecase.Execute( originalDB, restoringToDB, @@ -33,7 +37,16 @@ func (uc *RestoreBackupUsecase) Execute( storage, isExcludeExtensions, ) + case databases.DatabaseTypeMysql: + return uc.restoreMysqlBackupUsecase.Execute( + originalDB, + restoringToDB, + backupConfig, + restore, + backup, + storage, + ) + default: + return errors.New("database type not supported") } - - return errors.New("database type not supported") } diff --git a/backend/internal/features/tests/mysql_backup_restore_test.go b/backend/internal/features/tests/mysql_backup_restore_test.go new file mode 100644 index 0000000..7aee223 --- /dev/null +++ b/backend/internal/features/tests/mysql_backup_restore_test.go @@ -0,0 +1,671 @@ +package tests + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/gin-gonic/gin" + _ "github.com/go-sql-driver/mysql" + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/assert" + + "postgresus-backend/internal/config" + "postgresus-backend/internal/features/backups/backups" + backups_config "postgresus-backend/internal/features/backups/config" + "postgresus-backend/internal/features/databases" + mysqltypes "postgresus-backend/internal/features/databases/databases/mysql" + "postgresus-backend/internal/features/restores" + restores_enums "postgresus-backend/internal/features/restores/enums" + restores_models "postgresus-backend/internal/features/restores/models" + "postgresus-backend/internal/features/storages" + users_enums "postgresus-backend/internal/features/users/enums" + users_testing "postgresus-backend/internal/features/users/testing" + workspaces_testing "postgresus-backend/internal/features/workspaces/testing" + test_utils "postgresus-backend/internal/util/testing" + "postgresus-backend/internal/util/tools" +) + +const dropMysqlTestTableQuery = `DROP TABLE IF EXISTS test_data` + +const createMysqlTestTableQuery = ` +CREATE TABLE test_data ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + value INT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +)` + +const insertMysqlTestDataQuery = ` +INSERT INTO test_data (name, value) VALUES + ('test1', 100), + ('test2', 200), + ('test3', 300)` + +type MysqlContainer struct { + Host string + Port int + Username string + Password string + Database string + Version tools.MysqlVersion + DB *sqlx.DB +} + +type MysqlTestDataItem struct { + ID int `db:"id"` + Name string `db:"name"` + Value int `db:"value"` + CreatedAt time.Time `db:"created_at"` +} + +func Test_BackupAndRestoreMysql_RestoreIsSuccessful(t *testing.T) { + env := config.GetEnv() + cases := []struct { + name string + version tools.MysqlVersion + port string + }{ + {"MySQL 5.7", tools.MysqlVersion57, env.TestMysql57Port}, + {"MySQL 8.0", tools.MysqlVersion80, env.TestMysql80Port}, + {"MySQL 8.4", tools.MysqlVersion84, env.TestMysql84Port}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testMysqlBackupRestoreForVersion(t, tc.version, tc.port) + }) + } +} + +func Test_BackupAndRestoreMysqlWithEncryption_RestoreIsSuccessful(t *testing.T) { + env := config.GetEnv() + cases := []struct { + name string + version tools.MysqlVersion + port string + }{ + {"MySQL 5.7", tools.MysqlVersion57, env.TestMysql57Port}, + {"MySQL 8.0", tools.MysqlVersion80, env.TestMysql80Port}, + {"MySQL 8.4", tools.MysqlVersion84, env.TestMysql84Port}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testMysqlBackupRestoreWithEncryptionForVersion(t, tc.version, tc.port) + }) + } +} + +func Test_BackupAndRestoreMysql_WithReadOnlyUser_RestoreIsSuccessful(t *testing.T) { + env := config.GetEnv() + cases := []struct { + name string + version tools.MysqlVersion + port string + }{ + {"MySQL 5.7", tools.MysqlVersion57, env.TestMysql57Port}, + {"MySQL 8.0", tools.MysqlVersion80, env.TestMysql80Port}, + {"MySQL 8.4", tools.MysqlVersion84, env.TestMysql84Port}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testMysqlBackupRestoreWithReadOnlyUserForVersion(t, tc.version, tc.port) + }) + } +} + +func testMysqlBackupRestoreForVersion(t *testing.T, mysqlVersion tools.MysqlVersion, port string) { + container, err := connectToMysqlContainer(mysqlVersion, port) + if err != nil { + t.Skipf("Skipping MySQL %s test: %v", mysqlVersion, err) + return + } + defer func() { + if container.DB != nil { + container.DB.Close() + } + }() + + setupMysqlTestData(t, container.DB) + + router := createTestRouter() + user := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("MySQL Test Workspace", user, router) + + storage := storages.CreateTestStorage(workspace.ID) + + database := createMysqlDatabaseViaAPI( + t, router, "MySQL Test Database", workspace.ID, + container.Host, container.Port, + container.Username, container.Password, container.Database, + container.Version, + user.Token, + ) + + enableBackupsViaAPI( + t, router, database.ID, storage.ID, + backups_config.BackupEncryptionNone, user.Token, + ) + + createBackupViaAPI(t, router, database.ID, user.Token) + + backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute) + assert.Equal(t, backups.BackupStatusCompleted, backup.Status) + + newDBName := "restoreddb_mysql" + _, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName)) + assert.NoError(t, err) + + _, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName)) + assert.NoError(t, err) + + newDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + container.Username, container.Password, container.Host, container.Port, newDBName) + newDB, err := sqlx.Connect("mysql", newDSN) + assert.NoError(t, err) + defer newDB.Close() + + createMysqlRestoreViaAPI( + t, router, backup.ID, + container.Host, container.Port, + container.Username, container.Password, newDBName, + container.Version, + user.Token, + ) + + restore := waitForMysqlRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute) + assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status) + + var tableExists int + err = newDB.Get( + &tableExists, + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = ? AND table_name = 'test_data'", + newDBName, + ) + assert.NoError(t, err) + assert.Equal(t, 1, tableExists, "Table 'test_data' should exist in restored database") + + verifyMysqlDataIntegrity(t, container.DB, newDB) + + err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String())) + if err != nil { + t.Logf("Warning: Failed to delete backup file: %v", err) + } + + test_utils.MakeDeleteRequest( + t, + router, + "/api/v1/databases/"+database.ID.String(), + "Bearer "+user.Token, + http.StatusNoContent, + ) + storages.RemoveTestStorage(storage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) +} + +func testMysqlBackupRestoreWithEncryptionForVersion( + t *testing.T, + mysqlVersion tools.MysqlVersion, + port string, +) { + container, err := connectToMysqlContainer(mysqlVersion, port) + if err != nil { + t.Skipf("Skipping MySQL %s test: %v", mysqlVersion, err) + return + } + defer func() { + if container.DB != nil { + container.DB.Close() + } + }() + + setupMysqlTestData(t, container.DB) + + router := createTestRouter() + user := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace( + "MySQL Encrypted Test Workspace", + user, + router, + ) + + storage := storages.CreateTestStorage(workspace.ID) + + database := createMysqlDatabaseViaAPI( + t, router, "MySQL Encrypted Test Database", workspace.ID, + container.Host, container.Port, + container.Username, container.Password, container.Database, + container.Version, + user.Token, + ) + + enableBackupsViaAPI( + t, router, database.ID, storage.ID, + backups_config.BackupEncryptionEncrypted, user.Token, + ) + + createBackupViaAPI(t, router, database.ID, user.Token) + + backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute) + assert.Equal(t, backups.BackupStatusCompleted, backup.Status) + assert.Equal(t, backups_config.BackupEncryptionEncrypted, backup.Encryption) + + newDBName := "restoreddb_mysql_encrypted" + _, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName)) + assert.NoError(t, err) + + _, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName)) + assert.NoError(t, err) + + newDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + container.Username, container.Password, container.Host, container.Port, newDBName) + newDB, err := sqlx.Connect("mysql", newDSN) + assert.NoError(t, err) + defer newDB.Close() + + createMysqlRestoreViaAPI( + t, router, backup.ID, + container.Host, container.Port, + container.Username, container.Password, newDBName, + container.Version, + user.Token, + ) + + restore := waitForMysqlRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute) + assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status) + + var tableExists int + err = newDB.Get( + &tableExists, + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = ? AND table_name = 'test_data'", + newDBName, + ) + assert.NoError(t, err) + assert.Equal(t, 1, tableExists, "Table 'test_data' should exist in restored database") + + verifyMysqlDataIntegrity(t, container.DB, newDB) + + err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String())) + if err != nil { + t.Logf("Warning: Failed to delete backup file: %v", err) + } + + test_utils.MakeDeleteRequest( + t, + router, + "/api/v1/databases/"+database.ID.String(), + "Bearer "+user.Token, + http.StatusNoContent, + ) + storages.RemoveTestStorage(storage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) +} + +func testMysqlBackupRestoreWithReadOnlyUserForVersion( + t *testing.T, + mysqlVersion tools.MysqlVersion, + port string, +) { + container, err := connectToMysqlContainer(mysqlVersion, port) + if err != nil { + t.Skipf("Skipping MySQL %s test: %v", mysqlVersion, err) + return + } + defer func() { + if container.DB != nil { + container.DB.Close() + } + }() + + setupMysqlTestData(t, container.DB) + + router := createTestRouter() + user := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace( + "MySQL ReadOnly Test Workspace", + user, + router, + ) + + storage := storages.CreateTestStorage(workspace.ID) + + database := createMysqlDatabaseViaAPI( + t, router, "MySQL ReadOnly Test Database", workspace.ID, + container.Host, container.Port, + container.Username, container.Password, container.Database, + container.Version, + user.Token, + ) + + readOnlyUser := createMysqlReadOnlyUserViaAPI(t, router, database.ID, user.Token) + assert.NotEmpty(t, readOnlyUser.Username) + assert.NotEmpty(t, readOnlyUser.Password) + + updatedDatabase := updateMysqlDatabaseCredentialsViaAPI( + t, router, database, + readOnlyUser.Username, readOnlyUser.Password, + user.Token, + ) + + enableBackupsViaAPI( + t, router, updatedDatabase.ID, storage.ID, + backups_config.BackupEncryptionNone, user.Token, + ) + + createBackupViaAPI(t, router, updatedDatabase.ID, user.Token) + + backup := waitForBackupCompletion(t, router, updatedDatabase.ID, user.Token, 5*time.Minute) + assert.Equal(t, backups.BackupStatusCompleted, backup.Status) + + newDBName := "restoreddb_mysql_readonly" + _, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName)) + assert.NoError(t, err) + + _, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName)) + assert.NoError(t, err) + + newDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + container.Username, container.Password, container.Host, container.Port, newDBName) + newDB, err := sqlx.Connect("mysql", newDSN) + assert.NoError(t, err) + defer newDB.Close() + + createMysqlRestoreViaAPI( + t, router, backup.ID, + container.Host, container.Port, + container.Username, container.Password, newDBName, + container.Version, + user.Token, + ) + + restore := waitForMysqlRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute) + assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status) + + var tableExists int + err = newDB.Get( + &tableExists, + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = ? AND table_name = 'test_data'", + newDBName, + ) + assert.NoError(t, err) + assert.Equal(t, 1, tableExists, "Table 'test_data' should exist in restored database") + + verifyMysqlDataIntegrity(t, container.DB, newDB) + + err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String())) + if err != nil { + t.Logf("Warning: Failed to delete backup file: %v", err) + } + + test_utils.MakeDeleteRequest( + t, + router, + "/api/v1/databases/"+updatedDatabase.ID.String(), + "Bearer "+user.Token, + http.StatusNoContent, + ) + storages.RemoveTestStorage(storage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) +} + +func createMysqlDatabaseViaAPI( + t *testing.T, + router *gin.Engine, + name string, + workspaceID uuid.UUID, + host string, + port int, + username string, + password string, + database string, + version tools.MysqlVersion, + token string, +) *databases.Database { + request := databases.Database{ + Name: name, + WorkspaceID: &workspaceID, + Type: databases.DatabaseTypeMysql, + Mysql: &mysqltypes.MysqlDatabase{ + Host: host, + Port: port, + Username: username, + Password: password, + Database: &database, + Version: version, + }, + } + + w := workspaces_testing.MakeAPIRequest( + router, + "POST", + "/api/v1/databases/create", + "Bearer "+token, + request, + ) + + if w.Code != http.StatusCreated { + t.Fatalf("Failed to create MySQL database. Status: %d, Body: %s", w.Code, w.Body.String()) + } + + var createdDatabase databases.Database + if err := json.Unmarshal(w.Body.Bytes(), &createdDatabase); err != nil { + t.Fatalf("Failed to unmarshal database response: %v", err) + } + + return &createdDatabase +} + +func createMysqlRestoreViaAPI( + t *testing.T, + router *gin.Engine, + backupID uuid.UUID, + host string, + port int, + username string, + password string, + database string, + version tools.MysqlVersion, + token string, +) { + request := restores.RestoreBackupRequest{ + MysqlDatabase: &mysqltypes.MysqlDatabase{ + Host: host, + Port: port, + Username: username, + Password: password, + Database: &database, + Version: version, + }, + } + + test_utils.MakePostRequest( + t, + router, + fmt.Sprintf("/api/v1/restores/%s/restore", backupID.String()), + "Bearer "+token, + request, + http.StatusOK, + ) +} + +func waitForMysqlRestoreCompletion( + t *testing.T, + router *gin.Engine, + backupID uuid.UUID, + token string, + timeout time.Duration, +) *restores_models.Restore { + startTime := time.Now() + pollInterval := 500 * time.Millisecond + + for { + if time.Since(startTime) > timeout { + t.Fatalf("Timeout waiting for MySQL restore completion after %v", timeout) + } + + var restoresList []*restores_models.Restore + test_utils.MakeGetRequestAndUnmarshal( + t, + router, + fmt.Sprintf("/api/v1/restores/%s", backupID.String()), + "Bearer "+token, + http.StatusOK, + &restoresList, + ) + + for _, restore := range restoresList { + if restore.Status == restores_enums.RestoreStatusCompleted { + return restore + } + if restore.Status == restores_enums.RestoreStatusFailed { + t.Fatalf("MySQL restore failed: %v", restore.FailMessage) + } + } + + time.Sleep(pollInterval) + } +} + +func verifyMysqlDataIntegrity(t *testing.T, originalDB *sqlx.DB, restoredDB *sqlx.DB) { + var originalData []MysqlTestDataItem + var restoredData []MysqlTestDataItem + + err := originalDB.Select( + &originalData, + "SELECT id, name, value, created_at FROM test_data ORDER BY id", + ) + assert.NoError(t, err) + + err = restoredDB.Select( + &restoredData, + "SELECT id, name, value, created_at FROM test_data ORDER BY id", + ) + assert.NoError(t, err) + + assert.Equal(t, len(originalData), len(restoredData), "Should have same number of rows") + + if len(originalData) > 0 && len(restoredData) > 0 { + for i := range originalData { + assert.Equal(t, originalData[i].ID, restoredData[i].ID, "ID should match") + assert.Equal(t, originalData[i].Name, restoredData[i].Name, "Name should match") + assert.Equal(t, originalData[i].Value, restoredData[i].Value, "Value should match") + } + } +} + +func connectToMysqlContainer(version tools.MysqlVersion, port string) (*MysqlContainer, error) { + if port == "" { + return nil, fmt.Errorf("MySQL %s port not configured", version) + } + + dbName := "testdb" + password := "testpassword" + username := "testuser" + host := "localhost" + + portInt, err := strconv.Atoi(port) + if err != nil { + return nil, fmt.Errorf("failed to parse port: %w", err) + } + + dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + username, password, host, portInt, dbName) + + db, err := sqlx.Connect("mysql", dsn) + if err != nil { + return nil, fmt.Errorf("failed to connect to MySQL database: %w", err) + } + + return &MysqlContainer{ + Host: host, + Port: portInt, + Username: username, + Password: password, + Database: dbName, + Version: version, + DB: db, + }, nil +} + +func setupMysqlTestData(t *testing.T, db *sqlx.DB) { + _, err := db.Exec(dropMysqlTestTableQuery) + assert.NoError(t, err) + + _, err = db.Exec(createMysqlTestTableQuery) + assert.NoError(t, err) + + _, err = db.Exec(insertMysqlTestDataQuery) + assert.NoError(t, err) +} + +func createMysqlReadOnlyUserViaAPI( + t *testing.T, + router *gin.Engine, + databaseID uuid.UUID, + token string, +) *databases.CreateReadOnlyUserResponse { + var database databases.Database + test_utils.MakeGetRequestAndUnmarshal( + t, + router, + fmt.Sprintf("/api/v1/databases/%s", databaseID.String()), + "Bearer "+token, + http.StatusOK, + &database, + ) + + var response databases.CreateReadOnlyUserResponse + test_utils.MakePostRequestAndUnmarshal( + t, + router, + "/api/v1/databases/create-readonly-user", + "Bearer "+token, + database, + http.StatusOK, + &response, + ) + + return &response +} + +func updateMysqlDatabaseCredentialsViaAPI( + t *testing.T, + router *gin.Engine, + database *databases.Database, + username string, + password string, + token string, +) *databases.Database { + database.Mysql.Username = username + database.Mysql.Password = password + + w := workspaces_testing.MakeAPIRequest( + router, + "POST", + "/api/v1/databases/update", + "Bearer "+token, + database, + ) + + if w.Code != http.StatusOK { + t.Fatalf("Failed to update MySQL database. Status: %d, Body: %s", w.Code, w.Body.String()) + } + + var updatedDatabase databases.Database + if err := json.Unmarshal(w.Body.Bytes(), &updatedDatabase); err != nil { + t.Fatalf("Failed to unmarshal database response: %v", err) + } + + return &updatedDatabase +} diff --git a/backend/internal/features/tests/postgresql_backup_restore_test.go b/backend/internal/features/tests/postgresql_backup_restore_test.go index 20d28aa..b5aca0d 100644 --- a/backend/internal/features/tests/postgresql_backup_restore_test.go +++ b/backend/internal/features/tests/postgresql_backup_restore_test.go @@ -337,6 +337,30 @@ func Test_BackupPostgresql_SchemaSelection_OnlySpecifiedSchemas(t *testing.T) { } } +func Test_BackupAndRestorePostgresql_WithReadOnlyUser_RestoreIsSuccessful(t *testing.T) { + env := config.GetEnv() + cases := []struct { + name string + version string + port string + }{ + {"PostgreSQL 12", "12", env.TestPostgres12Port}, + {"PostgreSQL 13", "13", env.TestPostgres13Port}, + {"PostgreSQL 14", "14", env.TestPostgres14Port}, + {"PostgreSQL 15", "15", env.TestPostgres15Port}, + {"PostgreSQL 16", "16", env.TestPostgres16Port}, + {"PostgreSQL 17", "17", env.TestPostgres17Port}, + {"PostgreSQL 18", "18", env.TestPostgres18Port}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testBackupRestoreWithReadOnlyUserForVersion(t, tc.version, tc.port) + }) + } +} + func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) { container, err := connectToPostgresContainer(pgVersion, port) assert.NoError(t, err) @@ -811,6 +835,100 @@ func testBackupRestoreWithoutExcludeExtensionsForVersion( workspaces_testing.RemoveTestWorkspace(workspace, router) } +func testBackupRestoreWithReadOnlyUserForVersion(t *testing.T, pgVersion string, port string) { + container, err := connectToPostgresContainer(pgVersion, port) + assert.NoError(t, err) + defer func() { + if container.DB != nil { + container.DB.Close() + } + }() + + _, err = container.DB.Exec(createAndFillTableQuery) + assert.NoError(t, err) + + router := createTestRouter() + user := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("ReadOnly Test Workspace", user, router) + + storage := storages.CreateTestStorage(workspace.ID) + + database := createDatabaseViaAPI( + t, router, "ReadOnly Test Database", workspace.ID, + container.Host, container.Port, + container.Username, container.Password, container.Database, + user.Token, + ) + + readOnlyUser := createReadOnlyUserViaAPI(t, router, database.ID, user.Token) + assert.NotEmpty(t, readOnlyUser.Username) + assert.NotEmpty(t, readOnlyUser.Password) + + updatedDatabase := updateDatabaseCredentialsViaAPI( + t, router, database, + readOnlyUser.Username, readOnlyUser.Password, + user.Token, + ) + + enableBackupsViaAPI( + t, router, updatedDatabase.ID, storage.ID, + backups_config.BackupEncryptionNone, user.Token, + ) + + createBackupViaAPI(t, router, updatedDatabase.ID, user.Token) + + backup := waitForBackupCompletion(t, router, updatedDatabase.ID, user.Token, 5*time.Minute) + assert.Equal(t, backups.BackupStatusCompleted, backup.Status) + + newDBName := "restoreddb_readonly" + _, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName)) + assert.NoError(t, err) + + _, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName)) + assert.NoError(t, err) + + newDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", + container.Host, container.Port, container.Username, container.Password, newDBName) + newDB, err := sqlx.Connect("postgres", newDSN) + assert.NoError(t, err) + defer newDB.Close() + + createRestoreViaAPI( + t, router, backup.ID, + container.Host, container.Port, + container.Username, container.Password, newDBName, + user.Token, + ) + + restore := waitForRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute) + assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status) + + var tableExists bool + err = newDB.Get( + &tableExists, + "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'test_data')", + ) + assert.NoError(t, err) + assert.True(t, tableExists, "Table 'test_data' should exist in restored database") + + verifyDataIntegrity(t, container.DB, newDB) + + err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String())) + if err != nil { + t.Logf("Warning: Failed to delete backup file: %v", err) + } + + test_utils.MakeDeleteRequest( + t, + router, + "/api/v1/databases/"+updatedDatabase.ID.String(), + "Bearer "+user.Token, + http.StatusNoContent, + ) + storages.RemoveTestStorage(storage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) +} + func testSchemaSelectionOnlySpecifiedSchemasForVersion( t *testing.T, pgVersion string, @@ -1420,6 +1538,67 @@ func verifyDataIntegrity(t *testing.T, originalDB *sqlx.DB, restoredDB *sqlx.DB) } } +func createReadOnlyUserViaAPI( + t *testing.T, + router *gin.Engine, + databaseID uuid.UUID, + token string, +) *databases.CreateReadOnlyUserResponse { + var database databases.Database + test_utils.MakeGetRequestAndUnmarshal( + t, + router, + fmt.Sprintf("/api/v1/databases/%s", databaseID.String()), + "Bearer "+token, + http.StatusOK, + &database, + ) + + var response databases.CreateReadOnlyUserResponse + test_utils.MakePostRequestAndUnmarshal( + t, + router, + "/api/v1/databases/create-readonly-user", + "Bearer "+token, + database, + http.StatusOK, + &response, + ) + + return &response +} + +func updateDatabaseCredentialsViaAPI( + t *testing.T, + router *gin.Engine, + database *databases.Database, + username string, + password string, + token string, +) *databases.Database { + database.Postgresql.Username = username + database.Postgresql.Password = password + + w := workspaces_testing.MakeAPIRequest( + router, + "POST", + "/api/v1/databases/update", + "Bearer "+token, + database, + ) + + if w.Code != http.StatusOK { + t.Fatalf("Failed to update database. Status: %d, Body: %s", w.Code, w.Body.String()) + } + + var updatedDatabase databases.Database + if err := json.Unmarshal(w.Body.Bytes(), &updatedDatabase); err != nil { + t.Fatalf("Failed to unmarshal database response: %v", err) + } + + return &updatedDatabase +} + func connectToPostgresContainer(version string, port string) (*PostgresContainer, error) { dbName := "testdb" password := "testpassword" diff --git a/backend/internal/util/tools/mysql.go b/backend/internal/util/tools/mysql.go new file mode 100644 index 0000000..1cb7c50 --- /dev/null +++ b/backend/internal/util/tools/mysql.go @@ -0,0 +1,214 @@ +package tools + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "runtime" + "strings" + + env_utils "postgresus-backend/internal/util/env" +) + +type MysqlVersion string + +const ( + MysqlVersion57 MysqlVersion = "5.7" + MysqlVersion80 MysqlVersion = "8.0" + MysqlVersion84 MysqlVersion = "8.4" +) + +type MysqlExecutable string + +const ( + MysqlExecutableMysqldump MysqlExecutable = "mysqldump" + MysqlExecutableMysql MysqlExecutable = "mysql" +) + +// GetMysqlExecutable returns the full path to a specific MySQL executable +// for the given version. Common executables include: mysqldump, mysql. +// On Windows, automatically appends .exe extension. +func GetMysqlExecutable( + version MysqlVersion, + executable MysqlExecutable, + envMode env_utils.EnvMode, + mysqlInstallDir string, +) string { + basePath := getMysqlBasePath(version, envMode, mysqlInstallDir) + executableName := string(executable) + + if runtime.GOOS == "windows" { + executableName += ".exe" + } + + return filepath.Join(basePath, executableName) +} + +// VerifyMysqlInstallation verifies that MySQL versions 5.7, 8.0, 8.4 are installed +// in the current environment. Each version should be installed with the required +// client tools (mysqldump, mysql) available. +// In development: ./tools/mysql/mysql-{VERSION}/bin +// In production: /usr/local/mysql-{VERSION}/bin +func VerifyMysqlInstallation( + logger *slog.Logger, + envMode env_utils.EnvMode, + mysqlInstallDir string, +) { + versions := []MysqlVersion{ + MysqlVersion57, + MysqlVersion80, + MysqlVersion84, + } + + requiredCommands := []MysqlExecutable{ + MysqlExecutableMysqldump, + MysqlExecutableMysql, + } + + for _, version := range versions { + binDir := getMysqlBasePath(version, envMode, mysqlInstallDir) + + logger.Info( + "Verifying MySQL installation", + "version", + string(version), + "path", + binDir, + ) + + if _, err := os.Stat(binDir); os.IsNotExist(err) { + if envMode == env_utils.EnvModeDevelopment { + logger.Warn( + "MySQL bin directory not found. MySQL support will be disabled. Read ./tools/readme.md for details", + "version", + string(version), + "path", + binDir, + ) + } else { + logger.Warn( + "MySQL bin directory not found. MySQL support will be disabled.", + "version", + string(version), + "path", + binDir, + ) + } + continue + } + + for _, cmd := range requiredCommands { + cmdPath := GetMysqlExecutable( + version, + cmd, + envMode, + mysqlInstallDir, + ) + + logger.Info( + "Checking for MySQL command", + "command", + cmd, + "version", + string(version), + "path", + cmdPath, + ) + + if _, err := os.Stat(cmdPath); os.IsNotExist(err) { + if envMode == env_utils.EnvModeDevelopment { + logger.Warn( + "MySQL command not found. MySQL support for this version will be disabled. Read ./tools/readme.md for details", + "command", + cmd, + "version", + string(version), + "path", + cmdPath, + ) + } else { + logger.Warn( + "MySQL command not found. MySQL support for this version will be disabled.", + "command", + cmd, + "version", + string(version), + "path", + cmdPath, + ) + } + continue + } + + logger.Info( + "MySQL command found", + "command", + cmd, + "version", + string(version), + ) + } + + logger.Info( + "Installation of MySQL verified", + "version", + string(version), + "path", + binDir, + ) + } + + logger.Info("MySQL version-specific client tools verification completed!") +} + +// IsMysqlBackupVersionHigherThanRestoreVersion checks if backup was made with +// a newer MySQL version than the restore target +func IsMysqlBackupVersionHigherThanRestoreVersion( + backupVersion, restoreVersion MysqlVersion, +) bool { + versionOrder := map[MysqlVersion]int{ + MysqlVersion57: 1, + MysqlVersion80: 2, + MysqlVersion84: 3, + } + return versionOrder[backupVersion] > versionOrder[restoreVersion] +} + +// EscapeMysqlPassword escapes special characters for MySQL .my.cnf file format. +// In .my.cnf, passwords with special chars should be quoted. +// Escape backslash and quote characters. +func EscapeMysqlPassword(password string) string { + password = strings.ReplaceAll(password, "\\", "\\\\") + password = strings.ReplaceAll(password, "\"", "\\\"") + return password +} + +// GetMysqlVersionEnum converts a version string to MysqlVersion enum +func GetMysqlVersionEnum(version string) MysqlVersion { + switch version { + case "5.7": + return MysqlVersion57 + case "8.0": + return MysqlVersion80 + case "8.4": + return MysqlVersion84 + default: + panic(fmt.Sprintf("invalid mysql version: %s", version)) + } +} + +func getMysqlBasePath( + version MysqlVersion, + envMode env_utils.EnvMode, + mysqlInstallDir string, +) string { + if envMode == env_utils.EnvModeDevelopment { + return filepath.Join( + mysqlInstallDir, + fmt.Sprintf("mysql-%s", string(version)), + "bin", + ) + } + return fmt.Sprintf("/usr/local/mysql-%s/bin", string(version)) +} diff --git a/backend/migrations/20251220104453_add_mysql_databases_table.sql b/backend/migrations/20251220104453_add_mysql_databases_table.sql new file mode 100644 index 0000000..844a5e4 --- /dev/null +++ b/backend/migrations/20251220104453_add_mysql_databases_table.sql @@ -0,0 +1,27 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE mysql_databases ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + database_id UUID REFERENCES databases(id) ON DELETE CASCADE, + version TEXT NOT NULL, + host TEXT NOT NULL, + port INT NOT NULL, + username TEXT NOT NULL, + password TEXT NOT NULL, + database TEXT, + is_https BOOLEAN NOT NULL DEFAULT FALSE +); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE INDEX idx_mysql_databases_database_id ON mysql_databases(database_id); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP INDEX IF EXISTS idx_mysql_databases_database_id; +-- +goose StatementEnd + +-- +goose StatementBegin +DROP TABLE IF EXISTS mysql_databases; +-- +goose StatementEnd diff --git a/backend/tools/.gitignore b/backend/tools/.gitignore index 6f7d947..8afcbdd 100644 --- a/backend/tools/.gitignore +++ b/backend/tools/.gitignore @@ -1,2 +1,3 @@ postgresql +mysql downloads \ No newline at end of file diff --git a/backend/tools/download_linux.sh b/backend/tools/download_linux.sh index f94fcfa..a6dd55d 100644 --- a/backend/tools/download_linux.sh +++ b/backend/tools/download_linux.sh @@ -5,7 +5,7 @@ set -e # Exit on any error # Ensure non-interactive mode for apt export DEBIAN_FRONTEND=noninteractive -echo "Installing PostgreSQL client tools versions 12-18 for Linux (Debian/Ubuntu)..." +echo "Installing PostgreSQL and MySQL client tools for Linux (Debian/Ubuntu)..." echo # Check if running on supported system @@ -22,19 +22,27 @@ else echo "This script requires sudo privileges to install packages." fi -# Create postgresql directory +# Create directories mkdir -p postgresql +mkdir -p mysql -# Get absolute path +# Get absolute paths POSTGRES_DIR="$(pwd)/postgresql" +MYSQL_DIR="$(pwd)/mysql" echo "Installing PostgreSQL client tools to: $POSTGRES_DIR" +echo "Installing MySQL client tools to: $MYSQL_DIR" echo +# ========== PostgreSQL Installation ========== +echo "========================================" +echo "Installing PostgreSQL client tools (versions 12-18)..." +echo "========================================" + # Add PostgreSQL official APT repository echo "Adding PostgreSQL official APT repository..." $SUDO apt-get update -qq -y -$SUDO apt-get install -y -qq wget ca-certificates +$SUDO apt-get install -y -qq wget ca-certificates gnupg lsb-release # Add GPG key wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | $SUDO apt-key add - 2>/dev/null @@ -46,10 +54,10 @@ echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" echo "Updating package list..." $SUDO apt-get update -qq -y -# Install client tools for each version -versions="12 13 14 15 16 17 18" +# Install PostgreSQL client tools for each version +pg_versions="12 13 14 15 16 17 18" -for version in $versions; do +for version in $pg_versions; do echo "Installing PostgreSQL $version client tools..." # Install client tools only @@ -85,22 +93,116 @@ for version in $versions; do echo done +# ========== MySQL Installation ========== +echo "========================================" +echo "Installing MySQL client tools (versions 5.7, 8.0, 8.4)..." +echo "========================================" + +# Add MySQL APT repository +echo "Adding MySQL official APT repository..." +wget -q https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb -O /tmp/mysql-apt-config.deb +$SUDO DEBIAN_FRONTEND=noninteractive dpkg -i /tmp/mysql-apt-config.deb || true +rm /tmp/mysql-apt-config.deb + +# Update package list +$SUDO apt-get update -qq -y + +# MySQL versions and their package names +declare -A mysql_packages=( + ["5.7"]="mysql-community-client" + ["8.0"]="mysql-community-client" + ["8.4"]="mysql-community-client" +) + +# Download and extract MySQL client tools +mysql_versions="5.7 8.0 8.4" + +for version in $mysql_versions; do + echo "Installing MySQL $version client tools..." + + version_dir="$MYSQL_DIR/mysql-$version" + mkdir -p "$version_dir/bin" + + # Download MySQL client tools from official CDN + # Note: 5.7 is in Downloads, 8.0 and 8.4 specific versions are in archives + case $version in + "5.7") + MYSQL_URL="https://cdn.mysql.com/Downloads/MySQL-5.7/mysql-5.7.44-linux-glibc2.12-x86_64.tar.gz" + ;; + "8.0") + MYSQL_URL="https://cdn.mysql.com/archives/mysql-8.0/mysql-8.0.40-linux-glibc2.17-x86_64-minimal.tar.xz" + ;; + "8.4") + MYSQL_URL="https://cdn.mysql.com/archives/mysql-8.4/mysql-8.4.3-linux-glibc2.17-x86_64-minimal.tar.xz" + ;; + esac + + TEMP_DIR="/tmp/mysql_install_$version" + mkdir -p "$TEMP_DIR" + cd "$TEMP_DIR" + + echo " Downloading MySQL $version..." + wget -q "$MYSQL_URL" -O "mysql-$version.tar.gz" || wget -q "$MYSQL_URL" -O "mysql-$version.tar.xz" + + echo " Extracting MySQL $version..." + if [[ "$MYSQL_URL" == *.xz ]]; then + tar -xJf "mysql-$version.tar.xz" 2>/dev/null || tar -xJf "mysql-$version.tar.gz" 2>/dev/null + else + tar -xzf "mysql-$version.tar.gz" 2>/dev/null || tar -xzf "mysql-$version.tar.xz" 2>/dev/null + fi + + # Find extracted directory + EXTRACTED_DIR=$(ls -d mysql-*/ 2>/dev/null | head -1) + + if [ -d "$EXTRACTED_DIR" ] && [ -f "$EXTRACTED_DIR/bin/mysqldump" ]; then + # Copy client binaries + cp "$EXTRACTED_DIR/bin/mysql" "$version_dir/bin/" 2>/dev/null || true + cp "$EXTRACTED_DIR/bin/mysqldump" "$version_dir/bin/" 2>/dev/null || true + chmod +x "$version_dir/bin/"* + + echo " MySQL $version client tools installed successfully" + else + echo " Warning: Could not extract MySQL $version binaries" + echo " You may need to install MySQL $version client tools manually" + fi + + # Cleanup + cd - >/dev/null + rm -rf "$TEMP_DIR" + echo +done + +echo "========================================" echo "Installation completed!" +echo "========================================" +echo echo "PostgreSQL client tools are available in: $POSTGRES_DIR" +echo "MySQL client tools are available in: $MYSQL_DIR" echo -# List installed versions +# List installed PostgreSQL versions echo "Installed PostgreSQL client versions:" -for version in $versions; do +for version in $pg_versions; do version_dir="$POSTGRES_DIR/postgresql-$version" if [ -f "$version_dir/bin/pg_dump" ]; then echo " postgresql-$version: $version_dir/bin/" - # Verify the correct version version_output=$("$version_dir/bin/pg_dump" --version 2>/dev/null | grep -o "pg_dump (PostgreSQL) [0-9]\+\.[0-9]\+") echo " Version check: $version_output" fi done echo -echo "Usage example:" -echo " $POSTGRES_DIR/postgresql-15/bin/pg_dump --version" \ No newline at end of file +echo "Installed MySQL client versions:" +for version in $mysql_versions; do + version_dir="$MYSQL_DIR/mysql-$version" + if [ -f "$version_dir/bin/mysqldump" ]; then + echo " mysql-$version: $version_dir/bin/" + version_output=$("$version_dir/bin/mysqldump" --version 2>/dev/null | head -1) + echo " Version check: $version_output" + fi +done + +echo +echo "Usage examples:" +echo " $POSTGRES_DIR/postgresql-15/bin/pg_dump --version" +echo " $MYSQL_DIR/mysql-8.0/bin/mysqldump --version" \ No newline at end of file diff --git a/backend/tools/download_macos.sh b/backend/tools/download_macos.sh index 9ff11fe..c9cf590 100755 --- a/backend/tools/download_macos.sh +++ b/backend/tools/download_macos.sh @@ -2,7 +2,7 @@ set -e # Exit on any error -echo "Installing PostgreSQL client tools versions 12-18 for MacOS..." +echo "Installing PostgreSQL and MySQL client tools for MacOS..." echo # Check if Homebrew is installed @@ -12,13 +12,16 @@ if ! command -v brew &> /dev/null; then exit 1 fi -# Create postgresql directory +# Create directories mkdir -p postgresql +mkdir -p mysql -# Get absolute path +# Get absolute paths POSTGRES_DIR="$(pwd)/postgresql" +MYSQL_DIR="$(pwd)/mysql" echo "Installing PostgreSQL client tools to: $POSTGRES_DIR" +echo "Installing MySQL client tools to: $MYSQL_DIR" echo # Update Homebrew @@ -27,7 +30,12 @@ brew update # Install build dependencies echo "Installing build dependencies..." -brew install wget openssl readline zlib +brew install wget openssl readline zlib cmake + +# ========== PostgreSQL Installation ========== +echo "========================================" +echo "Building PostgreSQL client tools (versions 12-18)..." +echo "========================================" # PostgreSQL source URLs declare -A PG_URLS=( @@ -41,7 +49,7 @@ declare -A PG_URLS=( ) # Create temporary build directory -BUILD_DIR="/tmp/postgresql_build_$$" +BUILD_DIR="/tmp/db_tools_build_$$" mkdir -p "$BUILD_DIR" echo "Using temporary build directory: $BUILD_DIR" @@ -107,10 +115,10 @@ build_postgresql_client() { echo } -# Build each version -versions="12 13 14 15 16 17 18" +# Build each PostgreSQL version +pg_versions="12 13 14 15 16 17 18" -for version in $versions; do +for version in $pg_versions; do url=${PG_URLS[$version]} if [ -n "$url" ]; then build_postgresql_client "$version" "$url" @@ -119,17 +127,108 @@ for version in $versions; do fi done +# ========== MySQL Installation ========== +echo "========================================" +echo "Installing MySQL client tools (versions 5.7, 8.0, 8.4)..." +echo "========================================" + +# Detect architecture +ARCH=$(uname -m) +if [ "$ARCH" = "arm64" ]; then + MYSQL_ARCH="arm64" +else + MYSQL_ARCH="x86_64" +fi + +# MySQL download URLs for macOS (using CDN) +# Note: 5.7 is in Downloads, 8.0 and 8.4 specific versions are in archives +declare -A MYSQL_URLS=( + ["5.7"]="https://cdn.mysql.com/Downloads/MySQL-5.7/mysql-5.7.44-macos10.14-x86_64.tar.gz" + ["8.0"]="https://cdn.mysql.com/archives/mysql-8.0/mysql-8.0.40-macos14-${MYSQL_ARCH}.tar.gz" + ["8.4"]="https://cdn.mysql.com/archives/mysql-8.4/mysql-8.4.3-macos14-${MYSQL_ARCH}.tar.gz" +) + +# Function to install MySQL client tools +install_mysql_client() { + local version=$1 + local url=$2 + local version_dir="$MYSQL_DIR/mysql-$version" + + echo "Installing MySQL $version client tools..." + + # Skip if already exists + if [ -f "$version_dir/bin/mysqldump" ]; then + echo "MySQL $version already installed, skipping..." + return + fi + + mkdir -p "$version_dir/bin" + cd "$BUILD_DIR" + + # Download + echo " Downloading MySQL $version..." + wget -q "$url" -O "mysql-$version.tar.gz" || { + echo " Warning: Could not download MySQL $version for $MYSQL_ARCH" + echo " You may need to install MySQL $version client tools manually" + return + } + + # Extract + echo " Extracting MySQL $version..." + tar -xzf "mysql-$version.tar.gz" + + # Find extracted directory + EXTRACTED_DIR=$(ls -d mysql-*/ 2>/dev/null | head -1) + + if [ -d "$EXTRACTED_DIR" ] && [ -f "$EXTRACTED_DIR/bin/mysqldump" ]; then + # Copy client binaries + cp "$EXTRACTED_DIR/bin/mysql" "$version_dir/bin/" 2>/dev/null || true + cp "$EXTRACTED_DIR/bin/mysqldump" "$version_dir/bin/" 2>/dev/null || true + chmod +x "$version_dir/bin/"* + + echo " MySQL $version client tools installed successfully" + + # Test the installation + local mysql_version=$("$version_dir/bin/mysqldump" --version 2>/dev/null | head -1) + echo " Verified: $mysql_version" + else + echo " Warning: Could not extract MySQL $version binaries" + echo " You may need to install MySQL $version client tools manually" + fi + + # Clean up + rm -rf "mysql-$version.tar.gz" mysql-*/ + + echo +} + +# Install each MySQL version +mysql_versions="5.7 8.0 8.4" + +for version in $mysql_versions; do + url=${MYSQL_URLS[$version]} + if [ -n "$url" ]; then + install_mysql_client "$version" "$url" + else + echo "Warning: No URL defined for MySQL $version" + fi +done + # Clean up build directory echo "Cleaning up build directory..." rm -rf "$BUILD_DIR" +echo "========================================" echo "Installation completed!" +echo "========================================" +echo echo "PostgreSQL client tools are available in: $POSTGRES_DIR" +echo "MySQL client tools are available in: $MYSQL_DIR" echo -# List installed versions +# List installed PostgreSQL versions echo "Installed PostgreSQL client versions:" -for version in $versions; do +for version in $pg_versions; do version_dir="$POSTGRES_DIR/postgresql-$version" if [ -f "$version_dir/bin/pg_dump" ]; then pg_version=$("$version_dir/bin/pg_dump" --version | cut -d' ' -f3) @@ -138,8 +237,21 @@ for version in $versions; do done echo -echo "Usage example:" -echo " $POSTGRES_DIR/postgresql-15/bin/pg_dump --version" +echo "Installed MySQL client versions:" +for version in $mysql_versions; do + version_dir="$MYSQL_DIR/mysql-$version" + if [ -f "$version_dir/bin/mysqldump" ]; then + mysql_version=$("$version_dir/bin/mysqldump" --version 2>/dev/null | head -1) + echo " mysql-$version: $version_dir/bin/" + echo " $mysql_version" + fi +done + echo -echo "To add a specific version to your PATH temporarily:" -echo " export PATH=\"$POSTGRES_DIR/postgresql-15/bin:\$PATH\"" \ No newline at end of file +echo "Usage examples:" +echo " $POSTGRES_DIR/postgresql-15/bin/pg_dump --version" +echo " $MYSQL_DIR/mysql-8.0/bin/mysqldump --version" +echo +echo "To add specific versions to your PATH temporarily:" +echo " export PATH=\"$POSTGRES_DIR/postgresql-15/bin:\$PATH\"" +echo " export PATH=\"$MYSQL_DIR/mysql-8.0/bin:\$PATH\"" \ No newline at end of file diff --git a/backend/tools/download_windows.bat b/backend/tools/download_windows.bat index 8702e4f..5dcec23 100644 --- a/backend/tools/download_windows.bat +++ b/backend/tools/download_windows.bat @@ -1,22 +1,34 @@ @echo off setlocal enabledelayedexpansion -echo Downloading and installing PostgreSQL versions 12-18 for Windows... +echo Downloading and installing PostgreSQL and MySQL client tools for Windows... echo. -:: Create downloads and postgresql directories if they don't exist +:: Create directories if they don't exist if not exist "downloads" mkdir downloads if not exist "postgresql" mkdir postgresql +if not exist "mysql" mkdir mysql -:: Get the absolute path to the postgresql directory +:: Get the absolute paths set "POSTGRES_DIR=%cd%\postgresql" +set "MYSQL_DIR=%cd%\mysql" + +echo PostgreSQL will be installed to: %POSTGRES_DIR% +echo MySQL will be installed to: %MYSQL_DIR% +echo. cd downloads +:: ========== PostgreSQL Installation ========== +echo ======================================== +echo Installing PostgreSQL client tools (versions 12-18)... +echo ======================================== +echo. + :: PostgreSQL download URLs for Windows x64 set "BASE_URL=https://get.enterprisedb.com/postgresql" -:: Define versions and their corresponding download URLs +:: Define PostgreSQL versions and their corresponding download URLs set "PG12_URL=%BASE_URL%/postgresql-12.20-1-windows-x64.exe" set "PG13_URL=%BASE_URL%/postgresql-13.16-1-windows-x64.exe" set "PG14_URL=%BASE_URL%/postgresql-14.13-1-windows-x64.exe" @@ -25,11 +37,11 @@ set "PG16_URL=%BASE_URL%/postgresql-16.4-1-windows-x64.exe" set "PG17_URL=%BASE_URL%/postgresql-17.0-1-windows-x64.exe" set "PG18_URL=%BASE_URL%/postgresql-18.0-1-windows-x64.exe" -:: Array of versions -set "versions=12 13 14 15 16 17 18" +:: PostgreSQL versions +set "pg_versions=12 13 14 15 16 17 18" -:: Download and install each version -for %%v in (%versions%) do ( +:: Download and install each PostgreSQL version +for %%v in (%pg_versions%) do ( echo Processing PostgreSQL %%v... set "filename=postgresql-%%v-windows-x64.exe" set "install_dir=%POSTGRES_DIR%\postgresql-%%v" @@ -45,7 +57,7 @@ for %%v in (%versions%) do ( if !errorlevel! neq 0 ( echo Failed to download PostgreSQL %%v - goto :next_version + goto :next_pg_version ) echo PostgreSQL %%v downloaded successfully ) else ( @@ -83,13 +95,132 @@ for %%v in (%versions%) do ( ) ) - :next_version + :next_pg_version echo. ) +:: ========== MySQL Installation ========== +echo ======================================== +echo Installing MySQL client tools (versions 5.7, 8.0, 8.4)... +echo ======================================== echo. + +:: MySQL download URLs for Windows x64 (ZIP archives) - using CDN +:: Note: 5.7 is in Downloads, 8.0 and 8.4 specific versions are in archives +set "MYSQL57_URL=https://cdn.mysql.com/Downloads/MySQL-5.7/mysql-5.7.44-winx64.zip" +set "MYSQL80_URL=https://cdn.mysql.com/archives/mysql-8.0/mysql-8.0.40-winx64.zip" +set "MYSQL84_URL=https://cdn.mysql.com/archives/mysql-8.4/mysql-8.4.3-winx64.zip" + +:: MySQL versions +set "mysql_versions=5.7 8.0 8.4" + +:: Download and install each MySQL version +for %%v in (%mysql_versions%) do ( + echo Processing MySQL %%v... + set "version_underscore=%%v" + set "version_underscore=!version_underscore:.=!" + set "filename=mysql-%%v-winx64.zip" + set "install_dir=%MYSQL_DIR%\mysql-%%v" + + :: Build the URL variable name and get its value + call set "current_url=%%MYSQL!version_underscore!_URL%%" + + :: Check if already installed + if exist "!install_dir!\bin\mysqldump.exe" ( + echo MySQL %%v already installed, skipping... + ) else ( + :: Download if not exists + if not exist "!filename!" ( + echo Downloading MySQL %%v... + echo Downloading from: !current_url! + curl -L -o "!filename!" -A "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" "!current_url!" + if !errorlevel! neq 0 ( + echo ERROR: Download request failed + goto :next_mysql_version + ) + if not exist "!filename!" ( + echo ERROR: Download failed - file not created + goto :next_mysql_version + ) + for %%s in ("!filename!") do if %%~zs LSS 1000000 ( + echo ERROR: Download failed - file too small, likely error page + del "!filename!" 2>nul + goto :next_mysql_version + ) + echo MySQL %%v downloaded successfully + ) else ( + echo MySQL %%v already downloaded + ) + + :: Verify file exists before extraction + if not exist "!filename!" ( + echo Download file not found, skipping extraction... + goto :next_mysql_version + ) + + :: Extract MySQL + echo Extracting MySQL %%v... + mkdir "!install_dir!" 2>nul + + powershell -Command "Expand-Archive -Path '!filename!' -DestinationPath '!install_dir!_temp' -Force" + + :: Move files from nested directory to install_dir + for /d %%d in ("!install_dir!_temp\mysql-*") do ( + if exist "%%d\bin\mysqldump.exe" ( + mkdir "!install_dir!\bin" 2>nul + copy "%%d\bin\mysql.exe" "!install_dir!\bin\" >nul 2>&1 + copy "%%d\bin\mysqldump.exe" "!install_dir!\bin\" >nul 2>&1 + ) + ) + + :: Cleanup temp directory + rmdir /s /q "!install_dir!_temp" 2>nul + + :: Verify installation + if exist "!install_dir!\bin\mysqldump.exe" ( + echo MySQL %%v client tools installed successfully + ) else ( + echo Failed to install MySQL %%v - mysqldump.exe not found + ) + ) + + :next_mysql_version + echo. +) + +cd .. + +echo. +echo ======================================== echo Installation process completed! +echo ======================================== +echo. echo PostgreSQL versions are installed in: %POSTGRES_DIR% +echo MySQL versions are installed in: %MYSQL_DIR% +echo. + +:: List installed PostgreSQL versions +echo Installed PostgreSQL client versions: +for %%v in (%pg_versions%) do ( + set "version_dir=%POSTGRES_DIR%\postgresql-%%v" + if exist "!version_dir!\bin\pg_dump.exe" ( + echo postgresql-%%v: !version_dir!\bin\ + ) +) + +echo. +echo Installed MySQL client versions: +for %%v in (%mysql_versions%) do ( + set "version_dir=%MYSQL_DIR%\mysql-%%v" + if exist "!version_dir!\bin\mysqldump.exe" ( + echo mysql-%%v: !version_dir!\bin\ + ) +) + +echo. +echo Usage examples: +echo %POSTGRES_DIR%\postgresql-15\bin\pg_dump.exe --version +echo %MYSQL_DIR%\mysql-8.0\bin\mysqldump.exe --version echo. pause diff --git a/backend/tools/readme.md b/backend/tools/readme.md index 1e932a7..0ae5ae5 100644 --- a/backend/tools/readme.md +++ b/backend/tools/readme.md @@ -1,12 +1,14 @@ This directory is needed only for development and CI\CD. -We have to download and install all the PostgreSQL versions from 12 to 18 locally. -This is needed so we can call pg_dump, pg_dumpall, etc. on each version of the PostgreSQL database. +We have to download and install all the PostgreSQL versions from 12 to 18 and MySQL versions 5.7, 8.0, 8.4 locally. +This is needed so we can call pg_dump, pg_restore, mysqldump, mysql, etc. on each version of the database. -You do not need to install PostgreSQL fully with all the components. -We only need the client tools (pg_dump, pg_dumpall, psql, etc.) for each version. +You do not need to install the databases fully with all the components. +We only need the client tools for each version. -We have to install the following: +## Required Versions + +### PostgreSQL - PostgreSQL 12 - PostgreSQL 13 @@ -16,6 +18,12 @@ We have to install the following: - PostgreSQL 17 - PostgreSQL 18 +### MySQL + +- MySQL 5.7 +- MySQL 8.0 +- MySQL 8.4 + ## Installation Run the appropriate download script for your platform: @@ -45,12 +53,14 @@ chmod +x download_macos.sh ### Windows - Downloads official PostgreSQL installers from EnterpriseDB +- Downloads official MySQL ZIP archives from dev.mysql.com - Installs client tools only (no server components) -- May require administrator privileges during installation +- May require administrator privileges during PostgreSQL installation ### Linux (Debian/Ubuntu) - Uses the official PostgreSQL APT repository +- Downloads MySQL client tools from official archives - Requires sudo privileges to install packages - Creates symlinks in version-specific directories for consistency @@ -58,17 +68,22 @@ chmod +x download_macos.sh - Requires Homebrew to be installed - Compiles PostgreSQL from source (client tools only) -- Takes longer than other platforms due to compilation +- Downloads pre-built MySQL binaries from dev.mysql.com +- Takes longer than other platforms due to PostgreSQL compilation +- Supports both Intel (x86_64) and Apple Silicon (arm64) ## Manual Installation If something goes wrong with the automated scripts, install manually. The final directory structure should match: +### PostgreSQL + ``` ./tools/postgresql/postgresql-{version}/bin/pg_dump ./tools/postgresql/postgresql-{version}/bin/pg_dumpall ./tools/postgresql/postgresql-{version}/bin/psql +./tools/postgresql/postgresql-{version}/bin/pg_restore ``` For example: @@ -81,14 +96,69 @@ For example: - `./tools/postgresql/postgresql-17/bin/pg_dump` - `./tools/postgresql/postgresql-18/bin/pg_dump` +### MySQL + +``` +./tools/mysql/mysql-{version}/bin/mysqldump +./tools/mysql/mysql-{version}/bin/mysql +``` + +For example: + +- `./tools/mysql/mysql-5.7/bin/mysqldump` +- `./tools/mysql/mysql-8.0/bin/mysqldump` +- `./tools/mysql/mysql-8.4/bin/mysqldump` + ## Usage After installation, you can use version-specific tools: ```bash -# Windows +# Windows - PostgreSQL ./postgresql/postgresql-15/bin/pg_dump.exe --version -# Linux/MacOS +# Windows - MySQL +./mysql/mysql-8.0/bin/mysqldump.exe --version + +# Linux/MacOS - PostgreSQL ./postgresql/postgresql-15/bin/pg_dump --version + +# Linux/MacOS - MySQL +./mysql/mysql-8.0/bin/mysqldump --version ``` + +## Environment Variables + +The application expects these environment variables to be set (or uses defaults): + +```env +# PostgreSQL tools directory (default: ./tools/postgresql) +POSTGRES_INSTALL_DIR=C:\path\to\tools\postgresql + +# MySQL tools directory (default: ./tools/mysql) +MYSQL_INSTALL_DIR=C:\path\to\tools\mysql +``` + +## Troubleshooting + +### MySQL 5.7 on Apple Silicon (M1/M2/M3) + +MySQL 5.7 does not have native ARM64 binaries for macOS. The script will attempt to download the x86_64 version, which may work under Rosetta 2. If you encounter issues: + +1. Ensure Rosetta 2 is installed: `softwareupdate --install-rosetta` +2. Or skip MySQL 5.7 if you don't need to support that version + +### Permission Errors on Linux + +If you encounter permission errors, ensure you have sudo privileges: + +```bash +sudo ./download_linux.sh +``` + +### Download Failures + +If downloads fail, you can manually download the files: + +- PostgreSQL: https://www.postgresql.org/ftp/source/ +- MySQL: https://dev.mysql.com/downloads/mysql/ diff --git a/frontend/public/icons/databases/mysql.svg b/frontend/public/icons/databases/mysql.svg new file mode 100644 index 0000000..4c196ad --- /dev/null +++ b/frontend/public/icons/databases/mysql.svg @@ -0,0 +1,2 @@ + +file_type_mysql \ No newline at end of file diff --git a/frontend/src/entity/databases/index.ts b/frontend/src/entity/databases/index.ts index d9e6357..f62116d 100644 --- a/frontend/src/entity/databases/index.ts +++ b/frontend/src/entity/databases/index.ts @@ -1,8 +1,11 @@ export { databaseApi } from './api/databaseApi'; export { type Database } from './model/Database'; export { DatabaseType } from './model/DatabaseType'; +export { getDatabaseLogoFromType } from './model/getDatabaseLogoFromType'; export { Period } from './model/Period'; export { type PostgresqlDatabase } from './model/postgresql/PostgresqlDatabase'; export { PostgresqlVersion } from './model/postgresql/PostgresqlVersion'; +export { type MysqlDatabase } from './model/mysql/MysqlDatabase'; +export { MysqlVersion } from './model/mysql/MysqlVersion'; export { type IsReadOnlyResponse } from './model/IsReadOnlyResponse'; export { type CreateReadOnlyUserResponse } from './model/CreateReadOnlyUserResponse'; diff --git a/frontend/src/entity/databases/model/Database.ts b/frontend/src/entity/databases/model/Database.ts index fdc84a6..b46242b 100644 --- a/frontend/src/entity/databases/model/Database.ts +++ b/frontend/src/entity/databases/model/Database.ts @@ -1,6 +1,7 @@ import type { Notifier } from '../../notifiers'; import type { DatabaseType } from './DatabaseType'; import type { HealthStatus } from './HealthStatus'; +import type { MysqlDatabase } from './mysql/MysqlDatabase'; import type { PostgresqlDatabase } from './postgresql/PostgresqlDatabase'; export interface Database { @@ -10,6 +11,7 @@ export interface Database { type: DatabaseType; postgresql?: PostgresqlDatabase; + mysql?: MysqlDatabase; notifiers: Notifier[]; diff --git a/frontend/src/entity/databases/model/DatabaseType.ts b/frontend/src/entity/databases/model/DatabaseType.ts index 43c37a6..a36a2aa 100644 --- a/frontend/src/entity/databases/model/DatabaseType.ts +++ b/frontend/src/entity/databases/model/DatabaseType.ts @@ -1,3 +1,4 @@ export enum DatabaseType { POSTGRES = 'POSTGRES', + MYSQL = 'MYSQL', } diff --git a/frontend/src/entity/databases/model/getDatabaseLogoFromType.ts b/frontend/src/entity/databases/model/getDatabaseLogoFromType.ts new file mode 100644 index 0000000..6c81d40 --- /dev/null +++ b/frontend/src/entity/databases/model/getDatabaseLogoFromType.ts @@ -0,0 +1,12 @@ +import { DatabaseType } from './DatabaseType'; + +export const getDatabaseLogoFromType = (type: DatabaseType) => { + switch (type) { + case DatabaseType.POSTGRES: + return '/icons/databases/postgresql.svg'; + case DatabaseType.MYSQL: + return '/icons/databases/mysql.svg'; + default: + return ''; + } +}; diff --git a/frontend/src/entity/databases/model/mysql/MySqlConnectionStringParser.test.ts b/frontend/src/entity/databases/model/mysql/MySqlConnectionStringParser.test.ts new file mode 100644 index 0000000..354b273 --- /dev/null +++ b/frontend/src/entity/databases/model/mysql/MySqlConnectionStringParser.test.ts @@ -0,0 +1,484 @@ +import { describe, expect, it } from 'vitest'; + +import { + MySqlConnectionStringParser, + type ParseError, + type ParseResult, +} from './MySqlConnectionStringParser'; + +describe('MySqlConnectionStringParser', () => { + // Helper to assert successful parse + const expectSuccess = (result: ParseResult | ParseError): ParseResult => { + expect('error' in result).toBe(false); + return result as ParseResult; + }; + + // Helper to assert parse error + const expectError = (result: ParseResult | ParseError): ParseError => { + expect('error' in result).toBe(true); + return result as ParseError; + }; + + describe('Standard MySQL URI (mysql://)', () => { + it('should parse basic mysql:// connection string', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://myuser:mypassword@localhost:3306/mydb'), + ); + + expect(result.host).toBe('localhost'); + expect(result.port).toBe(3306); + expect(result.username).toBe('myuser'); + expect(result.password).toBe('mypassword'); + expect(result.database).toBe('mydb'); + expect(result.isHttps).toBe(false); + }); + + it('should default port to 3306 when not specified', () => { + const result = expectSuccess(MySqlConnectionStringParser.parse('mysql://user:pass@host/db')); + + expect(result.port).toBe(3306); + }); + + it('should handle URL-encoded passwords', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://user:p%40ss%23word@host:3306/db'), + ); + + expect(result.password).toBe('p@ss#word'); + }); + + it('should handle URL-encoded usernames', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://user%40domain:password@host:3306/db'), + ); + + expect(result.username).toBe('user@domain'); + }); + }); + + describe('AWS RDS Connection String', () => { + it('should parse AWS RDS connection string', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'mysql://rdsuser:rdspass@mydb.abc123xyz.us-east-1.rds.amazonaws.com:3306/mydb', + ), + ); + + expect(result.host).toBe('mydb.abc123xyz.us-east-1.rds.amazonaws.com'); + expect(result.port).toBe(3306); + expect(result.username).toBe('rdsuser'); + }); + }); + + describe('PlanetScale Connection String', () => { + it('should parse PlanetScale connection string with sslaccept', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'mysql://psuser:pspass@xxx.connect.psdb.cloud/mydb?sslaccept=strict', + ), + ); + + expect(result.host).toBe('xxx.connect.psdb.cloud'); + expect(result.username).toBe('psuser'); + expect(result.database).toBe('mydb'); + expect(result.isHttps).toBe(true); + }); + }); + + describe('DigitalOcean Connection String', () => { + it('should parse DigitalOcean connection string with ssl-mode', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'mysql://doadmin:dopassword@db-mysql-nyc1-12345-do-user-123456-0.b.db.ondigitalocean.com:25060/defaultdb?ssl-mode=REQUIRED', + ), + ); + + expect(result.host).toBe('db-mysql-nyc1-12345-do-user-123456-0.b.db.ondigitalocean.com'); + expect(result.port).toBe(25060); + expect(result.username).toBe('doadmin'); + expect(result.database).toBe('defaultdb'); + expect(result.isHttps).toBe(true); + }); + }); + + describe('Azure Database for MySQL Connection String', () => { + it('should parse Azure connection string with user@server format', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'mysql://myuser@myserver:mypassword@myserver.mysql.database.azure.com:3306/mydb?ssl-mode=REQUIRED', + ), + ); + + expect(result.host).toBe('myserver.mysql.database.azure.com'); + expect(result.port).toBe(3306); + expect(result.username).toBe('myuser'); + expect(result.password).toBe('mypassword'); + expect(result.database).toBe('mydb'); + expect(result.isHttps).toBe(true); + }); + }); + + describe('Railway Connection String', () => { + it('should parse Railway connection string', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'mysql://root:railwaypass@containers-us-west-123.railway.app:3306/railway', + ), + ); + + expect(result.host).toBe('containers-us-west-123.railway.app'); + expect(result.username).toBe('root'); + expect(result.database).toBe('railway'); + }); + }); + + describe('JDBC Connection String', () => { + it('should parse JDBC connection string with user and password params', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'jdbc:mysql://localhost:3306/mydb?user=admin&password=secret', + ), + ); + + expect(result.host).toBe('localhost'); + expect(result.port).toBe(3306); + expect(result.username).toBe('admin'); + expect(result.password).toBe('secret'); + expect(result.database).toBe('mydb'); + }); + + it('should parse JDBC connection string without port', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'jdbc:mysql://db.example.com/mydb?user=admin&password=secret', + ), + ); + + expect(result.host).toBe('db.example.com'); + expect(result.port).toBe(3306); + }); + + it('should parse JDBC with useSSL parameter', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'jdbc:mysql://host:3306/db?user=u&password=p&useSSL=true', + ), + ); + + expect(result.isHttps).toBe(true); + }); + + it('should parse JDBC with sslMode parameter', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'jdbc:mysql://host:3306/db?user=u&password=p&sslMode=REQUIRED', + ), + ); + + expect(result.isHttps).toBe(true); + }); + + it('should return error for JDBC without user parameter', () => { + const result = expectError( + MySqlConnectionStringParser.parse('jdbc:mysql://host:3306/db?password=secret'), + ); + + expect(result.error).toContain('user'); + expect(result.format).toBe('JDBC'); + }); + + it('should return error for JDBC without password parameter', () => { + const result = expectError( + MySqlConnectionStringParser.parse('jdbc:mysql://host:3306/db?user=admin'), + ); + + expect(result.error).toContain('Password'); + expect(result.format).toBe('JDBC'); + }); + }); + + describe('SSL Mode Handling', () => { + it('should set isHttps=true for ssl=true', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://u:p@host:3306/db?ssl=true'), + ); + + expect(result.isHttps).toBe(true); + }); + + it('should set isHttps=true for sslMode=REQUIRED', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://u:p@host:3306/db?sslMode=REQUIRED'), + ); + + expect(result.isHttps).toBe(true); + }); + + it('should set isHttps=true for ssl-mode=REQUIRED', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://u:p@host:3306/db?ssl-mode=REQUIRED'), + ); + + expect(result.isHttps).toBe(true); + }); + + it('should set isHttps=true for useSSL=true', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://u:p@host:3306/db?useSSL=true'), + ); + + expect(result.isHttps).toBe(true); + }); + + it('should set isHttps=true for sslMode=verify_ca', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://u:p@host:3306/db?sslMode=verify_ca'), + ); + + expect(result.isHttps).toBe(true); + }); + + it('should set isHttps=true for sslMode=verify_identity', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://u:p@host:3306/db?sslMode=verify_identity'), + ); + + expect(result.isHttps).toBe(true); + }); + + it('should set isHttps=false for ssl=false', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://u:p@host:3306/db?ssl=false'), + ); + + expect(result.isHttps).toBe(false); + }); + + it('should set isHttps=false when no ssl specified', () => { + const result = expectSuccess(MySqlConnectionStringParser.parse('mysql://u:p@host:3306/db')); + + expect(result.isHttps).toBe(false); + }); + }); + + describe('Key-Value Format', () => { + it('should parse key-value format connection string', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'host=localhost port=3306 database=mydb user=admin password=secret', + ), + ); + + expect(result.host).toBe('localhost'); + expect(result.port).toBe(3306); + expect(result.username).toBe('admin'); + expect(result.password).toBe('secret'); + expect(result.database).toBe('mydb'); + }); + + it('should parse key-value format with quoted password containing spaces', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + "host=localhost port=3306 database=mydb user=admin password='my secret pass'", + ), + ); + + expect(result.password).toBe('my secret pass'); + }); + + it('should default port to 3306 when not specified in key-value format', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'host=localhost database=mydb user=admin password=secret', + ), + ); + + expect(result.port).toBe(3306); + }); + + it('should handle hostaddr as alternative to host', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'hostaddr=192.168.1.1 port=3306 database=mydb user=admin password=secret', + ), + ); + + expect(result.host).toBe('192.168.1.1'); + }); + + it('should handle dbname as alternative to database', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'host=localhost port=3306 dbname=mydb user=admin password=secret', + ), + ); + + expect(result.database).toBe('mydb'); + }); + + it('should handle username as alternative to user', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'host=localhost port=3306 database=mydb username=admin password=secret', + ), + ); + + expect(result.username).toBe('admin'); + }); + + it('should parse ssl in key-value format', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'host=localhost database=mydb user=admin password=secret ssl=true', + ), + ); + + expect(result.isHttps).toBe(true); + }); + + it('should parse sslMode in key-value format', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'host=localhost database=mydb user=admin password=secret sslMode=REQUIRED', + ), + ); + + expect(result.isHttps).toBe(true); + }); + + it('should return error for key-value format missing host', () => { + const result = expectError( + MySqlConnectionStringParser.parse('port=3306 database=mydb user=admin password=secret'), + ); + + expect(result.error).toContain('Host'); + expect(result.format).toBe('key-value'); + }); + + it('should return error for key-value format missing user', () => { + const result = expectError( + MySqlConnectionStringParser.parse('host=localhost database=mydb password=secret'), + ); + + expect(result.error).toContain('Username'); + expect(result.format).toBe('key-value'); + }); + + it('should return error for key-value format missing password', () => { + const result = expectError( + MySqlConnectionStringParser.parse('host=localhost database=mydb user=admin'), + ); + + expect(result.error).toContain('Password'); + expect(result.format).toBe('key-value'); + }); + + it('should return error for key-value format missing database', () => { + const result = expectError( + MySqlConnectionStringParser.parse('host=localhost user=admin password=secret'), + ); + + expect(result.error).toContain('Database'); + expect(result.format).toBe('key-value'); + }); + }); + + describe('Error Cases', () => { + it('should return error for empty string', () => { + const result = expectError(MySqlConnectionStringParser.parse('')); + + expect(result.error).toContain('empty'); + }); + + it('should return error for whitespace-only string', () => { + const result = expectError(MySqlConnectionStringParser.parse(' ')); + + expect(result.error).toContain('empty'); + }); + + it('should return error for unrecognized format', () => { + const result = expectError(MySqlConnectionStringParser.parse('some random text')); + + expect(result.error).toContain('Unrecognized'); + }); + + it('should return error for missing username in URI', () => { + const result = expectError( + MySqlConnectionStringParser.parse('mysql://:password@host:3306/db'), + ); + + expect(result.error).toContain('Username'); + }); + + it('should return error for missing password in URI', () => { + const result = expectError(MySqlConnectionStringParser.parse('mysql://user@host:3306/db')); + + expect(result.error).toContain('Password'); + }); + + it('should return error for missing database in URI', () => { + const result = expectError(MySqlConnectionStringParser.parse('mysql://user:pass@host:3306/')); + + expect(result.error).toContain('Database'); + }); + + it('should return error for invalid JDBC format', () => { + const result = expectError(MySqlConnectionStringParser.parse('jdbc:mysql://invalid')); + + expect(result.format).toBe('JDBC'); + }); + + it('should return error for postgresql:// format (wrong database type)', () => { + const result = expectError( + MySqlConnectionStringParser.parse('postgresql://user:pass@host:5432/db'), + ); + + expect(result.error).toContain('Unrecognized'); + }); + }); + + describe('Edge Cases', () => { + it('should handle special characters in password', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://user:p%40ss%3Aw%2Ford@host:3306/db'), + ); + + expect(result.password).toBe('p@ss:w/ord'); + }); + + it('should handle numeric database names', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://user:pass@host:3306/12345'), + ); + + expect(result.database).toBe('12345'); + }); + + it('should handle hyphenated host names', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse('mysql://user:pass@my-database-host.example.com:3306/db'), + ); + + expect(result.host).toBe('my-database-host.example.com'); + }); + + it('should handle connection string with extra query parameters', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse( + 'mysql://user:pass@host:3306/db?ssl=true&connectTimeout=10&charset=utf8mb4', + ), + ); + + expect(result.isHttps).toBe(true); + expect(result.database).toBe('db'); + }); + + it('should trim whitespace from connection string', () => { + const result = expectSuccess( + MySqlConnectionStringParser.parse(' mysql://user:pass@host:3306/db '), + ); + + expect(result.host).toBe('host'); + }); + }); +}); diff --git a/frontend/src/entity/databases/model/mysql/MySqlConnectionStringParser.ts b/frontend/src/entity/databases/model/mysql/MySqlConnectionStringParser.ts new file mode 100644 index 0000000..c244b19 --- /dev/null +++ b/frontend/src/entity/databases/model/mysql/MySqlConnectionStringParser.ts @@ -0,0 +1,291 @@ +export type ParseResult = { + host: string; + port: number; + username: string; + password: string; + database: string; + isHttps: boolean; +}; + +export type ParseError = { + error: string; + format?: string; +}; + +export class MySqlConnectionStringParser { + /** + * Parses a MySQL connection string in various formats. + * + * Supported formats: + * 1. Standard MySQL URI: mysql://user:pass@host:port/db + * 2. JDBC format: jdbc:mysql://host:port/db?user=x&password=y + * 3. Key-value format: host=x port=3306 database=db user=u password=p + * 4. With SSL params: mysql://user:pass@host:port/db?ssl=true or ?sslMode=REQUIRED + * 5. AWS RDS: mysql://user:pass@xxx.rds.amazonaws.com:3306/db + * 6. PlanetScale: mysql://user:pass@xxx.connect.psdb.cloud/db?sslaccept=strict + * 7. DigitalOcean: mysql://user:pass@xxx.ondigitalocean.com:25060/db?ssl-mode=REQUIRED + * 8. Azure MySQL: mysql://user@servername:pass@xxx.mysql.database.azure.com:3306/db + * 9. Railway: mysql://user:pass@xxx.railway.app:port/db + */ + static parse(connectionString: string): ParseResult | ParseError { + const trimmed = connectionString.trim(); + + if (!trimmed) { + return { error: 'Connection string is empty' }; + } + + // Try JDBC format first (starts with jdbc:) + if (trimmed.startsWith('jdbc:mysql://')) { + return this.parseJdbc(trimmed); + } + + // Try key-value format (contains key=value pairs without ://) + if (this.isKeyValueFormat(trimmed)) { + return this.parseKeyValue(trimmed); + } + + // Try URI format (mysql://) + if (trimmed.startsWith('mysql://')) { + return this.parseUri(trimmed); + } + + return { + error: 'Unrecognized connection string format', + }; + } + + private static isKeyValueFormat(str: string): boolean { + // Key-value format has key=value pairs separated by spaces + // Must contain at least host= or database= to be considered key-value format + return ( + !str.includes('://') && + (str.includes('host=') || str.includes('database=')) && + str.includes('=') + ); + } + + private static parseUri(connectionString: string): ParseResult | ParseError { + try { + // Handle Azure format where username contains @: user@server:pass + // Azure format: mysql://user@servername:password@host:port/db + const azureMatch = connectionString.match( + /^mysql:\/\/([^@:]+)@([^:]+):([^@]+)@([^:/?]+):?(\d+)?\/([^?]+)(?:\?(.*))?$/, + ); + + if (azureMatch) { + const [, user, , password, host, port, database, queryString] = azureMatch; + const isHttps = this.checkSslMode(queryString); + + return { + host: host, + port: port ? parseInt(port, 10) : 3306, + username: decodeURIComponent(user), + password: decodeURIComponent(password), + database: decodeURIComponent(database), + isHttps, + }; + } + + // Standard URI parsing using URL API + const url = new URL(connectionString); + + const host = url.hostname; + const port = url.port ? parseInt(url.port, 10) : 3306; + const username = decodeURIComponent(url.username); + const password = decodeURIComponent(url.password); + const database = decodeURIComponent(url.pathname.slice(1)); // Remove leading / + const isHttps = this.checkSslMode(url.search); + + // Validate required fields + if (!host) { + return { error: 'Host is missing from connection string' }; + } + + if (!username) { + return { error: 'Username is missing from connection string' }; + } + + if (!password) { + return { error: 'Password is missing from connection string' }; + } + + if (!database) { + return { error: 'Database name is missing from connection string' }; + } + + return { + host, + port, + username, + password, + database, + isHttps, + }; + } catch (e) { + return { + error: `Failed to parse connection string: ${(e as Error).message}`, + format: 'URI', + }; + } + } + + private static parseJdbc(connectionString: string): ParseResult | ParseError { + try { + // JDBC format: jdbc:mysql://host:port/database?user=x&password=y + const jdbcRegex = /^jdbc:mysql:\/\/([^:/?]+):?(\d+)?\/([^?]+)(?:\?(.*))?$/; + const match = connectionString.match(jdbcRegex); + + if (!match) { + return { + error: + 'Invalid JDBC connection string format. Expected: jdbc:mysql://host:port/database?user=x&password=y', + format: 'JDBC', + }; + } + + const [, host, port, database, queryString] = match; + + if (!queryString) { + return { + error: 'JDBC connection string is missing query parameters (user and password)', + format: 'JDBC', + }; + } + + const params = new URLSearchParams(queryString); + const username = params.get('user'); + const password = params.get('password'); + const isHttps = this.checkSslMode(queryString); + + if (!username) { + return { + error: 'Username (user parameter) is missing from JDBC connection string', + format: 'JDBC', + }; + } + + if (!password) { + return { + error: 'Password parameter is missing from JDBC connection string', + format: 'JDBC', + }; + } + + return { + host, + port: port ? parseInt(port, 10) : 3306, + username: decodeURIComponent(username), + password: decodeURIComponent(password), + database: decodeURIComponent(database), + isHttps, + }; + } catch (e) { + return { + error: `Failed to parse JDBC connection string: ${(e as Error).message}`, + format: 'JDBC', + }; + } + } + + private static parseKeyValue(connectionString: string): ParseResult | ParseError { + try { + // Key-value format: host=x port=3306 database=db user=u password=p + // Values can be quoted with single quotes: password='my pass' + const params: Record = {}; + + // Match key=value or key='quoted value' + const regex = /(\w+)=(?:'([^']*)'|(\S+))/g; + let match; + + while ((match = regex.exec(connectionString)) !== null) { + const key = match[1]; + const value = match[2] !== undefined ? match[2] : match[3]; + params[key] = value; + } + + const host = params['host'] || params['hostaddr']; + const port = params['port']; + const database = params['database'] || params['dbname']; + const username = params['user'] || params['username']; + const password = params['password']; + const ssl = params['ssl'] || params['sslMode'] || params['ssl-mode'] || params['useSSL']; + + if (!host) { + return { + error: 'Host is missing from connection string. Use host=hostname', + format: 'key-value', + }; + } + + if (!username) { + return { + error: 'Username is missing from connection string. Use user=username', + format: 'key-value', + }; + } + + if (!password) { + return { + error: 'Password is missing from connection string. Use password=yourpassword', + format: 'key-value', + }; + } + + if (!database) { + return { + error: 'Database name is missing from connection string. Use database=database', + format: 'key-value', + }; + } + + const isHttps = this.isSslEnabled(ssl); + + return { + host, + port: port ? parseInt(port, 10) : 3306, + username, + password, + database, + isHttps, + }; + } catch (e) { + return { + error: `Failed to parse key-value connection string: ${(e as Error).message}`, + format: 'key-value', + }; + } + } + + private static checkSslMode(queryString: string | undefined | null): boolean { + if (!queryString) return false; + + const params = new URLSearchParams( + queryString.startsWith('?') ? queryString.slice(1) : queryString, + ); + + // Check various MySQL SSL parameter names + const ssl = params.get('ssl'); + const sslMode = params.get('sslMode'); + const sslModeHyphen = params.get('ssl-mode'); + const useSSL = params.get('useSSL'); + const sslaccept = params.get('sslaccept'); + + if (ssl) return this.isSslEnabled(ssl); + if (sslMode) return this.isSslEnabled(sslMode); + if (sslModeHyphen) return this.isSslEnabled(sslModeHyphen); + if (useSSL) return this.isSslEnabled(useSSL); + if (sslaccept) return sslaccept.toLowerCase() === 'strict'; + + return false; + } + + private static isSslEnabled(sslValue: string | null | undefined): boolean { + if (!sslValue) return false; + + const lowercased = sslValue.toLowerCase(); + + // These values indicate SSL is enabled + const enabledValues = ['true', 'required', 'verify_ca', 'verify_identity', 'yes', '1']; + return enabledValues.includes(lowercased); + } +} diff --git a/frontend/src/entity/databases/model/mysql/MysqlDatabase.ts b/frontend/src/entity/databases/model/mysql/MysqlDatabase.ts new file mode 100644 index 0000000..b1c324e --- /dev/null +++ b/frontend/src/entity/databases/model/mysql/MysqlDatabase.ts @@ -0,0 +1,13 @@ +import type { MysqlVersion } from './MysqlVersion'; + +export interface MysqlDatabase { + id: string; + version: MysqlVersion; + + host: string; + port: number; + username: string; + password: string; + database?: string; + isHttps: boolean; +} diff --git a/frontend/src/entity/databases/model/mysql/MysqlVersion.ts b/frontend/src/entity/databases/model/mysql/MysqlVersion.ts new file mode 100644 index 0000000..26d899e --- /dev/null +++ b/frontend/src/entity/databases/model/mysql/MysqlVersion.ts @@ -0,0 +1,5 @@ +export enum MysqlVersion { + MysqlVersion57 = '5.7', + MysqlVersion80 = '8.0', + MysqlVersion84 = '8.4', +} diff --git a/frontend/src/entity/restores/api/restoreApi.ts b/frontend/src/entity/restores/api/restoreApi.ts index 484c53d..ce6a7bf 100644 --- a/frontend/src/entity/restores/api/restoreApi.ts +++ b/frontend/src/entity/restores/api/restoreApi.ts @@ -1,7 +1,7 @@ import { getApplicationServer } from '../../../constants'; import RequestOptions from '../../../shared/api/RequestOptions'; import { apiHelper } from '../../../shared/api/apiHelper'; -import type { PostgresqlDatabase } from '../../databases'; +import type { MysqlDatabase, PostgresqlDatabase } from '../../databases'; import type { Restore } from '../model/Restore'; export const restoreApi = { @@ -16,14 +16,17 @@ export const restoreApi = { async restoreBackup({ backupId, postgresql, + mysql, }: { backupId: string; - postgresql: PostgresqlDatabase; + postgresql?: PostgresqlDatabase; + mysql?: MysqlDatabase; }) { const requestOptions: RequestOptions = new RequestOptions(); requestOptions.setBody( JSON.stringify({ postgresqlDatabase: postgresql, + mysqlDatabase: mysql, }), ); diff --git a/frontend/src/features/backups/ui/BackupsComponent.tsx b/frontend/src/features/backups/ui/BackupsComponent.tsx index c0bcd79..484ae96 100644 --- a/frontend/src/features/backups/ui/BackupsComponent.tsx +++ b/frontend/src/features/backups/ui/BackupsComponent.tsx @@ -22,7 +22,7 @@ import { backupConfigApi, backupsApi, } from '../../../entity/backups'; -import type { Database } from '../../../entity/databases'; +import { type Database, DatabaseType } from '../../../entity/databases'; import { getUserTimeFormat } from '../../../shared/time'; import { ConfirmationComponent } from '../../../shared/ui'; import { RestoresComponent } from '../../restores'; @@ -74,7 +74,8 @@ export const BackupsComponent = ({ database, isCanManageDBs, scrollContainerRef // Find the backup to get a meaningful filename const backup = backups.find((b) => b.id === backupId); const createdAt = backup ? dayjs(backup.createdAt).format('YYYY-MM-DD_HH-mm-ss') : 'backup'; - link.download = `${database.name}_backup_${createdAt}.dump`; + const extension = database.type === DatabaseType.MYSQL ? '.sql.zst' : '.dump.zst'; + link.download = `${database.name}_backup_${createdAt}${extension}`; // Trigger download document.body.appendChild(link); diff --git a/frontend/src/features/backups/ui/EditBackupConfigComponent.tsx b/frontend/src/features/backups/ui/EditBackupConfigComponent.tsx index 6489680..c75fefd 100644 --- a/frontend/src/features/backups/ui/EditBackupConfigComponent.tsx +++ b/frontend/src/features/backups/ui/EditBackupConfigComponent.tsx @@ -600,6 +600,7 @@ export const EditBackupConfigComponent = ({ setShowCreateStorage(false); setStorageSelectKey((prev) => prev + 1); }} + maskClosable={false} >
Storage - is a place where backups will be stored (local disk, S3, Google Drive, etc.) diff --git a/frontend/src/features/databases/ui/CreateDatabaseComponent.tsx b/frontend/src/features/databases/ui/CreateDatabaseComponent.tsx index 3360970..3f6dfac 100644 --- a/frontend/src/features/databases/ui/CreateDatabaseComponent.tsx +++ b/frontend/src/features/databases/ui/CreateDatabaseComponent.tsx @@ -4,6 +4,7 @@ import { type BackupConfig, backupConfigApi, backupsApi } from '../../../entity/ import { type Database, DatabaseType, + type MysqlDatabase, Period, type PostgresqlDatabase, databaseApi, @@ -21,18 +22,14 @@ interface Props { onClose: () => void; } -export const CreateDatabaseComponent = ({ workspaceId, onCreated, onClose }: Props) => { - const [isCreating, setIsCreating] = useState(false); - const [backupConfig, setBackupConfig] = useState(); - const [database, setDatabase] = useState({ +const createInitialDatabase = (workspaceId: string): Database => + ({ id: undefined as unknown as string, name: '', workspaceId, storePeriod: Period.MONTH, - postgresql: { - cpuCount: 1, - } as unknown as PostgresqlDatabase, + postgresql: {} as PostgresqlDatabase, type: DatabaseType.POSTGRES, @@ -40,7 +37,24 @@ export const CreateDatabaseComponent = ({ workspaceId, onCreated, onClose }: Pro notifiers: [], sendNotificationsOn: [], - } as Database); + }) as Database; + +const initializeDatabaseTypeData = (db: Database): Database => { + if (db.type === DatabaseType.POSTGRES && !db.postgresql) { + return { ...db, postgresql: {} as PostgresqlDatabase, mysql: undefined }; + } + + if (db.type === DatabaseType.MYSQL && !db.mysql) { + return { ...db, mysql: {} as MysqlDatabase, postgresql: undefined }; + } + + return db; +}; + +export const CreateDatabaseComponent = ({ workspaceId, onCreated, onClose }: Props) => { + const [isCreating, setIsCreating] = useState(false); + const [backupConfig, setBackupConfig] = useState(); + const [database, setDatabase] = useState(createInitialDatabase(workspaceId)); const [step, setStep] = useState< 'base-info' | 'db-settings' | 'create-readonly-user' | 'backup-config' | 'notifiers' @@ -74,11 +88,13 @@ export const CreateDatabaseComponent = ({ workspaceId, onCreated, onClose }: Pro onClose()} - onSaved={(database) => { - setDatabase({ ...database }); + onSaved={(db) => { + const initializedDb = initializeDatabaseTypeData(db); + setDatabase({ ...initializedDb }); setStep('db-settings'); }} /> diff --git a/frontend/src/features/databases/ui/DatabasesComponent.tsx b/frontend/src/features/databases/ui/DatabasesComponent.tsx index 2088615..0b3c351 100644 --- a/frontend/src/features/databases/ui/DatabasesComponent.tsx +++ b/frontend/src/features/databases/ui/DatabasesComponent.tsx @@ -179,6 +179,7 @@ export const DatabasesComponent = ({ contentHeight, workspace, isCanManageDBs }: footer={
} open={isShowAddDatabase} onCancel={() => setIsShowAddDatabase(false)} + maskClosable={false} width={420} >
diff --git a/frontend/src/features/databases/ui/edit/CreateReadOnlyComponent.tsx b/frontend/src/features/databases/ui/edit/CreateReadOnlyComponent.tsx index 0339855..b7ed52f 100644 --- a/frontend/src/features/databases/ui/edit/CreateReadOnlyComponent.tsx +++ b/frontend/src/features/databases/ui/edit/CreateReadOnlyComponent.tsx @@ -1,7 +1,7 @@ import { Button, Modal, Spin } from 'antd'; import { useEffect, useState } from 'react'; -import { type Database, databaseApi } from '../../../../entity/databases'; +import { type Database, DatabaseType, databaseApi } from '../../../../entity/databases'; interface Props { database: Database; @@ -21,6 +21,10 @@ export const CreateReadOnlyComponent = ({ const [isCreatingReadOnlyUser, setIsCreatingReadOnlyUser] = useState(false); const [isShowSkipConfirmation, setShowSkipConfirmation] = useState(false); + const isPostgres = database.type === DatabaseType.POSTGRES; + const isMysql = database.type === DatabaseType.MYSQL; + const databaseTypeName = isPostgres ? 'PostgreSQL' : isMysql ? 'MySQL' : 'database'; + const checkReadOnlyUser = async (): Promise => { try { const response = await databaseApi.isUserReadOnly(database); @@ -36,8 +40,15 @@ export const CreateReadOnlyComponent = ({ try { const response = await databaseApi.createReadOnlyUser(database); - database.postgresql!.username = response.username; - database.postgresql!.password = response.password; + + if (isPostgres && database.postgresql) { + database.postgresql.username = response.username; + database.postgresql.password = response.password; + } else if (isMysql && database.mysql) { + database.mysql.username = response.username; + database.mysql.password = response.password; + } + onReadOnlyUserUpdated(database); onContinue(); } catch (e) { @@ -62,7 +73,6 @@ export const CreateReadOnlyComponent = ({ const isReadOnly = await checkReadOnlyUser(); if (isReadOnly) { - // already has a read-only user onContinue(); } @@ -86,8 +96,8 @@ export const CreateReadOnlyComponent = ({

Create a read-only user for Postgresus?

- A read-only user is a PostgreSQL user with limited permissions that can only read data - from your database, not modify it. This is recommended for backup operations because: + A read-only user is a {databaseTypeName} user with limited permissions that can only read + data from your database, not modify it. This is recommended for backup operations because:

    diff --git a/frontend/src/features/databases/ui/edit/EditDatabaseBaseInfoComponent.tsx b/frontend/src/features/databases/ui/edit/EditDatabaseBaseInfoComponent.tsx index eee1301..852f64c 100644 --- a/frontend/src/features/databases/ui/edit/EditDatabaseBaseInfoComponent.tsx +++ b/frontend/src/features/databases/ui/edit/EditDatabaseBaseInfoComponent.tsx @@ -1,12 +1,20 @@ -import { Button, Input } from 'antd'; +import { Button, Input, Select } from 'antd'; import { useEffect, useState } from 'react'; -import { type Database, databaseApi } from '../../../../entity/databases'; +import { + type Database, + DatabaseType, + type MysqlDatabase, + type PostgresqlDatabase, + databaseApi, + getDatabaseLogoFromType, +} from '../../../../entity/databases'; interface Props { database: Database; isShowName?: boolean; + isShowType?: boolean; isShowCancelButton?: boolean; onCancel: () => void; @@ -15,9 +23,15 @@ interface Props { onSaved: (db: Database) => void; } +const databaseTypeOptions = [ + { value: DatabaseType.POSTGRES, label: 'PostgreSQL' }, + { value: DatabaseType.MYSQL, label: 'MySQL' }, +]; + export const EditDatabaseBaseInfoComponent = ({ database, isShowName, + isShowType, isShowCancelButton, onCancel, saveButtonText, @@ -33,6 +47,26 @@ export const EditDatabaseBaseInfoComponent = ({ setIsUnsaved(true); }; + const handleTypeChange = (newType: DatabaseType) => { + if (!editingDatabase) return; + + const updatedDatabase: Database = { + ...editingDatabase, + type: newType, + }; + + if (newType === DatabaseType.POSTGRES && !editingDatabase.postgresql) { + updatedDatabase.postgresql = {} as PostgresqlDatabase; + updatedDatabase.mysql = undefined; + } else if (newType === DatabaseType.MYSQL && !editingDatabase.mysql) { + updatedDatabase.mysql = {} as MysqlDatabase; + updatedDatabase.postgresql = undefined; + } + + setEditingDatabase(updatedDatabase); + setIsUnsaved(true); + }; + const saveDatabase = async () => { if (!editingDatabase) return; if (isSaveToApi) { @@ -59,7 +93,6 @@ export const EditDatabaseBaseInfoComponent = ({ if (!editingDatabase) return null; - // mandatory-field check const isAllFieldsFilled = !!editingDatabase.name?.trim(); return ( @@ -77,6 +110,28 @@ export const EditDatabaseBaseInfoComponent = ({
)} + {isShowType && ( +
+
Database type
+ +
+ { - if (!editingDatabase.postgresql) return; - - const updatedDatabase = { - ...editingDatabase, - postgresql: { - ...editingDatabase.postgresql, - host: e.target.value.trim().replace('https://', '').replace('http://', ''), - }, - }; - setEditingDatabase(autoAddPublicSchemaForSupabase(updatedDatabase)); - setIsConnectionTested(false); - }} - size="small" - className="max-w-[200px] grow" - placeholder="Enter PG host" - /> -
- - {isLocalhostDb && ( -
-
-
- Please{' '} - - read this document - {' '} - to study how to backup local database -
-
- )} - - {isSupabaseDb && ( -
-
-
- Please{' '} - - read this document - {' '} - to study how to backup Supabase database -
-
- )} - -
-
Port
- { - if (!editingDatabase.postgresql || e === null) return; - - setEditingDatabase({ - ...editingDatabase, - postgresql: { ...editingDatabase.postgresql, port: e }, - }); - setIsConnectionTested(false); - }} - size="small" - className="max-w-[200px] grow" - placeholder="Enter PG port" - /> -
- -
-
Username
- { - if (!editingDatabase.postgresql) return; - - const updatedDatabase = { - ...editingDatabase, - postgresql: { ...editingDatabase.postgresql, username: e.target.value.trim() }, - }; - setEditingDatabase(autoAddPublicSchemaForSupabase(updatedDatabase)); - setIsConnectionTested(false); - }} - size="small" - className="max-w-[200px] grow" - placeholder="Enter PG username" - /> -
- -
-
Password
- { - if (!editingDatabase.postgresql) return; - - setEditingDatabase({ - ...editingDatabase, - postgresql: { ...editingDatabase.postgresql, password: e.target.value.trim() }, - }); - setIsConnectionTested(false); - }} - size="small" - className="max-w-[200px] grow" - placeholder="Enter PG password" - /> -
- - {isShowDbName && ( -
-
DB name
- { - if (!editingDatabase.postgresql) return; - - setEditingDatabase({ - ...editingDatabase, - postgresql: { ...editingDatabase.postgresql, database: e.target.value.trim() }, - }); - setIsConnectionTested(false); - }} - size="small" - className="max-w-[200px] grow" - placeholder="Enter PG database name" - /> -
- )} - -
-
Use HTTPS
- { - if (!editingDatabase.postgresql) return; - - setEditingDatabase({ - ...editingDatabase, - postgresql: { ...editingDatabase.postgresql, isHttps: checked }, - }); - setIsConnectionTested(false); - }} - size="small" - /> -
- -
-
setShowAdvanced(!isShowAdvanced)} - > - Advanced settings - - {isShowAdvanced ? ( - - ) : ( - - )} -
-
- - {isShowAdvanced && ( - <> - {!isRestoreMode && ( -
-
Include schemas
- { + if (!editingDatabase.mysql) return; + + setEditingDatabase({ + ...editingDatabase, + mysql: { + ...editingDatabase.mysql, + host: e.target.value.trim().replace('https://', '').replace('http://', ''), + }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter MySQL host" + /> +
+ + {isLocalhostDb && ( +
+
+
+ Please{' '} + + read this document + {' '} + to study how to backup local database +
+
+ )} + +
+
Port
+ { + if (!editingDatabase.mysql || e === null) return; + + setEditingDatabase({ + ...editingDatabase, + mysql: { ...editingDatabase.mysql, port: e }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter MySQL port" + /> +
+ +
+
Username
+ { + if (!editingDatabase.mysql) return; + + setEditingDatabase({ + ...editingDatabase, + mysql: { ...editingDatabase.mysql, username: e.target.value.trim() }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter MySQL username" + /> +
+ +
+
Password
+ { + if (!editingDatabase.mysql) return; + + setEditingDatabase({ + ...editingDatabase, + mysql: { ...editingDatabase.mysql, password: e.target.value.trim() }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter MySQL password" + autoComplete="new-password" + /> +
+ + {isShowDbName && ( +
+
DB name
+ { + if (!editingDatabase.mysql) return; + + setEditingDatabase({ + ...editingDatabase, + mysql: { ...editingDatabase.mysql, database: e.target.value.trim() }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter MySQL database name" + /> +
+ )} + +
+
Use HTTPS
+ { + if (!editingDatabase.mysql) return; + + setEditingDatabase({ + ...editingDatabase, + mysql: { ...editingDatabase.mysql, isHttps: checked }, + }); + setIsConnectionTested(false); + }} + size="small" + /> +
+ +
+ {isShowCancelButton && ( + + )} + + {isShowBackButton && ( + + )} + + {!isConnectionTested && ( + + )} + + {isConnectionTested && ( + + )} +
+ + {isConnectionFailed && ( +
+ If your database uses IP whitelist, make sure Postgresus server IP is added to the allowed + list. +
+ )} +
+ ); +}; diff --git a/frontend/src/features/databases/ui/edit/EditPostgreSqlSpecificDataComponent.tsx b/frontend/src/features/databases/ui/edit/EditPostgreSqlSpecificDataComponent.tsx new file mode 100644 index 0000000..4ab2a77 --- /dev/null +++ b/frontend/src/features/databases/ui/edit/EditPostgreSqlSpecificDataComponent.tsx @@ -0,0 +1,473 @@ +import { CopyOutlined, DownOutlined, InfoCircleOutlined, UpOutlined } from '@ant-design/icons'; +import { App, Button, Checkbox, Input, InputNumber, Select, Switch, Tooltip } from 'antd'; +import { useEffect, useState } from 'react'; + +import { type Database, databaseApi } from '../../../../entity/databases'; +import { ConnectionStringParser } from '../../../../entity/databases/model/postgresql/ConnectionStringParser'; +import { ToastHelper } from '../../../../shared/toast'; + +interface Props { + database: Database; + + isShowCancelButton?: boolean; + onCancel: () => void; + + isShowBackButton: boolean; + onBack: () => void; + + saveButtonText?: string; + isSaveToApi: boolean; + onSaved: (database: Database) => void; + + isShowDbName?: boolean; + isRestoreMode?: boolean; +} + +export const EditPostgreSqlSpecificDataComponent = ({ + database, + + isShowCancelButton, + onCancel, + + isShowBackButton, + onBack, + + saveButtonText, + isSaveToApi, + onSaved, + isShowDbName = true, + isRestoreMode = false, +}: Props) => { + const { message } = App.useApp(); + + const [editingDatabase, setEditingDatabase] = useState(); + const [isSaving, setIsSaving] = useState(false); + + const [isConnectionTested, setIsConnectionTested] = useState(false); + const [isTestingConnection, setIsTestingConnection] = useState(false); + const [isConnectionFailed, setIsConnectionFailed] = useState(false); + + const hasAdvancedValues = + !!database.postgresql?.includeSchemas?.length || !!database.postgresql?.isExcludeExtensions; + const [isShowAdvanced, setShowAdvanced] = useState(hasAdvancedValues); + + const [hasAutoAddedPublicSchema, setHasAutoAddedPublicSchema] = useState(false); + + const parseFromClipboard = async () => { + try { + const text = await navigator.clipboard.readText(); + const trimmedText = text.trim(); + + if (!trimmedText) { + message.error('Clipboard is empty'); + return; + } + + const result = ConnectionStringParser.parse(trimmedText); + + if ('error' in result) { + message.error(result.error); + return; + } + + if (!editingDatabase?.postgresql) return; + + const updatedDatabase: Database = { + ...editingDatabase, + postgresql: { + ...editingDatabase.postgresql, + host: result.host, + port: result.port, + username: result.username, + password: result.password, + database: result.database, + isHttps: result.isHttps, + }, + }; + + setEditingDatabase(autoAddPublicSchemaForSupabase(updatedDatabase)); + setIsConnectionTested(false); + message.success('Connection string parsed successfully'); + } catch { + message.error('Failed to read clipboard. Please check browser permissions.'); + } + }; + + const autoAddPublicSchemaForSupabase = (updatedDatabase: Database): Database => { + if (hasAutoAddedPublicSchema) return updatedDatabase; + + const host = updatedDatabase.postgresql?.host || ''; + const username = updatedDatabase.postgresql?.username || ''; + const isSupabase = host.includes('supabase') || username.includes('supabase'); + + if (isSupabase && updatedDatabase.postgresql) { + setHasAutoAddedPublicSchema(true); + + const currentSchemas = updatedDatabase.postgresql.includeSchemas || []; + if (!currentSchemas.includes('public')) { + return { + ...updatedDatabase, + postgresql: { + ...updatedDatabase.postgresql, + includeSchemas: ['public', ...currentSchemas], + }, + }; + } + } + + return updatedDatabase; + }; + + const testConnection = async () => { + if (!editingDatabase) return; + setIsTestingConnection(true); + setIsConnectionFailed(false); + + try { + await databaseApi.testDatabaseConnectionDirect(editingDatabase); + setIsConnectionTested(true); + ToastHelper.showToast({ + title: 'Connection test passed', + description: 'You can continue with the next step', + }); + } catch (e) { + setIsConnectionFailed(true); + alert((e as Error).message); + } + + setIsTestingConnection(false); + }; + + const saveDatabase = async () => { + if (!editingDatabase) return; + + if (isSaveToApi) { + setIsSaving(true); + + try { + await databaseApi.updateDatabase(editingDatabase); + } catch (e) { + alert((e as Error).message); + } + + setIsSaving(false); + } + + onSaved(editingDatabase); + }; + + useEffect(() => { + setIsSaving(false); + setIsConnectionTested(false); + setIsTestingConnection(false); + setIsConnectionFailed(false); + + setEditingDatabase({ ...database }); + }, [database]); + + if (!editingDatabase) return null; + + let isAllFieldsFilled = true; + if (!editingDatabase.postgresql?.host) isAllFieldsFilled = false; + if (!editingDatabase.postgresql?.port) isAllFieldsFilled = false; + if (!editingDatabase.postgresql?.username) isAllFieldsFilled = false; + if (!editingDatabase.id && !editingDatabase.postgresql?.password) isAllFieldsFilled = false; + if (!editingDatabase.postgresql?.database) isAllFieldsFilled = false; + + const isLocalhostDb = + editingDatabase.postgresql?.host?.includes('localhost') || + editingDatabase.postgresql?.host?.includes('127.0.0.1'); + + const isSupabaseDb = + editingDatabase.postgresql?.host?.includes('supabase') || + editingDatabase.postgresql?.username?.includes('supabase'); + + return ( +
+
+
+
+ + Parse from clipboard +
+
+ +
+
Host
+ { + if (!editingDatabase.postgresql) return; + + const updatedDatabase = { + ...editingDatabase, + postgresql: { + ...editingDatabase.postgresql, + host: e.target.value.trim().replace('https://', '').replace('http://', ''), + }, + }; + setEditingDatabase(autoAddPublicSchemaForSupabase(updatedDatabase)); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter PG host" + /> +
+ + {isLocalhostDb && ( +
+
+
+ Please{' '} + + read this document + {' '} + to study how to backup local database +
+
+ )} + + {isSupabaseDb && ( +
+
+
+ Please{' '} + + read this document + {' '} + to study how to backup Supabase database +
+
+ )} + +
+
Port
+ { + if (!editingDatabase.postgresql || e === null) return; + + setEditingDatabase({ + ...editingDatabase, + postgresql: { ...editingDatabase.postgresql, port: e }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter PG port" + /> +
+ +
+
Username
+ { + if (!editingDatabase.postgresql) return; + + const updatedDatabase = { + ...editingDatabase, + postgresql: { ...editingDatabase.postgresql, username: e.target.value.trim() }, + }; + setEditingDatabase(autoAddPublicSchemaForSupabase(updatedDatabase)); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter PG username" + /> +
+ +
+
Password
+ { + if (!editingDatabase.postgresql) return; + + setEditingDatabase({ + ...editingDatabase, + postgresql: { ...editingDatabase.postgresql, password: e.target.value.trim() }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter PG password" + autoComplete="new-password" + /> +
+ + {isShowDbName && ( +
+
DB name
+ { + if (!editingDatabase.postgresql) return; + + setEditingDatabase({ + ...editingDatabase, + postgresql: { ...editingDatabase.postgresql, database: e.target.value.trim() }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter PG database name" + /> +
+ )} + +
+
Use HTTPS
+ { + if (!editingDatabase.postgresql) return; + + setEditingDatabase({ + ...editingDatabase, + postgresql: { ...editingDatabase.postgresql, isHttps: checked }, + }); + setIsConnectionTested(false); + }} + size="small" + /> +
+ +
+
setShowAdvanced(!isShowAdvanced)} + > + Advanced settings + + {isShowAdvanced ? ( + + ) : ( + + )} +
+
+ + {isShowAdvanced && ( + <> + {!isRestoreMode && ( +
+
Include schemas
+