From 3ae8761666cacb929bc1143f6d8f65e61891d2c3 Mon Sep 17 00:00:00 2001 From: Rostislav Dugin Date: Sun, 21 Dec 2025 14:53:53 +0300 Subject: [PATCH] FEATURE (databases): Add MariaDB support --- .dockerignore | 1 + .github/workflows/ci-release.yml | 64 +- Dockerfile | 43 +- backend/.env.development.example | 18 +- backend/.gitignore | 1 + backend/docker-compose.yml.example | 210 ++++++ backend/internal/config/config.go | 16 + .../backups/usecases/create_backup_uc.go | 12 + .../features/backups/backups/usecases/di.go | 2 + .../usecases/mariadb/create_backup_uc.go | 595 +++++++++++++++ .../backups/backups/usecases/mariadb/di.go | 17 + .../features/databases/controller_test.go | 52 +- .../databases/databases/mariadb/model.go | 412 ++++++++++ .../databases/mariadb/readonly_user_test.go | 387 ++++++++++ .../databases/databases/mysql/model.go | 7 + .../databases/databases/postgresql/model.go | 8 + backend/internal/features/databases/enums.go | 1 + backend/internal/features/databases/model.go | 19 + .../internal/features/databases/repository.go | 31 +- .../internal/features/databases/service.go | 26 + backend/internal/features/restores/dto.go | 2 + backend/internal/features/restores/service.go | 49 ++ .../internal/features/restores/usecases/di.go | 2 + .../features/restores/usecases/mariadb/di.go | 15 + .../usecases/mariadb/restore_backup_uc.go | 473 ++++++++++++ .../restores/usecases/restore_backup_uc.go | 11 + .../tests/mariadb_backup_restore_test.go | 706 ++++++++++++++++++ backend/internal/util/tools/mariadb.go | 241 ++++++ ...1221100000_add_mariadb_databases_table.sql | 28 + backend/tools/.gitignore | 3 +- backend/tools/download_linux.sh | 86 ++- backend/tools/download_macos.sh | 88 ++- backend/tools/download_windows.bat | 109 +++ backend/tools/readme.md | 58 +- frontend/public/icons/databases/mariadb.svg | 2 + frontend/src/entity/databases/index.ts | 2 + .../src/entity/databases/model/Database.ts | 2 + .../entity/databases/model/DatabaseType.ts | 1 + .../model/getDatabaseLogoFromType.ts | 2 + .../mariadb/MariadbConnectionStringParser.ts | 278 +++++++ .../model/mariadb/MariadbDatabase.ts | 12 + .../databases/model/mariadb/MariadbVersion.ts | 13 + .../src/entity/restores/api/restoreApi.ts | 5 +- .../features/backups/ui/BackupsComponent.tsx | 2 + .../databases/ui/CreateDatabaseComponent.tsx | 18 +- .../ui/edit/EditDatabaseBaseInfoComponent.tsx | 21 +- .../EditDatabaseSpecificDataComponent.tsx | 53 +- .../edit/EditMariaDbSpecificDataComponent.tsx | 347 +++++++++ .../edit/EditMySqlSpecificDataComponent.tsx | 5 +- .../EditPostgreSqlSpecificDataComponent.tsx | 5 +- .../ShowDatabaseSpecificDataComponent.tsx | 18 +- .../show/ShowMariaDbSpecificDataComponent.tsx | 60 ++ .../restores/ui/RestoresComponent.tsx | 79 +- .../EditAzureBlobStorageComponent.tsx | 10 +- .../edit/storages/EditFTPStorageComponent.tsx | 5 +- .../edit/storages/EditNASStorageComponent.tsx | 5 +- .../edit/storages/EditS3StorageComponent.tsx | 10 +- .../storages/EditSFTPStorageComponent.tsx | 5 +- 58 files changed, 4642 insertions(+), 111 deletions(-) create mode 100644 backend/internal/features/backups/backups/usecases/mariadb/create_backup_uc.go create mode 100644 backend/internal/features/backups/backups/usecases/mariadb/di.go create mode 100644 backend/internal/features/databases/databases/mariadb/model.go create mode 100644 backend/internal/features/databases/databases/mariadb/readonly_user_test.go create mode 100644 backend/internal/features/restores/usecases/mariadb/di.go create mode 100644 backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go create mode 100644 backend/internal/features/tests/mariadb_backup_restore_test.go create mode 100644 backend/internal/util/tools/mariadb.go create mode 100644 backend/migrations/20251221100000_add_mariadb_databases_table.sql create mode 100644 frontend/public/icons/databases/mariadb.svg create mode 100644 frontend/src/entity/databases/model/mariadb/MariadbConnectionStringParser.ts create mode 100644 frontend/src/entity/databases/model/mariadb/MariadbDatabase.ts create mode 100644 frontend/src/entity/databases/model/mariadb/MariadbVersion.ts create mode 100644 frontend/src/features/databases/ui/edit/EditMariaDbSpecificDataComponent.tsx create mode 100644 frontend/src/features/databases/ui/show/ShowMariaDbSpecificDataComponent.tsx diff --git a/.dockerignore b/.dockerignore index bff3ae3..3a15897 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,6 +11,7 @@ node_modules backend/tools backend/mysqldata backend/pgdata +backend/mariadbdata backend/temp backend/images backend/bin diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index 5d0271c..f5d47dd 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -173,6 +173,18 @@ jobs: TEST_MYSQL_57_PORT=33057 TEST_MYSQL_80_PORT=33080 TEST_MYSQL_84_PORT=33084 + # testing MariaDB + TEST_MARIADB_55_PORT=33055 + TEST_MARIADB_101_PORT=33101 + TEST_MARIADB_102_PORT=33102 + TEST_MARIADB_103_PORT=33103 + TEST_MARIADB_104_PORT=33104 + TEST_MARIADB_105_PORT=33105 + TEST_MARIADB_106_PORT=33106 + TEST_MARIADB_1011_PORT=33111 + TEST_MARIADB_114_PORT=33114 + TEST_MARIADB_118_PORT=33118 + TEST_MARIADB_120_PORT=33120 # testing Telegram TEST_TELEGRAM_BOT_TOKEN=${{ secrets.TEST_TELEGRAM_BOT_TOKEN }} TEST_TELEGRAM_CHAT_ID=${{ secrets.TEST_TELEGRAM_CHAT_ID }} @@ -222,6 +234,30 @@ jobs: echo "Waiting for MySQL 8.4..." timeout 120 bash -c 'until docker exec test-mysql-84 mysqladmin ping -h localhost -u root -prootpassword --silent 2>/dev/null; do sleep 2; done' + # Wait for MariaDB containers + echo "Waiting for MariaDB 5.5..." + timeout 120 bash -c 'until docker exec test-mariadb-55 mysqladmin ping -h localhost -prootpassword --silent 2>/dev/null; do sleep 2; done' + echo "Waiting for MariaDB 10.1..." + timeout 120 bash -c 'until docker exec test-mariadb-101 mysqladmin ping -h localhost -prootpassword --silent 2>/dev/null; do sleep 2; done' + echo "Waiting for MariaDB 10.2..." + timeout 120 bash -c 'until docker exec test-mariadb-102 mysqladmin ping -h localhost -prootpassword --silent 2>/dev/null; do sleep 2; done' + echo "Waiting for MariaDB 10.3..." + timeout 120 bash -c 'until docker exec test-mariadb-103 mysqladmin ping -h localhost -prootpassword --silent 2>/dev/null; do sleep 2; done' + echo "Waiting for MariaDB 10.4..." + timeout 120 bash -c 'until docker exec test-mariadb-104 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' + echo "Waiting for MariaDB 10.5..." + timeout 120 bash -c 'until docker exec test-mariadb-105 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' + echo "Waiting for MariaDB 10.6..." + timeout 120 bash -c 'until docker exec test-mariadb-106 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' + echo "Waiting for MariaDB 10.11..." + timeout 120 bash -c 'until docker exec test-mariadb-1011 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' + echo "Waiting for MariaDB 11.4..." + timeout 120 bash -c 'until docker exec test-mariadb-114 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' + echo "Waiting for MariaDB 11.8..." + timeout 120 bash -c 'until docker exec test-mariadb-118 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' + echo "Waiting for MariaDB 12.0..." + timeout 120 bash -c 'until docker exec test-mariadb-120 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' + - name: Create data and temp directories run: | # Create directories that are used for backups and restore @@ -243,6 +279,13 @@ jobs: path: backend/tools/mysql key: mysql-clients-57-80-84-v1 + - name: Cache MariaDB client tools + id: cache-mariadb + uses: actions/cache@v4 + with: + path: backend/tools/mariadb + key: mariadb-clients-106-121-v1 + - name: Install MySQL dependencies run: | sudo apt-get update -qq @@ -250,8 +293,8 @@ jobs: sudo ln -sf /usr/lib/x86_64-linux-gnu/libncurses.so.6 /usr/lib/x86_64-linux-gnu/libncurses.so.5 sudo ln -sf /usr/lib/x86_64-linux-gnu/libtinfo.so.6 /usr/lib/x86_64-linux-gnu/libtinfo.so.5 - - name: Install PostgreSQL and MySQL client tools - if: steps.cache-postgres.outputs.cache-hit != 'true' || steps.cache-mysql.outputs.cache-hit != 'true' + - name: Install PostgreSQL, MySQL and MariaDB client tools + if: steps.cache-postgres.outputs.cache-hit != 'true' || steps.cache-mysql.outputs.cache-hit != 'true' || steps.cache-mariadb.outputs.cache-hit != 'true' run: | chmod +x backend/tools/download_linux.sh cd backend/tools @@ -276,6 +319,23 @@ jobs: fi done + - name: Verify MariaDB client tools exist + run: | + cd backend/tools + echo "Checking MariaDB client tools..." + if [ -f "mariadb/mariadb-10.6/bin/mariadb-dump" ]; then + echo "MariaDB 10.6 client tools found" + ls -la mariadb/mariadb-10.6/bin/ + else + echo "MariaDB 10.6 client tools NOT found" + fi + if [ -f "mariadb/mariadb-12.1/bin/mariadb-dump" ]; then + echo "MariaDB 12.1 client tools found" + ls -la mariadb/mariadb-12.1/bin/ + else + echo "MariaDB 12.1 client tools NOT found" + fi + - name: Run database migrations run: | cd backend diff --git a/Dockerfile b/Dockerfile index 6b3ff0a..5ef78fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,9 +77,10 @@ ENV APP_VERSION=$APP_VERSION # Set production mode for Docker containers ENV ENV_MODE=production -# Install PostgreSQL server and client tools (versions 12-18), MySQL client tools (5.7, 8.0, 8.4), and rclone +# Install PostgreSQL server and client tools (versions 12-18), MySQL client tools (5.7, 8.0, 8.4), MariaDB client tools, and rclone # Note: MySQL 5.7 is only available for x86_64, MySQL 8.0+ supports both x86_64 and ARM64 # Note: MySQL binaries require libncurses5 for terminal handling +# Note: MariaDB uses a single client version (12.1) that is backward compatible with all server versions ARG TARGETARCH RUN apt-get update && apt-get install -y --no-install-recommends \ wget ca-certificates gnupg lsb-release sudo gosu curl unzip xz-utils libncurses5 && \ @@ -127,6 +128,46 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ rm -rf /tmp/mysql-8.4.* /tmp/mysql84.tar.xz && \ # Make MySQL binaries executable (ignore errors for empty dirs on ARM64) chmod +x /usr/local/mysql-*/bin/* 2>/dev/null || true && \ + # Create MariaDB directories for both versions + # MariaDB uses two client versions: + # - 10.6 (legacy): For older servers (5.5, 10.1) that don't have generation_expression column + # - 12.1 (modern): For newer servers (10.2+) + mkdir -p /usr/local/mariadb-10.6/bin /usr/local/mariadb-12.1/bin && \ + # Download and install MariaDB 10.6 client tools (legacy - for older servers) + if [ "$TARGETARCH" = "amd64" ]; then \ + wget -q https://archive.mariadb.org/mariadb-10.6.21/bintar-linux-systemd-x86_64/mariadb-10.6.21-linux-systemd-x86_64.tar.gz -O /tmp/mariadb106.tar.gz && \ + tar -xzf /tmp/mariadb106.tar.gz -C /tmp && \ + cp /tmp/mariadb-10.6.*/bin/mariadb /usr/local/mariadb-10.6/bin/ && \ + cp /tmp/mariadb-10.6.*/bin/mariadb-dump /usr/local/mariadb-10.6/bin/ && \ + rm -rf /tmp/mariadb-10.6.* /tmp/mariadb106.tar.gz; \ + elif [ "$TARGETARCH" = "arm64" ]; then \ + # For ARM64, install MariaDB 10.6 client from official repository + curl -fsSL https://mariadb.org/mariadb_release_signing_key.asc | gpg --dearmor -o /usr/share/keyrings/mariadb-keyring.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/mariadb-keyring.gpg] https://mirror.mariadb.org/repo/10.6/debian $(lsb_release -cs) main" > /etc/apt/sources.list.d/mariadb106.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends mariadb-client && \ + cp /usr/bin/mariadb /usr/local/mariadb-10.6/bin/mariadb && \ + cp /usr/bin/mariadb-dump /usr/local/mariadb-10.6/bin/mariadb-dump && \ + apt-get remove -y mariadb-client && \ + rm /etc/apt/sources.list.d/mariadb106.list; \ + fi && \ + # Download and install MariaDB 12.1 client tools (modern - for newer servers) + if [ "$TARGETARCH" = "amd64" ]; then \ + wget -q https://archive.mariadb.org/mariadb-12.1.2/bintar-linux-systemd-x86_64/mariadb-12.1.2-linux-systemd-x86_64.tar.gz -O /tmp/mariadb121.tar.gz && \ + tar -xzf /tmp/mariadb121.tar.gz -C /tmp && \ + cp /tmp/mariadb-12.1.*/bin/mariadb /usr/local/mariadb-12.1/bin/ && \ + cp /tmp/mariadb-12.1.*/bin/mariadb-dump /usr/local/mariadb-12.1/bin/ && \ + rm -rf /tmp/mariadb-12.1.* /tmp/mariadb121.tar.gz; \ + elif [ "$TARGETARCH" = "arm64" ]; then \ + # For ARM64, install MariaDB 12.1 client from official repository + echo "deb [signed-by=/usr/share/keyrings/mariadb-keyring.gpg] https://mirror.mariadb.org/repo/12.1/debian $(lsb_release -cs) main" > /etc/apt/sources.list.d/mariadb121.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends mariadb-client && \ + cp /usr/bin/mariadb /usr/local/mariadb-12.1/bin/mariadb && \ + cp /usr/bin/mariadb-dump /usr/local/mariadb-12.1/bin/mariadb-dump; \ + fi && \ + # Make MariaDB binaries executable + chmod +x /usr/local/mariadb-*/bin/* 2>/dev/null || true && \ # Cleanup rm -rf /var/lib/apt/lists/* diff --git a/backend/.env.development.example b/backend/.env.development.example index 2672d9e..3093a91 100644 --- a/backend/.env.development.example +++ b/backend/.env.development.example @@ -43,4 +43,20 @@ TEST_SUPABASE_DATABASE= # FTP TEST_FTP_PORT=7007 # SFTP -TEST_SFTP_PORT=7008 \ No newline at end of file +TEST_SFTP_PORT=7008 +# MySQL Test Ports +TEST_MYSQL_57_PORT=33057 +TEST_MYSQL_80_PORT=33080 +TEST_MYSQL_84_PORT=33084 +# testing MariaDB +TEST_MARIADB_55_PORT=33055 +TEST_MARIADB_101_PORT=33101 +TEST_MARIADB_102_PORT=33102 +TEST_MARIADB_103_PORT=33103 +TEST_MARIADB_104_PORT=33104 +TEST_MARIADB_105_PORT=33105 +TEST_MARIADB_106_PORT=33106 +TEST_MARIADB_1011_PORT=33111 +TEST_MARIADB_114_PORT=33114 +TEST_MARIADB_118_PORT=33118 +TEST_MARIADB_120_PORT=33120 \ No newline at end of file diff --git a/backend/.gitignore b/backend/.gitignore index 35a6c2f..0b0b08d 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -4,6 +4,7 @@ docker-compose.yml pgdata pgdata_test/ mysqldata/ +mariadbdata/ main.exe swagger/ swagger/* diff --git a/backend/docker-compose.yml.example b/backend/docker-compose.yml.example index 2b1d777..e4b03bd 100644 --- a/backend/docker-compose.yml.example +++ b/backend/docker-compose.yml.example @@ -212,3 +212,213 @@ services: interval: 5s timeout: 5s retries: 10 + + # Test MariaDB containers + test-mariadb-55: + image: mariadb:5.5 + ports: + - "${TEST_MARIADB_55_PORT:-33055}:3306" + environment: + - MYSQL_ROOT_PASSWORD=rootpassword + - MYSQL_DATABASE=testdb + - MYSQL_USER=testuser + - MYSQL_PASSWORD=testpassword + command: --character-set-server=utf8 --collation-server=utf8_unicode_ci + volumes: + - ./mariadbdata/mariadb-55:/var/lib/mysql + container_name: test-mariadb-55 + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-prootpassword"] + interval: 5s + timeout: 5s + retries: 10 + + test-mariadb-101: + image: mariadb:10.1 + ports: + - "${TEST_MARIADB_101_PORT:-33101}:3306" + environment: + - MYSQL_ROOT_PASSWORD=rootpassword + - MYSQL_DATABASE=testdb + - MYSQL_USER=testuser + - MYSQL_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - ./mariadbdata/mariadb-101:/var/lib/mysql + container_name: test-mariadb-101 + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-prootpassword"] + interval: 5s + timeout: 5s + retries: 10 + + test-mariadb-102: + image: mariadb:10.2 + ports: + - "${TEST_MARIADB_102_PORT:-33102}:3306" + environment: + - MYSQL_ROOT_PASSWORD=rootpassword + - MYSQL_DATABASE=testdb + - MYSQL_USER=testuser + - MYSQL_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - ./mariadbdata/mariadb-102:/var/lib/mysql + container_name: test-mariadb-102 + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-prootpassword"] + interval: 5s + timeout: 5s + retries: 10 + + test-mariadb-103: + image: mariadb:10.3 + ports: + - "${TEST_MARIADB_103_PORT:-33103}:3306" + environment: + - MYSQL_ROOT_PASSWORD=rootpassword + - MYSQL_DATABASE=testdb + - MYSQL_USER=testuser + - MYSQL_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - ./mariadbdata/mariadb-103:/var/lib/mysql + container_name: test-mariadb-103 + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-prootpassword"] + interval: 5s + timeout: 5s + retries: 10 + + test-mariadb-104: + image: mariadb:10.4 + ports: + - "${TEST_MARIADB_104_PORT:-33104}:3306" + environment: + - MARIADB_ROOT_PASSWORD=rootpassword + - MARIADB_DATABASE=testdb + - MARIADB_USER=testuser + - MARIADB_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - ./mariadbdata/mariadb-104:/var/lib/mysql + container_name: test-mariadb-104 + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 5s + retries: 10 + + test-mariadb-105: + image: mariadb:10.5 + ports: + - "${TEST_MARIADB_105_PORT:-33105}:3306" + environment: + - MARIADB_ROOT_PASSWORD=rootpassword + - MARIADB_DATABASE=testdb + - MARIADB_USER=testuser + - MARIADB_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - ./mariadbdata/mariadb-105:/var/lib/mysql + container_name: test-mariadb-105 + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 5s + retries: 10 + + test-mariadb-106: + image: mariadb:10.6 + ports: + - "${TEST_MARIADB_106_PORT:-33106}:3306" + environment: + - MARIADB_ROOT_PASSWORD=rootpassword + - MARIADB_DATABASE=testdb + - MARIADB_USER=testuser + - MARIADB_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - ./mariadbdata/mariadb-106:/var/lib/mysql + container_name: test-mariadb-106 + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 5s + retries: 10 + + test-mariadb-1011: + image: mariadb:10.11 + ports: + - "${TEST_MARIADB_1011_PORT:-33111}:3306" + environment: + - MARIADB_ROOT_PASSWORD=rootpassword + - MARIADB_DATABASE=testdb + - MARIADB_USER=testuser + - MARIADB_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - ./mariadbdata/mariadb-1011:/var/lib/mysql + container_name: test-mariadb-1011 + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 5s + retries: 10 + + test-mariadb-114: + image: mariadb:11.4 + ports: + - "${TEST_MARIADB_114_PORT:-33114}:3306" + environment: + - MARIADB_ROOT_PASSWORD=rootpassword + - MARIADB_DATABASE=testdb + - MARIADB_USER=testuser + - MARIADB_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - ./mariadbdata/mariadb-114:/var/lib/mysql + container_name: test-mariadb-114 + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 5s + retries: 10 + + test-mariadb-118: + image: mariadb:11.8 + ports: + - "${TEST_MARIADB_118_PORT:-33118}:3306" + environment: + - MARIADB_ROOT_PASSWORD=rootpassword + - MARIADB_DATABASE=testdb + - MARIADB_USER=testuser + - MARIADB_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - ./mariadbdata/mariadb-118:/var/lib/mysql + container_name: test-mariadb-118 + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 5s + retries: 10 + + test-mariadb-120: + image: mariadb:12.0 + ports: + - "${TEST_MARIADB_120_PORT:-33120}:3306" + environment: + - MARIADB_ROOT_PASSWORD=rootpassword + - MARIADB_DATABASE=testdb + - MARIADB_USER=testuser + - MARIADB_PASSWORD=testpassword + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - ./mariadbdata/mariadb-120:/var/lib/mysql + container_name: test-mariadb-120 + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 5s + timeout: 5s + retries: 10 diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 20e3ff3..73f8781 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -26,6 +26,7 @@ type EnvVariables struct { EnvMode env_utils.EnvMode `env:"ENV_MODE" required:"true"` PostgresesInstallDir string `env:"POSTGRES_INSTALL_DIR"` MysqlInstallDir string `env:"MYSQL_INSTALL_DIR"` + MariadbInstallDir string `env:"MARIADB_INSTALL_DIR"` DataFolder string TempFolder string @@ -56,6 +57,18 @@ type EnvVariables struct { TestMysql80Port string `env:"TEST_MYSQL_80_PORT"` TestMysql84Port string `env:"TEST_MYSQL_84_PORT"` + TestMariadb55Port string `env:"TEST_MARIADB_55_PORT"` + TestMariadb101Port string `env:"TEST_MARIADB_101_PORT"` + TestMariadb102Port string `env:"TEST_MARIADB_102_PORT"` + TestMariadb103Port string `env:"TEST_MARIADB_103_PORT"` + TestMariadb104Port string `env:"TEST_MARIADB_104_PORT"` + TestMariadb105Port string `env:"TEST_MARIADB_105_PORT"` + TestMariadb106Port string `env:"TEST_MARIADB_106_PORT"` + TestMariadb1011Port string `env:"TEST_MARIADB_1011_PORT"` + TestMariadb114Port string `env:"TEST_MARIADB_114_PORT"` + TestMariadb118Port string `env:"TEST_MARIADB_118_PORT"` + TestMariadb120Port string `env:"TEST_MARIADB_120_PORT"` + // oauth GitHubClientID string `env:"GITHUB_CLIENT_ID"` GitHubClientSecret string `env:"GITHUB_CLIENT_SECRET"` @@ -160,6 +173,9 @@ func loadEnvVariables() { env.MysqlInstallDir = filepath.Join(backendRoot, "tools", "mysql") tools.VerifyMysqlInstallation(log, env.EnvMode, env.MysqlInstallDir) + env.MariadbInstallDir = filepath.Join(backendRoot, "tools", "mariadb") + tools.VerifyMariadbInstallation(log, env.EnvMode, env.MariadbInstallDir) + // Store the data and temp folders one level below the root // (projectRoot/postgresus-data -> /postgresus-data) env.DataFolder = filepath.Join(filepath.Dir(backendRoot), "postgresus-data", "backups") diff --git a/backend/internal/features/backups/backups/usecases/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/create_backup_uc.go index 33c575e..c8c3bf1 100644 --- a/backend/internal/features/backups/backups/usecases/create_backup_uc.go +++ b/backend/internal/features/backups/backups/usecases/create_backup_uc.go @@ -5,6 +5,7 @@ import ( "errors" usecases_common "postgresus-backend/internal/features/backups/backups/usecases/common" + usecases_mariadb "postgresus-backend/internal/features/backups/backups/usecases/mariadb" usecases_mysql "postgresus-backend/internal/features/backups/backups/usecases/mysql" usecases_postgresql "postgresus-backend/internal/features/backups/backups/usecases/postgresql" backups_config "postgresus-backend/internal/features/backups/config" @@ -17,6 +18,7 @@ import ( type CreateBackupUsecase struct { CreatePostgresqlBackupUsecase *usecases_postgresql.CreatePostgresqlBackupUsecase CreateMysqlBackupUsecase *usecases_mysql.CreateMysqlBackupUsecase + CreateMariadbBackupUsecase *usecases_mariadb.CreateMariadbBackupUsecase } func (uc *CreateBackupUsecase) Execute( @@ -48,6 +50,16 @@ func (uc *CreateBackupUsecase) Execute( backupProgressListener, ) + case databases.DatabaseTypeMariadb: + return uc.CreateMariadbBackupUsecase.Execute( + ctx, + backupID, + backupConfig, + database, + storage, + backupProgressListener, + ) + default: return nil, errors.New("database type not supported") } diff --git a/backend/internal/features/backups/backups/usecases/di.go b/backend/internal/features/backups/backups/usecases/di.go index edbcaf8..cd678e1 100644 --- a/backend/internal/features/backups/backups/usecases/di.go +++ b/backend/internal/features/backups/backups/usecases/di.go @@ -1,6 +1,7 @@ package usecases import ( + usecases_mariadb "postgresus-backend/internal/features/backups/backups/usecases/mariadb" usecases_mysql "postgresus-backend/internal/features/backups/backups/usecases/mysql" usecases_postgresql "postgresus-backend/internal/features/backups/backups/usecases/postgresql" ) @@ -8,6 +9,7 @@ import ( var createBackupUsecase = &CreateBackupUsecase{ usecases_postgresql.GetCreatePostgresqlBackupUsecase(), usecases_mysql.GetCreateMysqlBackupUsecase(), + usecases_mariadb.GetCreateMariadbBackupUsecase(), } func GetCreateBackupUsecase() *CreateBackupUsecase { diff --git a/backend/internal/features/backups/backups/usecases/mariadb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/mariadb/create_backup_uc.go new file mode 100644 index 0000000..31b3e40 --- /dev/null +++ b/backend/internal/features/backups/backups/usecases/mariadb/create_backup_uc.go @@ -0,0 +1,595 @@ +package usecases_mariadb + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/klauspost/compress/zstd" + + "postgresus-backend/internal/config" + backup_encryption "postgresus-backend/internal/features/backups/backups/encryption" + usecases_common "postgresus-backend/internal/features/backups/backups/usecases/common" + backups_config "postgresus-backend/internal/features/backups/config" + "postgresus-backend/internal/features/databases" + mariadbtypes "postgresus-backend/internal/features/databases/databases/mariadb" + encryption_secrets "postgresus-backend/internal/features/encryption/secrets" + "postgresus-backend/internal/features/storages" + "postgresus-backend/internal/util/encryption" + "postgresus-backend/internal/util/tools" +) + +const ( + backupTimeout = 23 * time.Hour + shutdownCheckInterval = 1 * time.Second + copyBufferSize = 8 * 1024 * 1024 + progressReportIntervalMB = 1.0 + zstdStorageCompressionLevel = 3 + exitCodeGenericError = 1 + exitCodeConnectionError = 2 +) + +type CreateMariadbBackupUsecase struct { + logger *slog.Logger + secretKeyService *encryption_secrets.SecretKeyService + fieldEncryptor encryption.FieldEncryptor +} + +type writeResult struct { + bytesWritten int + writeErr error +} + +func (uc *CreateMariadbBackupUsecase) Execute( + ctx context.Context, + backupID uuid.UUID, + backupConfig *backups_config.BackupConfig, + db *databases.Database, + storage *storages.Storage, + backupProgressListener func(completedMBs float64), +) (*usecases_common.BackupMetadata, error) { + uc.logger.Info( + "Creating MariaDB backup via mariadb-dump", + "databaseId", db.ID, + "storageId", storage.ID, + ) + + if !backupConfig.IsBackupsEnabled { + return nil, fmt.Errorf("backups are not enabled for this database: \"%s\"", db.Name) + } + + mdb := db.Mariadb + if mdb == nil { + return nil, fmt.Errorf("mariadb database configuration is required") + } + + if mdb.Database == nil || *mdb.Database == "" { + return nil, fmt.Errorf("database name is required for mariadb-dump backups") + } + + decryptedPassword, err := uc.fieldEncryptor.Decrypt(db.ID, mdb.Password) + if err != nil { + return nil, fmt.Errorf("failed to decrypt database password: %w", err) + } + + args := uc.buildMariadbDumpArgs(mdb) + + return uc.streamToStorage( + ctx, + backupID, + backupConfig, + tools.GetMariadbExecutable( + tools.MariadbExecutableMariadbDump, + mdb.Version, + config.GetEnv().EnvMode, + config.GetEnv().MariadbInstallDir, + ), + args, + decryptedPassword, + storage, + backupProgressListener, + mdb, + ) +} + +func (uc *CreateMariadbBackupUsecase) buildMariadbDumpArgs( + mdb *mariadbtypes.MariadbDatabase, +) []string { + args := []string{ + "--host=" + mdb.Host, + "--port=" + strconv.Itoa(mdb.Port), + "--user=" + mdb.Username, + "--single-transaction", + "--routines", + "--triggers", + "--events", + "--quick", + "--verbose", + } + + args = append(args, "--compress") + + if mdb.IsHttps { + args = append(args, "--ssl") + } + + if mdb.Database != nil && *mdb.Database != "" { + args = append(args, *mdb.Database) + } + + return args +} + +func (uc *CreateMariadbBackupUsecase) streamToStorage( + parentCtx context.Context, + backupID uuid.UUID, + backupConfig *backups_config.BackupConfig, + mariadbBin string, + args []string, + password string, + storage *storages.Storage, + backupProgressListener func(completedMBs float64), + mdbConfig *mariadbtypes.MariadbDatabase, +) (*usecases_common.BackupMetadata, error) { + uc.logger.Info("Streaming MariaDB backup to storage", "mariadbBin", mariadbBin) + + ctx, cancel := uc.createBackupContext(parentCtx) + defer cancel() + + myCnfFile, err := uc.createTempMyCnfFile(mdbConfig, password) + if err != nil { + return nil, fmt.Errorf("failed to create .my.cnf: %w", err) + } + defer func() { _ = os.RemoveAll(filepath.Dir(myCnfFile)) }() + + fullArgs := append([]string{"--defaults-file=" + myCnfFile}, args...) + + cmd := exec.CommandContext(ctx, mariadbBin, fullArgs...) + uc.logger.Info("Executing MariaDB backup command", "command", cmd.String()) + + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, + "MYSQL_PWD=", + "LC_ALL=C.UTF-8", + "LANG=C.UTF-8", + ) + + pgStdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("stdout pipe: %w", err) + } + + pgStderr, err := cmd.StderrPipe() + if err != nil { + return nil, fmt.Errorf("stderr pipe: %w", err) + } + + stderrCh := make(chan []byte, 1) + go func() { + stderrOutput, _ := io.ReadAll(pgStderr) + stderrCh <- stderrOutput + }() + + storageReader, storageWriter := io.Pipe() + + finalWriter, encryptionWriter, backupMetadata, err := uc.setupBackupEncryption( + backupID, + backupConfig, + storageWriter, + ) + if err != nil { + return nil, err + } + + zstdWriter, err := zstd.NewWriter(finalWriter, + zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(zstdStorageCompressionLevel))) + if err != nil { + return nil, fmt.Errorf("failed to create zstd writer: %w", err) + } + countingWriter := usecases_common.NewCountingWriter(zstdWriter) + + saveErrCh := make(chan error, 1) + go func() { + saveErr := storage.SaveFile(ctx, uc.fieldEncryptor, uc.logger, backupID, storageReader) + saveErrCh <- saveErr + }() + + if err = cmd.Start(); err != nil { + return nil, fmt.Errorf("start %s: %w", filepath.Base(mariadbBin), err) + } + + copyResultCh := make(chan error, 1) + bytesWrittenCh := make(chan int64, 1) + go func() { + bytesWritten, err := uc.copyWithShutdownCheck( + ctx, + countingWriter, + pgStdout, + backupProgressListener, + ) + bytesWrittenCh <- bytesWritten + copyResultCh <- err + }() + + copyErr := <-copyResultCh + bytesWritten := <-bytesWrittenCh + waitErr := cmd.Wait() + + select { + case <-ctx.Done(): + uc.cleanupOnCancellation(zstdWriter, encryptionWriter, storageWriter, saveErrCh) + return nil, uc.checkCancellationReason() + default: + } + + if err := zstdWriter.Close(); err != nil { + uc.logger.Error("Failed to close zstd writer", "error", err) + } + if err := uc.closeWriters(encryptionWriter, storageWriter); err != nil { + <-saveErrCh + return nil, err + } + + saveErr := <-saveErrCh + stderrOutput := <-stderrCh + + if waitErr == nil && copyErr == nil && saveErr == nil && backupProgressListener != nil { + sizeMB := float64(bytesWritten) / (1024 * 1024) + backupProgressListener(sizeMB) + } + + switch { + case waitErr != nil: + return nil, uc.buildMariadbDumpErrorMessage(waitErr, stderrOutput, mariadbBin) + case copyErr != nil: + return nil, fmt.Errorf("copy to storage: %w", copyErr) + case saveErr != nil: + return nil, fmt.Errorf("save to storage: %w", saveErr) + } + + return &backupMetadata, nil +} + +func (uc *CreateMariadbBackupUsecase) createTempMyCnfFile( + mdbConfig *mariadbtypes.MariadbDatabase, + password string, +) (string, error) { + tempDir, err := os.MkdirTemp("", "mycnf") + if err != nil { + return "", fmt.Errorf("failed to create temp directory: %w", err) + } + + myCnfFile := filepath.Join(tempDir, ".my.cnf") + + content := fmt.Sprintf(`[client] +user=%s +password="%s" +host=%s +port=%d +`, mdbConfig.Username, tools.EscapeMariadbPassword(password), mdbConfig.Host, mdbConfig.Port) + + if mdbConfig.IsHttps { + content += "ssl=true\n" + } else { + content += "ssl=false\n" + } + + err = os.WriteFile(myCnfFile, []byte(content), 0600) + if err != nil { + return "", fmt.Errorf("failed to write .my.cnf: %w", err) + } + + return myCnfFile, nil +} + +func (uc *CreateMariadbBackupUsecase) copyWithShutdownCheck( + ctx context.Context, + dst io.Writer, + src io.Reader, + backupProgressListener func(completedMBs float64), +) (int64, error) { + buf := make([]byte, copyBufferSize) + var totalBytesWritten int64 + var lastReportedMB float64 + + for { + select { + case <-ctx.Done(): + return totalBytesWritten, fmt.Errorf("copy cancelled: %w", ctx.Err()) + default: + } + + if config.IsShouldShutdown() { + return totalBytesWritten, fmt.Errorf("copy cancelled due to shutdown") + } + + bytesRead, readErr := src.Read(buf) + if bytesRead > 0 { + writeResultCh := make(chan writeResult, 1) + go func() { + bytesWritten, writeErr := dst.Write(buf[0:bytesRead]) + writeResultCh <- writeResult{bytesWritten, writeErr} + }() + + var bytesWritten int + var writeErr error + + select { + case <-ctx.Done(): + return totalBytesWritten, fmt.Errorf("copy cancelled during write: %w", ctx.Err()) + case result := <-writeResultCh: + bytesWritten = result.bytesWritten + writeErr = result.writeErr + } + + if bytesWritten < 0 || bytesRead < bytesWritten { + bytesWritten = 0 + if writeErr == nil { + writeErr = fmt.Errorf("invalid write result") + } + } + + if writeErr != nil { + return totalBytesWritten, writeErr + } + + if bytesRead != bytesWritten { + return totalBytesWritten, io.ErrShortWrite + } + + totalBytesWritten += int64(bytesWritten) + + if backupProgressListener != nil { + currentSizeMB := float64(totalBytesWritten) / (1024 * 1024) + if currentSizeMB >= lastReportedMB+progressReportIntervalMB { + backupProgressListener(currentSizeMB) + lastReportedMB = currentSizeMB + } + } + } + + if readErr != nil { + if readErr != io.EOF { + return totalBytesWritten, readErr + } + break + } + } + + return totalBytesWritten, nil +} + +func (uc *CreateMariadbBackupUsecase) createBackupContext( + parentCtx context.Context, +) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithTimeout(parentCtx, backupTimeout) + + go func() { + ticker := time.NewTicker(shutdownCheckInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-parentCtx.Done(): + cancel() + return + case <-ticker.C: + if config.IsShouldShutdown() { + cancel() + return + } + } + } + }() + + return ctx, cancel +} + +func (uc *CreateMariadbBackupUsecase) setupBackupEncryption( + backupID uuid.UUID, + backupConfig *backups_config.BackupConfig, + storageWriter io.WriteCloser, +) (io.Writer, *backup_encryption.EncryptionWriter, usecases_common.BackupMetadata, error) { + metadata := usecases_common.BackupMetadata{} + + if backupConfig.Encryption != backups_config.BackupEncryptionEncrypted { + metadata.Encryption = backups_config.BackupEncryptionNone + uc.logger.Info("Encryption disabled for backup", "backupId", backupID) + return storageWriter, nil, metadata, nil + } + + salt, err := backup_encryption.GenerateSalt() + if err != nil { + return nil, nil, metadata, fmt.Errorf("failed to generate salt: %w", err) + } + + nonce, err := backup_encryption.GenerateNonce() + if err != nil { + return nil, nil, metadata, fmt.Errorf("failed to generate nonce: %w", err) + } + + masterKey, err := uc.secretKeyService.GetSecretKey() + if err != nil { + return nil, nil, metadata, fmt.Errorf("failed to get master key: %w", err) + } + + encWriter, err := backup_encryption.NewEncryptionWriter( + storageWriter, + masterKey, + backupID, + salt, + nonce, + ) + if err != nil { + return nil, nil, metadata, fmt.Errorf("failed to create encrypting writer: %w", err) + } + + saltBase64 := base64.StdEncoding.EncodeToString(salt) + nonceBase64 := base64.StdEncoding.EncodeToString(nonce) + metadata.EncryptionSalt = &saltBase64 + metadata.EncryptionIV = &nonceBase64 + metadata.Encryption = backups_config.BackupEncryptionEncrypted + + uc.logger.Info("Encryption enabled for backup", "backupId", backupID) + return encWriter, encWriter, metadata, nil +} + +func (uc *CreateMariadbBackupUsecase) cleanupOnCancellation( + zstdWriter *zstd.Encoder, + encryptionWriter *backup_encryption.EncryptionWriter, + storageWriter io.WriteCloser, + saveErrCh chan error, +) { + if zstdWriter != nil { + go func() { + if closeErr := zstdWriter.Close(); closeErr != nil { + uc.logger.Error( + "Failed to close zstd writer during cancellation", + "error", + closeErr, + ) + } + }() + } + + if encryptionWriter != nil { + go func() { + if closeErr := encryptionWriter.Close(); closeErr != nil { + uc.logger.Error( + "Failed to close encrypting writer during cancellation", + "error", + closeErr, + ) + } + }() + } + + if err := storageWriter.Close(); err != nil { + uc.logger.Error("Failed to close pipe writer during cancellation", "error", err) + } + + <-saveErrCh +} + +func (uc *CreateMariadbBackupUsecase) closeWriters( + encryptionWriter *backup_encryption.EncryptionWriter, + storageWriter io.WriteCloser, +) error { + encryptionCloseErrCh := make(chan error, 1) + if encryptionWriter != nil { + go func() { + closeErr := encryptionWriter.Close() + if closeErr != nil { + uc.logger.Error("Failed to close encrypting writer", "error", closeErr) + } + encryptionCloseErrCh <- closeErr + }() + } else { + encryptionCloseErrCh <- nil + } + + encryptionCloseErr := <-encryptionCloseErrCh + if encryptionCloseErr != nil { + if err := storageWriter.Close(); err != nil { + uc.logger.Error("Failed to close pipe writer after encryption error", "error", err) + } + return fmt.Errorf("failed to close encryption writer: %w", encryptionCloseErr) + } + + if err := storageWriter.Close(); err != nil { + uc.logger.Error("Failed to close pipe writer", "error", err) + return err + } + + return nil +} + +func (uc *CreateMariadbBackupUsecase) checkCancellationReason() error { + if config.IsShouldShutdown() { + return fmt.Errorf("backup cancelled due to shutdown") + } + return fmt.Errorf("backup cancelled") +} + +func (uc *CreateMariadbBackupUsecase) buildMariadbDumpErrorMessage( + waitErr error, + stderrOutput []byte, + mariadbBin string, +) error { + stderrStr := string(stderrOutput) + errorMsg := fmt.Sprintf( + "%s failed: %v – stderr: %s", + filepath.Base(mariadbBin), + waitErr, + stderrStr, + ) + + exitErr, ok := waitErr.(*exec.ExitError) + if !ok { + return errors.New(errorMsg) + } + + exitCode := exitErr.ExitCode() + + if exitCode == exitCodeGenericError || exitCode == exitCodeConnectionError { + return uc.handleConnectionErrors(stderrStr) + } + + return errors.New(errorMsg) +} + +func (uc *CreateMariadbBackupUsecase) handleConnectionErrors(stderrStr string) error { + if containsIgnoreCase(stderrStr, "access denied") { + return fmt.Errorf( + "MariaDB access denied. Check username and password. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "can't connect") || + containsIgnoreCase(stderrStr, "connection refused") { + return fmt.Errorf( + "MariaDB connection refused. Check if the server is running and accessible. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "unknown database") { + return fmt.Errorf( + "MariaDB database does not exist. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "ssl") { + return fmt.Errorf( + "MariaDB SSL connection failed. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "timeout") { + return fmt.Errorf( + "MariaDB connection timeout. stderr: %s", + stderrStr, + ) + } + + return fmt.Errorf("MariaDB connection or authentication error. stderr: %s", stderrStr) +} + +func containsIgnoreCase(str, substr string) bool { + return strings.Contains(strings.ToLower(str), strings.ToLower(substr)) +} diff --git a/backend/internal/features/backups/backups/usecases/mariadb/di.go b/backend/internal/features/backups/backups/usecases/mariadb/di.go new file mode 100644 index 0000000..c62c00b --- /dev/null +++ b/backend/internal/features/backups/backups/usecases/mariadb/di.go @@ -0,0 +1,17 @@ +package usecases_mariadb + +import ( + "postgresus-backend/internal/features/encryption/secrets" + "postgresus-backend/internal/util/encryption" + "postgresus-backend/internal/util/logger" +) + +var createMariadbBackupUsecase = &CreateMariadbBackupUsecase{ + logger.GetLogger(), + secrets.GetSecretKeyService(), + encryption.GetFieldEncryptor(), +} + +func GetCreateMariadbBackupUsecase() *CreateMariadbBackupUsecase { + return createMariadbBackupUsecase +} diff --git a/backend/internal/features/databases/controller_test.go b/backend/internal/features/databases/controller_test.go index a2dbae8..5b7c207 100644 --- a/backend/internal/features/databases/controller_test.go +++ b/backend/internal/features/databases/controller_test.go @@ -11,6 +11,7 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" + "postgresus-backend/internal/features/databases/databases/mariadb" "postgresus-backend/internal/features/databases/databases/postgresql" users_enums "postgresus-backend/internal/features/users/enums" users_testing "postgresus-backend/internal/features/users/testing" @@ -881,11 +882,9 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) { } }, verifySensitiveData: func(t *testing.T, database *Database) { - // Verify password is encrypted assert.True(t, strings.HasPrefix(database.Postgresql.Password, "enc:"), "Password should be encrypted in database") - // Verify it can be decrypted back to original encryptor := encryption.GetFieldEncryptor() decrypted, err := encryptor.Decrypt(database.ID, database.Postgresql.Password) assert.NoError(t, err) @@ -895,6 +894,55 @@ func Test_DatabaseSensitiveDataLifecycle_AllTypes(t *testing.T) { assert.Equal(t, "", database.Postgresql.Password) }, }, + { + name: "MariaDB Database", + databaseType: DatabaseTypeMariadb, + createDatabase: func(workspaceID uuid.UUID) *Database { + testDbName := "test_db" + return &Database{ + WorkspaceID: &workspaceID, + Name: "Test MariaDB Database", + Type: DatabaseTypeMariadb, + Mariadb: &mariadb.MariadbDatabase{ + Version: tools.MariadbVersion1011, + Host: "localhost", + Port: 3306, + Username: "root", + Password: "original-password-secret", + Database: &testDbName, + }, + } + }, + updateDatabase: func(workspaceID uuid.UUID, databaseID uuid.UUID) *Database { + testDbName := "updated_test_db" + return &Database{ + ID: databaseID, + WorkspaceID: &workspaceID, + Name: "Updated MariaDB Database", + Type: DatabaseTypeMariadb, + Mariadb: &mariadb.MariadbDatabase{ + Version: tools.MariadbVersion114, + Host: "updated-host", + Port: 3307, + Username: "updated_user", + Password: "", + Database: &testDbName, + }, + } + }, + verifySensitiveData: func(t *testing.T, database *Database) { + assert.True(t, strings.HasPrefix(database.Mariadb.Password, "enc:"), + "Password should be encrypted in database") + + encryptor := encryption.GetFieldEncryptor() + decrypted, err := encryptor.Decrypt(database.ID, database.Mariadb.Password) + assert.NoError(t, err) + assert.Equal(t, "original-password-secret", decrypted) + }, + verifyHiddenData: func(t *testing.T, database *Database) { + assert.Equal(t, "", database.Mariadb.Password) + }, + }, } for _, tc := range testCases { diff --git a/backend/internal/features/databases/databases/mariadb/model.go b/backend/internal/features/databases/databases/mariadb/model.go new file mode 100644 index 0000000..4bd61f9 --- /dev/null +++ b/backend/internal/features/databases/databases/mariadb/model.go @@ -0,0 +1,412 @@ +package mariadb + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "regexp" + "strings" + "time" + + "postgresus-backend/internal/util/encryption" + "postgresus-backend/internal/util/tools" + + _ "github.com/go-sql-driver/mysql" + "github.com/google/uuid" +) + +type MariadbDatabase struct { + ID uuid.UUID `json:"id" gorm:"primaryKey;type:uuid;default:gen_random_uuid()"` + DatabaseID *uuid.UUID `json:"databaseId" gorm:"type:uuid;column:database_id"` + + Version tools.MariadbVersion `json:"version" gorm:"type:text;not null"` + + Host string `json:"host" gorm:"type:text;not null"` + Port int `json:"port" gorm:"type:int;not null"` + Username string `json:"username" gorm:"type:text;not null"` + Password string `json:"password" gorm:"type:text;not null"` + Database *string `json:"database" gorm:"type:text"` + IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"` +} + +func (m *MariadbDatabase) TableName() string { + return "mariadb_databases" +} + +func (m *MariadbDatabase) Validate() error { + if m.Host == "" { + return errors.New("host is required") + } + if m.Port == 0 { + return errors.New("port is required") + } + if m.Username == "" { + return errors.New("username is required") + } + if m.Password == "" { + return errors.New("password is required") + } + return nil +} + +func (m *MariadbDatabase) TestConnection( + logger *slog.Logger, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) error { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + if m.Database == nil || *m.Database == "" { + return errors.New("database name is required for MariaDB backup") + } + + password, err := decryptPasswordIfNeeded(m.Password, encryptor, databaseID) + if err != nil { + return fmt.Errorf("failed to decrypt password: %w", err) + } + + dsn := m.buildDSN(password, *m.Database) + + db, err := sql.Open("mysql", dsn) + if err != nil { + return fmt.Errorf("failed to connect to MariaDB database '%s': %w", *m.Database, err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + logger.Error("Failed to close MariaDB connection", "error", closeErr) + } + }() + + db.SetConnMaxLifetime(15 * time.Second) + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + + if err := db.PingContext(ctx); err != nil { + return fmt.Errorf("failed to ping MariaDB database '%s': %w", *m.Database, err) + } + + detectedVersion, err := detectMariadbVersion(ctx, db) + if err != nil { + return err + } + m.Version = detectedVersion + + return nil +} + +func (m *MariadbDatabase) HideSensitiveData() { + if m == nil { + return + } + m.Password = "" +} + +func (m *MariadbDatabase) Update(incoming *MariadbDatabase) { + m.Version = incoming.Version + m.Host = incoming.Host + m.Port = incoming.Port + m.Username = incoming.Username + m.Database = incoming.Database + m.IsHttps = incoming.IsHttps + + if incoming.Password != "" { + m.Password = incoming.Password + } +} + +func (m *MariadbDatabase) EncryptSensitiveFields( + databaseID uuid.UUID, + encryptor encryption.FieldEncryptor, +) error { + if m.Password != "" { + encrypted, err := encryptor.Encrypt(databaseID, m.Password) + if err != nil { + return err + } + m.Password = encrypted + } + return nil +} + +func (m *MariadbDatabase) PopulateVersionIfEmpty( + logger *slog.Logger, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) error { + if m.Version != "" { + return nil + } + return m.PopulateVersion(logger, encryptor, databaseID) +} + +func (m *MariadbDatabase) PopulateVersion( + logger *slog.Logger, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) error { + if m.Database == nil || *m.Database == "" { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + password, err := decryptPasswordIfNeeded(m.Password, encryptor, databaseID) + if err != nil { + return fmt.Errorf("failed to decrypt password: %w", err) + } + + dsn := m.buildDSN(password, *m.Database) + + db, err := sql.Open("mysql", dsn) + if err != nil { + return fmt.Errorf("failed to connect to database: %w", err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + logger.Error("Failed to close connection", "error", closeErr) + } + }() + + detectedVersion, err := detectMariadbVersion(ctx, db) + if err != nil { + return err + } + + m.Version = detectedVersion + return nil +} + +func (m *MariadbDatabase) IsUserReadOnly( + ctx context.Context, + logger *slog.Logger, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) (bool, error) { + password, err := decryptPasswordIfNeeded(m.Password, encryptor, databaseID) + if err != nil { + return false, fmt.Errorf("failed to decrypt password: %w", err) + } + + dsn := m.buildDSN(password, *m.Database) + + db, err := sql.Open("mysql", dsn) + if err != nil { + return false, fmt.Errorf("failed to connect to database: %w", err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + logger.Error("Failed to close connection", "error", closeErr) + } + }() + + rows, err := db.QueryContext(ctx, "SHOW GRANTS FOR CURRENT_USER()") + if err != nil { + return false, fmt.Errorf("failed to check grants: %w", err) + } + defer func() { _ = rows.Close() }() + + writePrivileges := []string{ + "INSERT", "UPDATE", "DELETE", "CREATE", "DROP", "ALTER", + "INDEX", "GRANT OPTION", "ALL PRIVILEGES", "SUPER", + } + + for rows.Next() { + var grant string + if err := rows.Scan(&grant); err != nil { + return false, fmt.Errorf("failed to scan grant: %w", err) + } + + for _, priv := range writePrivileges { + if regexp.MustCompile(`(?i)\b` + priv + `\b`).MatchString(grant) { + return false, nil + } + } + } + + if err := rows.Err(); err != nil { + return false, fmt.Errorf("error iterating grants: %w", err) + } + + return true, nil +} + +func (m *MariadbDatabase) CreateReadOnlyUser( + ctx context.Context, + logger *slog.Logger, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) (string, string, error) { + password, err := decryptPasswordIfNeeded(m.Password, encryptor, databaseID) + if err != nil { + return "", "", fmt.Errorf("failed to decrypt password: %w", err) + } + + dsn := m.buildDSN(password, *m.Database) + + db, err := sql.Open("mysql", dsn) + if err != nil { + return "", "", fmt.Errorf("failed to connect to database: %w", err) + } + defer func() { + if closeErr := db.Close(); closeErr != nil { + logger.Error("Failed to close connection", "error", closeErr) + } + }() + + maxRetries := 3 + for attempt := range maxRetries { + // MariaDB 5.5 has a 16-character username limit, use shorter prefix + newUsername := fmt.Sprintf("pgs-%s", uuid.New().String()[:8]) + newPassword := uuid.New().String() + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return "", "", fmt.Errorf("failed to begin transaction: %w", err) + } + + success := false + defer func() { + if !success { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + logger.Error("Failed to rollback transaction", "error", rollbackErr) + } + } + }() + + _, err = tx.ExecContext(ctx, fmt.Sprintf( + "CREATE USER '%s'@'%%' IDENTIFIED BY '%s'", + newUsername, + newPassword, + )) + if err != nil { + if attempt < maxRetries-1 { + continue + } + return "", "", fmt.Errorf("failed to create user: %w", err) + } + + _, err = tx.ExecContext(ctx, fmt.Sprintf( + "GRANT SELECT, SHOW VIEW, LOCK TABLES, TRIGGER, EVENT ON `%s`.* TO '%s'@'%%'", + *m.Database, + newUsername, + )) + if err != nil { + return "", "", fmt.Errorf("failed to grant database privileges: %w", err) + } + + _, err = tx.ExecContext(ctx, fmt.Sprintf( + "GRANT PROCESS ON *.* TO '%s'@'%%'", + newUsername, + )) + if err != nil { + return "", "", fmt.Errorf("failed to grant PROCESS privilege: %w", err) + } + + _, err = tx.ExecContext(ctx, "FLUSH PRIVILEGES") + if err != nil { + return "", "", fmt.Errorf("failed to flush privileges: %w", err) + } + + if err := tx.Commit(); err != nil { + return "", "", fmt.Errorf("failed to commit transaction: %w", err) + } + + success = true + logger.Info( + "Read-only MariaDB user created successfully", + "username", newUsername, + ) + return newUsername, newPassword, nil + } + + return "", "", errors.New("failed to generate unique username after 3 attempts") +} + +func (m *MariadbDatabase) buildDSN(password string, database string) string { + tlsConfig := "false" + if m.IsHttps { + tlsConfig = "true" + } + + return fmt.Sprintf( + "%s:%s@tcp(%s:%d)/%s?parseTime=true&timeout=15s&tls=%s&charset=utf8mb4", + m.Username, + password, + m.Host, + m.Port, + database, + tlsConfig, + ) +} + +// detectMariadbVersion parses VERSION() output to detect MariaDB version +// MariaDB returns strings like "10.11.6-MariaDB" or "11.4.2-MariaDB-1:11.4.2+maria~ubu2204" +func detectMariadbVersion(ctx context.Context, db *sql.DB) (tools.MariadbVersion, error) { + var versionStr string + err := db.QueryRowContext(ctx, "SELECT VERSION()").Scan(&versionStr) + if err != nil { + return "", fmt.Errorf("failed to query MariaDB version: %w", err) + } + + if !strings.Contains(strings.ToLower(versionStr), "mariadb") { + return "", fmt.Errorf( + "not a MariaDB server (version: %s). Use MySQL database type instead", + versionStr, + ) + } + + re := regexp.MustCompile(`^(\d+)\.(\d+)`) + matches := re.FindStringSubmatch(versionStr) + if len(matches) < 3 { + return "", fmt.Errorf("could not parse MariaDB version: %s", versionStr) + } + + major := matches[1] + minor := matches[2] + versionKey := fmt.Sprintf("%s.%s", major, minor) + + switch versionKey { + case "5.5": + return tools.MariadbVersion55, nil + case "10.1": + return tools.MariadbVersion101, nil + case "10.2": + return tools.MariadbVersion102, nil + case "10.3": + return tools.MariadbVersion103, nil + case "10.4": + return tools.MariadbVersion104, nil + case "10.5": + return tools.MariadbVersion105, nil + case "10.6": + return tools.MariadbVersion106, nil + case "10.11": + return tools.MariadbVersion1011, nil + case "11.4": + return tools.MariadbVersion114, nil + case "11.8": + return tools.MariadbVersion118, nil + case "12.0": + return tools.MariadbVersion120, nil + default: + return "", fmt.Errorf( + "unsupported MariaDB version: %s (supported: 5.5, 10.1-10.6, 10.11, 11.4, 11.8, 12.0)", + versionKey, + ) + } +} + +func decryptPasswordIfNeeded( + password string, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) (string, error) { + if encryptor == nil { + return password, nil + } + return encryptor.Decrypt(databaseID, password) +} diff --git a/backend/internal/features/databases/databases/mariadb/readonly_user_test.go b/backend/internal/features/databases/databases/mariadb/readonly_user_test.go new file mode 100644 index 0000000..2d0254c --- /dev/null +++ b/backend/internal/features/databases/databases/mariadb/readonly_user_test.go @@ -0,0 +1,387 @@ +package mariadb + +import ( + "context" + "fmt" + "log/slog" + "os" + "strconv" + "strings" + "testing" + + _ "github.com/go-sql-driver/mysql" + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/assert" + + "postgresus-backend/internal/config" + "postgresus-backend/internal/util/tools" +) + +func Test_IsUserReadOnly_AdminUser_ReturnsFalse(t *testing.T) { + env := config.GetEnv() + cases := []struct { + name string + version tools.MariadbVersion + port string + }{ + {"MariaDB 5.5", tools.MariadbVersion55, env.TestMariadb55Port}, + {"MariaDB 10.1", tools.MariadbVersion101, env.TestMariadb101Port}, + {"MariaDB 10.2", tools.MariadbVersion102, env.TestMariadb102Port}, + {"MariaDB 10.3", tools.MariadbVersion103, env.TestMariadb103Port}, + {"MariaDB 10.4", tools.MariadbVersion104, env.TestMariadb104Port}, + {"MariaDB 10.5", tools.MariadbVersion105, env.TestMariadb105Port}, + {"MariaDB 10.6", tools.MariadbVersion106, env.TestMariadb106Port}, + {"MariaDB 10.11", tools.MariadbVersion1011, env.TestMariadb1011Port}, + {"MariaDB 11.4", tools.MariadbVersion114, env.TestMariadb114Port}, + {"MariaDB 11.8", tools.MariadbVersion118, env.TestMariadb118Port}, + {"MariaDB 12.0", tools.MariadbVersion120, env.TestMariadb120Port}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + container := connectToMariadbContainer(t, tc.port, tc.version) + defer container.DB.Close() + + mariadbModel := createMariadbModel(container) + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + ctx := context.Background() + + isReadOnly, err := mariadbModel.IsUserReadOnly(ctx, logger, nil, uuid.New()) + assert.NoError(t, err) + assert.False(t, isReadOnly, "Root user should not be read-only") + }) + } +} + +func Test_CreateReadOnlyUser_UserCanReadButNotWrite(t *testing.T) { + env := config.GetEnv() + cases := []struct { + name string + version tools.MariadbVersion + port string + }{ + {"MariaDB 5.5", tools.MariadbVersion55, env.TestMariadb55Port}, + {"MariaDB 10.1", tools.MariadbVersion101, env.TestMariadb101Port}, + {"MariaDB 10.2", tools.MariadbVersion102, env.TestMariadb102Port}, + {"MariaDB 10.3", tools.MariadbVersion103, env.TestMariadb103Port}, + {"MariaDB 10.4", tools.MariadbVersion104, env.TestMariadb104Port}, + {"MariaDB 10.5", tools.MariadbVersion105, env.TestMariadb105Port}, + {"MariaDB 10.6", tools.MariadbVersion106, env.TestMariadb106Port}, + {"MariaDB 10.11", tools.MariadbVersion1011, env.TestMariadb1011Port}, + {"MariaDB 11.4", tools.MariadbVersion114, env.TestMariadb114Port}, + {"MariaDB 11.8", tools.MariadbVersion118, env.TestMariadb118Port}, + {"MariaDB 12.0", tools.MariadbVersion120, env.TestMariadb120Port}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + container := connectToMariadbContainer(t, tc.port, tc.version) + defer container.DB.Close() + + _, err := container.DB.Exec(`DROP TABLE IF EXISTS readonly_test`) + assert.NoError(t, err) + _, err = container.DB.Exec(`DROP TABLE IF EXISTS hack_table`) + assert.NoError(t, err) + _, err = container.DB.Exec(`DROP TABLE IF EXISTS future_table`) + assert.NoError(t, err) + + _, err = container.DB.Exec(` + CREATE TABLE readonly_test ( + id INT AUTO_INCREMENT PRIMARY KEY, + data VARCHAR(255) NOT NULL + ) + `) + assert.NoError(t, err) + + _, err = container.DB.Exec( + `INSERT INTO readonly_test (data) VALUES ('test1'), ('test2')`, + ) + assert.NoError(t, err) + + mariadbModel := createMariadbModel(container) + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + ctx := context.Background() + + username, password, err := mariadbModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New()) + assert.NoError(t, err) + assert.NotEmpty(t, username) + assert.NotEmpty(t, password) + assert.True(t, strings.HasPrefix(username, "pgs-")) + + if err != nil { + return + } + + readOnlyModel := &MariadbDatabase{ + Version: mariadbModel.Version, + Host: mariadbModel.Host, + Port: mariadbModel.Port, + Username: username, + Password: password, + Database: mariadbModel.Database, + IsHttps: false, + } + + isReadOnly, err := readOnlyModel.IsUserReadOnly(ctx, logger, nil, uuid.New()) + assert.NoError(t, err) + assert.True(t, isReadOnly, "Created user should be read-only") + + readOnlyDSN := fmt.Sprintf( + "%s:%s@tcp(%s:%d)/%s?parseTime=true", + username, + password, + container.Host, + container.Port, + container.Database, + ) + readOnlyConn, err := sqlx.Connect("mysql", readOnlyDSN) + assert.NoError(t, err) + defer readOnlyConn.Close() + + var count int + err = readOnlyConn.Get(&count, "SELECT COUNT(*) FROM readonly_test") + assert.NoError(t, err) + assert.Equal(t, 2, count) + + _, err = readOnlyConn.Exec("INSERT INTO readonly_test (data) VALUES ('should-fail')") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = readOnlyConn.Exec("UPDATE readonly_test SET data = 'hacked' WHERE id = 1") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = readOnlyConn.Exec("DELETE FROM readonly_test WHERE id = 1") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = readOnlyConn.Exec("CREATE TABLE hack_table (id INT)") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + dropUserSafe(container.DB, username) + }) + } +} + +func Test_ReadOnlyUser_FutureTables_NoSelectPermission(t *testing.T) { + env := config.GetEnv() + container := connectToMariadbContainer(t, env.TestMariadb1011Port, tools.MariadbVersion1011) + defer container.DB.Close() + + mariadbModel := createMariadbModel(container) + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + ctx := context.Background() + + username, password, err := mariadbModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New()) + assert.NoError(t, err) + + _, err = container.DB.Exec(`DROP TABLE IF EXISTS future_table`) + assert.NoError(t, err) + _, err = container.DB.Exec(` + CREATE TABLE future_table ( + id INT AUTO_INCREMENT PRIMARY KEY, + data VARCHAR(255) NOT NULL + ) + `) + assert.NoError(t, err) + _, err = container.DB.Exec(`INSERT INTO future_table (data) VALUES ('future_data')`) + assert.NoError(t, err) + + readOnlyDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + username, password, container.Host, container.Port, container.Database) + readOnlyConn, err := sqlx.Connect("mysql", readOnlyDSN) + assert.NoError(t, err) + defer readOnlyConn.Close() + + var data string + err = readOnlyConn.Get(&data, "SELECT data FROM future_table LIMIT 1") + assert.NoError(t, err) + assert.Equal(t, "future_data", data) + + dropUserSafe(container.DB, username) +} + +func Test_CreateReadOnlyUser_DatabaseNameWithDash_Success(t *testing.T) { + env := config.GetEnv() + container := connectToMariadbContainer(t, env.TestMariadb1011Port, tools.MariadbVersion1011) + defer container.DB.Close() + + dashDbName := "test-db-with-dash" + + _, err := container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", dashDbName)) + assert.NoError(t, err) + + _, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE `%s`", dashDbName)) + assert.NoError(t, err) + + defer func() { + _, _ = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", dashDbName)) + }() + + dashDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + container.Username, container.Password, container.Host, container.Port, dashDbName) + dashDB, err := sqlx.Connect("mysql", dashDSN) + assert.NoError(t, err) + defer dashDB.Close() + + _, err = dashDB.Exec(` + CREATE TABLE dash_test ( + id INT AUTO_INCREMENT PRIMARY KEY, + data VARCHAR(255) NOT NULL + ) + `) + assert.NoError(t, err) + + _, err = dashDB.Exec(`INSERT INTO dash_test (data) VALUES ('test1'), ('test2')`) + assert.NoError(t, err) + + mariadbModel := &MariadbDatabase{ + Version: tools.MariadbVersion1011, + Host: container.Host, + Port: container.Port, + Username: container.Username, + Password: container.Password, + Database: &dashDbName, + IsHttps: false, + } + + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + ctx := context.Background() + + username, password, err := mariadbModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New()) + assert.NoError(t, err) + assert.NotEmpty(t, username) + assert.NotEmpty(t, password) + assert.True(t, strings.HasPrefix(username, "pgs-")) + + readOnlyDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + username, password, container.Host, container.Port, dashDbName) + readOnlyConn, err := sqlx.Connect("mysql", readOnlyDSN) + assert.NoError(t, err) + defer readOnlyConn.Close() + + var count int + err = readOnlyConn.Get(&count, "SELECT COUNT(*) FROM dash_test") + assert.NoError(t, err) + assert.Equal(t, 2, count) + + _, err = readOnlyConn.Exec("INSERT INTO dash_test (data) VALUES ('should-fail')") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + dropUserSafe(dashDB, username) +} + +func Test_ReadOnlyUser_CannotDropOrAlterTables(t *testing.T) { + env := config.GetEnv() + container := connectToMariadbContainer(t, env.TestMariadb1011Port, tools.MariadbVersion1011) + defer container.DB.Close() + + _, err := container.DB.Exec(`DROP TABLE IF EXISTS drop_test`) + assert.NoError(t, err) + _, err = container.DB.Exec(` + CREATE TABLE drop_test ( + id INT AUTO_INCREMENT PRIMARY KEY, + data VARCHAR(255) NOT NULL + ) + `) + assert.NoError(t, err) + _, err = container.DB.Exec(`INSERT INTO drop_test (data) VALUES ('test1')`) + assert.NoError(t, err) + + mariadbModel := createMariadbModel(container) + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + ctx := context.Background() + + username, password, err := mariadbModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New()) + assert.NoError(t, err) + + readOnlyDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + username, password, container.Host, container.Port, container.Database) + readOnlyConn, err := sqlx.Connect("mysql", readOnlyDSN) + assert.NoError(t, err) + defer readOnlyConn.Close() + + _, err = readOnlyConn.Exec("DROP TABLE drop_test") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = readOnlyConn.Exec("ALTER TABLE drop_test ADD COLUMN new_col VARCHAR(100)") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + _, err = readOnlyConn.Exec("TRUNCATE TABLE drop_test") + assert.Error(t, err) + assert.Contains(t, strings.ToLower(err.Error()), "denied") + + dropUserSafe(container.DB, username) +} + +type MariadbContainer struct { + Host string + Port int + Username string + Password string + Database string + Version tools.MariadbVersion + DB *sqlx.DB +} + +func connectToMariadbContainer( + t *testing.T, + port string, + version tools.MariadbVersion, +) *MariadbContainer { + if port == "" { + t.Skipf("MariaDB port not configured for version %s", version) + } + + dbName := "testdb" + host := "127.0.0.1" + username := "root" + password := "rootpassword" + + portInt, err := strconv.Atoi(port) + assert.NoError(t, err) + + dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + username, password, host, portInt, dbName) + + db, err := sqlx.Connect("mysql", dsn) + if err != nil { + t.Skipf("Failed to connect to MariaDB %s: %v", version, err) + } + + return &MariadbContainer{ + Host: host, + Port: portInt, + Username: username, + Password: password, + Database: dbName, + Version: version, + DB: db, + } +} + +func createMariadbModel(container *MariadbContainer) *MariadbDatabase { + return &MariadbDatabase{ + Version: container.Version, + Host: container.Host, + Port: container.Port, + Username: container.Username, + Password: container.Password, + Database: &container.Database, + IsHttps: false, + } +} + +func dropUserSafe(db *sqlx.DB, username string) { + // MariaDB 5.5 doesn't support DROP USER IF EXISTS, so we ignore errors + _, _ = db.Exec(fmt.Sprintf("DROP USER '%s'@'%%'", username)) +} diff --git a/backend/internal/features/databases/databases/mysql/model.go b/backend/internal/features/databases/databases/mysql/model.go index ee1f32d..f8a0864 100644 --- a/backend/internal/features/databases/databases/mysql/model.go +++ b/backend/internal/features/databases/databases/mysql/model.go @@ -138,7 +138,14 @@ func (m *MysqlDatabase) PopulateVersionIfEmpty( if m.Version != "" { return nil } + return m.PopulateVersion(logger, encryptor, databaseID) +} +func (m *MysqlDatabase) PopulateVersion( + logger *slog.Logger, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) error { if m.Database == nil || *m.Database == "" { return nil } diff --git a/backend/internal/features/databases/databases/postgresql/model.go b/backend/internal/features/databases/databases/postgresql/model.go index 8314388..e2c45d3 100644 --- a/backend/internal/features/databases/databases/postgresql/model.go +++ b/backend/internal/features/databases/databases/postgresql/model.go @@ -141,7 +141,15 @@ func (p *PostgresqlDatabase) PopulateVersionIfEmpty( if p.Version != "" { return nil } + return p.PopulateVersion(logger, encryptor, databaseID) +} +// PopulateVersion detects and sets the PostgreSQL version by querying the database. +func (p *PostgresqlDatabase) PopulateVersion( + logger *slog.Logger, + encryptor encryption.FieldEncryptor, + databaseID uuid.UUID, +) error { if p.Database == nil || *p.Database == "" { return nil } diff --git a/backend/internal/features/databases/enums.go b/backend/internal/features/databases/enums.go index 8a65530..291de1a 100644 --- a/backend/internal/features/databases/enums.go +++ b/backend/internal/features/databases/enums.go @@ -5,6 +5,7 @@ type DatabaseType string const ( DatabaseTypePostgres DatabaseType = "POSTGRES" DatabaseTypeMysql DatabaseType = "MYSQL" + DatabaseTypeMariadb DatabaseType = "MARIADB" ) type HealthStatus string diff --git a/backend/internal/features/databases/model.go b/backend/internal/features/databases/model.go index fc6b167..fc0a3c9 100644 --- a/backend/internal/features/databases/model.go +++ b/backend/internal/features/databases/model.go @@ -3,6 +3,7 @@ package databases import ( "errors" "log/slog" + "postgresus-backend/internal/features/databases/databases/mariadb" "postgresus-backend/internal/features/databases/databases/mysql" "postgresus-backend/internal/features/databases/databases/postgresql" "postgresus-backend/internal/features/notifiers" @@ -23,6 +24,7 @@ type Database struct { Postgresql *postgresql.PostgresqlDatabase `json:"postgresql,omitempty" gorm:"foreignKey:DatabaseID"` Mysql *mysql.MysqlDatabase `json:"mysql,omitempty" gorm:"foreignKey:DatabaseID"` + Mariadb *mariadb.MariadbDatabase `json:"mariadb,omitempty" gorm:"foreignKey:DatabaseID"` Notifiers []notifiers.Notifier `json:"notifiers" gorm:"many2many:database_notifiers;"` @@ -50,6 +52,11 @@ func (d *Database) Validate() error { return errors.New("mysql database is required") } return d.Mysql.Validate() + case DatabaseTypeMariadb: + if d.Mariadb == nil { + return errors.New("mariadb database is required") + } + return d.Mariadb.Validate() default: return errors.New("invalid database type: " + string(d.Type)) } @@ -81,6 +88,9 @@ func (d *Database) EncryptSensitiveFields(encryptor encryption.FieldEncryptor) e if d.Mysql != nil { return d.Mysql.EncryptSensitiveFields(d.ID, encryptor) } + if d.Mariadb != nil { + return d.Mariadb.EncryptSensitiveFields(d.ID, encryptor) + } return nil } @@ -94,6 +104,9 @@ func (d *Database) PopulateVersionIfEmpty( if d.Mysql != nil { return d.Mysql.PopulateVersionIfEmpty(logger, encryptor, d.ID) } + if d.Mariadb != nil { + return d.Mariadb.PopulateVersionIfEmpty(logger, encryptor, d.ID) + } return nil } @@ -111,6 +124,10 @@ func (d *Database) Update(incoming *Database) { if d.Mysql != nil && incoming.Mysql != nil { d.Mysql.Update(incoming.Mysql) } + case DatabaseTypeMariadb: + if d.Mariadb != nil && incoming.Mariadb != nil { + d.Mariadb.Update(incoming.Mariadb) + } } } @@ -120,6 +137,8 @@ func (d *Database) getSpecificDatabase() DatabaseConnector { return d.Postgresql case DatabaseTypeMysql: return d.Mysql + case DatabaseTypeMariadb: + return d.Mariadb } panic("invalid database type: " + string(d.Type)) diff --git a/backend/internal/features/databases/repository.go b/backend/internal/features/databases/repository.go index 42621ab..6268ac6 100644 --- a/backend/internal/features/databases/repository.go +++ b/backend/internal/features/databases/repository.go @@ -2,6 +2,7 @@ package databases import ( "errors" + "postgresus-backend/internal/features/databases/databases/mariadb" "postgresus-backend/internal/features/databases/databases/mysql" "postgresus-backend/internal/features/databases/databases/postgresql" "postgresus-backend/internal/storage" @@ -32,17 +33,22 @@ func (r *DatabaseRepository) Save(database *Database) (*Database, error) { return errors.New("mysql configuration is required for MySQL database") } database.Mysql.DatabaseID = &database.ID + case DatabaseTypeMariadb: + if database.Mariadb == nil { + return errors.New("mariadb configuration is required for MariaDB database") + } + database.Mariadb.DatabaseID = &database.ID } if isNew { if err := tx.Create(database). - Omit("Postgresql", "Mysql", "Notifiers"). + Omit("Postgresql", "Mysql", "Mariadb", "Notifiers"). Error; err != nil { return err } } else { if err := tx.Save(database). - Omit("Postgresql", "Mysql", "Notifiers"). + Omit("Postgresql", "Mysql", "Mariadb", "Notifiers"). Error; err != nil { return err } @@ -73,6 +79,18 @@ func (r *DatabaseRepository) Save(database *Database) (*Database, error) { return err } } + case DatabaseTypeMariadb: + database.Mariadb.DatabaseID = &database.ID + if database.Mariadb.ID == uuid.Nil { + database.Mariadb.ID = uuid.New() + if err := tx.Create(database.Mariadb).Error; err != nil { + return err + } + } else { + if err := tx.Save(database.Mariadb).Error; err != nil { + return err + } + } } if err := tx. @@ -99,6 +117,7 @@ func (r *DatabaseRepository) FindByID(id uuid.UUID) (*Database, error) { GetDb(). Preload("Postgresql"). Preload("Mysql"). + Preload("Mariadb"). Preload("Notifiers"). Where("id = ?", id). First(&database).Error; err != nil { @@ -115,6 +134,7 @@ func (r *DatabaseRepository) FindByWorkspaceID(workspaceID uuid.UUID) ([]*Databa GetDb(). Preload("Postgresql"). Preload("Mysql"). + Preload("Mariadb"). Preload("Notifiers"). Where("workspace_id = ?", workspaceID). Order("CASE WHEN health_status = 'UNAVAILABLE' THEN 1 WHEN health_status = 'AVAILABLE' THEN 2 WHEN health_status IS NULL THEN 3 ELSE 4 END, name ASC"). @@ -151,6 +171,12 @@ func (r *DatabaseRepository) Delete(id uuid.UUID) error { Delete(&mysql.MysqlDatabase{}).Error; err != nil { return err } + case DatabaseTypeMariadb: + if err := tx. + Where("database_id = ?", id). + Delete(&mariadb.MariadbDatabase{}).Error; err != nil { + return err + } } if err := tx.Delete(&Database{}, id).Error; err != nil { @@ -182,6 +208,7 @@ func (r *DatabaseRepository) GetAllDatabases() ([]*Database, error) { GetDb(). Preload("Postgresql"). Preload("Mysql"). + Preload("Mariadb"). Preload("Notifiers"). Find(&databases).Error; err != nil { return nil, err diff --git a/backend/internal/features/databases/service.go b/backend/internal/features/databases/service.go index 51b7a60..d65757d 100644 --- a/backend/internal/features/databases/service.go +++ b/backend/internal/features/databases/service.go @@ -8,6 +8,7 @@ import ( "time" audit_logs "postgresus-backend/internal/features/audit_logs" + "postgresus-backend/internal/features/databases/databases/mariadb" "postgresus-backend/internal/features/databases/databases/mysql" "postgresus-backend/internal/features/databases/databases/postgresql" "postgresus-backend/internal/features/notifiers" @@ -419,6 +420,20 @@ func (s *DatabaseService) CopyDatabase( IsHttps: existingDatabase.Mysql.IsHttps, } } + case DatabaseTypeMariadb: + if existingDatabase.Mariadb != nil { + newDatabase.Mariadb = &mariadb.MariadbDatabase{ + ID: uuid.Nil, + DatabaseID: nil, + Version: existingDatabase.Mariadb.Version, + Host: existingDatabase.Mariadb.Host, + Port: existingDatabase.Mariadb.Port, + Username: existingDatabase.Mariadb.Username, + Password: existingDatabase.Mariadb.Password, + Database: existingDatabase.Mariadb.Database, + IsHttps: existingDatabase.Mariadb.IsHttps, + } + } } if err := newDatabase.Validate(); err != nil { @@ -551,6 +566,13 @@ func (s *DatabaseService) IsUserReadOnly( s.fieldEncryptor, usingDatabase.ID, ) + case DatabaseTypeMariadb: + return usingDatabase.Mariadb.IsUserReadOnly( + ctx, + s.logger, + s.fieldEncryptor, + usingDatabase.ID, + ) default: return false, errors.New("read-only check not supported for this database type") } @@ -620,6 +642,10 @@ func (s *DatabaseService) CreateReadOnlyUser( username, password, err = usingDatabase.Mysql.CreateReadOnlyUser( ctx, s.logger, s.fieldEncryptor, usingDatabase.ID, ) + case DatabaseTypeMariadb: + username, password, err = usingDatabase.Mariadb.CreateReadOnlyUser( + ctx, s.logger, s.fieldEncryptor, usingDatabase.ID, + ) default: return "", "", errors.New("read-only user creation not supported for this database type") } diff --git a/backend/internal/features/restores/dto.go b/backend/internal/features/restores/dto.go index acd8e4a..caf4383 100644 --- a/backend/internal/features/restores/dto.go +++ b/backend/internal/features/restores/dto.go @@ -1,6 +1,7 @@ package restores import ( + "postgresus-backend/internal/features/databases/databases/mariadb" "postgresus-backend/internal/features/databases/databases/mysql" "postgresus-backend/internal/features/databases/databases/postgresql" ) @@ -8,4 +9,5 @@ import ( type RestoreBackupRequest struct { PostgresqlDatabase *postgresql.PostgresqlDatabase `json:"postgresqlDatabase"` MysqlDatabase *mysql.MysqlDatabase `json:"mysqlDatabase"` + MariadbDatabase *mariadb.MariadbDatabase `json:"mariadbDatabase"` } diff --git a/backend/internal/features/restores/service.go b/backend/internal/features/restores/service.go index 7011f59..85f03ae 100644 --- a/backend/internal/features/restores/service.go +++ b/backend/internal/features/restores/service.go @@ -167,6 +167,10 @@ func (s *RestoreService) RestoreBackup( if requestDTO.MysqlDatabase == nil { return errors.New("mysql database is required") } + case databases.DatabaseTypeMariadb: + if requestDTO.MariadbDatabase == nil { + return errors.New("mariadb database is required") + } } restore := models.Restore{ @@ -210,6 +214,7 @@ func (s *RestoreService) RestoreBackup( Type: database.Type, Postgresql: requestDTO.PostgresqlDatabase, Mysql: requestDTO.MysqlDatabase, + Mariadb: requestDTO.MariadbDatabase, } if err := restoringToDB.PopulateVersionIfEmpty(s.logger, s.fieldEncryptor); err != nil { @@ -257,6 +262,38 @@ func (s *RestoreService) validateVersionCompatibility( backupDatabase *databases.Database, requestDTO RestoreBackupRequest, ) error { + // populate version + if requestDTO.MariadbDatabase != nil { + err := requestDTO.MariadbDatabase.PopulateVersion( + s.logger, + s.fieldEncryptor, + backupDatabase.ID, + ) + if err != nil { + return err + } + } + if requestDTO.MysqlDatabase != nil { + err := requestDTO.MysqlDatabase.PopulateVersion( + s.logger, + s.fieldEncryptor, + backupDatabase.ID, + ) + if err != nil { + return err + } + } + if requestDTO.PostgresqlDatabase != nil { + err := requestDTO.PostgresqlDatabase.PopulateVersion( + s.logger, + s.fieldEncryptor, + backupDatabase.ID, + ) + if err != nil { + return err + } + } + switch backupDatabase.Type { case databases.DatabaseTypePostgres: if requestDTO.PostgresqlDatabase == nil { @@ -282,6 +319,18 @@ func (s *RestoreService) validateVersionCompatibility( `Should be restored to the same version as the backup database or higher. ` + `For example, you can restore MySQL 8.0 backup to MySQL 8.0, 8.4 or higher. But cannot restore to 5.7`) } + case databases.DatabaseTypeMariadb: + if requestDTO.MariadbDatabase == nil { + return errors.New("mariadb database configuration is required for restore") + } + if tools.IsMariadbBackupVersionHigherThanRestoreVersion( + backupDatabase.Mariadb.Version, + requestDTO.MariadbDatabase.Version, + ) { + return errors.New(`backup database version is higher than restore database version. ` + + `Should be restored to the same version as the backup database or higher. ` + + `For example, you can restore MariaDB 10.11 backup to MariaDB 10.11, 11.4 or higher. But cannot restore to 10.6`) + } } return nil } diff --git a/backend/internal/features/restores/usecases/di.go b/backend/internal/features/restores/usecases/di.go index 764755d..2851ab2 100644 --- a/backend/internal/features/restores/usecases/di.go +++ b/backend/internal/features/restores/usecases/di.go @@ -1,6 +1,7 @@ package usecases import ( + usecases_mariadb "postgresus-backend/internal/features/restores/usecases/mariadb" usecases_mysql "postgresus-backend/internal/features/restores/usecases/mysql" usecases_postgresql "postgresus-backend/internal/features/restores/usecases/postgresql" ) @@ -8,6 +9,7 @@ import ( var restoreBackupUsecase = &RestoreBackupUsecase{ usecases_postgresql.GetRestorePostgresqlBackupUsecase(), usecases_mysql.GetRestoreMysqlBackupUsecase(), + usecases_mariadb.GetRestoreMariadbBackupUsecase(), } func GetRestoreBackupUsecase() *RestoreBackupUsecase { diff --git a/backend/internal/features/restores/usecases/mariadb/di.go b/backend/internal/features/restores/usecases/mariadb/di.go new file mode 100644 index 0000000..f86e904 --- /dev/null +++ b/backend/internal/features/restores/usecases/mariadb/di.go @@ -0,0 +1,15 @@ +package usecases_mariadb + +import ( + "postgresus-backend/internal/features/encryption/secrets" + "postgresus-backend/internal/util/logger" +) + +var restoreMariadbBackupUsecase = &RestoreMariadbBackupUsecase{ + logger.GetLogger(), + secrets.GetSecretKeyService(), +} + +func GetRestoreMariadbBackupUsecase() *RestoreMariadbBackupUsecase { + return restoreMariadbBackupUsecase +} diff --git a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go new file mode 100644 index 0000000..20704f7 --- /dev/null +++ b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go @@ -0,0 +1,473 @@ +package usecases_mariadb + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/klauspost/compress/zstd" + + "postgresus-backend/internal/config" + "postgresus-backend/internal/features/backups/backups" + "postgresus-backend/internal/features/backups/backups/encryption" + backups_config "postgresus-backend/internal/features/backups/config" + "postgresus-backend/internal/features/databases" + mariadbtypes "postgresus-backend/internal/features/databases/databases/mariadb" + encryption_secrets "postgresus-backend/internal/features/encryption/secrets" + "postgresus-backend/internal/features/restores/models" + "postgresus-backend/internal/features/storages" + util_encryption "postgresus-backend/internal/util/encryption" + files_utils "postgresus-backend/internal/util/files" + "postgresus-backend/internal/util/tools" +) + +type RestoreMariadbBackupUsecase struct { + logger *slog.Logger + secretKeyService *encryption_secrets.SecretKeyService +} + +func (uc *RestoreMariadbBackupUsecase) Execute( + originalDB *databases.Database, + restoringToDB *databases.Database, + backupConfig *backups_config.BackupConfig, + restore models.Restore, + backup *backups.Backup, + storage *storages.Storage, +) error { + if originalDB.Type != databases.DatabaseTypeMariadb { + return errors.New("database type not supported") + } + + uc.logger.Info( + "Restoring MariaDB backup via mariadb client", + "restoreId", restore.ID, + "backupId", backup.ID, + ) + + mdb := restoringToDB.Mariadb + if mdb == nil { + return fmt.Errorf("mariadb configuration is required for restore") + } + + if mdb.Database == nil || *mdb.Database == "" { + return fmt.Errorf("target database name is required for mariadb restore") + } + + args := []string{ + "--host=" + mdb.Host, + "--port=" + strconv.Itoa(mdb.Port), + "--user=" + mdb.Username, + "--verbose", + } + + if mdb.IsHttps { + args = append(args, "--ssl") + } + + if mdb.Database != nil && *mdb.Database != "" { + args = append(args, *mdb.Database) + } + + return uc.restoreFromStorage( + originalDB, + tools.GetMariadbExecutable( + tools.MariadbExecutableMariadb, + mdb.Version, + config.GetEnv().EnvMode, + config.GetEnv().MariadbInstallDir, + ), + args, + mdb.Password, + backup, + storage, + mdb, + ) +} + +func (uc *RestoreMariadbBackupUsecase) restoreFromStorage( + database *databases.Database, + mariadbBin string, + args []string, + password string, + backup *backups.Backup, + storage *storages.Storage, + mdbConfig *mariadbtypes.MariadbDatabase, +) error { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() + + go func() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if config.IsShouldShutdown() { + cancel() + return + } + } + } + }() + + fieldEncryptor := util_encryption.GetFieldEncryptor() + decryptedPassword, err := fieldEncryptor.Decrypt(database.ID, password) + if err != nil { + return fmt.Errorf("failed to decrypt password: %w", err) + } + + myCnfFile, err := uc.createTempMyCnfFile(mdbConfig, decryptedPassword) + if err != nil { + return fmt.Errorf("failed to create .my.cnf: %w", err) + } + defer func() { _ = os.RemoveAll(filepath.Dir(myCnfFile)) }() + + tempBackupFile, cleanupFunc, err := uc.downloadBackupToTempFile(ctx, backup, storage) + if err != nil { + return fmt.Errorf("failed to download backup: %w", err) + } + defer cleanupFunc() + + return uc.executeMariadbRestore( + ctx, + database, + mariadbBin, + args, + myCnfFile, + tempBackupFile, + backup, + ) +} + +func (uc *RestoreMariadbBackupUsecase) executeMariadbRestore( + ctx context.Context, + database *databases.Database, + mariadbBin string, + args []string, + myCnfFile string, + backupFile string, + backup *backups.Backup, +) error { + fullArgs := append([]string{"--defaults-file=" + myCnfFile}, args...) + + cmd := exec.CommandContext(ctx, mariadbBin, fullArgs...) + uc.logger.Info("Executing MariaDB restore command", "command", cmd.String()) + + backupFileHandle, err := os.Open(backupFile) + if err != nil { + return fmt.Errorf("failed to open backup file: %w", err) + } + defer func() { _ = backupFileHandle.Close() }() + + var inputReader io.Reader = backupFileHandle + + if backup.Encryption == backups_config.BackupEncryptionEncrypted { + decryptReader, err := uc.setupDecryption(backupFileHandle, backup) + if err != nil { + return fmt.Errorf("failed to setup decryption: %w", err) + } + inputReader = decryptReader + } + + zstdReader, err := zstd.NewReader(inputReader) + if err != nil { + return fmt.Errorf("failed to create zstd reader: %w", err) + } + defer zstdReader.Close() + + cmd.Stdin = zstdReader + + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, + "MYSQL_PWD=", + "LC_ALL=C.UTF-8", + "LANG=C.UTF-8", + ) + + stderrPipe, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("stderr pipe: %w", err) + } + + stderrCh := make(chan []byte, 1) + go func() { + output, _ := io.ReadAll(stderrPipe) + stderrCh <- output + }() + + if err = cmd.Start(); err != nil { + return fmt.Errorf("start mariadb: %w", err) + } + + waitErr := cmd.Wait() + stderrOutput := <-stderrCh + + if config.IsShouldShutdown() { + return fmt.Errorf("restore cancelled due to shutdown") + } + + if waitErr != nil { + return uc.handleMariadbRestoreError(database, waitErr, stderrOutput, mariadbBin) + } + + return nil +} + +func (uc *RestoreMariadbBackupUsecase) downloadBackupToTempFile( + ctx context.Context, + backup *backups.Backup, + storage *storages.Storage, +) (string, func(), error) { + err := files_utils.EnsureDirectories([]string{ + config.GetEnv().TempFolder, + }) + if err != nil { + return "", nil, fmt.Errorf("failed to ensure directories: %w", err) + } + + tempDir, err := os.MkdirTemp(config.GetEnv().TempFolder, "restore_"+uuid.New().String()) + if err != nil { + return "", nil, fmt.Errorf("failed to create temporary directory: %w", err) + } + + cleanupFunc := func() { + _ = os.RemoveAll(tempDir) + } + + tempBackupFile := filepath.Join(tempDir, "backup.sql.zst") + + uc.logger.Info( + "Downloading backup file from storage to temporary file", + "backupId", backup.ID, + "tempFile", tempBackupFile, + "encrypted", backup.Encryption == backups_config.BackupEncryptionEncrypted, + ) + + fieldEncryptor := util_encryption.GetFieldEncryptor() + rawReader, err := storage.GetFile(fieldEncryptor, backup.ID) + if err != nil { + cleanupFunc() + return "", nil, fmt.Errorf("failed to get backup file from storage: %w", err) + } + defer func() { + if err := rawReader.Close(); err != nil { + uc.logger.Error("Failed to close backup reader", "error", err) + } + }() + + tempFile, err := os.Create(tempBackupFile) + if err != nil { + cleanupFunc() + return "", nil, fmt.Errorf("failed to create temporary backup file: %w", err) + } + defer func() { + if err := tempFile.Close(); err != nil { + uc.logger.Error("Failed to close temporary file", "error", err) + } + }() + + _, err = uc.copyWithShutdownCheck(ctx, tempFile, rawReader) + if err != nil { + cleanupFunc() + return "", nil, fmt.Errorf("failed to write backup to temporary file: %w", err) + } + + uc.logger.Info("Backup file written to temporary location", "tempFile", tempBackupFile) + return tempBackupFile, cleanupFunc, nil +} + +func (uc *RestoreMariadbBackupUsecase) setupDecryption( + reader io.Reader, + backup *backups.Backup, +) (io.Reader, error) { + if backup.EncryptionSalt == nil || backup.EncryptionIV == nil { + return nil, fmt.Errorf("backup is encrypted but missing encryption metadata") + } + + masterKey, err := uc.secretKeyService.GetSecretKey() + if err != nil { + return nil, fmt.Errorf("failed to get master key for decryption: %w", err) + } + + salt, err := base64.StdEncoding.DecodeString(*backup.EncryptionSalt) + if err != nil { + return nil, fmt.Errorf("failed to decode encryption salt: %w", err) + } + + iv, err := base64.StdEncoding.DecodeString(*backup.EncryptionIV) + if err != nil { + return nil, fmt.Errorf("failed to decode encryption IV: %w", err) + } + + decryptReader, err := encryption.NewDecryptionReader( + reader, + masterKey, + backup.ID, + salt, + iv, + ) + if err != nil { + return nil, fmt.Errorf("failed to create decryption reader: %w", err) + } + + uc.logger.Info("Using decryption for encrypted backup", "backupId", backup.ID) + return decryptReader, nil +} + +func (uc *RestoreMariadbBackupUsecase) createTempMyCnfFile( + mdbConfig *mariadbtypes.MariadbDatabase, + password string, +) (string, error) { + tempDir, err := os.MkdirTemp("", "mycnf") + if err != nil { + return "", fmt.Errorf("failed to create temp directory: %w", err) + } + + myCnfFile := filepath.Join(tempDir, ".my.cnf") + + content := fmt.Sprintf(`[client] +user=%s +password="%s" +host=%s +port=%d +`, mdbConfig.Username, tools.EscapeMariadbPassword(password), mdbConfig.Host, mdbConfig.Port) + + if mdbConfig.IsHttps { + content += "ssl=true\n" + } else { + content += "ssl=false\n" + } + + err = os.WriteFile(myCnfFile, []byte(content), 0600) + if err != nil { + return "", fmt.Errorf("failed to write .my.cnf: %w", err) + } + + return myCnfFile, nil +} + +func (uc *RestoreMariadbBackupUsecase) copyWithShutdownCheck( + ctx context.Context, + dst io.Writer, + src io.Reader, +) (int64, error) { + buf := make([]byte, 16*1024*1024) + var totalBytesWritten int64 + + for { + select { + case <-ctx.Done(): + return totalBytesWritten, fmt.Errorf("copy cancelled: %w", ctx.Err()) + default: + } + + if config.IsShouldShutdown() { + return totalBytesWritten, fmt.Errorf("copy cancelled due to shutdown") + } + + bytesRead, readErr := src.Read(buf) + if bytesRead > 0 { + bytesWritten, writeErr := dst.Write(buf[0:bytesRead]) + if bytesWritten < 0 || bytesRead < bytesWritten { + bytesWritten = 0 + if writeErr == nil { + writeErr = fmt.Errorf("invalid write result") + } + } + + if writeErr != nil { + return totalBytesWritten, writeErr + } + + if bytesRead != bytesWritten { + return totalBytesWritten, io.ErrShortWrite + } + + totalBytesWritten += int64(bytesWritten) + } + + if readErr != nil { + if readErr != io.EOF { + return totalBytesWritten, readErr + } + break + } + } + + return totalBytesWritten, nil +} + +func (uc *RestoreMariadbBackupUsecase) handleMariadbRestoreError( + database *databases.Database, + waitErr error, + stderrOutput []byte, + mariadbBin string, +) error { + stderrStr := string(stderrOutput) + errorMsg := fmt.Sprintf( + "%s failed: %v – stderr: %s", + filepath.Base(mariadbBin), + waitErr, + stderrStr, + ) + + if containsIgnoreCase(stderrStr, "access denied") { + return fmt.Errorf( + "MariaDB access denied. Check username and password. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "can't connect") || + containsIgnoreCase(stderrStr, "connection refused") { + return fmt.Errorf( + "MariaDB connection refused. Check if the server is running and accessible. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "unknown database") { + backupDbName := "unknown" + if database.Mariadb != nil && database.Mariadb.Database != nil { + backupDbName = *database.Mariadb.Database + } + + return fmt.Errorf( + "target database does not exist (backup db %s). Create the database before restoring. stderr: %s", + backupDbName, + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "ssl") { + return fmt.Errorf( + "MariaDB SSL connection failed. stderr: %s", + stderrStr, + ) + } + + if containsIgnoreCase(stderrStr, "timeout") { + return fmt.Errorf( + "MariaDB connection timeout. stderr: %s", + stderrStr, + ) + } + + return errors.New(errorMsg) +} + +func containsIgnoreCase(str, substr string) bool { + return strings.Contains(strings.ToLower(str), strings.ToLower(substr)) +} diff --git a/backend/internal/features/restores/usecases/restore_backup_uc.go b/backend/internal/features/restores/usecases/restore_backup_uc.go index 9be2fcc..eb334bb 100644 --- a/backend/internal/features/restores/usecases/restore_backup_uc.go +++ b/backend/internal/features/restores/usecases/restore_backup_uc.go @@ -7,6 +7,7 @@ import ( backups_config "postgresus-backend/internal/features/backups/config" "postgresus-backend/internal/features/databases" "postgresus-backend/internal/features/restores/models" + usecases_mariadb "postgresus-backend/internal/features/restores/usecases/mariadb" usecases_mysql "postgresus-backend/internal/features/restores/usecases/mysql" usecases_postgresql "postgresus-backend/internal/features/restores/usecases/postgresql" "postgresus-backend/internal/features/storages" @@ -15,6 +16,7 @@ import ( type RestoreBackupUsecase struct { restorePostgresqlBackupUsecase *usecases_postgresql.RestorePostgresqlBackupUsecase restoreMysqlBackupUsecase *usecases_mysql.RestoreMysqlBackupUsecase + restoreMariadbBackupUsecase *usecases_mariadb.RestoreMariadbBackupUsecase } func (uc *RestoreBackupUsecase) Execute( @@ -46,6 +48,15 @@ func (uc *RestoreBackupUsecase) Execute( backup, storage, ) + case databases.DatabaseTypeMariadb: + return uc.restoreMariadbBackupUsecase.Execute( + originalDB, + restoringToDB, + backupConfig, + restore, + backup, + storage, + ) default: return errors.New("database type not supported") } diff --git a/backend/internal/features/tests/mariadb_backup_restore_test.go b/backend/internal/features/tests/mariadb_backup_restore_test.go new file mode 100644 index 0000000..8964eb7 --- /dev/null +++ b/backend/internal/features/tests/mariadb_backup_restore_test.go @@ -0,0 +1,706 @@ +package tests + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/gin-gonic/gin" + _ "github.com/go-sql-driver/mysql" + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/assert" + + "postgresus-backend/internal/config" + "postgresus-backend/internal/features/backups/backups" + backups_config "postgresus-backend/internal/features/backups/config" + "postgresus-backend/internal/features/databases" + mariadbtypes "postgresus-backend/internal/features/databases/databases/mariadb" + "postgresus-backend/internal/features/restores" + restores_enums "postgresus-backend/internal/features/restores/enums" + restores_models "postgresus-backend/internal/features/restores/models" + "postgresus-backend/internal/features/storages" + users_enums "postgresus-backend/internal/features/users/enums" + users_testing "postgresus-backend/internal/features/users/testing" + workspaces_testing "postgresus-backend/internal/features/workspaces/testing" + test_utils "postgresus-backend/internal/util/testing" + "postgresus-backend/internal/util/tools" +) + +const dropMariadbTestTableQuery = `DROP TABLE IF EXISTS test_data` + +const createMariadbTestTableQuery = ` +CREATE TABLE test_data ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + value INT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +)` + +const insertMariadbTestDataQuery = ` +INSERT INTO test_data (name, value) VALUES + ('test1', 100), + ('test2', 200), + ('test3', 300)` + +type MariadbContainer struct { + Host string + Port int + Username string + Password string + Database string + Version tools.MariadbVersion + DB *sqlx.DB +} + +type MariadbTestDataItem struct { + ID int `db:"id"` + Name string `db:"name"` + Value int `db:"value"` + CreatedAt time.Time `db:"created_at"` +} + +func Test_BackupAndRestoreMariadb_RestoreIsSuccessful(t *testing.T) { + env := config.GetEnv() + cases := []struct { + name string + version tools.MariadbVersion + port string + }{ + {"MariaDB 5.5", tools.MariadbVersion55, env.TestMariadb55Port}, + {"MariaDB 10.1", tools.MariadbVersion101, env.TestMariadb101Port}, + {"MariaDB 10.2", tools.MariadbVersion102, env.TestMariadb102Port}, + {"MariaDB 10.3", tools.MariadbVersion103, env.TestMariadb103Port}, + {"MariaDB 10.4", tools.MariadbVersion104, env.TestMariadb104Port}, + {"MariaDB 10.5", tools.MariadbVersion105, env.TestMariadb105Port}, + {"MariaDB 10.6", tools.MariadbVersion106, env.TestMariadb106Port}, + {"MariaDB 10.11", tools.MariadbVersion1011, env.TestMariadb1011Port}, + {"MariaDB 11.4", tools.MariadbVersion114, env.TestMariadb114Port}, + {"MariaDB 11.8", tools.MariadbVersion118, env.TestMariadb118Port}, + {"MariaDB 12.0", tools.MariadbVersion120, env.TestMariadb120Port}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testMariadbBackupRestoreForVersion(t, tc.version, tc.port) + }) + } +} + +func Test_BackupAndRestoreMariadbWithEncryption_RestoreIsSuccessful(t *testing.T) { + env := config.GetEnv() + cases := []struct { + name string + version tools.MariadbVersion + port string + }{ + {"MariaDB 5.5", tools.MariadbVersion55, env.TestMariadb55Port}, + {"MariaDB 10.1", tools.MariadbVersion101, env.TestMariadb101Port}, + {"MariaDB 10.2", tools.MariadbVersion102, env.TestMariadb102Port}, + {"MariaDB 10.3", tools.MariadbVersion103, env.TestMariadb103Port}, + {"MariaDB 10.4", tools.MariadbVersion104, env.TestMariadb104Port}, + {"MariaDB 10.5", tools.MariadbVersion105, env.TestMariadb105Port}, + {"MariaDB 10.6", tools.MariadbVersion106, env.TestMariadb106Port}, + {"MariaDB 10.11", tools.MariadbVersion1011, env.TestMariadb1011Port}, + {"MariaDB 11.4", tools.MariadbVersion114, env.TestMariadb114Port}, + {"MariaDB 11.8", tools.MariadbVersion118, env.TestMariadb118Port}, + {"MariaDB 12.0", tools.MariadbVersion120, env.TestMariadb120Port}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testMariadbBackupRestoreWithEncryptionForVersion(t, tc.version, tc.port) + }) + } +} + +func Test_BackupAndRestoreMariadb_WithReadOnlyUser_RestoreIsSuccessful(t *testing.T) { + env := config.GetEnv() + cases := []struct { + name string + version tools.MariadbVersion + port string + }{ + {"MariaDB 5.5", tools.MariadbVersion55, env.TestMariadb55Port}, + {"MariaDB 10.1", tools.MariadbVersion101, env.TestMariadb101Port}, + {"MariaDB 10.2", tools.MariadbVersion102, env.TestMariadb102Port}, + {"MariaDB 10.3", tools.MariadbVersion103, env.TestMariadb103Port}, + {"MariaDB 10.4", tools.MariadbVersion104, env.TestMariadb104Port}, + {"MariaDB 10.5", tools.MariadbVersion105, env.TestMariadb105Port}, + {"MariaDB 10.6", tools.MariadbVersion106, env.TestMariadb106Port}, + {"MariaDB 10.11", tools.MariadbVersion1011, env.TestMariadb1011Port}, + {"MariaDB 11.4", tools.MariadbVersion114, env.TestMariadb114Port}, + {"MariaDB 11.8", tools.MariadbVersion118, env.TestMariadb118Port}, + {"MariaDB 12.0", tools.MariadbVersion120, env.TestMariadb120Port}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + testMariadbBackupRestoreWithReadOnlyUserForVersion(t, tc.version, tc.port) + }) + } +} + +func testMariadbBackupRestoreForVersion( + t *testing.T, + mariadbVersion tools.MariadbVersion, + port string, +) { + container, err := connectToMariadbContainer(mariadbVersion, port) + if err != nil { + t.Skipf("Skipping MariaDB %s test: %v", mariadbVersion, err) + return + } + defer func() { + if container.DB != nil { + container.DB.Close() + } + }() + + setupMariadbTestData(t, container.DB) + + router := createTestRouter() + user := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("MariaDB Test Workspace", user, router) + + storage := storages.CreateTestStorage(workspace.ID) + + database := createMariadbDatabaseViaAPI( + t, router, "MariaDB Test Database", workspace.ID, + container.Host, container.Port, + container.Username, container.Password, container.Database, + container.Version, + user.Token, + ) + + enableBackupsViaAPI( + t, router, database.ID, storage.ID, + backups_config.BackupEncryptionNone, user.Token, + ) + + createBackupViaAPI(t, router, database.ID, user.Token) + + backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute) + assert.Equal(t, backups.BackupStatusCompleted, backup.Status) + + newDBName := "restoreddb_mariadb" + _, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName)) + assert.NoError(t, err) + + _, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName)) + assert.NoError(t, err) + + newDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + container.Username, container.Password, container.Host, container.Port, newDBName) + newDB, err := sqlx.Connect("mysql", newDSN) + assert.NoError(t, err) + defer newDB.Close() + + createMariadbRestoreViaAPI( + t, router, backup.ID, + container.Host, container.Port, + container.Username, container.Password, newDBName, + container.Version, + user.Token, + ) + + restore := waitForMariadbRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute) + assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status) + + var tableExists int + err = newDB.Get( + &tableExists, + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = ? AND table_name = 'test_data'", + newDBName, + ) + assert.NoError(t, err) + assert.Equal(t, 1, tableExists, "Table 'test_data' should exist in restored database") + + verifyMariadbDataIntegrity(t, container.DB, newDB) + + err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String())) + if err != nil { + t.Logf("Warning: Failed to delete backup file: %v", err) + } + + test_utils.MakeDeleteRequest( + t, + router, + "/api/v1/databases/"+database.ID.String(), + "Bearer "+user.Token, + http.StatusNoContent, + ) + storages.RemoveTestStorage(storage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) +} + +func testMariadbBackupRestoreWithEncryptionForVersion( + t *testing.T, + mariadbVersion tools.MariadbVersion, + port string, +) { + container, err := connectToMariadbContainer(mariadbVersion, port) + if err != nil { + t.Skipf("Skipping MariaDB %s test: %v", mariadbVersion, err) + return + } + defer func() { + if container.DB != nil { + container.DB.Close() + } + }() + + setupMariadbTestData(t, container.DB) + + router := createTestRouter() + user := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace( + "MariaDB Encrypted Test Workspace", + user, + router, + ) + + storage := storages.CreateTestStorage(workspace.ID) + + database := createMariadbDatabaseViaAPI( + t, router, "MariaDB Encrypted Test Database", workspace.ID, + container.Host, container.Port, + container.Username, container.Password, container.Database, + container.Version, + user.Token, + ) + + enableBackupsViaAPI( + t, router, database.ID, storage.ID, + backups_config.BackupEncryptionEncrypted, user.Token, + ) + + createBackupViaAPI(t, router, database.ID, user.Token) + + backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute) + assert.Equal(t, backups.BackupStatusCompleted, backup.Status) + assert.Equal(t, backups_config.BackupEncryptionEncrypted, backup.Encryption) + + newDBName := "restoreddb_mariadb_encrypted" + _, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName)) + assert.NoError(t, err) + + _, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName)) + assert.NoError(t, err) + + newDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + container.Username, container.Password, container.Host, container.Port, newDBName) + newDB, err := sqlx.Connect("mysql", newDSN) + assert.NoError(t, err) + defer newDB.Close() + + createMariadbRestoreViaAPI( + t, router, backup.ID, + container.Host, container.Port, + container.Username, container.Password, newDBName, + container.Version, + user.Token, + ) + + restore := waitForMariadbRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute) + assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status) + + var tableExists int + err = newDB.Get( + &tableExists, + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = ? AND table_name = 'test_data'", + newDBName, + ) + assert.NoError(t, err) + assert.Equal(t, 1, tableExists, "Table 'test_data' should exist in restored database") + + verifyMariadbDataIntegrity(t, container.DB, newDB) + + err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String())) + if err != nil { + t.Logf("Warning: Failed to delete backup file: %v", err) + } + + test_utils.MakeDeleteRequest( + t, + router, + "/api/v1/databases/"+database.ID.String(), + "Bearer "+user.Token, + http.StatusNoContent, + ) + storages.RemoveTestStorage(storage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) +} + +func testMariadbBackupRestoreWithReadOnlyUserForVersion( + t *testing.T, + mariadbVersion tools.MariadbVersion, + port string, +) { + container, err := connectToMariadbContainer(mariadbVersion, port) + if err != nil { + t.Skipf("Skipping MariaDB %s test: %v", mariadbVersion, err) + return + } + defer func() { + if container.DB != nil { + container.DB.Close() + } + }() + + setupMariadbTestData(t, container.DB) + + router := createTestRouter() + user := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace( + "MariaDB ReadOnly Test Workspace", + user, + router, + ) + + storage := storages.CreateTestStorage(workspace.ID) + + database := createMariadbDatabaseViaAPI( + t, router, "MariaDB ReadOnly Test Database", workspace.ID, + container.Host, container.Port, + container.Username, container.Password, container.Database, + container.Version, + user.Token, + ) + + readOnlyUser := createMariadbReadOnlyUserViaAPI(t, router, database.ID, user.Token) + assert.NotEmpty(t, readOnlyUser.Username) + assert.NotEmpty(t, readOnlyUser.Password) + + updatedDatabase := updateMariadbDatabaseCredentialsViaAPI( + t, router, database, + readOnlyUser.Username, readOnlyUser.Password, + user.Token, + ) + + enableBackupsViaAPI( + t, router, updatedDatabase.ID, storage.ID, + backups_config.BackupEncryptionNone, user.Token, + ) + + createBackupViaAPI(t, router, updatedDatabase.ID, user.Token) + + backup := waitForBackupCompletion(t, router, updatedDatabase.ID, user.Token, 5*time.Minute) + assert.Equal(t, backups.BackupStatusCompleted, backup.Status) + + newDBName := "restoreddb_mariadb_readonly" + _, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName)) + assert.NoError(t, err) + + _, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName)) + assert.NoError(t, err) + + newDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + container.Username, container.Password, container.Host, container.Port, newDBName) + newDB, err := sqlx.Connect("mysql", newDSN) + assert.NoError(t, err) + defer newDB.Close() + + createMariadbRestoreViaAPI( + t, router, backup.ID, + container.Host, container.Port, + container.Username, container.Password, newDBName, + container.Version, + user.Token, + ) + + restore := waitForMariadbRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute) + assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status) + + var tableExists int + err = newDB.Get( + &tableExists, + "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = ? AND table_name = 'test_data'", + newDBName, + ) + assert.NoError(t, err) + assert.Equal(t, 1, tableExists, "Table 'test_data' should exist in restored database") + + verifyMariadbDataIntegrity(t, container.DB, newDB) + + err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String())) + if err != nil { + t.Logf("Warning: Failed to delete backup file: %v", err) + } + + test_utils.MakeDeleteRequest( + t, + router, + "/api/v1/databases/"+updatedDatabase.ID.String(), + "Bearer "+user.Token, + http.StatusNoContent, + ) + storages.RemoveTestStorage(storage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) +} + +func createMariadbDatabaseViaAPI( + t *testing.T, + router *gin.Engine, + name string, + workspaceID uuid.UUID, + host string, + port int, + username string, + password string, + database string, + version tools.MariadbVersion, + token string, +) *databases.Database { + request := databases.Database{ + Name: name, + WorkspaceID: &workspaceID, + Type: databases.DatabaseTypeMariadb, + Mariadb: &mariadbtypes.MariadbDatabase{ + Host: host, + Port: port, + Username: username, + Password: password, + Database: &database, + Version: version, + }, + } + + w := workspaces_testing.MakeAPIRequest( + router, + "POST", + "/api/v1/databases/create", + "Bearer "+token, + request, + ) + + if w.Code != http.StatusCreated { + t.Fatalf("Failed to create MariaDB database. Status: %d, Body: %s", w.Code, w.Body.String()) + } + + var createdDatabase databases.Database + if err := json.Unmarshal(w.Body.Bytes(), &createdDatabase); err != nil { + t.Fatalf("Failed to unmarshal database response: %v", err) + } + + return &createdDatabase +} + +func createMariadbRestoreViaAPI( + t *testing.T, + router *gin.Engine, + backupID uuid.UUID, + host string, + port int, + username string, + password string, + database string, + version tools.MariadbVersion, + token string, +) { + request := restores.RestoreBackupRequest{ + MariadbDatabase: &mariadbtypes.MariadbDatabase{ + Host: host, + Port: port, + Username: username, + Password: password, + Database: &database, + Version: version, + }, + } + + test_utils.MakePostRequest( + t, + router, + fmt.Sprintf("/api/v1/restores/%s/restore", backupID.String()), + "Bearer "+token, + request, + http.StatusOK, + ) +} + +func waitForMariadbRestoreCompletion( + t *testing.T, + router *gin.Engine, + backupID uuid.UUID, + token string, + timeout time.Duration, +) *restores_models.Restore { + startTime := time.Now() + pollInterval := 500 * time.Millisecond + + for { + if time.Since(startTime) > timeout { + t.Fatalf("Timeout waiting for MariaDB restore completion after %v", timeout) + } + + var restoresList []*restores_models.Restore + test_utils.MakeGetRequestAndUnmarshal( + t, + router, + fmt.Sprintf("/api/v1/restores/%s", backupID.String()), + "Bearer "+token, + http.StatusOK, + &restoresList, + ) + + for _, restore := range restoresList { + if restore.Status == restores_enums.RestoreStatusCompleted { + return restore + } + if restore.Status == restores_enums.RestoreStatusFailed { + failMsg := "unknown error" + if restore.FailMessage != nil { + failMsg = *restore.FailMessage + } + t.Fatalf("MariaDB restore failed: %s", failMsg) + } + } + + time.Sleep(pollInterval) + } +} + +func verifyMariadbDataIntegrity(t *testing.T, originalDB *sqlx.DB, restoredDB *sqlx.DB) { + var originalData []MariadbTestDataItem + var restoredData []MariadbTestDataItem + + err := originalDB.Select( + &originalData, + "SELECT id, name, value, created_at FROM test_data ORDER BY id", + ) + assert.NoError(t, err) + + err = restoredDB.Select( + &restoredData, + "SELECT id, name, value, created_at FROM test_data ORDER BY id", + ) + assert.NoError(t, err) + + assert.Equal(t, len(originalData), len(restoredData), "Should have same number of rows") + + if len(originalData) > 0 && len(restoredData) > 0 { + for i := range originalData { + assert.Equal(t, originalData[i].ID, restoredData[i].ID, "ID should match") + assert.Equal(t, originalData[i].Name, restoredData[i].Name, "Name should match") + assert.Equal(t, originalData[i].Value, restoredData[i].Value, "Value should match") + } + } +} + +func connectToMariadbContainer( + version tools.MariadbVersion, + port string, +) (*MariadbContainer, error) { + if port == "" { + return nil, fmt.Errorf("MariaDB %s port not configured", version) + } + + dbName := "testdb" + password := "rootpassword" + username := "root" + host := "127.0.0.1" + + portInt, err := strconv.Atoi(port) + if err != nil { + return nil, fmt.Errorf("failed to parse port: %w", err) + } + + dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + username, password, host, portInt, dbName) + + db, err := sqlx.Connect("mysql", dsn) + if err != nil { + return nil, fmt.Errorf("failed to connect to MariaDB database: %w", err) + } + + return &MariadbContainer{ + Host: host, + Port: portInt, + Username: username, + Password: password, + Database: dbName, + Version: version, + DB: db, + }, nil +} + +func setupMariadbTestData(t *testing.T, db *sqlx.DB) { + _, err := db.Exec(dropMariadbTestTableQuery) + assert.NoError(t, err) + + _, err = db.Exec(createMariadbTestTableQuery) + assert.NoError(t, err) + + _, err = db.Exec(insertMariadbTestDataQuery) + assert.NoError(t, err) +} + +func createMariadbReadOnlyUserViaAPI( + t *testing.T, + router *gin.Engine, + databaseID uuid.UUID, + token string, +) *databases.CreateReadOnlyUserResponse { + var database databases.Database + test_utils.MakeGetRequestAndUnmarshal( + t, + router, + fmt.Sprintf("/api/v1/databases/%s", databaseID.String()), + "Bearer "+token, + http.StatusOK, + &database, + ) + + var response databases.CreateReadOnlyUserResponse + test_utils.MakePostRequestAndUnmarshal( + t, + router, + "/api/v1/databases/create-readonly-user", + "Bearer "+token, + database, + http.StatusOK, + &response, + ) + + return &response +} + +func updateMariadbDatabaseCredentialsViaAPI( + t *testing.T, + router *gin.Engine, + database *databases.Database, + username string, + password string, + token string, +) *databases.Database { + database.Mariadb.Username = username + database.Mariadb.Password = password + + w := workspaces_testing.MakeAPIRequest( + router, + "POST", + "/api/v1/databases/update", + "Bearer "+token, + database, + ) + + if w.Code != http.StatusOK { + t.Fatalf("Failed to update MariaDB database. Status: %d, Body: %s", w.Code, w.Body.String()) + } + + var updatedDatabase databases.Database + if err := json.Unmarshal(w.Body.Bytes(), &updatedDatabase); err != nil { + t.Fatalf("Failed to unmarshal database response: %v", err) + } + + return &updatedDatabase +} diff --git a/backend/internal/util/tools/mariadb.go b/backend/internal/util/tools/mariadb.go new file mode 100644 index 0000000..cf2bfe8 --- /dev/null +++ b/backend/internal/util/tools/mariadb.go @@ -0,0 +1,241 @@ +package tools + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "runtime" + "strings" + + env_utils "postgresus-backend/internal/util/env" +) + +type MariadbVersion string + +const ( + MariadbVersion55 MariadbVersion = "5.5" + MariadbVersion101 MariadbVersion = "10.1" + MariadbVersion102 MariadbVersion = "10.2" + MariadbVersion103 MariadbVersion = "10.3" + MariadbVersion104 MariadbVersion = "10.4" + MariadbVersion105 MariadbVersion = "10.5" + MariadbVersion106 MariadbVersion = "10.6" + MariadbVersion1011 MariadbVersion = "10.11" + MariadbVersion114 MariadbVersion = "11.4" + MariadbVersion118 MariadbVersion = "11.8" + MariadbVersion120 MariadbVersion = "12.0" +) + +// MariadbClientVersion represents the client tool version to use +type MariadbClientVersion string + +const ( + // MariadbClientLegacy is used for older MariaDB servers (5.5, 10.1) that don't support + // the generation_expression column in information_schema.columns + MariadbClientLegacy MariadbClientVersion = "10.6" + // MariadbClientModern is used for newer MariaDB servers (10.2+) + MariadbClientModern MariadbClientVersion = "12.1" +) + +type MariadbExecutable string + +const ( + MariadbExecutableMariadbDump MariadbExecutable = "mariadb-dump" + MariadbExecutableMariadb MariadbExecutable = "mariadb" +) + +// GetMariadbClientVersionForServer returns the appropriate client version to use +// for a given server version. MariaDB 12.1 client uses SQL queries that reference +// the generation_expression column which was added in MariaDB 10.2, so older +// servers (5.5, 10.1) need the legacy 10.6 client. +func GetMariadbClientVersionForServer(serverVersion MariadbVersion) MariadbClientVersion { + switch serverVersion { + case MariadbVersion55, MariadbVersion101: + return MariadbClientLegacy + default: + return MariadbClientModern + } +} + +// GetMariadbExecutable returns the full path to a MariaDB executable. +// The serverVersion parameter determines which client tools to use: +// - For MariaDB 5.5 and 10.1: uses legacy 10.6 client (compatible with older servers) +// - For MariaDB 10.2+: uses modern 12.1 client +func GetMariadbExecutable( + executable MariadbExecutable, + serverVersion MariadbVersion, + envMode env_utils.EnvMode, + mariadbInstallDir string, +) string { + clientVersion := GetMariadbClientVersionForServer(serverVersion) + basePath := getMariadbBasePath(clientVersion, envMode, mariadbInstallDir) + executableName := string(executable) + + if runtime.GOOS == "windows" { + executableName += ".exe" + } + + return filepath.Join(basePath, executableName) +} + +// VerifyMariadbInstallation verifies that MariaDB client tools are installed. +// MariaDB uses two client versions: +// - Legacy (10.6) for older servers (5.5, 10.1) +// - Modern (12.1) for newer servers (10.2+) +func VerifyMariadbInstallation( + logger *slog.Logger, + envMode env_utils.EnvMode, + mariadbInstallDir string, +) { + clientVersions := []MariadbClientVersion{MariadbClientLegacy, MariadbClientModern} + + for _, clientVersion := range clientVersions { + binDir := getMariadbBasePath(clientVersion, envMode, mariadbInstallDir) + + logger.Info( + "Verifying MariaDB installation", + "clientVersion", clientVersion, + "path", binDir, + ) + + if _, err := os.Stat(binDir); os.IsNotExist(err) { + if envMode == env_utils.EnvModeDevelopment { + logger.Warn( + "MariaDB bin directory not found. Some MariaDB versions may not be supported. Read ./tools/readme.md for details", + "clientVersion", + clientVersion, + "path", + binDir, + ) + } else { + logger.Warn( + "MariaDB bin directory not found. Some MariaDB versions may not be supported.", + "clientVersion", clientVersion, + "path", binDir, + ) + } + continue + } + + requiredCommands := []MariadbExecutable{ + MariadbExecutableMariadbDump, + MariadbExecutableMariadb, + } + + for _, cmd := range requiredCommands { + // Use a dummy server version that maps to this client version + var dummyServerVersion MariadbVersion + if clientVersion == MariadbClientLegacy { + dummyServerVersion = MariadbVersion55 + } else { + dummyServerVersion = MariadbVersion102 + } + cmdPath := GetMariadbExecutable(cmd, dummyServerVersion, envMode, mariadbInstallDir) + + logger.Info( + "Checking for MariaDB command", + "clientVersion", clientVersion, + "command", cmd, + "path", cmdPath, + ) + + if _, err := os.Stat(cmdPath); os.IsNotExist(err) { + if envMode == env_utils.EnvModeDevelopment { + logger.Warn( + "MariaDB command not found. Some MariaDB versions may not be supported. Read ./tools/readme.md for details", + "clientVersion", + clientVersion, + "command", + cmd, + "path", + cmdPath, + ) + } else { + logger.Warn( + "MariaDB command not found. Some MariaDB versions may not be supported.", + "clientVersion", clientVersion, + "command", cmd, + "path", cmdPath, + ) + } + continue + } + + logger.Info("MariaDB command found", "clientVersion", clientVersion, "command", cmd) + } + } + + logger.Info("MariaDB client tools verification completed!") +} + +// IsMariadbBackupVersionHigherThanRestoreVersion checks if backup was made with +// a newer MariaDB version than the restore target +func IsMariadbBackupVersionHigherThanRestoreVersion( + backupVersion, restoreVersion MariadbVersion, +) bool { + versionOrder := map[MariadbVersion]int{ + MariadbVersion55: 1, + MariadbVersion101: 2, + MariadbVersion102: 3, + MariadbVersion103: 4, + MariadbVersion104: 5, + MariadbVersion105: 6, + MariadbVersion106: 7, + MariadbVersion1011: 8, + MariadbVersion114: 9, + MariadbVersion118: 10, + MariadbVersion120: 11, + } + return versionOrder[backupVersion] > versionOrder[restoreVersion] +} + +// GetMariadbVersionEnum converts a version string to MariadbVersion enum +func GetMariadbVersionEnum(version string) MariadbVersion { + switch version { + case "5.5": + return MariadbVersion55 + case "10.1": + return MariadbVersion101 + case "10.2": + return MariadbVersion102 + case "10.3": + return MariadbVersion103 + case "10.4": + return MariadbVersion104 + case "10.5": + return MariadbVersion105 + case "10.6": + return MariadbVersion106 + case "10.11": + return MariadbVersion1011 + case "11.4": + return MariadbVersion114 + case "11.8": + return MariadbVersion118 + case "12.0": + return MariadbVersion120 + default: + panic(fmt.Sprintf("invalid mariadb version: %s", version)) + } +} + +// EscapeMariadbPassword escapes special characters for MariaDB .my.cnf file format. +func EscapeMariadbPassword(password string) string { + password = strings.ReplaceAll(password, "\\", "\\\\") + password = strings.ReplaceAll(password, "\"", "\\\"") + return password +} + +func getMariadbBasePath( + clientVersion MariadbClientVersion, + envMode env_utils.EnvMode, + mariadbInstallDir string, +) string { + if envMode == env_utils.EnvModeDevelopment { + // Development: tools/mariadb/mariadb-{version}/bin + return filepath.Join(mariadbInstallDir, fmt.Sprintf("mariadb-%s", clientVersion), "bin") + } + // Production: /usr/local/mariadb-{version}/bin + return fmt.Sprintf("/usr/local/mariadb-%s/bin", clientVersion) +} diff --git a/backend/migrations/20251221100000_add_mariadb_databases_table.sql b/backend/migrations/20251221100000_add_mariadb_databases_table.sql new file mode 100644 index 0000000..c088df8 --- /dev/null +++ b/backend/migrations/20251221100000_add_mariadb_databases_table.sql @@ -0,0 +1,28 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE mariadb_databases ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + database_id UUID REFERENCES databases(id) ON DELETE CASCADE, + version TEXT NOT NULL, + host TEXT NOT NULL, + port INT NOT NULL, + username TEXT NOT NULL, + password TEXT NOT NULL, + database TEXT, + is_https BOOLEAN NOT NULL DEFAULT FALSE +); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE INDEX idx_mariadb_databases_database_id ON mariadb_databases(database_id); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP INDEX IF EXISTS idx_mariadb_databases_database_id; +-- +goose StatementEnd + +-- +goose StatementBegin +DROP TABLE IF EXISTS mariadb_databases; +-- +goose StatementEnd + diff --git a/backend/tools/.gitignore b/backend/tools/.gitignore index 8afcbdd..6510ae2 100644 --- a/backend/tools/.gitignore +++ b/backend/tools/.gitignore @@ -1,3 +1,4 @@ postgresql mysql -downloads \ No newline at end of file +downloads +mariadb \ No newline at end of file diff --git a/backend/tools/download_linux.sh b/backend/tools/download_linux.sh index a26de08..f3e6f19 100644 --- a/backend/tools/download_linux.sh +++ b/backend/tools/download_linux.sh @@ -156,12 +156,84 @@ for version in $mysql_versions; do echo done +# ========== MariaDB Installation ========== +echo "========================================" +echo "Installing MariaDB client tools (versions 10.6 and 12.1)..." +echo "========================================" + +# MariaDB uses two client versions: +# - 10.6 (legacy): For older servers (5.5, 10.1) that don't have generation_expression column +# - 12.1 (modern): For newer servers (10.2+) + +MARIADB_DIR="$(pwd)/mariadb" + +echo "Installing MariaDB client tools to: $MARIADB_DIR" + +# Install dependencies +$SUDO apt-get install -y -qq apt-transport-https curl + +# MariaDB versions to install with their URLs +declare -A MARIADB_URLS=( + ["10.6"]="https://archive.mariadb.org/mariadb-10.6.21/bintar-linux-systemd-x86_64/mariadb-10.6.21-linux-systemd-x86_64.tar.gz" + ["12.1"]="https://archive.mariadb.org/mariadb-12.1.2/bintar-linux-systemd-x86_64/mariadb-12.1.2-linux-systemd-x86_64.tar.gz" +) + +mariadb_versions="10.6 12.1" + +for version in $mariadb_versions; do + echo "Installing MariaDB $version client tools..." + + version_dir="$MARIADB_DIR/mariadb-$version" + mkdir -p "$version_dir/bin" + + # Skip if already exists + if [ -f "$version_dir/bin/mariadb-dump" ]; then + echo "MariaDB $version already installed, skipping..." + continue + fi + + url=${MARIADB_URLS[$version]} + + TEMP_DIR="/tmp/mariadb_install_$version" + mkdir -p "$TEMP_DIR" + cd "$TEMP_DIR" + + echo " Downloading MariaDB $version from official archive..." + wget -q "$url" -O "mariadb-$version.tar.gz" || { + echo " Warning: Could not download MariaDB $version binaries" + cd - >/dev/null + rm -rf "$TEMP_DIR" + continue + } + + echo " Extracting MariaDB $version..." + tar -xzf "mariadb-$version.tar.gz" + EXTRACTED_DIR=$(ls -d mariadb-*/ 2>/dev/null | head -1) + + if [ -d "$EXTRACTED_DIR" ] && [ -f "$EXTRACTED_DIR/bin/mariadb-dump" ]; then + cp "$EXTRACTED_DIR/bin/mariadb" "$version_dir/bin/" 2>/dev/null || true + cp "$EXTRACTED_DIR/bin/mariadb-dump" "$version_dir/bin/" 2>/dev/null || true + chmod +x "$version_dir/bin/"* + echo " MariaDB $version client tools installed successfully" + else + echo " Warning: Could not extract MariaDB $version binaries" + fi + + # Cleanup + cd - >/dev/null + rm -rf "$TEMP_DIR" + echo +done + +echo + echo "========================================" echo "Installation completed!" echo "========================================" echo echo "PostgreSQL client tools are available in: $POSTGRES_DIR" echo "MySQL client tools are available in: $MYSQL_DIR" +echo "MariaDB client tools are available in: $MARIADB_DIR" echo # List installed PostgreSQL versions @@ -186,7 +258,19 @@ for version in $mysql_versions; do fi done +echo +echo "Installed MariaDB client versions:" +for version in $mariadb_versions; do + version_dir="$MARIADB_DIR/mariadb-$version" + if [ -f "$version_dir/bin/mariadb-dump" ]; then + echo " mariadb-$version: $version_dir/bin/" + version_output=$("$version_dir/bin/mariadb-dump" --version 2>/dev/null | head -1) + echo " Version check: $version_output" + fi +done + echo echo "Usage examples:" echo " $POSTGRES_DIR/postgresql-15/bin/pg_dump --version" -echo " $MYSQL_DIR/mysql-8.0/bin/mysqldump --version" \ No newline at end of file +echo " $MYSQL_DIR/mysql-8.0/bin/mysqldump --version" +echo " $MARIADB_DIR/mariadb-12.1/bin/mariadb-dump --version" \ No newline at end of file diff --git a/backend/tools/download_macos.sh b/backend/tools/download_macos.sh index c9cf590..f6217e7 100755 --- a/backend/tools/download_macos.sh +++ b/backend/tools/download_macos.sh @@ -214,6 +214,78 @@ for version in $mysql_versions; do fi done +# ========== MariaDB Installation ========== +echo "========================================" +echo "Installing MariaDB client tools (versions 10.6 and 12.1)..." +echo "========================================" + +# MariaDB uses two client versions: +# - 10.6 (legacy): For older servers (5.5, 10.1) that don't have generation_expression column +# - 12.1 (modern): For newer servers (10.2+) + +MARIADB_DIR="$(pwd)/mariadb" + +echo "Installing MariaDB client tools to: $MARIADB_DIR" + +# MariaDB versions to install +# Note: MariaDB doesn't provide pre-built macOS binaries for older versions +# We install via Homebrew and use the same version for both (Homebrew only has latest) +# For production macOS use, the latest client should work with older servers for basic operations + +mariadb_versions="10.6 12.1" + +# Install MariaDB via Homebrew first (we'll use it for the modern version) +echo " Installing MariaDB via Homebrew..." +brew install mariadb 2>/dev/null || { + echo " Warning: Could not install mariadb via Homebrew" + brew install mariadb-connector-c 2>/dev/null || true +} + +# Find Homebrew MariaDB path +BREW_MARIADB="" +if [ -f "/opt/homebrew/bin/mariadb-dump" ]; then + BREW_MARIADB="/opt/homebrew/bin" +elif [ -f "/usr/local/bin/mariadb-dump" ]; then + BREW_MARIADB="/usr/local/bin" +else + BREW_PREFIX=$(brew --prefix mariadb 2>/dev/null || echo "") + if [ -n "$BREW_PREFIX" ] && [ -f "$BREW_PREFIX/bin/mariadb-dump" ]; then + BREW_MARIADB="$BREW_PREFIX/bin" + fi +fi + +for version in $mariadb_versions; do + echo "Setting up MariaDB $version client tools..." + + version_dir="$MARIADB_DIR/mariadb-$version" + mkdir -p "$version_dir/bin" + + # Skip if already exists + if [ -f "$version_dir/bin/mariadb-dump" ]; then + echo " MariaDB $version already installed, skipping..." + continue + fi + + if [ -n "$BREW_MARIADB" ]; then + # Link from Homebrew + # Note: On macOS, we use the same Homebrew version for both paths + # The Homebrew version (latest) should handle both old and new servers + ln -sf "$BREW_MARIADB/mariadb" "$version_dir/bin/mariadb" + ln -sf "$BREW_MARIADB/mariadb-dump" "$version_dir/bin/mariadb-dump" + echo " MariaDB $version client tools linked from Homebrew" + + # Test the installation + mariadb_ver=$("$version_dir/bin/mariadb-dump" --version 2>/dev/null | head -1) + echo " Verified: $mariadb_ver" + else + echo " Warning: Could not find MariaDB binaries for $version" + echo " Please install MariaDB manually: brew install mariadb" + fi + echo +done + +echo + # Clean up build directory echo "Cleaning up build directory..." rm -rf "$BUILD_DIR" @@ -224,6 +296,7 @@ echo "========================================" echo echo "PostgreSQL client tools are available in: $POSTGRES_DIR" echo "MySQL client tools are available in: $MYSQL_DIR" +echo "MariaDB client tools are available in: $MARIADB_DIR" echo # List installed PostgreSQL versions @@ -247,11 +320,24 @@ for version in $mysql_versions; do fi done +echo +echo "Installed MariaDB client versions:" +for version in $mariadb_versions; do + version_dir="$MARIADB_DIR/mariadb-$version" + if [ -f "$version_dir/bin/mariadb-dump" ]; then + mariadb_ver=$("$version_dir/bin/mariadb-dump" --version 2>/dev/null | head -1) + echo " mariadb-$version: $version_dir/bin/" + echo " $mariadb_ver" + fi +done + echo echo "Usage examples:" echo " $POSTGRES_DIR/postgresql-15/bin/pg_dump --version" echo " $MYSQL_DIR/mysql-8.0/bin/mysqldump --version" +echo " $MARIADB_DIR/mariadb-12.1/bin/mariadb-dump --version" echo echo "To add specific versions to your PATH temporarily:" echo " export PATH=\"$POSTGRES_DIR/postgresql-15/bin:\$PATH\"" -echo " export PATH=\"$MYSQL_DIR/mysql-8.0/bin:\$PATH\"" \ No newline at end of file +echo " export PATH=\"$MYSQL_DIR/mysql-8.0/bin:\$PATH\"" +echo " export PATH=\"$MARIADB_DIR/mariadb-12.1/bin:\$PATH\"" \ No newline at end of file diff --git a/backend/tools/download_windows.bat b/backend/tools/download_windows.bat index 5dcec23..9fe9e17 100644 --- a/backend/tools/download_windows.bat +++ b/backend/tools/download_windows.bat @@ -8,13 +8,16 @@ echo. if not exist "downloads" mkdir downloads if not exist "postgresql" mkdir postgresql if not exist "mysql" mkdir mysql +if not exist "mariadb" mkdir mariadb :: Get the absolute paths set "POSTGRES_DIR=%cd%\postgresql" set "MYSQL_DIR=%cd%\mysql" +set "MARIADB_DIR=%cd%\mariadb" echo PostgreSQL will be installed to: %POSTGRES_DIR% echo MySQL will be installed to: %MYSQL_DIR% +echo MariaDB will be installed to: %MARIADB_DIR% echo. cd downloads @@ -188,6 +191,101 @@ for %%v in (%mysql_versions%) do ( echo. ) +:: ========== MariaDB Installation ========== +echo ======================================== +echo Installing MariaDB client tools (versions 10.6 and 12.1)... +echo ======================================== +echo. + +:: MariaDB uses two client versions: +:: - 10.6 (legacy): For older servers (5.5, 10.1) that don't have generation_expression column +:: - 12.1 (modern): For newer servers (10.2+) + +:: MariaDB download URLs +set "MARIADB106_URL=https://archive.mariadb.org/mariadb-10.6.21/winx64-packages/mariadb-10.6.21-winx64.zip" +set "MARIADB121_URL=https://archive.mariadb.org/mariadb-12.1.2/winx64-packages/mariadb-12.1.2-winx64.zip" + +:: MariaDB versions to install +set "mariadb_versions=10.6 12.1" + +:: Download and install each MariaDB version +for %%v in (%mariadb_versions%) do ( + echo Processing MariaDB %%v... + set "version_underscore=%%v" + set "version_underscore=!version_underscore:.=!" + set "mariadb_install_dir=%MARIADB_DIR%\mariadb-%%v" + + :: Build the URL variable name and get its value + call set "current_url=%%MARIADB!version_underscore!_URL%%" + + :: Check if already installed + if exist "!mariadb_install_dir!\bin\mariadb-dump.exe" ( + echo MariaDB %%v already installed, skipping... + ) else ( + :: Extract version number from URL for filename + for %%u in ("!current_url!") do set "mariadb_filename=%%~nxu" + + if not exist "!mariadb_filename!" ( + echo Downloading MariaDB %%v... + echo Downloading from: !current_url! + curl -L -o "!mariadb_filename!" -A "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" "!current_url!" + if !errorlevel! neq 0 ( + echo ERROR: Download request failed + goto :next_mariadb_version + ) + if not exist "!mariadb_filename!" ( + echo ERROR: Download failed - file not created + goto :next_mariadb_version + ) + for %%s in ("!mariadb_filename!") do if %%~zs LSS 1000000 ( + echo ERROR: Download failed - file too small, likely error page + del "!mariadb_filename!" 2>nul + goto :next_mariadb_version + ) + echo MariaDB %%v downloaded successfully + ) else ( + echo MariaDB %%v already downloaded + ) + + :: Verify file exists before extraction + if not exist "!mariadb_filename!" ( + echo Download file not found, skipping extraction... + goto :next_mariadb_version + ) + + :: Extract MariaDB + echo Extracting MariaDB %%v... + mkdir "!mariadb_install_dir!" 2>nul + mkdir "!mariadb_install_dir!\bin" 2>nul + + powershell -Command "Expand-Archive -Path '!mariadb_filename!' -DestinationPath '!mariadb_install_dir!_temp' -Force" + + :: Move files from nested directory to install_dir + for /d %%d in ("!mariadb_install_dir!_temp\mariadb-*") do ( + if exist "%%d\bin\mariadb-dump.exe" ( + copy "%%d\bin\mariadb.exe" "!mariadb_install_dir!\bin\" >nul 2>&1 + copy "%%d\bin\mariadb-dump.exe" "!mariadb_install_dir!\bin\" >nul 2>&1 + ) + ) + + :: Cleanup temp directory + rmdir /s /q "!mariadb_install_dir!_temp" 2>nul + + :: Verify installation + if exist "!mariadb_install_dir!\bin\mariadb-dump.exe" ( + echo MariaDB %%v client tools installed successfully + ) else ( + echo Failed to install MariaDB %%v - mariadb-dump.exe not found + ) + ) + + :next_mariadb_version + echo. +) + +:skip_mariadb +echo. + cd .. echo. @@ -197,6 +295,7 @@ echo ======================================== echo. echo PostgreSQL versions are installed in: %POSTGRES_DIR% echo MySQL versions are installed in: %MYSQL_DIR% +echo MariaDB is installed in: %MARIADB_DIR% echo. :: List installed PostgreSQL versions @@ -217,10 +316,20 @@ for %%v in (%mysql_versions%) do ( ) ) +echo. +echo Installed MariaDB client versions: +for %%v in (%mariadb_versions%) do ( + set "version_dir=%MARIADB_DIR%\mariadb-%%v" + if exist "!version_dir!\bin\mariadb-dump.exe" ( + echo mariadb-%%v: !version_dir!\bin\ + ) +) + echo. echo Usage examples: echo %POSTGRES_DIR%\postgresql-15\bin\pg_dump.exe --version echo %MYSQL_DIR%\mysql-8.0\bin\mysqldump.exe --version +echo %MARIADB_DIR%\mariadb-12.1\bin\mariadb-dump.exe --version echo. pause diff --git a/backend/tools/readme.md b/backend/tools/readme.md index 0ae5ae5..4b83bb2 100644 --- a/backend/tools/readme.md +++ b/backend/tools/readme.md @@ -1,7 +1,7 @@ This directory is needed only for development and CI\CD. -We have to download and install all the PostgreSQL versions from 12 to 18 and MySQL versions 5.7, 8.0, 8.4 locally. -This is needed so we can call pg_dump, pg_restore, mysqldump, mysql, etc. on each version of the database. +We have to download and install all the PostgreSQL versions from 12 to 18, MySQL versions 5.7, 8.0, 8.4 and MariaDB client tools locally. +This is needed so we can call pg_dump, pg_restore, mysqldump, mysql, mariadb-dump, mariadb, etc. on each version of the database. You do not need to install the databases fully with all the components. We only need the client tools for each version. @@ -24,6 +24,15 @@ We only need the client tools for each version. - MySQL 8.0 - MySQL 8.4 +### MariaDB + +MariaDB uses two client versions to support all server versions: + +- MariaDB 10.6 (legacy client - for older servers 5.5 and 10.1) +- MariaDB 12.1 (modern client - for servers 10.2+) + +The reason for two versions is that MariaDB 12.1 client uses SQL queries that reference the `generation_expression` column in `information_schema.columns`, which was only added in MariaDB 10.2. Older servers (5.5, 10.1) don't have this column and fail with newer clients. + ## Installation Run the appropriate download script for your platform: @@ -61,6 +70,7 @@ chmod +x download_macos.sh - Uses the official PostgreSQL APT repository - Downloads MySQL client tools from official archives +- Installs MariaDB client from official MariaDB repository - Requires sudo privileges to install packages - Creates symlinks in version-specific directories for consistency @@ -69,6 +79,7 @@ chmod +x download_macos.sh - Requires Homebrew to be installed - Compiles PostgreSQL from source (client tools only) - Downloads pre-built MySQL binaries from dev.mysql.com +- Downloads pre-built MariaDB binaries or installs via Homebrew - Takes longer than other platforms due to PostgreSQL compilation - Supports both Intel (x86_64) and Apple Silicon (arm64) @@ -109,6 +120,20 @@ For example: - `./tools/mysql/mysql-8.0/bin/mysqldump` - `./tools/mysql/mysql-8.4/bin/mysqldump` +### MariaDB + +MariaDB uses two client versions to handle compatibility with all server versions: + +``` +./tools/mariadb/mariadb-{client-version}/bin/mariadb-dump +./tools/mariadb/mariadb-{client-version}/bin/mariadb +``` + +For example: + +- `./tools/mariadb/mariadb-10.6/bin/mariadb-dump` (legacy - for servers 5.5, 10.1) +- `./tools/mariadb/mariadb-12.1/bin/mariadb-dump` (modern - for servers 10.2+) + ## Usage After installation, you can use version-specific tools: @@ -120,11 +145,17 @@ After installation, you can use version-specific tools: # Windows - MySQL ./mysql/mysql-8.0/bin/mysqldump.exe --version +# Windows - MariaDB +./mariadb/mariadb-12.1/bin/mariadb-dump.exe --version + # Linux/MacOS - PostgreSQL ./postgresql/postgresql-15/bin/pg_dump --version # Linux/MacOS - MySQL ./mysql/mysql-8.0/bin/mysqldump --version + +# Linux/MacOS - MariaDB +./mariadb/mariadb-12.1/bin/mariadb-dump --version ``` ## Environment Variables @@ -137,6 +168,10 @@ POSTGRES_INSTALL_DIR=C:\path\to\tools\postgresql # MySQL tools directory (default: ./tools/mysql) MYSQL_INSTALL_DIR=C:\path\to\tools\mysql + +# MariaDB tools directory (default: ./tools/mariadb) +# Contains subdirectories: mariadb-10.6 and mariadb-12.1 +MARIADB_INSTALL_DIR=C:\path\to\tools\mariadb ``` ## Troubleshooting @@ -162,3 +197,22 @@ If downloads fail, you can manually download the files: - PostgreSQL: https://www.postgresql.org/ftp/source/ - MySQL: https://dev.mysql.com/downloads/mysql/ +- MariaDB: https://mariadb.org/download/ or https://cdn.mysql.com/archives/mariadb-12.0/ + +### MariaDB Client Compatibility + +MariaDB client tools require different versions depending on the server: + +**Legacy client (10.6)** - Required for: + +- MariaDB 5.5 +- MariaDB 10.1 + +**Modern client (12.1)** - Works with: + +- MariaDB 10.2 - 10.6 +- MariaDB 10.11 +- MariaDB 11.4, 11.8 +- MariaDB 12.0 + +The reason is that MariaDB 12.1 client uses SQL queries referencing the `generation_expression` column in `information_schema.columns`, which was added in MariaDB 10.2. The application automatically selects the appropriate client version based on the target server version. diff --git a/frontend/public/icons/databases/mariadb.svg b/frontend/public/icons/databases/mariadb.svg new file mode 100644 index 0000000..7ada63a --- /dev/null +++ b/frontend/public/icons/databases/mariadb.svg @@ -0,0 +1,2 @@ + +file_type_mariadb \ No newline at end of file diff --git a/frontend/src/entity/databases/index.ts b/frontend/src/entity/databases/index.ts index f62116d..385d8ba 100644 --- a/frontend/src/entity/databases/index.ts +++ b/frontend/src/entity/databases/index.ts @@ -7,5 +7,7 @@ export { type PostgresqlDatabase } from './model/postgresql/PostgresqlDatabase'; export { PostgresqlVersion } from './model/postgresql/PostgresqlVersion'; export { type MysqlDatabase } from './model/mysql/MysqlDatabase'; export { MysqlVersion } from './model/mysql/MysqlVersion'; +export { type MariadbDatabase } from './model/mariadb/MariadbDatabase'; +export { MariadbVersion } from './model/mariadb/MariadbVersion'; export { type IsReadOnlyResponse } from './model/IsReadOnlyResponse'; export { type CreateReadOnlyUserResponse } from './model/CreateReadOnlyUserResponse'; diff --git a/frontend/src/entity/databases/model/Database.ts b/frontend/src/entity/databases/model/Database.ts index b46242b..458e72e 100644 --- a/frontend/src/entity/databases/model/Database.ts +++ b/frontend/src/entity/databases/model/Database.ts @@ -1,6 +1,7 @@ import type { Notifier } from '../../notifiers'; import type { DatabaseType } from './DatabaseType'; import type { HealthStatus } from './HealthStatus'; +import type { MariadbDatabase } from './mariadb/MariadbDatabase'; import type { MysqlDatabase } from './mysql/MysqlDatabase'; import type { PostgresqlDatabase } from './postgresql/PostgresqlDatabase'; @@ -12,6 +13,7 @@ export interface Database { postgresql?: PostgresqlDatabase; mysql?: MysqlDatabase; + mariadb?: MariadbDatabase; notifiers: Notifier[]; diff --git a/frontend/src/entity/databases/model/DatabaseType.ts b/frontend/src/entity/databases/model/DatabaseType.ts index a36a2aa..59f666a 100644 --- a/frontend/src/entity/databases/model/DatabaseType.ts +++ b/frontend/src/entity/databases/model/DatabaseType.ts @@ -1,4 +1,5 @@ export enum DatabaseType { POSTGRES = 'POSTGRES', MYSQL = 'MYSQL', + MARIADB = 'MARIADB', } diff --git a/frontend/src/entity/databases/model/getDatabaseLogoFromType.ts b/frontend/src/entity/databases/model/getDatabaseLogoFromType.ts index 6c81d40..d0ccd97 100644 --- a/frontend/src/entity/databases/model/getDatabaseLogoFromType.ts +++ b/frontend/src/entity/databases/model/getDatabaseLogoFromType.ts @@ -6,6 +6,8 @@ export const getDatabaseLogoFromType = (type: DatabaseType) => { return '/icons/databases/postgresql.svg'; case DatabaseType.MYSQL: return '/icons/databases/mysql.svg'; + case DatabaseType.MARIADB: + return '/icons/databases/mariadb.svg'; default: return ''; } diff --git a/frontend/src/entity/databases/model/mariadb/MariadbConnectionStringParser.ts b/frontend/src/entity/databases/model/mariadb/MariadbConnectionStringParser.ts new file mode 100644 index 0000000..182ae39 --- /dev/null +++ b/frontend/src/entity/databases/model/mariadb/MariadbConnectionStringParser.ts @@ -0,0 +1,278 @@ +export type ParseResult = { + host: string; + port: number; + username: string; + password: string; + database: string; + isHttps: boolean; +}; + +export type ParseError = { + error: string; + format?: string; +}; + +export class MariadbConnectionStringParser { + /** + * Parses a MariaDB connection string in various formats. + * + * Supported formats: + * 1. Standard MariaDB URI: mariadb://user:pass@host:port/db + * 2. MySQL URI (compatible): mysql://user:pass@host:port/db + * 3. JDBC format: jdbc:mariadb://host:port/db?user=x&password=y + * 4. JDBC MySQL format: jdbc:mysql://host:port/db?user=x&password=y + * 5. Key-value format: host=x port=3306 database=db user=u password=p + * 6. With SSL params: mariadb://user:pass@host:port/db?ssl=true or ?sslMode=REQUIRED + * 7. SkySQL: mariadb://user:pass@xxx.skysql.net:5001/db?ssl=true + */ + static parse(connectionString: string): ParseResult | ParseError { + const trimmed = connectionString.trim(); + + if (!trimmed) { + return { error: 'Connection string is empty' }; + } + + // Try JDBC format first (starts with jdbc:) + if (trimmed.startsWith('jdbc:mariadb://') || trimmed.startsWith('jdbc:mysql://')) { + return this.parseJdbc(trimmed); + } + + // Try key-value format (contains key=value pairs without ://) + if (this.isKeyValueFormat(trimmed)) { + return this.parseKeyValue(trimmed); + } + + // Try URI format (mariadb:// or mysql://) + if (trimmed.startsWith('mariadb://') || trimmed.startsWith('mysql://')) { + return this.parseUri(trimmed); + } + + return { + error: 'Unrecognized connection string format', + }; + } + + private static isKeyValueFormat(str: string): boolean { + return ( + !str.includes('://') && + (str.includes('host=') || str.includes('database=')) && + str.includes('=') + ); + } + + private static parseUri(connectionString: string): ParseResult | ParseError { + try { + // Handle Azure format where username contains @: user@server:pass + const azureMatch = connectionString.match( + /^(?:mariadb|mysql):\/\/([^@:]+)@([^:]+):([^@]+)@([^:/?]+):?(\d+)?\/([^?]+)(?:\?(.*))?$/, + ); + + if (azureMatch) { + const [, user, , password, host, port, database, queryString] = azureMatch; + const isHttps = this.checkSslMode(queryString); + + return { + host: host, + port: port ? parseInt(port, 10) : 3306, + username: decodeURIComponent(user), + password: decodeURIComponent(password), + database: decodeURIComponent(database), + isHttps, + }; + } + + // Standard URI parsing using URL API + const url = new URL(connectionString); + + const host = url.hostname; + const port = url.port ? parseInt(url.port, 10) : 3306; + const username = decodeURIComponent(url.username); + const password = decodeURIComponent(url.password); + const database = decodeURIComponent(url.pathname.slice(1)); + const isHttps = this.checkSslMode(url.search); + + if (!host) { + return { error: 'Host is missing from connection string' }; + } + + if (!username) { + return { error: 'Username is missing from connection string' }; + } + + if (!password) { + return { error: 'Password is missing from connection string' }; + } + + if (!database) { + return { error: 'Database name is missing from connection string' }; + } + + return { + host, + port, + username, + password, + database, + isHttps, + }; + } catch (e) { + return { + error: `Failed to parse connection string: ${(e as Error).message}`, + format: 'URI', + }; + } + } + + private static parseJdbc(connectionString: string): ParseResult | ParseError { + try { + const jdbcRegex = /^jdbc:(?:mariadb|mysql):\/\/([^:/?]+):?(\d+)?\/([^?]+)(?:\?(.*))?$/; + const match = connectionString.match(jdbcRegex); + + if (!match) { + return { + error: + 'Invalid JDBC connection string format. Expected: jdbc:mariadb://host:port/database?user=x&password=y', + format: 'JDBC', + }; + } + + const [, host, port, database, queryString] = match; + + if (!queryString) { + return { + error: 'JDBC connection string is missing query parameters (user and password)', + format: 'JDBC', + }; + } + + const params = new URLSearchParams(queryString); + const username = params.get('user'); + const password = params.get('password'); + const isHttps = this.checkSslMode(queryString); + + if (!username) { + return { + error: 'Username (user parameter) is missing from JDBC connection string', + format: 'JDBC', + }; + } + + if (!password) { + return { + error: 'Password parameter is missing from JDBC connection string', + format: 'JDBC', + }; + } + + return { + host, + port: port ? parseInt(port, 10) : 3306, + username: decodeURIComponent(username), + password: decodeURIComponent(password), + database: decodeURIComponent(database), + isHttps, + }; + } catch (e) { + return { + error: `Failed to parse JDBC connection string: ${(e as Error).message}`, + format: 'JDBC', + }; + } + } + + private static parseKeyValue(connectionString: string): ParseResult | ParseError { + try { + const params: Record = {}; + + const regex = /(\w+)=(?:'([^']*)'|(\S+))/g; + let match; + + while ((match = regex.exec(connectionString)) !== null) { + const key = match[1]; + const value = match[2] !== undefined ? match[2] : match[3]; + params[key] = value; + } + + const host = params['host'] || params['hostaddr']; + const port = params['port']; + const database = params['database'] || params['dbname']; + const username = params['user'] || params['username']; + const password = params['password']; + const ssl = params['ssl'] || params['sslMode'] || params['ssl-mode'] || params['useSSL']; + + if (!host) { + return { + error: 'Host is missing from connection string. Use host=hostname', + format: 'key-value', + }; + } + + if (!username) { + return { + error: 'Username is missing from connection string. Use user=username', + format: 'key-value', + }; + } + + if (!password) { + return { + error: 'Password is missing from connection string. Use password=yourpassword', + format: 'key-value', + }; + } + + if (!database) { + return { + error: 'Database name is missing from connection string. Use database=database', + format: 'key-value', + }; + } + + const isHttps = this.isSslEnabled(ssl); + + return { + host, + port: port ? parseInt(port, 10) : 3306, + username, + password, + database, + isHttps, + }; + } catch (e) { + return { + error: `Failed to parse key-value connection string: ${(e as Error).message}`, + format: 'key-value', + }; + } + } + + private static checkSslMode(queryString: string | undefined | null): boolean { + if (!queryString) return false; + + const params = new URLSearchParams( + queryString.startsWith('?') ? queryString.slice(1) : queryString, + ); + + const ssl = params.get('ssl'); + const sslMode = params.get('sslMode'); + const sslModeHyphen = params.get('ssl-mode'); + const useSSL = params.get('useSSL'); + const sslaccept = params.get('sslaccept'); + + if (ssl) return this.isSslEnabled(ssl); + if (sslMode) return this.isSslEnabled(sslMode); + if (sslModeHyphen) return this.isSslEnabled(sslModeHyphen); + if (useSSL) return this.isSslEnabled(useSSL); + if (sslaccept) return sslaccept.toLowerCase() === 'strict'; + + return false; + } + + private static isSslEnabled(sslValue: string | null | undefined): boolean { + if (!sslValue) return false; + + const lowercased = sslValue.toLowerCase(); + const enabledValues = ['true', 'required', 'verify_ca', 'verify_identity', 'yes', '1']; + return enabledValues.includes(lowercased); + } +} diff --git a/frontend/src/entity/databases/model/mariadb/MariadbDatabase.ts b/frontend/src/entity/databases/model/mariadb/MariadbDatabase.ts new file mode 100644 index 0000000..e2cb282 --- /dev/null +++ b/frontend/src/entity/databases/model/mariadb/MariadbDatabase.ts @@ -0,0 +1,12 @@ +import type { MariadbVersion } from './MariadbVersion'; + +export interface MariadbDatabase { + id: string; + version: MariadbVersion; + host: string; + port: number; + username: string; + password: string; + database?: string; + isHttps: boolean; +} diff --git a/frontend/src/entity/databases/model/mariadb/MariadbVersion.ts b/frontend/src/entity/databases/model/mariadb/MariadbVersion.ts new file mode 100644 index 0000000..9ed5ae4 --- /dev/null +++ b/frontend/src/entity/databases/model/mariadb/MariadbVersion.ts @@ -0,0 +1,13 @@ +export enum MariadbVersion { + MariadbVersion55 = '5.5', + MariadbVersion101 = '10.1', + MariadbVersion102 = '10.2', + MariadbVersion103 = '10.3', + MariadbVersion104 = '10.4', + MariadbVersion105 = '10.5', + MariadbVersion106 = '10.6', + MariadbVersion1011 = '10.11', + MariadbVersion114 = '11.4', + MariadbVersion118 = '11.8', + MariadbVersion120 = '12.0', +} diff --git a/frontend/src/entity/restores/api/restoreApi.ts b/frontend/src/entity/restores/api/restoreApi.ts index ce6a7bf..4bef134 100644 --- a/frontend/src/entity/restores/api/restoreApi.ts +++ b/frontend/src/entity/restores/api/restoreApi.ts @@ -1,7 +1,7 @@ import { getApplicationServer } from '../../../constants'; import RequestOptions from '../../../shared/api/RequestOptions'; import { apiHelper } from '../../../shared/api/apiHelper'; -import type { MysqlDatabase, PostgresqlDatabase } from '../../databases'; +import type { MariadbDatabase, MysqlDatabase, PostgresqlDatabase } from '../../databases'; import type { Restore } from '../model/Restore'; export const restoreApi = { @@ -17,16 +17,19 @@ export const restoreApi = { backupId, postgresql, mysql, + mariadb, }: { backupId: string; postgresql?: PostgresqlDatabase; mysql?: MysqlDatabase; + mariadb?: MariadbDatabase; }) { const requestOptions: RequestOptions = new RequestOptions(); requestOptions.setBody( JSON.stringify({ postgresqlDatabase: postgresql, mysqlDatabase: mysql, + mariadbDatabase: mariadb, }), ); diff --git a/frontend/src/features/backups/ui/BackupsComponent.tsx b/frontend/src/features/backups/ui/BackupsComponent.tsx index 484ae96..da2b6de 100644 --- a/frontend/src/features/backups/ui/BackupsComponent.tsx +++ b/frontend/src/features/backups/ui/BackupsComponent.tsx @@ -655,6 +655,7 @@ export const BackupsComponent = ({ database, isCanManageDBs, scrollContainerRef onCancel={() => setShowingRestoresBackupId(undefined)} title="Restore from backup" footer={null} + maskClosable={false} > setShowingBackupError(undefined)} + maskClosable={false} footer={null} >
{showingBackupError.failMessage}
diff --git a/frontend/src/features/databases/ui/CreateDatabaseComponent.tsx b/frontend/src/features/databases/ui/CreateDatabaseComponent.tsx index 3f6dfac..dcab9c7 100644 --- a/frontend/src/features/databases/ui/CreateDatabaseComponent.tsx +++ b/frontend/src/features/databases/ui/CreateDatabaseComponent.tsx @@ -4,6 +4,7 @@ import { type BackupConfig, backupConfigApi, backupsApi } from '../../../entity/ import { type Database, DatabaseType, + type MariadbDatabase, type MysqlDatabase, Period, type PostgresqlDatabase, @@ -40,15 +41,18 @@ const createInitialDatabase = (workspaceId: string): Database => }) as Database; const initializeDatabaseTypeData = (db: Database): Database => { - if (db.type === DatabaseType.POSTGRES && !db.postgresql) { - return { ...db, postgresql: {} as PostgresqlDatabase, mysql: undefined }; - } + const base = { ...db, postgresql: undefined, mysql: undefined, mariadb: undefined }; - if (db.type === DatabaseType.MYSQL && !db.mysql) { - return { ...db, mysql: {} as MysqlDatabase, postgresql: undefined }; + switch (db.type) { + case DatabaseType.POSTGRES: + return { ...base, postgresql: db.postgresql ?? ({} as PostgresqlDatabase) }; + case DatabaseType.MYSQL: + return { ...base, mysql: db.mysql ?? ({} as MysqlDatabase) }; + case DatabaseType.MARIADB: + return { ...base, mariadb: db.mariadb ?? ({} as MariadbDatabase) }; + default: + return db; } - - return db; }; export const CreateDatabaseComponent = ({ workspaceId, onCreated, onClose }: Props) => { diff --git a/frontend/src/features/databases/ui/edit/EditDatabaseBaseInfoComponent.tsx b/frontend/src/features/databases/ui/edit/EditDatabaseBaseInfoComponent.tsx index 852f64c..3de4e9e 100644 --- a/frontend/src/features/databases/ui/edit/EditDatabaseBaseInfoComponent.tsx +++ b/frontend/src/features/databases/ui/edit/EditDatabaseBaseInfoComponent.tsx @@ -4,6 +4,7 @@ import { useEffect, useState } from 'react'; import { type Database, DatabaseType, + type MariadbDatabase, type MysqlDatabase, type PostgresqlDatabase, databaseApi, @@ -26,6 +27,7 @@ interface Props { const databaseTypeOptions = [ { value: DatabaseType.POSTGRES, label: 'PostgreSQL' }, { value: DatabaseType.MYSQL, label: 'MySQL' }, + { value: DatabaseType.MARIADB, label: 'MariaDB' }, ]; export const EditDatabaseBaseInfoComponent = ({ @@ -53,14 +55,21 @@ export const EditDatabaseBaseInfoComponent = ({ const updatedDatabase: Database = { ...editingDatabase, type: newType, + postgresql: undefined, + mysql: undefined, + mariadb: undefined, }; - if (newType === DatabaseType.POSTGRES && !editingDatabase.postgresql) { - updatedDatabase.postgresql = {} as PostgresqlDatabase; - updatedDatabase.mysql = undefined; - } else if (newType === DatabaseType.MYSQL && !editingDatabase.mysql) { - updatedDatabase.mysql = {} as MysqlDatabase; - updatedDatabase.postgresql = undefined; + switch (newType) { + case DatabaseType.POSTGRES: + updatedDatabase.postgresql = editingDatabase.postgresql ?? ({} as PostgresqlDatabase); + break; + case DatabaseType.MYSQL: + updatedDatabase.mysql = editingDatabase.mysql ?? ({} as MysqlDatabase); + break; + case DatabaseType.MARIADB: + updatedDatabase.mariadb = editingDatabase.mariadb ?? ({} as MariadbDatabase); + break; } setEditingDatabase(updatedDatabase); diff --git a/frontend/src/features/databases/ui/edit/EditDatabaseSpecificDataComponent.tsx b/frontend/src/features/databases/ui/edit/EditDatabaseSpecificDataComponent.tsx index a385638..bd952fd 100644 --- a/frontend/src/features/databases/ui/edit/EditDatabaseSpecificDataComponent.tsx +++ b/frontend/src/features/databases/ui/edit/EditDatabaseSpecificDataComponent.tsx @@ -1,4 +1,5 @@ import { type Database, DatabaseType } from '../../../../entity/databases'; +import { EditMariaDbSpecificDataComponent } from './EditMariaDbSpecificDataComponent'; import { EditMySqlSpecificDataComponent } from './EditMySqlSpecificDataComponent'; import { EditPostgreSqlSpecificDataComponent } from './EditPostgreSqlSpecificDataComponent'; @@ -34,38 +35,26 @@ export const EditDatabaseSpecificDataComponent = ({ isShowDbName = true, isRestoreMode = false, }: Props) => { - if (database.type === DatabaseType.POSTGRES) { - return ( - - ); - } + const commonProps = { + database, + isShowCancelButton, + onCancel, + isShowBackButton, + onBack, + saveButtonText, + isSaveToApi, + onSaved, + isShowDbName, + }; - if (database.type === DatabaseType.MYSQL) { - return ( - - ); + switch (database.type) { + case DatabaseType.POSTGRES: + return ; + case DatabaseType.MYSQL: + return ; + case DatabaseType.MARIADB: + return ; + default: + return null; } - - return null; }; diff --git a/frontend/src/features/databases/ui/edit/EditMariaDbSpecificDataComponent.tsx b/frontend/src/features/databases/ui/edit/EditMariaDbSpecificDataComponent.tsx new file mode 100644 index 0000000..d24247d --- /dev/null +++ b/frontend/src/features/databases/ui/edit/EditMariaDbSpecificDataComponent.tsx @@ -0,0 +1,347 @@ +import { CopyOutlined } from '@ant-design/icons'; +import { App, Button, Input, InputNumber, Switch } from 'antd'; +import { useEffect, useState } from 'react'; + +import { type Database, databaseApi } from '../../../../entity/databases'; +import { MariadbConnectionStringParser } from '../../../../entity/databases/model/mariadb/MariadbConnectionStringParser'; +import { ToastHelper } from '../../../../shared/toast'; + +interface Props { + database: Database; + + isShowCancelButton?: boolean; + onCancel: () => void; + + isShowBackButton: boolean; + onBack: () => void; + + saveButtonText?: string; + isSaveToApi: boolean; + onSaved: (database: Database) => void; + + isShowDbName?: boolean; +} + +export const EditMariaDbSpecificDataComponent = ({ + database, + + isShowCancelButton, + onCancel, + + isShowBackButton, + onBack, + + saveButtonText, + isSaveToApi, + onSaved, + isShowDbName = true, +}: Props) => { + const { message } = App.useApp(); + + const [editingDatabase, setEditingDatabase] = useState(); + const [isSaving, setIsSaving] = useState(false); + + const [isConnectionTested, setIsConnectionTested] = useState(false); + const [isTestingConnection, setIsTestingConnection] = useState(false); + const [isConnectionFailed, setIsConnectionFailed] = useState(false); + + const parseFromClipboard = async () => { + try { + const text = await navigator.clipboard.readText(); + const trimmedText = text.trim(); + + if (!trimmedText) { + message.error('Clipboard is empty'); + return; + } + + const result = MariadbConnectionStringParser.parse(trimmedText); + + if ('error' in result) { + message.error(result.error); + return; + } + + if (!editingDatabase?.mariadb) return; + + const updatedDatabase: Database = { + ...editingDatabase, + mariadb: { + ...editingDatabase.mariadb, + host: result.host, + port: result.port, + username: result.username, + password: result.password, + database: result.database, + isHttps: result.isHttps, + }, + }; + + setEditingDatabase(updatedDatabase); + setIsConnectionTested(false); + message.success('Connection string parsed successfully'); + } catch { + message.error('Failed to read clipboard. Please check browser permissions.'); + } + }; + + const testConnection = async () => { + if (!editingDatabase) return; + setIsTestingConnection(true); + setIsConnectionFailed(false); + + try { + await databaseApi.testDatabaseConnectionDirect(editingDatabase); + setIsConnectionTested(true); + ToastHelper.showToast({ + title: 'Connection test passed', + description: 'You can continue with the next step', + }); + } catch (e) { + setIsConnectionFailed(true); + alert((e as Error).message); + } + + setIsTestingConnection(false); + }; + + const saveDatabase = async () => { + if (!editingDatabase) return; + + if (isSaveToApi) { + setIsSaving(true); + + try { + await databaseApi.updateDatabase(editingDatabase); + } catch (e) { + alert((e as Error).message); + } + + setIsSaving(false); + } + + onSaved(editingDatabase); + }; + + useEffect(() => { + setIsSaving(false); + setIsConnectionTested(false); + setIsTestingConnection(false); + setIsConnectionFailed(false); + + setEditingDatabase({ ...database }); + }, [database]); + + if (!editingDatabase) return null; + + let isAllFieldsFilled = true; + if (!editingDatabase.mariadb?.host) isAllFieldsFilled = false; + if (!editingDatabase.mariadb?.port) isAllFieldsFilled = false; + if (!editingDatabase.mariadb?.username) isAllFieldsFilled = false; + if (!editingDatabase.id && !editingDatabase.mariadb?.password) isAllFieldsFilled = false; + if (!editingDatabase.mariadb?.database) isAllFieldsFilled = false; + + const isLocalhostDb = + editingDatabase.mariadb?.host?.includes('localhost') || + editingDatabase.mariadb?.host?.includes('127.0.0.1'); + + return ( +
+
+
+
+ + Parse from clipboard +
+
+ +
+
Host
+ { + if (!editingDatabase.mariadb) return; + + setEditingDatabase({ + ...editingDatabase, + mariadb: { + ...editingDatabase.mariadb, + host: e.target.value.trim().replace('https://', '').replace('http://', ''), + }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter MariaDB host" + /> +
+ + {isLocalhostDb && ( +
+
+
+ Please{' '} + + read this document + {' '} + to study how to backup local database +
+
+ )} + +
+
Port
+ { + if (!editingDatabase.mariadb || e === null) return; + + setEditingDatabase({ + ...editingDatabase, + mariadb: { ...editingDatabase.mariadb, port: e }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter MariaDB port" + /> +
+ +
+
Username
+ { + if (!editingDatabase.mariadb) return; + + setEditingDatabase({ + ...editingDatabase, + mariadb: { ...editingDatabase.mariadb, username: e.target.value.trim() }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter MariaDB username" + /> +
+ +
+
Password
+ { + if (!editingDatabase.mariadb) return; + + setEditingDatabase({ + ...editingDatabase, + mariadb: { ...editingDatabase.mariadb, password: e.target.value.trim() }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter MariaDB password" + autoComplete="off" + data-1p-ignore + data-lpignore="true" + data-form-type="other" + /> +
+ + {isShowDbName && ( +
+
DB name
+ { + if (!editingDatabase.mariadb) return; + + setEditingDatabase({ + ...editingDatabase, + mariadb: { ...editingDatabase.mariadb, database: e.target.value.trim() }, + }); + setIsConnectionTested(false); + }} + size="small" + className="max-w-[200px] grow" + placeholder="Enter MariaDB database name" + /> +
+ )} + +
+
Use HTTPS
+ { + if (!editingDatabase.mariadb) return; + + setEditingDatabase({ + ...editingDatabase, + mariadb: { ...editingDatabase.mariadb, isHttps: checked }, + }); + setIsConnectionTested(false); + }} + size="small" + /> +
+ +
+ {isShowCancelButton && ( + + )} + + {isShowBackButton && ( + + )} + + {!isConnectionTested && ( + + )} + + {isConnectionTested && ( + + )} +
+ + {isConnectionFailed && ( +
+ If your database uses IP whitelist, make sure Postgresus server IP is added to the allowed + list. +
+ )} +
+ ); +}; diff --git a/frontend/src/features/databases/ui/edit/EditMySqlSpecificDataComponent.tsx b/frontend/src/features/databases/ui/edit/EditMySqlSpecificDataComponent.tsx index 07988df..a997a8b 100644 --- a/frontend/src/features/databases/ui/edit/EditMySqlSpecificDataComponent.tsx +++ b/frontend/src/features/databases/ui/edit/EditMySqlSpecificDataComponent.tsx @@ -253,7 +253,10 @@ export const EditMySqlSpecificDataComponent = ({ size="small" className="max-w-[200px] grow" placeholder="Enter MySQL password" - autoComplete="new-password" + autoComplete="off" + data-1p-ignore + data-lpignore="true" + data-form-type="other" />
diff --git a/frontend/src/features/databases/ui/edit/EditPostgreSqlSpecificDataComponent.tsx b/frontend/src/features/databases/ui/edit/EditPostgreSqlSpecificDataComponent.tsx index 4ab2a77..4dc6fe1 100644 --- a/frontend/src/features/databases/ui/edit/EditPostgreSqlSpecificDataComponent.tsx +++ b/frontend/src/features/databases/ui/edit/EditPostgreSqlSpecificDataComponent.tsx @@ -310,7 +310,10 @@ export const EditPostgreSqlSpecificDataComponent = ({ size="small" className="max-w-[200px] grow" placeholder="Enter PG password" - autoComplete="new-password" + autoComplete="off" + data-1p-ignore + data-lpignore="true" + data-form-type="other" />
diff --git a/frontend/src/features/databases/ui/show/ShowDatabaseSpecificDataComponent.tsx b/frontend/src/features/databases/ui/show/ShowDatabaseSpecificDataComponent.tsx index 3d6421f..6fbfdc2 100644 --- a/frontend/src/features/databases/ui/show/ShowDatabaseSpecificDataComponent.tsx +++ b/frontend/src/features/databases/ui/show/ShowDatabaseSpecificDataComponent.tsx @@ -1,4 +1,5 @@ import { type Database, DatabaseType } from '../../../../entity/databases'; +import { ShowMariaDbSpecificDataComponent } from './ShowMariaDbSpecificDataComponent'; import { ShowMySqlSpecificDataComponent } from './ShowMySqlSpecificDataComponent'; import { ShowPostgreSqlSpecificDataComponent } from './ShowPostgreSqlSpecificDataComponent'; @@ -7,13 +8,14 @@ interface Props { } export const ShowDatabaseSpecificDataComponent = ({ database }: Props) => { - if (database.type === DatabaseType.POSTGRES) { - return ; + switch (database.type) { + case DatabaseType.POSTGRES: + return ; + case DatabaseType.MYSQL: + return ; + case DatabaseType.MARIADB: + return ; + default: + return null; } - - if (database.type === DatabaseType.MYSQL) { - return ; - } - - return null; }; diff --git a/frontend/src/features/databases/ui/show/ShowMariaDbSpecificDataComponent.tsx b/frontend/src/features/databases/ui/show/ShowMariaDbSpecificDataComponent.tsx new file mode 100644 index 0000000..2725455 --- /dev/null +++ b/frontend/src/features/databases/ui/show/ShowMariaDbSpecificDataComponent.tsx @@ -0,0 +1,60 @@ +import { type Database, MariadbVersion } from '../../../../entity/databases'; + +interface Props { + database: Database; +} + +const mariadbVersionLabels: Record = { + [MariadbVersion.MariadbVersion55]: '5.5', + [MariadbVersion.MariadbVersion101]: '10.1', + [MariadbVersion.MariadbVersion102]: '10.2', + [MariadbVersion.MariadbVersion103]: '10.3', + [MariadbVersion.MariadbVersion104]: '10.4', + [MariadbVersion.MariadbVersion105]: '10.5', + [MariadbVersion.MariadbVersion106]: '10.6', + [MariadbVersion.MariadbVersion1011]: '10.11', + [MariadbVersion.MariadbVersion114]: '11.4', + [MariadbVersion.MariadbVersion118]: '11.8', + [MariadbVersion.MariadbVersion120]: '12.0', +}; + +export const ShowMariaDbSpecificDataComponent = ({ database }: Props) => { + return ( +
+
+
MariaDB version
+
{database.mariadb?.version ? mariadbVersionLabels[database.mariadb.version] : ''}
+
+ +
+
Host
+
{database.mariadb?.host || ''}
+
+ +
+
Port
+
{database.mariadb?.port || ''}
+
+ +
+
Username
+
{database.mariadb?.username || ''}
+
+ +
+
Password
+
{'*************'}
+
+ +
+
DB name
+
{database.mariadb?.database || ''}
+
+ +
+
Use HTTPS
+
{database.mariadb?.isHttps ? 'Yes' : 'No'}
+
+
+ ); +}; diff --git a/frontend/src/features/restores/ui/RestoresComponent.tsx b/frontend/src/features/restores/ui/RestoresComponent.tsx index cb32e4a..18b8cb9 100644 --- a/frontend/src/features/restores/ui/RestoresComponent.tsx +++ b/frontend/src/features/restores/ui/RestoresComponent.tsx @@ -5,12 +5,7 @@ import dayjs from 'dayjs'; import { useEffect, useRef, useState } from 'react'; import type { Backup } from '../../../entity/backups'; -import { - type Database, - DatabaseType, - type MysqlDatabase, - type PostgresqlDatabase, -} from '../../../entity/databases'; +import { type Database, DatabaseType } from '../../../entity/databases'; import { type Restore, RestoreStatus, restoreApi } from '../../../entity/restores'; import { getUserTimeFormat } from '../../../shared/time'; import { EditDatabaseSpecificDataComponent } from '../../databases/ui/edit/EditDatabaseSpecificDataComponent'; @@ -20,30 +15,50 @@ interface Props { backup: Backup; } +type DatabaseCredentials = { + username?: string; + host?: string; + port?: number; + password?: string; +}; + +const clearCredentials = (db: T | undefined): T | undefined => { + if (!db) return undefined; + return { + ...db, + username: undefined, + host: undefined, + port: undefined, + password: undefined, + } as T; +}; + +const createInitialEditingDatabase = (database: Database): Database => ({ + ...database, + postgresql: clearCredentials(database.postgresql), + mysql: clearCredentials(database.mysql), + mariadb: clearCredentials(database.mariadb), +}); + +const getRestorePayload = (database: Database, editingDatabase: Database) => { + switch (database.type) { + case DatabaseType.POSTGRES: + return { postgresql: editingDatabase.postgresql }; + case DatabaseType.MYSQL: + return { mysql: editingDatabase.mysql }; + case DatabaseType.MARIADB: + return { mariadb: editingDatabase.mariadb }; + default: + return {}; + } +}; + export const RestoresComponent = ({ database, backup }: Props) => { const { message } = App.useApp(); - const [editingDatabase, setEditingDatabase] = useState({ - ...database, - postgresql: database.postgresql - ? ({ - ...database.postgresql, - username: undefined, - host: undefined, - port: undefined, - password: undefined, - } as unknown as PostgresqlDatabase) - : undefined, - mysql: database.mysql - ? ({ - ...database.mysql, - username: undefined, - host: undefined, - port: undefined, - password: undefined, - } as unknown as MysqlDatabase) - : undefined, - }); + const [editingDatabase, setEditingDatabase] = useState( + createInitialEditingDatabase(database), + ); const [restores, setRestores] = useState([]); const [isLoading, setIsLoading] = useState(false); @@ -75,14 +90,7 @@ export const RestoresComponent = ({ database, backup }: Props) => { try { await restoreApi.restoreBackup({ backupId: backup.id, - postgresql: - database.type === DatabaseType.POSTGRES - ? (editingDatabase.postgresql as PostgresqlDatabase) - : undefined, - mysql: - database.type === DatabaseType.MYSQL - ? (editingDatabase.mysql as MysqlDatabase) - : undefined, + ...getRestorePayload(database, editingDatabase), }); await loadRestores(); @@ -253,6 +261,7 @@ export const RestoresComponent = ({ database, backup }: Props) => { title="Restore error details" open={!!showingRestoreError} onCancel={() => setShowingRestoreError(undefined)} + maskClosable={false} footer={