From 04621b9b2d66a582e68883f90481c878dd482744 Mon Sep 17 00:00:00 2001 From: Rostislav Dugin Date: Sat, 17 Jan 2026 23:45:39 +0300 Subject: [PATCH] FEATURE (ci \ cd): Adjust CI \ CD to run heavy jobs on self hosted performant runner --- .github/workflows/ci-release.yml | 189 +++++++++++------- backend/.env.development.example | 2 + backend/internal/config/config.go | 6 + .../backups/backups/controller_test.go | 2 +- .../backups/config/controller_test.go | 2 +- .../features/databases/controller_test.go | 8 +- .../databases/databases/mariadb/model_test.go | 2 +- .../databases/databases/mongodb/model_test.go | 16 +- .../databases/databases/mysql/model_test.go | 2 +- .../databases/postgresql/model_test.go | 2 +- .../internal/features/databases/testing.go | 6 +- .../features/restores/controller_test.go | 22 +- .../restores/restoring/restorer_test.go | 3 +- .../restores/restoring/scheduler_test.go | 9 +- .../features/restores/restoring/testing.go | 3 +- .../internal/features/storages/model_test.go | 18 +- .../tests/mariadb_backup_restore_test.go | 2 +- .../tests/mongodb_backup_restore_test.go | 11 +- .../tests/mysql_backup_restore_test.go | 2 +- .../tests/postgresql_backup_restore_test.go | 2 +- 20 files changed, 193 insertions(+), 116 deletions(-) diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index 90b239f..ac98942 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -9,15 +9,26 @@ on: jobs: lint-backend: - runs-on: ubuntu-latest + runs-on: self-hosted + container: + image: golang:1.24.9 + volumes: + - /runner-cache/go-pkg:/go/pkg/mod + - /runner-cache/go-build:/root/.cache/go-build + - /runner-cache/golangci-lint:/root/.cache/golangci-lint + - /runner-cache/apt-archives:/var/cache/apt/archives steps: - name: Check out code uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: "1.24.9" + - name: Configure Git for container + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Download Go modules + run: | + cd backend + go mod download - name: Install golangci-lint run: | @@ -93,34 +104,32 @@ jobs: npm run test test-backend: - runs-on: ubuntu-latest + runs-on: self-hosted needs: [lint-backend] + container: + image: golang:1.24.9 + options: --privileged -v /var/run/docker.sock:/var/run/docker.sock --add-host=host.docker.internal:host-gateway + volumes: + - /runner-cache/go-pkg:/go/pkg/mod + - /runner-cache/go-build:/root/.cache/go-build + - /runner-cache/apt-archives:/var/cache/apt/archives steps: - - name: Free up disk space + - name: Install Docker CLI run: | - echo "Disk space before cleanup:" - df -h - # Remove unnecessary pre-installed software - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - sudo rm -rf /usr/local/share/boost - sudo rm -rf /usr/share/swift - # Clean apt cache - sudo apt-get clean - # Clean docker images (if any pre-installed) - docker system prune -af --volumes || true - echo "Disk space after cleanup:" - df -h + apt-get update -qq + apt-get install -y -qq docker.io docker-compose netcat-openbsd wget - name: Check out code uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: "1.24.9" + - name: Configure Git for container + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Download Go modules + run: | + cd backend + go mod download - name: Create .env file for testing run: | @@ -132,14 +141,15 @@ jobs: DEV_DB_PASSWORD=Q1234567 #app ENV_MODE=development - # db - DATABASE_DSN=host=localhost user=postgres password=Q1234567 dbname=databasus port=5437 sslmode=disable - DATABASE_URL=postgres://postgres:Q1234567@localhost:5437/databasus?sslmode=disable + # db - using 172.17.0.1 to access host from container + DATABASE_DSN=host=172.17.0.1 user=postgres password=Q1234567 dbname=databasus port=5437 sslmode=disable + DATABASE_URL=postgres://postgres:Q1234567@172.17.0.1:5437/databasus?sslmode=disable # migrations GOOSE_DRIVER=postgres - GOOSE_DBSTRING=postgres://postgres:Q1234567@localhost:5437/databasus?sslmode=disable + GOOSE_DBSTRING=postgres://postgres:Q1234567@172.17.0.1:5437/databasus?sslmode=disable GOOSE_MIGRATION_DIR=./migrations - # testing + # testing + TEST_LOCALHOST=172.17.0.1 # to get Google Drive env variables: add storage in UI and copy data from added storage here TEST_GOOGLE_DRIVE_CLIENT_ID=${{ secrets.TEST_GOOGLE_DRIVE_CLIENT_ID }} TEST_GOOGLE_DRIVE_CLIENT_SECRET=${{ secrets.TEST_GOOGLE_DRIVE_CLIENT_SECRET }} @@ -197,12 +207,14 @@ jobs: TEST_MONGODB_60_PORT=27060 TEST_MONGODB_70_PORT=27070 TEST_MONGODB_82_PORT=27082 - # Valkey (cache) - VALKEY_HOST=localhost + # Valkey (cache) - using 172.17.0.1 + VALKEY_HOST=172.17.0.1 VALKEY_PORT=6379 VALKEY_USERNAME= VALKEY_PASSWORD= VALKEY_IS_SSL=false + # Host for test databases (container -> host) + TEST_DB_HOST=172.17.0.1 EOF - name: Start test containers @@ -220,25 +232,25 @@ jobs: timeout 60 bash -c 'until docker exec dev-valkey valkey-cli ping 2>/dev/null | grep -q PONG; do sleep 2; done' echo "Valkey is ready!" - # Wait for test databases - timeout 60 bash -c 'until nc -z localhost 5000; do sleep 2; done' - timeout 60 bash -c 'until nc -z localhost 5001; do sleep 2; done' - timeout 60 bash -c 'until nc -z localhost 5002; do sleep 2; done' - timeout 60 bash -c 'until nc -z localhost 5003; do sleep 2; done' - timeout 60 bash -c 'until nc -z localhost 5004; do sleep 2; done' - timeout 60 bash -c 'until nc -z localhost 5005; do sleep 2; done' + # Wait for test databases (using 172.17.0.1 from container) + timeout 60 bash -c 'until nc -z 172.17.0.1 5000; do sleep 2; done' + timeout 60 bash -c 'until nc -z 172.17.0.1 5001; do sleep 2; done' + timeout 60 bash -c 'until nc -z 172.17.0.1 5002; do sleep 2; done' + timeout 60 bash -c 'until nc -z 172.17.0.1 5003; do sleep 2; done' + timeout 60 bash -c 'until nc -z 172.17.0.1 5004; do sleep 2; done' + timeout 60 bash -c 'until nc -z 172.17.0.1 5005; do sleep 2; done' # Wait for MinIO - timeout 60 bash -c 'until nc -z localhost 9000; do sleep 2; done' + timeout 60 bash -c 'until nc -z 172.17.0.1 9000; do sleep 2; done' # Wait for Azurite - timeout 60 bash -c 'until nc -z localhost 10000; do sleep 2; done' + timeout 60 bash -c 'until nc -z 172.17.0.1 10000; do sleep 2; done' # Wait for FTP - timeout 60 bash -c 'until nc -z localhost 7007; do sleep 2; done' + timeout 60 bash -c 'until nc -z 172.17.0.1 7007; do sleep 2; done' # Wait for SFTP - timeout 60 bash -c 'until nc -z localhost 7008; do sleep 2; done' + timeout 60 bash -c 'until nc -z 172.17.0.1 7008; do sleep 2; done' # Wait for MySQL containers echo "Waiting for MySQL 5.7..." @@ -297,63 +309,63 @@ jobs: mkdir -p databasus-data/backups mkdir -p databasus-data/temp - - name: Install MySQL dependencies + - name: Install database client dependencies run: | - sudo apt-get update -qq - sudo apt-get install -y -qq libncurses6 - sudo ln -sf /usr/lib/x86_64-linux-gnu/libncurses.so.6 /usr/lib/x86_64-linux-gnu/libncurses.so.5 - sudo ln -sf /usr/lib/x86_64-linux-gnu/libtinfo.so.6 /usr/lib/x86_64-linux-gnu/libtinfo.so.5 + apt-get update -qq + apt-get install -y -qq libncurses6 libpq5 + ln -sf /usr/lib/x86_64-linux-gnu/libncurses.so.6 /usr/lib/x86_64-linux-gnu/libncurses.so.5 || true + ln -sf /usr/lib/x86_64-linux-gnu/libtinfo.so.6 /usr/lib/x86_64-linux-gnu/libtinfo.so.5 || true - name: Setup PostgreSQL, MySQL and MariaDB client tools from pre-built assets run: | cd backend/tools - + # Create directory structure mkdir -p postgresql mysql mariadb mongodb/bin - + # Copy PostgreSQL client tools (12-18) from pre-built assets for version in 12 13 14 15 16 17 18; do mkdir -p postgresql/postgresql-$version cp -r ../../assets/tools/x64/postgresql/postgresql-$version/bin postgresql/postgresql-$version/ done - + # Copy MySQL client tools (5.7, 8.0, 8.4, 9) from pre-built assets for version in 5.7 8.0 8.4 9; do mkdir -p mysql/mysql-$version cp -r ../../assets/tools/x64/mysql/mysql-$version/bin mysql/mysql-$version/ done - + # Copy MariaDB client tools (10.6, 12.1) from pre-built assets for version in 10.6 12.1; do mkdir -p mariadb/mariadb-$version cp -r ../../assets/tools/x64/mariadb/mariadb-$version/bin mariadb/mariadb-$version/ done - + # Make all binaries executable chmod +x postgresql/*/bin/* chmod +x mysql/*/bin/* chmod +x mariadb/*/bin/* - + echo "Pre-built client tools setup complete" - name: Install MongoDB Database Tools run: | cd backend/tools - + # MongoDB Database Tools must be downloaded (not in pre-built assets) # They are backward compatible - single version supports all servers (4.0-8.0) MONGODB_TOOLS_URL="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-debian12-x86_64-100.10.0.deb" - + echo "Downloading MongoDB Database Tools..." wget -q "$MONGODB_TOOLS_URL" -O /tmp/mongodb-database-tools.deb - + echo "Installing MongoDB Database Tools..." - sudo dpkg -i /tmp/mongodb-database-tools.deb || sudo apt-get install -f -y --no-install-recommends - + dpkg -i /tmp/mongodb-database-tools.deb || apt-get install -f -y --no-install-recommends + # Create symlinks to tools directory ln -sf /usr/bin/mongodump mongodb/bin/mongodump ln -sf /usr/bin/mongorestore mongodb/bin/mongorestore - + rm -f /tmp/mongodb-database-tools.deb echo "MongoDB Database Tools installed successfully" @@ -401,10 +413,28 @@ jobs: if: always() run: | cd backend + # Stop and remove containers (keeping images for next run) docker compose -f docker-compose.yml.example down -v + # Clean up all data directories created by docker-compose + echo "Cleaning up data directories..." + rm -rf pgdata || true + rm -rf valkey-data || true + rm -rf mysqldata || true + rm -rf mariadbdata || true + rm -rf temp/nas || true + rm -rf databasus-data || true + + # Also clean root-level databasus-data if exists + cd .. + rm -rf databasus-data || true + + echo "Cleanup complete" + determine-version: - runs-on: ubuntu-latest + runs-on: self-hosted + container: + image: node:20 needs: [test-backend, test-frontend] if: ${{ github.ref == 'refs/heads/main' && !contains(github.event.head_commit.message, '[skip-release]') }} outputs: @@ -417,10 +447,9 @@ jobs: with: fetch-depth: 0 - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: "20" + - name: Configure Git for container + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Install semver run: npm install -g semver @@ -497,7 +526,7 @@ jobs: fi build-only: - runs-on: ubuntu-latest + runs-on: self-hosted needs: [test-backend, test-frontend] if: ${{ github.ref == 'refs/heads/main' && contains(github.event.head_commit.message, '[skip-release]') }} steps: @@ -529,7 +558,7 @@ jobs: databasus/databasus:${{ github.sha }} build-and-push: - runs-on: ubuntu-latest + runs-on: self-hosted needs: [determine-version] if: ${{ needs.determine-version.outputs.should_release == 'true' }} permissions: @@ -564,7 +593,9 @@ jobs: databasus/databasus:${{ github.sha }} release: - runs-on: ubuntu-latest + runs-on: self-hosted + container: + image: node:20 needs: [determine-version, build-and-push] if: ${{ needs.determine-version.outputs.should_release == 'true' }} permissions: @@ -577,6 +608,10 @@ jobs: fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} + - name: Configure Git for container + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Generate changelog id: changelog run: | @@ -670,16 +705,28 @@ jobs: prerelease: false publish-helm-chart: - runs-on: ubuntu-latest + runs-on: self-hosted + container: + image: alpine:3.19 + volumes: + - /runner-cache/apk-cache:/etc/apk/cache needs: [determine-version, build-and-push] if: ${{ needs.determine-version.outputs.should_release == 'true' }} permissions: contents: read packages: write steps: + - name: Install dependencies + run: | + apk add --no-cache git bash curl + - name: Check out code uses: actions/checkout@v4 + - name: Configure Git for container + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Set up Helm uses: azure/setup-helm@v4 with: @@ -701,4 +748,4 @@ jobs: - name: Push Helm chart to GHCR run: | VERSION="${{ needs.determine-version.outputs.new_version }}" - helm push databasus-${VERSION}.tgz oci://ghcr.io/databasus/charts \ No newline at end of file + helm push databasus-${VERSION}.tgz oci://ghcr.io/databasus/charts diff --git a/backend/.env.development.example b/backend/.env.development.example index 8d96aee..d3b818c 100644 --- a/backend/.env.development.example +++ b/backend/.env.development.example @@ -6,6 +6,8 @@ DEV_DB_PASSWORD=Q1234567 ENV_MODE=development # logging SHOW_DB_INSTALLATION_VERIFICATION_LOGS=true +# tests +TEST_LOCALHOST=localhost # db DATABASE_DSN=host=dev-db user=postgres password=Q1234567 dbname=databasus port=5437 sslmode=disable DATABASE_URL=postgres://postgres:Q1234567@dev-db:5437/databasus?sslmode=disable diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 38e5906..d05f690 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -29,6 +29,8 @@ type EnvVariables struct { MariadbInstallDir string `env:"MARIADB_INSTALL_DIR"` MongodbInstallDir string `env:"MONGODB_INSTALL_DIR"` + TestLocalhost string `env:"TEST_LOCALHOST"` + ShowDbInstallationVerificationLogs bool `env:"SHOW_DB_INSTALLATION_VERIFICATION_LOGS"` IsManyNodesMode bool `env:"IS_MANY_NODES_MODE"` @@ -237,6 +239,10 @@ func loadEnvVariables() { env.IsProcessingNode = true } + if env.TestLocalhost == "" { + env.TestLocalhost = "localhost" + } + // Valkey if env.ValkeyHost == "" { log.Error("VALKEY_HOST is empty") diff --git a/backend/internal/features/backups/backups/controller_test.go b/backend/internal/features/backups/backups/controller_test.go index 369a988..140e472 100644 --- a/backend/internal/features/backups/backups/controller_test.go +++ b/backend/internal/features/backups/backups/controller_test.go @@ -1156,7 +1156,7 @@ func createTestDatabase( Type: databases.DatabaseTypePostgres, Postgresql: &postgresql.PostgresqlDatabase{ Version: tools.PostgresqlVersion16, - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: port, Username: "testuser", Password: "testpassword", diff --git a/backend/internal/features/backups/config/controller_test.go b/backend/internal/features/backups/config/controller_test.go index 379717b..ea89b92 100644 --- a/backend/internal/features/backups/config/controller_test.go +++ b/backend/internal/features/backups/config/controller_test.go @@ -1462,7 +1462,7 @@ func createTestDatabaseViaAPI( Type: databases.DatabaseTypePostgres, Postgresql: &postgresql.PostgresqlDatabase{ Version: tools.PostgresqlVersion16, - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: port, Username: "testuser", Password: "testpassword", diff --git a/backend/internal/features/databases/controller_test.go b/backend/internal/features/databases/controller_test.go index 8c632ba..157eabe 100644 --- a/backend/internal/features/databases/controller_test.go +++ b/backend/internal/features/databases/controller_test.go @@ -44,7 +44,7 @@ func getTestPostgresConfig() *postgresql.PostgresqlDatabase { testDbName := "testdb" return &postgresql.PostgresqlDatabase{ Version: tools.PostgresqlVersion16, - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: port, Username: "testuser", Password: "testpassword", @@ -67,7 +67,7 @@ func getTestMariadbConfig() *mariadb.MariadbDatabase { testDbName := "testdb" return &mariadb.MariadbDatabase{ Version: tools.MariadbVersion1011, - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: port, Username: "testuser", Password: "testpassword", @@ -88,7 +88,7 @@ func getTestMongodbConfig() *mongodb.MongodbDatabase { return &mongodb.MongodbDatabase{ Version: tools.MongodbVersion7, - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: port, Username: "root", Password: "rootpassword", @@ -829,7 +829,7 @@ func createTestDatabaseViaAPI( Type: DatabaseTypePostgres, Postgresql: &postgresql.PostgresqlDatabase{ Version: tools.PostgresqlVersion16, - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: port, Username: "testuser", Password: "testpassword", diff --git a/backend/internal/features/databases/databases/mariadb/model_test.go b/backend/internal/features/databases/databases/mariadb/model_test.go index 40fb043..ec4bd8e 100644 --- a/backend/internal/features/databases/databases/mariadb/model_test.go +++ b/backend/internal/features/databases/databases/mariadb/model_test.go @@ -714,7 +714,7 @@ func connectToMariadbContainer( } dbName := "testdb" - host := "127.0.0.1" + host := config.GetEnv().TestLocalhost username := "root" password := "rootpassword" diff --git a/backend/internal/features/databases/databases/mongodb/model_test.go b/backend/internal/features/databases/databases/mongodb/model_test.go index 5497e4e..4e4c3f8 100644 --- a/backend/internal/features/databases/databases/mongodb/model_test.go +++ b/backend/internal/features/databases/databases/mongodb/model_test.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" "testing" + "time" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -397,7 +398,7 @@ func connectToMongodbContainer( } dbName := "testdb" - host := "127.0.0.1" + host := config.GetEnv().TestLocalhost username := "root" password := "rootpassword" authDatabase := "admin" @@ -406,11 +407,18 @@ func connectToMongodbContainer( assert.NoError(t, err) uri := fmt.Sprintf( - "mongodb://%s:%s@%s:%d/%s?authSource=%s", - username, password, host, portInt, dbName, authDatabase, + "mongodb://%s:%s@%s:%d/%s?authSource=%s&serverSelectionTimeoutMS=5000&connectTimeoutMS=5000", + username, + password, + host, + portInt, + dbName, + authDatabase, ) - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + clientOptions := options.Client().ApplyURI(uri) client, err := mongo.Connect(ctx, clientOptions) if err != nil { diff --git a/backend/internal/features/databases/databases/mysql/model_test.go b/backend/internal/features/databases/databases/mysql/model_test.go index 50dbb2f..b5d62d4 100644 --- a/backend/internal/features/databases/databases/mysql/model_test.go +++ b/backend/internal/features/databases/databases/mysql/model_test.go @@ -694,7 +694,7 @@ func connectToMysqlContainer( } dbName := "testdb" - host := "127.0.0.1" + host := config.GetEnv().TestLocalhost username := "root" password := "rootpassword" diff --git a/backend/internal/features/databases/databases/postgresql/model_test.go b/backend/internal/features/databases/databases/postgresql/model_test.go index f210113..e3d277c 100644 --- a/backend/internal/features/databases/databases/postgresql/model_test.go +++ b/backend/internal/features/databases/databases/postgresql/model_test.go @@ -981,7 +981,7 @@ func connectToPostgresContainer(t *testing.T, port string) *PostgresContainer { dbName := "testdb" password := "testpassword" username := "testuser" - host := "localhost" + host := config.GetEnv().TestLocalhost portInt, err := strconv.Atoi(port) assert.NoError(t, err) diff --git a/backend/internal/features/databases/testing.go b/backend/internal/features/databases/testing.go index bab070d..c71868e 100644 --- a/backend/internal/features/databases/testing.go +++ b/backend/internal/features/databases/testing.go @@ -25,7 +25,7 @@ func GetTestPostgresConfig() *postgresql.PostgresqlDatabase { testDbName := "testdb" return &postgresql.PostgresqlDatabase{ Version: tools.PostgresqlVersion16, - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: port, Username: "testuser", Password: "testpassword", @@ -48,7 +48,7 @@ func GetTestMariadbConfig() *mariadb.MariadbDatabase { testDbName := "testdb" return &mariadb.MariadbDatabase{ Version: tools.MariadbVersion1011, - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: port, Username: "testuser", Password: "testpassword", @@ -69,7 +69,7 @@ func GetTestMongodbConfig() *mongodb.MongodbDatabase { return &mongodb.MongodbDatabase{ Version: tools.MongodbVersion7, - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: port, Username: "root", Password: "rootpassword", diff --git a/backend/internal/features/restores/controller_test.go b/backend/internal/features/restores/controller_test.go index c437364..33790b7 100644 --- a/backend/internal/features/restores/controller_test.go +++ b/backend/internal/features/restores/controller_test.go @@ -16,7 +16,7 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" - "databasus-backend/internal/config" + env_config "databasus-backend/internal/config" audit_logs "databasus-backend/internal/features/audit_logs" "databasus-backend/internal/features/backups/backups" backups_core "databasus-backend/internal/features/backups/backups/core" @@ -120,7 +120,7 @@ func Test_RestoreBackup_WhenUserIsWorkspaceMember_RestoreInitiated(t *testing.T) request := restores_core.RestoreBackupRequest{ PostgresqlDatabase: &postgresql.PostgresqlDatabase{ Version: tools.PostgresqlVersion16, - Host: "localhost", + Host: env_config.GetEnv().TestLocalhost, Port: 5432, Username: "postgres", Password: "postgres", @@ -151,7 +151,7 @@ func Test_RestoreBackup_WhenUserIsNotWorkspaceMember_ReturnsForbidden(t *testing request := restores_core.RestoreBackupRequest{ PostgresqlDatabase: &postgresql.PostgresqlDatabase{ Version: tools.PostgresqlVersion16, - Host: "localhost", + Host: env_config.GetEnv().TestLocalhost, Port: 5432, Username: "postgres", Password: "postgres", @@ -184,7 +184,7 @@ func Test_RestoreBackup_WithIsExcludeExtensions_FlagPassedCorrectly(t *testing.T request := restores_core.RestoreBackupRequest{ PostgresqlDatabase: &postgresql.PostgresqlDatabase{ Version: tools.PostgresqlVersion16, - Host: "localhost", + Host: env_config.GetEnv().TestLocalhost, Port: 5432, Username: "postgres", Password: "postgres", @@ -218,7 +218,7 @@ func Test_RestoreBackup_AuditLogWritten(t *testing.T) { request := restores_core.RestoreBackupRequest{ PostgresqlDatabase: &postgresql.PostgresqlDatabase{ Version: tools.PostgresqlVersion16, - Host: "localhost", + Host: env_config.GetEnv().TestLocalhost, Port: 5432, Username: "postgres", Password: "postgres", @@ -305,7 +305,7 @@ func Test_RestoreBackup_DiskSpaceValidation(t *testing.T) { request = restores_core.RestoreBackupRequest{ PostgresqlDatabase: &postgresql.PostgresqlDatabase{ Version: tools.PostgresqlVersion16, - Host: "localhost", + Host: env_config.GetEnv().TestLocalhost, Port: 5432, Username: "postgres", Password: "postgres", @@ -335,7 +335,7 @@ func Test_RestoreBackup_DiskSpaceValidation(t *testing.T) { request = restores_core.RestoreBackupRequest{ MysqlDatabase: &mysql.MysqlDatabase{ Version: tools.MysqlVersion80, - Host: "localhost", + Host: env_config.GetEnv().TestLocalhost, Port: 3306, Username: "root", Password: "password", @@ -428,7 +428,7 @@ func Test_CancelRestore_InProgressRestore_SuccessfullyCancelled(t *testing.T) { restoreRequest := restores_core.RestoreBackupRequest{ PostgresqlDatabase: &postgresql.PostgresqlDatabase{ Version: tools.PostgresqlVersion16, - Host: "localhost", + Host: env_config.GetEnv().TestLocalhost, Port: 5432, Username: "postgres", Password: "postgres", @@ -525,7 +525,7 @@ func Test_RestoreBackup_WithParallelRestoreInProgress_ReturnsError(t *testing.T) request := restores_core.RestoreBackupRequest{ PostgresqlDatabase: &postgresql.PostgresqlDatabase{ Version: tools.PostgresqlVersion16, - Host: "localhost", + Host: env_config.GetEnv().TestLocalhost, Port: 5432, Username: "postgres", Password: "postgres", @@ -626,7 +626,7 @@ func createTestMySQLDatabase( token string, router *gin.Engine, ) *databases.Database { - env := config.GetEnv() + env := env_config.GetEnv() portStr := env.TestMysql80Port if portStr == "" { portStr = "33080" @@ -644,7 +644,7 @@ func createTestMySQLDatabase( Type: databases.DatabaseTypeMysql, Mysql: &mysql.MysqlDatabase{ Version: tools.MysqlVersion80, - Host: "localhost", + Host: env_config.GetEnv().TestLocalhost, Port: port, Username: "testuser", Password: "testpassword", diff --git a/backend/internal/features/restores/restoring/restorer_test.go b/backend/internal/features/restores/restoring/restorer_test.go index 2024565..65137f1 100644 --- a/backend/internal/features/restores/restoring/restorer_test.go +++ b/backend/internal/features/restores/restoring/restorer_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" + "databasus-backend/internal/config" "databasus-backend/internal/features/backups/backups" backups_core "databasus-backend/internal/features/backups/backups/core" backups_config "databasus-backend/internal/features/backups/config" @@ -139,7 +140,7 @@ func Test_MakeRestore_WhenTaskStarts_CacheDeletedImmediately(t *testing.T) { // Cache DB credentials separately dbCache := &RestoreDatabaseCache{ PostgresqlDatabase: &postgresql.PostgresqlDatabase{ - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: 5432, Username: "test", Password: "test", diff --git a/backend/internal/features/restores/restoring/scheduler_test.go b/backend/internal/features/restores/restoring/scheduler_test.go index 874e4b7..3b6f353 100644 --- a/backend/internal/features/restores/restoring/scheduler_test.go +++ b/backend/internal/features/restores/restoring/scheduler_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "databasus-backend/internal/config" "databasus-backend/internal/features/backups/backups" backups_core "databasus-backend/internal/features/backups/backups/core" backups_config "databasus-backend/internal/features/backups/config" @@ -681,7 +682,7 @@ func Test_StartRestore_CredentialsStoredEncryptedInCache(t *testing.T) { // Create PostgreSQL database credentials with plaintext password postgresDB := &postgresql.PostgresqlDatabase{ - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: 5432, Username: "testuser", Password: plaintextPassword, @@ -719,7 +720,7 @@ func Test_StartRestore_CredentialsStoredEncryptedInCache(t *testing.T) { "Cached password should match the encrypted version") // Verify other fields are present - assert.Equal(t, "localhost", cachedData.PostgresqlDatabase.Host) + assert.Equal(t, config.GetEnv().TestLocalhost, cachedData.PostgresqlDatabase.Host) assert.Equal(t, 5432, cachedData.PostgresqlDatabase.Port) assert.Equal(t, "testuser", cachedData.PostgresqlDatabase.Username) assert.Equal(t, "testdb", *cachedData.PostgresqlDatabase.Database) @@ -792,7 +793,7 @@ func Test_StartRestore_CredentialsRemovedAfterRestoreStarts(t *testing.T) { // Create PostgreSQL database credentials // Database field is nil to avoid PopulateDbData trying to connect postgresDB := &postgresql.PostgresqlDatabase{ - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: 5432, Username: "testuser", Password: plaintextPassword, @@ -832,7 +833,7 @@ func Test_StartRestore_CredentialsRemovedAfterRestoreStarts(t *testing.T) { // Verify mock received valid credentials assert.NotNil(t, capturedDB, "Captured database should not be nil") assert.NotNil(t, capturedDB.Postgresql, "PostgreSQL credentials should be provided to usecase") - assert.Equal(t, "localhost", capturedDB.Postgresql.Host) + assert.Equal(t, config.GetEnv().TestLocalhost, capturedDB.Postgresql.Host) assert.Equal(t, 5432, capturedDB.Postgresql.Port) assert.Equal(t, "testuser", capturedDB.Postgresql.Username) assert.NotEmpty(t, capturedDB.Postgresql.Password, "Password should be provided to usecase") diff --git a/backend/internal/features/restores/restoring/testing.go b/backend/internal/features/restores/restoring/testing.go index dc9a059..d37c6f4 100644 --- a/backend/internal/features/restores/restoring/testing.go +++ b/backend/internal/features/restores/restoring/testing.go @@ -11,6 +11,7 @@ import ( "github.com/gin-gonic/gin" "github.com/google/uuid" + "databasus-backend/internal/config" "databasus-backend/internal/features/backups/backups" backups_core "databasus-backend/internal/features/backups/backups/core" backups_config "databasus-backend/internal/features/backups/config" @@ -319,7 +320,7 @@ func CreateTestRestore( BackupID: backup.ID, Status: status, PostgresqlDatabase: &postgresql.PostgresqlDatabase{ - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: 5432, Username: "test", Password: "test", diff --git a/backend/internal/features/storages/model_test.go b/backend/internal/features/storages/model_test.go index 89a6282..ac83500 100644 --- a/backend/internal/features/storages/model_test.go +++ b/backend/internal/features/storages/model_test.go @@ -113,7 +113,7 @@ func Test_Storage_BasicOperations(t *testing.T) { name: "NASStorage", storage: &nas_storage.NASStorage{ StorageID: uuid.New(), - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: nasPort, Share: "backups", Username: "testuser", @@ -147,7 +147,7 @@ func Test_Storage_BasicOperations(t *testing.T) { name: "FTPStorage", storage: &ftp_storage.FTPStorage{ StorageID: uuid.New(), - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: ftpPort, Username: "testuser", Password: "testpassword", @@ -159,7 +159,7 @@ func Test_Storage_BasicOperations(t *testing.T) { name: "SFTPStorage", storage: &sftp_storage.SFTPStorage{ StorageID: uuid.New(), - Host: "localhost", + Host: config.GetEnv().TestLocalhost, Port: sftpPort, Username: "testuser", Password: "testpassword", @@ -297,7 +297,7 @@ func setupS3Container(ctx context.Context) (*S3Container, error) { secretKey := "testpassword" bucketName := "test-bucket" region := "us-east-1" - endpoint := fmt.Sprintf("127.0.0.1:%s", env.TestMinioPort) + endpoint := fmt.Sprintf("%s:%s", env.TestLocalhost, env.TestMinioPort) // Create MinIO client and ensure bucket exists minioClient, err := minio.New(endpoint, &minio.Options{ @@ -343,15 +343,21 @@ func setupAzuriteContainer(ctx context.Context) (*AzuriteContainer, error) { accountName := "devstoreaccount1" // this is real testing key for azurite, it's not a real key accountKey := "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" - serviceURL := fmt.Sprintf("http://127.0.0.1:%s/%s", env.TestAzuriteBlobPort, accountName) + serviceURL := fmt.Sprintf( + "http://%s:%s/%s", + env.TestLocalhost, + env.TestAzuriteBlobPort, + accountName, + ) containerNameKey := "test-container-key" containerNameStr := "test-container-connstr" // Build explicit connection string for Azurite connectionString := fmt.Sprintf( - "DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=http://127.0.0.1:%s/%s", + "DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=http://%s:%s/%s", accountName, accountKey, + env.TestLocalhost, env.TestAzuriteBlobPort, accountName, ) diff --git a/backend/internal/features/tests/mariadb_backup_restore_test.go b/backend/internal/features/tests/mariadb_backup_restore_test.go index eaf6b5f..03f1010 100644 --- a/backend/internal/features/tests/mariadb_backup_restore_test.go +++ b/backend/internal/features/tests/mariadb_backup_restore_test.go @@ -605,7 +605,7 @@ func connectToMariadbContainer( dbName := "testdb" password := "rootpassword" username := "root" - host := "127.0.0.1" + host := config.GetEnv().TestLocalhost portInt, err := strconv.Atoi(port) if err != nil { diff --git a/backend/internal/features/tests/mongodb_backup_restore_test.go b/backend/internal/features/tests/mongodb_backup_restore_test.go index 2abd505..fffa140 100644 --- a/backend/internal/features/tests/mongodb_backup_restore_test.go +++ b/backend/internal/features/tests/mongodb_backup_restore_test.go @@ -550,7 +550,7 @@ func connectToMongodbContainer( password := "rootpassword" username := "root" authDatabase := "admin" - host := "127.0.0.1" + host := config.GetEnv().TestLocalhost portInt, err := strconv.Atoi(port) if err != nil { @@ -558,8 +558,13 @@ func connectToMongodbContainer( } uri := fmt.Sprintf( - "mongodb://%s:%s@%s:%d/%s?authSource=%s", - username, password, host, portInt, dbName, authDatabase, + "mongodb://%s:%s@%s:%d/%s?authSource=%s&serverSelectionTimeoutMS=5000&connectTimeoutMS=5000", + username, + password, + host, + portInt, + dbName, + authDatabase, ) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) diff --git a/backend/internal/features/tests/mysql_backup_restore_test.go b/backend/internal/features/tests/mysql_backup_restore_test.go index 27b8089..71d09b5 100644 --- a/backend/internal/features/tests/mysql_backup_restore_test.go +++ b/backend/internal/features/tests/mysql_backup_restore_test.go @@ -577,7 +577,7 @@ func connectToMysqlContainer(version tools.MysqlVersion, port string) (*MysqlCon dbName := "testdb" password := "rootpassword" username := "root" - host := "127.0.0.1" + host := config.GetEnv().TestLocalhost portInt, err := strconv.Atoi(port) if err != nil { diff --git a/backend/internal/features/tests/postgresql_backup_restore_test.go b/backend/internal/features/tests/postgresql_backup_restore_test.go index dbc19ac..aefe4dc 100644 --- a/backend/internal/features/tests/postgresql_backup_restore_test.go +++ b/backend/internal/features/tests/postgresql_backup_restore_test.go @@ -1754,7 +1754,7 @@ func connectToPostgresContainer(version string, port string) (*PostgresContainer dbName := "testdb" password := "testpassword" username := "testuser" - host := "localhost" + host := config.GetEnv().TestLocalhost portInt, err := strconv.Atoi(port) if err != nil {