Compare commits

...

13 Commits

Author SHA1 Message Date
Rostislav Dugin
8217906c7a Merge pull request #139 from RostislavDugin/develop
Merge develop into main
2025-12-11 20:02:32 +03:00
Rostislav Dugin
db71a5ef7b FIX (databases): Add support dashed databases for read only users creation 2025-12-11 19:57:49 +03:00
Rostislav Dugin
df78e296b3 FEATURE (s3): Allow to skip TLS verification 2025-12-11 19:50:59 +03:00
Rostislav Dugin
fda3bf9b98 FEATURE (supabase): Add support of Supabase, schemas excluding and get rid of version in UI 2025-12-11 19:27:45 +03:00
pv-create
e19f449c60 FIX (readme): Fix typos and links
* fix typos

* fix link

* fix email param

---------

Co-authored-by: pavelvilkov <vilkovpy@mi-broker.ru>
2025-12-10 19:44:49 +03:00
Leonardo Flores
5944d7c4b6 feat(postgresus): Add schema filter for pg_dump and pg_restore (#131)
Add optional "Schemas" field to PostgreSQL database settings allowing
users to specify which schemas to include in backups (comma-separated).

This solves permission issues when backing up some of databases that
have restricted internal schemas (auth, storage, realtime).

Changes:
- Add schemas column to postgresql_databases table (migration)
- Update PostgresqlDatabase model with Schemas field
- Modify buildPgDumpArgs() to append --schema flags for each schema
- Modify pg_restore args to support --schema filtering on restore
- Add Schemas input field to frontend edit form with tooltip
- Display schemas in read-only database view

Example usage: Setting schemas to "public,drizzle" generates:
  pg_dump ... --schema public --schema drizzle
  pg_restore ... --schema public --schema drizzle
2025-12-10 13:19:15 +03:00
Unicorn-Zombie-Apocalypse
1f5c9d3d01 feat: Add support for custom Root CA configuration in Helm chart (#129)
* feat: Add support for custom Root CA configuration in Helm chart

* fix: Remove default value for customRootCA in Helm chart
2025-12-09 19:36:52 +03:00
Rostislav Dugin
d27b885fc1 FIX (postgresql): Fix version detection without minor version after major 2025-12-09 10:36:07 +03:00
Rostislav Dugin
45054bc4b5 FIX (readme): Update README about PITR 2025-12-08 22:20:41 +03:00
Rostislav Dugin
09f27019e8 FIX (postgresql): Use UTF-8 encoding for DB connection by default 2025-12-08 17:40:37 +03:00
Rostislav Dugin
cba8fdf49c FEATURE (core)!: Release 2.0 2025-12-08 10:41:36 +03:00
Rostislav Dugin
41c72cf7b6 FIX (buffering): Simplify buffering logic for localstorage 2025-12-07 19:40:40 +03:00
Rostislav Dugin
f04a8b7a82 FIX (backup): Add double buffering for local storange 2025-12-07 19:02:44 +03:00
28 changed files with 1158 additions and 182 deletions

View File

@@ -144,6 +144,12 @@ jobs:
# testing Telegram
TEST_TELEGRAM_BOT_TOKEN=${{ secrets.TEST_TELEGRAM_BOT_TOKEN }}
TEST_TELEGRAM_CHAT_ID=${{ secrets.TEST_TELEGRAM_CHAT_ID }}
# supabase
TEST_SUPABASE_HOST=${{ secrets.TEST_SUPABASE_HOST }}
TEST_SUPABASE_PORT=${{ secrets.TEST_SUPABASE_PORT }}
TEST_SUPABASE_USERNAME=${{ secrets.TEST_SUPABASE_USERNAME }}
TEST_SUPABASE_PASSWORD=${{ secrets.TEST_SUPABASE_PASSWORD }}
TEST_SUPABASE_DATABASE=${{ secrets.TEST_SUPABASE_DATABASE }}
EOF
- name: Start test containers

3
.gitignore vendored
View File

@@ -7,4 +7,5 @@ node_modules/
.idea
/articles
.DS_Store
.DS_Store
/scripts

View File

@@ -80,6 +80,15 @@
- **Dark & light themes**: Choose the look that suits your workflow
- **Mobile adaptive**: Check your backups from anywhere on any device
### ☁️ **Works with Self-Hosted & Cloud Databases**
Postgresus works seamlessly with both self-hosted PostgreSQL and cloud-managed databases:
- **Cloud support**: AWS RDS, Google Cloud SQL, Azure Database for PostgreSQL
- **Self-hosted**: Any PostgreSQL instance you manage yourself
- **Why no PITR?**: Cloud providers already offer native PITR, and external PITR backups cannot be restored to managed cloud databases — making them impractical for cloud-hosted PostgreSQL
- **Practical granularity**: Hourly and daily backups are sufficient for 99% of projects without the operational complexity of WAL archiving
### 🐳 **Self-Hosted & Secure**
- **Docker-based**: Easy deployment and management
@@ -88,7 +97,7 @@
### 📦 Installation <a href="https://postgresus.com/installation">(docs)</a>
You have three ways to install Postgresus:
You have several ways to install Postgresus:
- Script (recommended)
- Simple Docker run
@@ -106,7 +115,7 @@ You have three ways to install Postgresus: automated script (recommended), simpl
The installation script will:
- ✅ Install Docker with Docker Compose(if not already installed)
- ✅ Install Docker with Docker Compose (if not already installed)
- ✅ Set up Postgresus
- ✅ Configure automatic startup on system reboot
@@ -229,4 +238,4 @@ This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENS
## 🤝 Contributing
Contributions are welcome! Read <a href="https://postgresus.com/contributing">contributing guide</a> for more details, prioerities and rules are specified there. If you want to contribute, but don't know what and how - message me on Telegram [@rostislav_dugin](https://t.me/rostislav_dugin)
Contributions are welcome! Read <a href="https://postgresus.com/contribute">contributing guide</a> for more details, priorities and rules are specified there. If you want to contribute, but don't know what and how - message me on Telegram [@rostislav_dugin](https://t.me/rostislav_dugin)

View File

@@ -33,4 +33,10 @@ TEST_NAS_PORT=7006
TEST_TELEGRAM_BOT_TOKEN=
TEST_TELEGRAM_CHAT_ID=
# testing Azure Blob Storage
TEST_AZURITE_BLOB_PORT=10000
TEST_AZURITE_BLOB_PORT=10000
# supabase
TEST_SUPABASE_HOST=
TEST_SUPABASE_PORT=
TEST_SUPABASE_USERNAME=
TEST_SUPABASE_PASSWORD=
TEST_SUPABASE_DATABASE=

View File

@@ -58,6 +58,13 @@ type EnvVariables struct {
// testing Telegram
TestTelegramBotToken string `env:"TEST_TELEGRAM_BOT_TOKEN"`
TestTelegramChatID string `env:"TEST_TELEGRAM_CHAT_ID"`
// testing Supabase
TestSupabaseHost string `env:"TEST_SUPABASE_HOST"`
TestSupabasePort string `env:"TEST_SUPABASE_PORT"`
TestSupabaseUsername string `env:"TEST_SUPABASE_USERNAME"`
TestSupabasePassword string `env:"TEST_SUPABASE_PASSWORD"`
TestSupabaseDatabase string `env:"TEST_SUPABASE_DATABASE"`
}
var (

View File

@@ -30,7 +30,7 @@ import (
const (
backupTimeout = 23 * time.Hour
shutdownCheckInterval = 1 * time.Second
copyBufferSize = 16 * 1024 * 1024
copyBufferSize = 8 * 1024 * 1024
progressReportIntervalMB = 1.0
pgConnectTimeout = 30
compressionLevel = 5
@@ -334,6 +334,10 @@ func (uc *CreatePostgresqlBackupUsecase) buildPgDumpArgs(pg *pgtypes.PostgresqlD
"--verbose",
}
for _, schema := range pg.IncludeSchemas {
args = append(args, "-n", schema)
}
compressionArgs := uc.getCompressionArgs(pg.Version)
return append(args, compressionArgs...)
}

View File

@@ -13,6 +13,7 @@ import (
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"gorm.io/gorm"
)
type PostgresqlDatabase struct {
@@ -29,17 +30,37 @@ type PostgresqlDatabase struct {
Password string `json:"password" gorm:"type:text;not null"`
Database *string `json:"database" gorm:"type:text"`
IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`
// backup settings
IncludeSchemas []string `json:"includeSchemas" gorm:"-"`
IncludeSchemasString string `json:"-" gorm:"column:include_schemas;type:text;not null;default:''"`
}
func (p *PostgresqlDatabase) TableName() string {
return "postgresql_databases"
}
func (p *PostgresqlDatabase) Validate() error {
if p.Version == "" {
return errors.New("version is required")
func (p *PostgresqlDatabase) BeforeSave(_ *gorm.DB) error {
if len(p.IncludeSchemas) > 0 {
p.IncludeSchemasString = strings.Join(p.IncludeSchemas, ",")
} else {
p.IncludeSchemasString = ""
}
return nil
}
func (p *PostgresqlDatabase) AfterFind(_ *gorm.DB) error {
if p.IncludeSchemasString != "" {
p.IncludeSchemas = strings.Split(p.IncludeSchemasString, ",")
} else {
p.IncludeSchemas = []string{}
}
return nil
}
func (p *PostgresqlDatabase) Validate() error {
if p.Host == "" {
return errors.New("host is required")
}
@@ -85,6 +106,7 @@ func (p *PostgresqlDatabase) Update(incoming *PostgresqlDatabase) {
p.Username = incoming.Username
p.Database = incoming.Database
p.IsHttps = incoming.IsHttps
p.IncludeSchemas = incoming.IncludeSchemas
if incoming.Password != "" {
p.Password = incoming.Password
@@ -106,6 +128,50 @@ func (p *PostgresqlDatabase) EncryptSensitiveFields(
return nil
}
// PopulateVersionIfEmpty detects and sets the PostgreSQL version if not already set.
// This should be called before encrypting sensitive fields.
func (p *PostgresqlDatabase) PopulateVersionIfEmpty(
logger *slog.Logger,
encryptor encryption.FieldEncryptor,
databaseID uuid.UUID,
) error {
if p.Version != "" {
return nil
}
if p.Database == nil || *p.Database == "" {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
password, err := decryptPasswordIfNeeded(p.Password, encryptor, databaseID)
if err != nil {
return fmt.Errorf("failed to decrypt password: %w", err)
}
connStr := buildConnectionStringForDB(p, *p.Database, password)
conn, err := pgx.Connect(ctx, connStr)
if err != nil {
return fmt.Errorf("failed to connect to database: %w", err)
}
defer func() {
if closeErr := conn.Close(ctx); closeErr != nil {
logger.Error("Failed to close connection", "error", closeErr)
}
}()
detectedVersion, err := detectDatabaseVersion(ctx, conn)
if err != nil {
return err
}
p.Version = detectedVersion
return nil
}
// IsUserReadOnly checks if the database user has read-only privileges.
//
// This method performs a comprehensive security check by examining:
@@ -286,8 +352,20 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
// Retry logic for username collision
maxRetries := 3
for attempt := 0; attempt < maxRetries; attempt++ {
username := fmt.Sprintf("postgresus-%s", uuid.New().String()[:8])
for attempt := range maxRetries {
// Generate base username for PostgreSQL user creation
baseUsername := fmt.Sprintf("postgresus-%s", uuid.New().String()[:8])
// For Supabase session pooler, the username format for connection is "username.projectid"
// but the actual PostgreSQL user must be created with just the base name.
// The pooler will strip the ".projectid" suffix when authenticating.
connectionUsername := baseUsername
if isSupabaseConnection(p.Host, p.Username) {
if supabaseProjectID := extractSupabaseProjectID(p.Username); supabaseProjectID != "" {
connectionUsername = fmt.Sprintf("%s.%s", baseUsername, supabaseProjectID)
}
}
newPassword := uuid.New().String()
tx, err := conn.Begin(ctx)
@@ -305,9 +383,10 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
}()
// Step 1: Create PostgreSQL user with LOGIN privilege
// Note: We use baseUsername for the actual PostgreSQL user name if Supabase is used
_, err = tx.Exec(
ctx,
fmt.Sprintf(`CREATE USER "%s" WITH PASSWORD '%s' LOGIN`, username, newPassword),
fmt.Sprintf(`CREATE USER "%s" WITH PASSWORD '%s' LOGIN`, baseUsername, newPassword),
)
if err != nil {
if err.Error() != "" && attempt < maxRetries-1 {
@@ -331,28 +410,28 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
}
// Now revoke from the specific user as well (belt and suspenders)
_, err = tx.Exec(ctx, fmt.Sprintf(`REVOKE CREATE ON SCHEMA public FROM "%s"`, username))
_, err = tx.Exec(ctx, fmt.Sprintf(`REVOKE CREATE ON SCHEMA public FROM "%s"`, baseUsername))
if err != nil {
logger.Error(
"Failed to revoke CREATE on public schema from user",
"error",
err,
"username",
username,
baseUsername,
)
}
// Step 2: Grant database connection privilege and revoke TEMP
_, err = tx.Exec(
ctx,
fmt.Sprintf(`GRANT CONNECT ON DATABASE %s TO "%s"`, *p.Database, username),
fmt.Sprintf(`GRANT CONNECT ON DATABASE "%s" TO "%s"`, *p.Database, baseUsername),
)
if err != nil {
return "", "", fmt.Errorf("failed to grant connect privilege: %w", err)
}
// Revoke TEMP privilege from PUBLIC role (like CREATE on public schema, TEMP is granted to PUBLIC by default)
_, err = tx.Exec(ctx, fmt.Sprintf(`REVOKE TEMP ON DATABASE %s FROM PUBLIC`, *p.Database))
_, err = tx.Exec(ctx, fmt.Sprintf(`REVOKE TEMP ON DATABASE "%s" FROM PUBLIC`, *p.Database))
if err != nil {
logger.Warn("Failed to revoke TEMP from PUBLIC", "error", err)
}
@@ -360,10 +439,10 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
// Also revoke from the specific user (belt and suspenders)
_, err = tx.Exec(
ctx,
fmt.Sprintf(`REVOKE TEMP ON DATABASE %s FROM "%s"`, *p.Database, username),
fmt.Sprintf(`REVOKE TEMP ON DATABASE "%s" FROM "%s"`, *p.Database, baseUsername),
)
if err != nil {
logger.Warn("Failed to revoke TEMP privilege", "error", err, "username", username)
logger.Warn("Failed to revoke TEMP privilege", "error", err, "username", baseUsername)
}
// Step 3: Discover all user-created schemas
@@ -396,7 +475,7 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
// Revoke CREATE specifically (handles inheritance from PUBLIC role)
_, err = tx.Exec(
ctx,
fmt.Sprintf(`REVOKE CREATE ON SCHEMA "%s" FROM "%s"`, schema, username),
fmt.Sprintf(`REVOKE CREATE ON SCHEMA "%s" FROM "%s"`, schema, baseUsername),
)
if err != nil {
logger.Warn(
@@ -406,14 +485,14 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
"schema",
schema,
"username",
username,
baseUsername,
)
}
// Grant only USAGE (not CREATE)
_, err = tx.Exec(
ctx,
fmt.Sprintf(`GRANT USAGE ON SCHEMA "%s" TO "%s"`, schema, username),
fmt.Sprintf(`GRANT USAGE ON SCHEMA "%s" TO "%s"`, schema, baseUsername),
)
if err != nil {
return "", "", fmt.Errorf("failed to grant usage on schema %s: %w", schema, err)
@@ -435,7 +514,7 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
EXECUTE format('GRANT SELECT ON ALL SEQUENCES IN SCHEMA %%I TO "%s"', schema_rec.schema_name);
END LOOP;
END $$;
`, username, username)
`, baseUsername, baseUsername)
_, err = tx.Exec(ctx, grantSelectSQL)
if err != nil {
@@ -457,7 +536,7 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %%I GRANT SELECT ON SEQUENCES TO "%s"', schema_rec.schema_name);
END LOOP;
END $$;
`, username, username)
`, baseUsername, baseUsername)
_, err = tx.Exec(ctx, defaultPrivilegesSQL)
if err != nil {
@@ -466,7 +545,7 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
// Step 7: Verify user creation before committing
var verifyUsername string
err = tx.QueryRow(ctx, fmt.Sprintf(`SELECT rolname FROM pg_roles WHERE rolname = '%s'`, username)).
err = tx.QueryRow(ctx, fmt.Sprintf(`SELECT rolname FROM pg_roles WHERE rolname = '%s'`, baseUsername)).
Scan(&verifyUsername)
if err != nil {
return "", "", fmt.Errorf("failed to verify user creation: %w", err)
@@ -477,8 +556,15 @@ func (p *PostgresqlDatabase) CreateReadOnlyUser(
}
success = true
logger.Info("Read-only user created successfully", "username", username)
return username, newPassword, nil
// Return connectionUsername (with project ID suffix for Supabase) for the caller to use when connecting
logger.Info(
"Read-only user created successfully",
"username",
baseUsername,
"connectionUsername",
connectionUsername,
)
return connectionUsername, newPassword, nil
}
return "", "", errors.New("failed to generate unique username after 3 attempts")
@@ -521,10 +607,12 @@ func testSingleDatabaseConnection(
}
}()
// Check version after successful connection
if err := verifyDatabaseVersion(ctx, conn, postgresDb.Version); err != nil {
// Detect and set the database version automatically
detectedVersion, err := detectDatabaseVersion(ctx, conn)
if err != nil {
return err
}
postgresDb.Version = detectedVersion
// Test if we can perform basic operations (like pg_dump would need)
if err := testBasicOperations(ctx, conn, *postgresDb.Database); err != nil {
@@ -538,35 +626,31 @@ func testSingleDatabaseConnection(
return nil
}
// verifyDatabaseVersion checks if the actual database version matches the specified version
func verifyDatabaseVersion(
ctx context.Context,
conn *pgx.Conn,
expectedVersion tools.PostgresqlVersion,
) error {
// detectDatabaseVersion queries and returns the PostgreSQL major version
func detectDatabaseVersion(ctx context.Context, conn *pgx.Conn) (tools.PostgresqlVersion, error) {
var versionStr string
err := conn.QueryRow(ctx, "SELECT version()").Scan(&versionStr)
if err != nil {
return fmt.Errorf("failed to query database version: %w", err)
return "", fmt.Errorf("failed to query database version: %w", err)
}
// Parse version from string like "PostgreSQL 14.2 on x86_64-pc-linux-gnu..."
re := regexp.MustCompile(`PostgreSQL (\d+)\.`)
// or "PostgreSQL 16 maintained by Postgre BY..." (some builds omit minor version)
re := regexp.MustCompile(`PostgreSQL (\d+)`)
matches := re.FindStringSubmatch(versionStr)
if len(matches) < 2 {
return fmt.Errorf("could not parse version from: %s", versionStr)
return "", fmt.Errorf("could not parse version from: %s", versionStr)
}
actualVersion := tools.GetPostgresqlVersionEnum(matches[1])
if actualVersion != expectedVersion {
return fmt.Errorf(
"you specified wrong version. Real version is %s, but you specified %s",
actualVersion,
expectedVersion,
)
}
majorVersion := matches[1]
return nil
// Map to known PostgresqlVersion enum values
switch majorVersion {
case "12", "13", "14", "15", "16", "17", "18":
return tools.PostgresqlVersion(majorVersion), nil
default:
return "", fmt.Errorf("unsupported PostgreSQL version: %s", majorVersion)
}
}
// testBasicOperations tests basic operations that backup tools need
@@ -594,7 +678,7 @@ func buildConnectionStringForDB(p *PostgresqlDatabase, dbName string, password s
}
return fmt.Sprintf(
"host=%s port=%d user=%s password=%s dbname=%s sslmode=%s default_query_exec_mode=simple_protocol standard_conforming_strings=on",
"host=%s port=%d user=%s password=%s dbname=%s sslmode=%s default_query_exec_mode=simple_protocol standard_conforming_strings=on client_encoding=UTF8",
p.Host,
p.Port,
p.Username,
@@ -614,3 +698,15 @@ func decryptPasswordIfNeeded(
}
return encryptor.Decrypt(databaseID, password)
}
func isSupabaseConnection(host, username string) bool {
return strings.Contains(strings.ToLower(host), "supabase") ||
strings.Contains(strings.ToLower(username), "supabase")
}
func extractSupabaseProjectID(username string) string {
if idx := strings.Index(username, "."); idx != -1 {
return username[idx+1:]
}
return ""
}

View File

@@ -246,6 +246,188 @@ func Test_ReadOnlyUser_MultipleSchemas_AllAccessible(t *testing.T) {
assert.NoError(t, err)
}
func Test_CreateReadOnlyUser_DatabaseNameWithDash_Success(t *testing.T) {
env := config.GetEnv()
container := connectToPostgresContainer(t, env.TestPostgres16Port)
defer container.DB.Close()
dashDbName := "test-db-with-dash"
_, err := container.DB.Exec(fmt.Sprintf(`DROP DATABASE IF EXISTS "%s"`, dashDbName))
assert.NoError(t, err)
_, err = container.DB.Exec(fmt.Sprintf(`CREATE DATABASE "%s"`, dashDbName))
assert.NoError(t, err)
defer func() {
_, _ = container.DB.Exec(fmt.Sprintf(`DROP DATABASE IF EXISTS "%s"`, dashDbName))
}()
dashDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
container.Host, container.Port, container.Username, container.Password, dashDbName)
dashDB, err := sqlx.Connect("postgres", dashDSN)
assert.NoError(t, err)
defer dashDB.Close()
_, err = dashDB.Exec(`
CREATE TABLE dash_test (
id SERIAL PRIMARY KEY,
data TEXT NOT NULL
);
INSERT INTO dash_test (data) VALUES ('test1'), ('test2');
`)
assert.NoError(t, err)
pgModel := &PostgresqlDatabase{
Version: tools.GetPostgresqlVersionEnum("16"),
Host: container.Host,
Port: container.Port,
Username: container.Username,
Password: container.Password,
Database: &dashDbName,
IsHttps: false,
}
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
ctx := context.Background()
username, password, err := pgModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New())
assert.NoError(t, err)
assert.NotEmpty(t, username)
assert.NotEmpty(t, password)
assert.True(t, strings.HasPrefix(username, "postgresus-"))
readOnlyDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
container.Host, container.Port, username, password, dashDbName)
readOnlyConn, err := sqlx.Connect("postgres", readOnlyDSN)
assert.NoError(t, err)
defer readOnlyConn.Close()
var count int
err = readOnlyConn.Get(&count, "SELECT COUNT(*) FROM dash_test")
assert.NoError(t, err)
assert.Equal(t, 2, count)
_, err = readOnlyConn.Exec("INSERT INTO dash_test (data) VALUES ('should-fail')")
assert.Error(t, err)
assert.Contains(t, err.Error(), "permission denied")
_, err = dashDB.Exec(fmt.Sprintf(`DROP OWNED BY "%s" CASCADE`, username))
if err != nil {
t.Logf("Warning: Failed to drop owned objects: %v", err)
}
_, err = dashDB.Exec(fmt.Sprintf(`DROP USER IF EXISTS "%s"`, username))
assert.NoError(t, err)
}
func Test_CreateReadOnlyUser_Supabase_UserCanReadButNotWrite(t *testing.T) {
env := config.GetEnv()
if env.TestSupabaseHost == "" {
t.Skip("Skipping Supabase test: missing environment variables")
}
portInt, err := strconv.Atoi(env.TestSupabasePort)
assert.NoError(t, err)
dsn := fmt.Sprintf(
"host=%s port=%d user=%s password=%s dbname=%s sslmode=require",
env.TestSupabaseHost,
portInt,
env.TestSupabaseUsername,
env.TestSupabasePassword,
env.TestSupabaseDatabase,
)
adminDB, err := sqlx.Connect("postgres", dsn)
assert.NoError(t, err)
defer adminDB.Close()
tableName := fmt.Sprintf(
"readonly_test_%s",
strings.ReplaceAll(uuid.New().String()[:8], "-", ""),
)
_, err = adminDB.Exec(fmt.Sprintf(`
DROP TABLE IF EXISTS public.%s CASCADE;
CREATE TABLE public.%s (
id SERIAL PRIMARY KEY,
data TEXT NOT NULL
);
INSERT INTO public.%s (data) VALUES ('test1'), ('test2');
`, tableName, tableName, tableName))
assert.NoError(t, err)
defer func() {
_, _ = adminDB.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS public.%s CASCADE`, tableName))
}()
pgModel := &PostgresqlDatabase{
Host: env.TestSupabaseHost,
Port: portInt,
Username: env.TestSupabaseUsername,
Password: env.TestSupabasePassword,
Database: &env.TestSupabaseDatabase,
IsHttps: true,
}
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
ctx := context.Background()
connectionUsername, newPassword, err := pgModel.CreateReadOnlyUser(ctx, logger, nil, uuid.New())
assert.NoError(t, err)
assert.NotEmpty(t, connectionUsername)
assert.NotEmpty(t, newPassword)
assert.True(t, strings.HasPrefix(connectionUsername, "postgresus-"))
baseUsername := connectionUsername
if idx := strings.Index(connectionUsername, "."); idx != -1 {
baseUsername = connectionUsername[:idx]
}
defer func() {
_, _ = adminDB.Exec(fmt.Sprintf(`DROP OWNED BY "%s" CASCADE`, baseUsername))
_, _ = adminDB.Exec(fmt.Sprintf(`DROP USER IF EXISTS "%s"`, baseUsername))
}()
readOnlyDSN := fmt.Sprintf(
"host=%s port=%d user=%s password=%s dbname=%s sslmode=require",
env.TestSupabaseHost,
portInt,
connectionUsername,
newPassword,
env.TestSupabaseDatabase,
)
readOnlyConn, err := sqlx.Connect("postgres", readOnlyDSN)
assert.NoError(t, err)
defer readOnlyConn.Close()
var count int
err = readOnlyConn.Get(&count, fmt.Sprintf("SELECT COUNT(*) FROM public.%s", tableName))
assert.NoError(t, err)
assert.Equal(t, 2, count)
_, err = readOnlyConn.Exec(
fmt.Sprintf("INSERT INTO public.%s (data) VALUES ('should-fail')", tableName),
)
assert.Error(t, err)
assert.Contains(t, err.Error(), "permission denied")
_, err = readOnlyConn.Exec(
fmt.Sprintf("UPDATE public.%s SET data = 'hacked' WHERE id = 1", tableName),
)
assert.Error(t, err)
assert.Contains(t, err.Error(), "permission denied")
_, err = readOnlyConn.Exec(fmt.Sprintf("DELETE FROM public.%s WHERE id = 1", tableName))
assert.Error(t, err)
assert.Contains(t, err.Error(), "permission denied")
_, err = readOnlyConn.Exec("CREATE TABLE public.hack_table (id INT)")
assert.Error(t, err)
assert.Contains(t, err.Error(), "permission denied")
}
type PostgresContainer struct {
Host string
Port int

View File

@@ -75,6 +75,16 @@ func (d *Database) EncryptSensitiveFields(encryptor encryption.FieldEncryptor) e
return nil
}
func (d *Database) PopulateVersionIfEmpty(
logger *slog.Logger,
encryptor encryption.FieldEncryptor,
) error {
if d.Postgresql != nil {
return d.Postgresql.PopulateVersionIfEmpty(logger, encryptor, d.ID)
}
return nil
}
func (d *Database) Update(incoming *Database) {
d.Name = incoming.Name
d.Type = incoming.Type

View File

@@ -68,6 +68,10 @@ func (s *DatabaseService) CreateDatabase(
return nil, err
}
if err := database.PopulateVersionIfEmpty(s.logger, s.fieldEncryptor); err != nil {
return nil, fmt.Errorf("failed to auto-detect database version: %w", err)
}
if err := database.EncryptSensitiveFields(s.fieldEncryptor); err != nil {
return nil, fmt.Errorf("failed to encrypt sensitive fields: %w", err)
}
@@ -125,6 +129,10 @@ func (s *DatabaseService) UpdateDatabase(
return err
}
if err := existingDatabase.PopulateVersionIfEmpty(s.logger, s.fieldEncryptor); err != nil {
return fmt.Errorf("failed to auto-detect database version: %w", err)
}
if err := existingDatabase.EncryptSensitiveFields(s.fieldEncryptor); err != nil {
return fmt.Errorf("failed to encrypt sensitive fields: %w", err)
}

View File

@@ -8,6 +8,7 @@ import (
"postgresus-backend/internal/features/restores/usecases"
"postgresus-backend/internal/features/storages"
workspaces_services "postgresus-backend/internal/features/workspaces/services"
"postgresus-backend/internal/util/encryption"
"postgresus-backend/internal/util/logger"
)
@@ -22,6 +23,7 @@ var restoreService = &RestoreService{
logger.GetLogger(),
workspaces_services.GetWorkspaceService(),
audit_logs.GetAuditLogService(),
encryption.GetFieldEncryptor(),
}
var restoreController = &RestoreController{
restoreService,

View File

@@ -14,6 +14,7 @@ import (
"postgresus-backend/internal/features/storages"
users_models "postgresus-backend/internal/features/users/models"
workspaces_services "postgresus-backend/internal/features/workspaces/services"
"postgresus-backend/internal/util/encryption"
"postgresus-backend/internal/util/tools"
"time"
@@ -30,6 +31,7 @@ type RestoreService struct {
logger *slog.Logger
workspaceService *workspaces_services.WorkspaceService
auditLogService *audit_logs.AuditLogService
fieldEncryptor encryption.FieldEncryptor
}
func (s *RestoreService) OnBeforeBackupRemove(backup *backups.Backup) error {
@@ -120,12 +122,6 @@ func (s *RestoreService) RestoreBackupWithAuth(
return err
}
fmt.Printf(
"restore from %s to %s\n",
backupDatabase.Postgresql.Version,
requestDTO.PostgresqlDatabase.Version,
)
if tools.IsBackupDbVersionHigherThanRestoreDbVersion(
backupDatabase.Postgresql.Version,
requestDTO.PostgresqlDatabase.Version,
@@ -214,6 +210,10 @@ func (s *RestoreService) RestoreBackup(
Postgresql: requestDTO.PostgresqlDatabase,
}
if err := restoringToDB.PopulateVersionIfEmpty(s.logger, s.fieldEncryptor); err != nil {
return fmt.Errorf("failed to auto-detect database version: %w", err)
}
err = s.restoreBackupUsecase.Execute(
backupConfig,
restore,

View File

@@ -2,7 +2,6 @@ package local_storage
import (
"context"
"errors"
"fmt"
"io"
"log/slog"
@@ -16,10 +15,10 @@ import (
)
const (
// Chunk size for local storage writes - 16MB provides good balance between
// memory usage and write efficiency. This creates backpressure to pg_dump
// by only reading one chunk at a time and waiting for disk to confirm receipt.
localChunkSize = 16 * 1024 * 1024
// Chunk size for local storage writes - 8MB per buffer with double-buffering
// allows overlapped I/O while keeping total memory under 32MB.
// Two 8MB buffers = 16MB for local storage, plus 8MB for pg_dump buffer = ~25MB total.
localChunkSize = 8 * 1024 * 1024
)
// LocalStorage uses ./postgresus_local_backups folder as a
@@ -192,11 +191,6 @@ func (l *LocalStorage) EncryptSensitiveData(encryptor encryption.FieldEncryptor)
func (l *LocalStorage) Update(incoming *LocalStorage) {
}
type writeResult struct {
bytesWritten int
writeErr error
}
func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) {
buf := make([]byte, localChunkSize)
var written int64
@@ -208,54 +202,23 @@ func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (int64,
default:
}
nr, readErr := io.ReadFull(src, buf)
if nr == 0 && readErr == io.EOF {
break
}
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
return written, readErr
}
writeResultCh := make(chan writeResult, 1)
go func() {
nw, writeErr := dst.Write(buf[0:nr])
writeResultCh <- writeResult{nw, writeErr}
}()
var nw int
var writeErr error
select {
case <-ctx.Done():
return written, ctx.Err()
case result := <-writeResultCh:
nw = result.bytesWritten
writeErr = result.writeErr
}
if nw < 0 || nr < nw {
nw = 0
if writeErr == nil {
writeErr = errors.New("invalid write result")
nr, readErr := src.Read(buf)
if nr > 0 {
nw, writeErr := dst.Write(buf[:nr])
written += int64(nw)
if writeErr != nil {
return written, writeErr
}
if nr != nw {
return written, io.ErrShortWrite
}
}
if writeErr != nil {
return written, writeErr
if readErr == io.EOF {
return written, nil
}
if nr != nw {
return written, io.ErrShortWrite
}
written += int64(nw)
if readErr == io.EOF || readErr == io.ErrUnexpectedEOF {
break
if readErr != nil {
return written, readErr
}
}
return written, nil
}

View File

@@ -3,6 +3,7 @@ package s3_storage
import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
@@ -40,6 +41,7 @@ type S3Storage struct {
S3Prefix string `json:"s3Prefix" gorm:"type:text;column:s3_prefix"`
S3UseVirtualHostedStyle bool `json:"s3UseVirtualHostedStyle" gorm:"default:false;column:s3_use_virtual_hosted_style"`
SkipTLSVerify bool `json:"skipTLSVerify" gorm:"default:false;column:skip_tls_verify"`
}
func (s *S3Storage) TableName() string {
@@ -331,6 +333,7 @@ func (s *S3Storage) Update(incoming *S3Storage) {
s.S3Region = incoming.S3Region
s.S3Endpoint = incoming.S3Endpoint
s.S3UseVirtualHostedStyle = incoming.S3UseVirtualHostedStyle
s.SkipTLSVerify = incoming.SkipTLSVerify
if incoming.S3AccessKey != "" {
s.S3AccessKey = incoming.S3AccessKey
@@ -442,6 +445,9 @@ func (s *S3Storage) getClientParams(
TLSHandshakeTimeout: s3TLSHandshakeTimeout,
ResponseHeaderTimeout: s3ResponseTimeout,
IdleConnTimeout: s3IdleConnTimeout,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: s.SkipTLSVerify,
},
}
return endpoint, useSSL, accessKey, secretKey, bucketLookup, transport, nil

View File

@@ -30,7 +30,6 @@ import (
workspaces_controllers "postgresus-backend/internal/features/workspaces/controllers"
workspaces_testing "postgresus-backend/internal/features/workspaces/testing"
test_utils "postgresus-backend/internal/util/testing"
"postgresus-backend/internal/util/tools"
)
const createAndFillTableQuery = `
@@ -114,6 +113,382 @@ func Test_BackupAndRestorePostgresqlWithEncryption_RestoreIsSuccessful(t *testin
}
}
func Test_BackupAndRestoreSupabase_PublicSchemaOnly_RestoreIsSuccessful(t *testing.T) {
env := config.GetEnv()
if env.TestSupabaseHost == "" {
t.Skip("Skipping Supabase test: missing environment variables")
}
portInt, err := strconv.Atoi(env.TestSupabasePort)
assert.NoError(t, err)
dsn := fmt.Sprintf(
"host=%s port=%d user=%s password=%s dbname=%s sslmode=require",
env.TestSupabaseHost,
portInt,
env.TestSupabaseUsername,
env.TestSupabasePassword,
env.TestSupabaseDatabase,
)
supabaseDB, err := sqlx.Connect("postgres", dsn)
assert.NoError(t, err)
defer supabaseDB.Close()
tableName := fmt.Sprintf("backup_test_%s", uuid.New().String()[:8])
createTableQuery := fmt.Sprintf(`
DROP TABLE IF EXISTS public.%s;
CREATE TABLE public.%s (
id SERIAL PRIMARY KEY,
name TEXT NOT NULL,
value INTEGER NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
INSERT INTO public.%s (name, value) VALUES
('test1', 100),
('test2', 200),
('test3', 300);
`, tableName, tableName, tableName)
_, err = supabaseDB.Exec(createTableQuery)
assert.NoError(t, err)
defer func() {
_, _ = supabaseDB.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS public.%s`, tableName))
}()
router := createTestRouter()
user := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Supabase Test Workspace", user, router)
storage := storages.CreateTestStorage(workspace.ID)
database := createSupabaseDatabaseViaAPI(
t, router, "Supabase Test Database", workspace.ID,
env.TestSupabaseHost, portInt,
env.TestSupabaseUsername, env.TestSupabasePassword, env.TestSupabaseDatabase,
[]string{"public"},
user.Token,
)
enableBackupsViaAPI(
t, router, database.ID, storage.ID,
backups_config.BackupEncryptionNone, user.Token,
)
createBackupViaAPI(t, router, database.ID, user.Token)
backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute)
assert.Equal(t, backups.BackupStatusCompleted, backup.Status)
_, err = supabaseDB.Exec(fmt.Sprintf(`DELETE FROM public.%s`, tableName))
assert.NoError(t, err)
var countAfterDelete int
err = supabaseDB.Get(
&countAfterDelete,
fmt.Sprintf(`SELECT COUNT(*) FROM public.%s`, tableName),
)
assert.NoError(t, err)
assert.Equal(t, 0, countAfterDelete, "Table should be empty after delete")
createSupabaseRestoreViaAPI(
t, router, backup.ID,
env.TestSupabaseHost, portInt,
env.TestSupabaseUsername, env.TestSupabasePassword, env.TestSupabaseDatabase,
user.Token,
)
restore := waitForRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute)
assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status)
var countAfterRestore int
err = supabaseDB.Get(
&countAfterRestore,
fmt.Sprintf(`SELECT COUNT(*) FROM public.%s`, tableName),
)
assert.NoError(t, err)
assert.Equal(t, 3, countAfterRestore, "Table should have 3 rows after restore")
var restoredData []TestDataItem
err = supabaseDB.Select(
&restoredData,
fmt.Sprintf(`SELECT id, name, value, created_at FROM public.%s ORDER BY id`, tableName),
)
assert.NoError(t, err)
assert.Len(t, restoredData, 3)
assert.Equal(t, "test1", restoredData[0].Name)
assert.Equal(t, 100, restoredData[0].Value)
assert.Equal(t, "test2", restoredData[1].Name)
assert.Equal(t, 200, restoredData[1].Value)
assert.Equal(t, "test3", restoredData[2].Name)
assert.Equal(t, 300, restoredData[2].Value)
err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))
if err != nil {
t.Logf("Warning: Failed to delete backup file: %v", err)
}
test_utils.MakeDeleteRequest(
t,
router,
"/api/v1/databases/"+database.ID.String(),
"Bearer "+user.Token,
http.StatusNoContent,
)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}
func Test_BackupPostgresql_SchemaSelection_AllSchemasWhenNoneSpecified(t *testing.T) {
env := config.GetEnv()
container, err := connectToPostgresContainer("16", env.TestPostgres16Port)
assert.NoError(t, err)
defer container.DB.Close()
_, err = container.DB.Exec(`
DROP SCHEMA IF EXISTS schema_a CASCADE;
DROP SCHEMA IF EXISTS schema_b CASCADE;
CREATE SCHEMA schema_a;
CREATE SCHEMA schema_b;
CREATE TABLE public.public_table (id SERIAL PRIMARY KEY, data TEXT);
CREATE TABLE schema_a.table_a (id SERIAL PRIMARY KEY, data TEXT);
CREATE TABLE schema_b.table_b (id SERIAL PRIMARY KEY, data TEXT);
INSERT INTO public.public_table (data) VALUES ('public_data');
INSERT INTO schema_a.table_a (data) VALUES ('schema_a_data');
INSERT INTO schema_b.table_b (data) VALUES ('schema_b_data');
`)
assert.NoError(t, err)
defer func() {
_, _ = container.DB.Exec(`
DROP TABLE IF EXISTS public.public_table;
DROP SCHEMA IF EXISTS schema_a CASCADE;
DROP SCHEMA IF EXISTS schema_b CASCADE;
`)
}()
router := createTestRouter()
user := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Schema Test Workspace", user, router)
storage := storages.CreateTestStorage(workspace.ID)
database := createDatabaseWithSchemasViaAPI(
t, router, "All Schemas Database", workspace.ID,
container.Host, container.Port,
container.Username, container.Password, container.Database,
nil,
user.Token,
)
enableBackupsViaAPI(
t, router, database.ID, storage.ID,
backups_config.BackupEncryptionNone, user.Token,
)
createBackupViaAPI(t, router, database.ID, user.Token)
backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute)
assert.Equal(t, backups.BackupStatusCompleted, backup.Status)
newDBName := "restored_all_schemas"
_, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName))
assert.NoError(t, err)
_, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName))
assert.NoError(t, err)
newDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
container.Host, container.Port, container.Username, container.Password, newDBName)
newDB, err := sqlx.Connect("postgres", newDSN)
assert.NoError(t, err)
defer newDB.Close()
createRestoreViaAPI(
t, router, backup.ID,
container.Host, container.Port,
container.Username, container.Password, newDBName,
user.Token,
)
restore := waitForRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute)
assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status)
var publicTableExists bool
err = newDB.Get(&publicTableExists, `
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'public' AND table_name = 'public_table'
)
`)
assert.NoError(t, err)
assert.True(t, publicTableExists, "public.public_table should exist in restored database")
var schemaATableExists bool
err = newDB.Get(&schemaATableExists, `
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'schema_a' AND table_name = 'table_a'
)
`)
assert.NoError(t, err)
assert.True(t, schemaATableExists, "schema_a.table_a should exist in restored database")
var schemaBTableExists bool
err = newDB.Get(&schemaBTableExists, `
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'schema_b' AND table_name = 'table_b'
)
`)
assert.NoError(t, err)
assert.True(t, schemaBTableExists, "schema_b.table_b should exist in restored database")
err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))
if err != nil {
t.Logf("Warning: Failed to delete backup file: %v", err)
}
test_utils.MakeDeleteRequest(
t,
router,
"/api/v1/databases/"+database.ID.String(),
"Bearer "+user.Token,
http.StatusNoContent,
)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}
func Test_BackupPostgresql_SchemaSelection_OnlySpecifiedSchemas(t *testing.T) {
env := config.GetEnv()
container, err := connectToPostgresContainer("16", env.TestPostgres16Port)
assert.NoError(t, err)
defer container.DB.Close()
_, err = container.DB.Exec(`
DROP SCHEMA IF EXISTS schema_a CASCADE;
DROP SCHEMA IF EXISTS schema_b CASCADE;
CREATE SCHEMA schema_a;
CREATE SCHEMA schema_b;
CREATE TABLE public.public_table (id SERIAL PRIMARY KEY, data TEXT);
CREATE TABLE schema_a.table_a (id SERIAL PRIMARY KEY, data TEXT);
CREATE TABLE schema_b.table_b (id SERIAL PRIMARY KEY, data TEXT);
INSERT INTO public.public_table (data) VALUES ('public_data');
INSERT INTO schema_a.table_a (data) VALUES ('schema_a_data');
INSERT INTO schema_b.table_b (data) VALUES ('schema_b_data');
`)
assert.NoError(t, err)
defer func() {
_, _ = container.DB.Exec(`
DROP TABLE IF EXISTS public.public_table;
DROP SCHEMA IF EXISTS schema_a CASCADE;
DROP SCHEMA IF EXISTS schema_b CASCADE;
`)
}()
router := createTestRouter()
user := users_testing.CreateTestUser(users_enums.UserRoleMember)
workspace := workspaces_testing.CreateTestWorkspace("Schema Test Workspace", user, router)
storage := storages.CreateTestStorage(workspace.ID)
database := createDatabaseWithSchemasViaAPI(
t, router, "Specific Schemas Database", workspace.ID,
container.Host, container.Port,
container.Username, container.Password, container.Database,
[]string{"public", "schema_a"},
user.Token,
)
enableBackupsViaAPI(
t, router, database.ID, storage.ID,
backups_config.BackupEncryptionNone, user.Token,
)
createBackupViaAPI(t, router, database.ID, user.Token)
backup := waitForBackupCompletion(t, router, database.ID, user.Token, 5*time.Minute)
assert.Equal(t, backups.BackupStatusCompleted, backup.Status)
newDBName := "restored_specific_schemas"
_, err = container.DB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s;", newDBName))
assert.NoError(t, err)
_, err = container.DB.Exec(fmt.Sprintf("CREATE DATABASE %s;", newDBName))
assert.NoError(t, err)
newDSN := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
container.Host, container.Port, container.Username, container.Password, newDBName)
newDB, err := sqlx.Connect("postgres", newDSN)
assert.NoError(t, err)
defer newDB.Close()
createRestoreViaAPI(
t, router, backup.ID,
container.Host, container.Port,
container.Username, container.Password, newDBName,
user.Token,
)
restore := waitForRestoreCompletion(t, router, backup.ID, user.Token, 5*time.Minute)
assert.Equal(t, restores_enums.RestoreStatusCompleted, restore.Status)
var publicTableExists bool
err = newDB.Get(&publicTableExists, `
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'public' AND table_name = 'public_table'
)
`)
assert.NoError(t, err)
assert.True(t, publicTableExists, "public.public_table should exist (was included)")
var schemaATableExists bool
err = newDB.Get(&schemaATableExists, `
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'schema_a' AND table_name = 'table_a'
)
`)
assert.NoError(t, err)
assert.True(t, schemaATableExists, "schema_a.table_a should exist (was included)")
var schemaBTableExists bool
err = newDB.Get(&schemaBTableExists, `
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'schema_b' AND table_name = 'table_b'
)
`)
assert.NoError(t, err)
assert.False(t, schemaBTableExists, "schema_b.table_b should NOT exist (was excluded)")
err = os.Remove(filepath.Join(config.GetEnv().DataFolder, backup.ID.String()))
if err != nil {
t.Logf("Warning: Failed to delete backup file: %v", err)
}
test_utils.MakeDeleteRequest(
t,
router,
"/api/v1/databases/"+database.ID.String(),
"Bearer "+user.Token,
http.StatusNoContent,
)
storages.RemoveTestStorage(storage.ID)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}
func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {
container, err := connectToPostgresContainer(pgVersion, port)
assert.NoError(t, err)
@@ -132,10 +507,9 @@ func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {
storage := storages.CreateTestStorage(workspace.ID)
pgVersionEnum := tools.GetPostgresqlVersionEnum(pgVersion)
database := createDatabaseViaAPI(
t, router, "Test Database", workspace.ID,
pgVersionEnum, container.Host, container.Port,
container.Host, container.Port,
container.Username, container.Password, container.Database,
user.Token,
)
@@ -164,7 +538,7 @@ func testBackupRestoreForVersion(t *testing.T, pgVersion string, port string) {
defer newDB.Close()
createRestoreViaAPI(
t, router, backup.ID, pgVersionEnum,
t, router, backup.ID,
container.Host, container.Port,
container.Username, container.Password, newDBName,
user.Token,
@@ -217,10 +591,9 @@ func testBackupRestoreWithEncryptionForVersion(t *testing.T, pgVersion string, p
storage := storages.CreateTestStorage(workspace.ID)
pgVersionEnum := tools.GetPostgresqlVersionEnum(pgVersion)
database := createDatabaseViaAPI(
t, router, "Test Database", workspace.ID,
pgVersionEnum, container.Host, container.Port,
container.Host, container.Port,
container.Username, container.Password, container.Database,
user.Token,
)
@@ -250,7 +623,7 @@ func testBackupRestoreWithEncryptionForVersion(t *testing.T, pgVersion string, p
defer newDB.Close()
createRestoreViaAPI(
t, router, backup.ID, pgVersionEnum,
t, router, backup.ID,
container.Host, container.Port,
container.Username, container.Password, newDBName,
user.Token,
@@ -379,7 +752,6 @@ func createDatabaseViaAPI(
router *gin.Engine,
name string,
workspaceID uuid.UUID,
pgVersion tools.PostgresqlVersion,
host string,
port int,
username string,
@@ -392,7 +764,6 @@ func createDatabaseViaAPI(
WorkspaceID: &workspaceID,
Type: databases.DatabaseTypePostgres,
Postgresql: &pgtypes.PostgresqlDatabase{
Version: pgVersion,
Host: host,
Port: port,
Username: username,
@@ -475,7 +846,6 @@ func createRestoreViaAPI(
t *testing.T,
router *gin.Engine,
backupID uuid.UUID,
pgVersion tools.PostgresqlVersion,
host string,
port int,
username string,
@@ -485,7 +855,6 @@ func createRestoreViaAPI(
) {
request := restores.RestoreBackupRequest{
PostgresqlDatabase: &pgtypes.PostgresqlDatabase{
Version: pgVersion,
Host: host,
Port: port,
Username: username,
@@ -504,6 +873,141 @@ func createRestoreViaAPI(
)
}
func createDatabaseWithSchemasViaAPI(
t *testing.T,
router *gin.Engine,
name string,
workspaceID uuid.UUID,
host string,
port int,
username string,
password string,
database string,
includeSchemas []string,
token string,
) *databases.Database {
request := databases.Database{
Name: name,
WorkspaceID: &workspaceID,
Type: databases.DatabaseTypePostgres,
Postgresql: &pgtypes.PostgresqlDatabase{
Host: host,
Port: port,
Username: username,
Password: password,
Database: &database,
IncludeSchemas: includeSchemas,
},
}
w := workspaces_testing.MakeAPIRequest(
router,
"POST",
"/api/v1/databases/create",
"Bearer "+token,
request,
)
if w.Code != http.StatusCreated {
t.Fatalf(
"Failed to create database with schemas. Status: %d, Body: %s",
w.Code,
w.Body.String(),
)
}
var createdDatabase databases.Database
if err := json.Unmarshal(w.Body.Bytes(), &createdDatabase); err != nil {
t.Fatalf("Failed to unmarshal database response: %v", err)
}
return &createdDatabase
}
func createSupabaseDatabaseViaAPI(
t *testing.T,
router *gin.Engine,
name string,
workspaceID uuid.UUID,
host string,
port int,
username string,
password string,
database string,
includeSchemas []string,
token string,
) *databases.Database {
request := databases.Database{
Name: name,
WorkspaceID: &workspaceID,
Type: databases.DatabaseTypePostgres,
Postgresql: &pgtypes.PostgresqlDatabase{
Host: host,
Port: port,
Username: username,
Password: password,
Database: &database,
IsHttps: true,
IncludeSchemas: includeSchemas,
},
}
w := workspaces_testing.MakeAPIRequest(
router,
"POST",
"/api/v1/databases/create",
"Bearer "+token,
request,
)
if w.Code != http.StatusCreated {
t.Fatalf(
"Failed to create Supabase database. Status: %d, Body: %s",
w.Code,
w.Body.String(),
)
}
var createdDatabase databases.Database
if err := json.Unmarshal(w.Body.Bytes(), &createdDatabase); err != nil {
t.Fatalf("Failed to unmarshal database response: %v", err)
}
return &createdDatabase
}
func createSupabaseRestoreViaAPI(
t *testing.T,
router *gin.Engine,
backupID uuid.UUID,
host string,
port int,
username string,
password string,
database string,
token string,
) {
request := restores.RestoreBackupRequest{
PostgresqlDatabase: &pgtypes.PostgresqlDatabase{
Host: host,
Port: port,
Username: username,
Password: password,
Database: &database,
IsHttps: true,
},
}
test_utils.MakePostRequest(
t,
router,
fmt.Sprintf("/api/v1/restores/%s/restore", backupID.String()),
"Bearer "+token,
request,
http.StatusOK,
)
}
func verifyDataIntegrity(t *testing.T, originalDB *sqlx.DB, restoredDB *sqlx.DB) {
var originalData []TestDataItem
var restoredData []TestDataItem
@@ -550,7 +1054,6 @@ func connectToPostgresContainer(version string, port string) (*PostgresContainer
Username: username,
Password: password,
Database: dbName,
Version: version,
DB: db,
}, nil
}

View File

@@ -0,0 +1,11 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE postgresql_databases
ADD COLUMN include_schemas TEXT NOT NULL DEFAULT '';
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE postgresql_databases
DROP COLUMN include_schemas;
-- +goose StatementEnd

View File

@@ -0,0 +1,11 @@
-- +goose Up
-- +goose StatementBegin
ALTER TABLE s3_storages
ADD COLUMN skip_tls_verify BOOLEAN NOT NULL DEFAULT FALSE;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
ALTER TABLE s3_storages
DROP COLUMN skip_tls_verify;
-- +goose StatementEnd

View File

@@ -32,6 +32,29 @@ Then open `http://localhost:4005` in your browser.
| `image.pullPolicy` | Image pull policy | `Always` |
| `replicaCount` | Number of replicas | `1` |
### Custom Root CA
| Parameter | Description | Default Value |
| -------------- | ---------------------------------------- | ------------- |
| `customRootCA` | Name of Secret containing CA certificate | `""` |
To trust a custom CA certificate (e.g., for internal services with self-signed certificates):
1. Create a Secret with your CA certificate:
```bash
kubectl create secret generic my-root-ca \
--from-file=ca.crt=./path/to/ca-certificate.crt
```
2. Reference it in values:
```yaml
customRootCA: my-root-ca
```
The certificate will be mounted to `/etc/ssl/certs/custom-root-ca.crt` and the `SSL_CERT_FILE` environment variable will be set automatically.
### Service
| Parameter | Description | Default Value |

View File

@@ -39,6 +39,11 @@ spec:
- name: {{ .Chart.Name }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- if .Values.customRootCA }}
env:
- name: SSL_CERT_FILE
value: /etc/ssl/certs/custom-root-ca.crt
{{- end }}
ports:
- name: http
containerPort: {{ .Values.service.targetPort }}
@@ -46,6 +51,12 @@ spec:
volumeMounts:
- name: postgresus-storage
mountPath: {{ .Values.persistence.mountPath }}
{{- if .Values.customRootCA }}
- name: custom-root-ca
mountPath: /etc/ssl/certs/custom-root-ca.crt
subPath: ca.crt
readOnly: true
{{- end }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- if .Values.livenessProbe.enabled }}
@@ -66,6 +77,12 @@ spec:
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
{{- end }}
{{- if .Values.customRootCA }}
volumes:
- name: custom-root-ca
secret:
secretName: {{ .Values.customRootCA }}
{{- end }}
{{- if .Values.persistence.enabled }}
volumeClaimTemplates:
- metadata:

View File

@@ -9,6 +9,9 @@ image:
# StatefulSet configuration
replicaCount: 1
# RootCA setup, need name of secret in same namespace
customRootCA: ""
# Service configuration
service:
type: ClusterIP

View File

@@ -11,4 +11,7 @@ export interface PostgresqlDatabase {
password: string;
database?: string;
isHttps: boolean;
// backup settings
includeSchemas?: string[];
}

View File

@@ -6,4 +6,5 @@ export interface S3Storage {
s3Endpoint?: string;
s3Prefix?: string;
s3UseVirtualHostedStyle?: boolean;
skipTLSVerify?: boolean;
}

View File

@@ -1,13 +1,8 @@
import { InfoCircleOutlined } from '@ant-design/icons';
import { Button, Input, InputNumber, Select, Switch, Tooltip } from 'antd';
import { DownOutlined, UpOutlined } from '@ant-design/icons';
import { Button, Input, InputNumber, Select, Switch } from 'antd';
import { useEffect, useState } from 'react';
import {
type Database,
DatabaseType,
PostgresqlVersion,
databaseApi,
} from '../../../../entity/databases';
import { type Database, DatabaseType, databaseApi } from '../../../../entity/databases';
import { ToastHelper } from '../../../../shared/toast';
interface Props {
@@ -23,7 +18,6 @@ interface Props {
isSaveToApi: boolean;
onSaved: (database: Database) => void;
isShowDbVersionHint?: boolean;
isShowDbName?: boolean;
}
@@ -39,8 +33,6 @@ export const EditDatabaseSpecificDataComponent = ({
saveButtonText,
isSaveToApi,
onSaved,
isShowDbVersionHint = true,
isShowDbName = true,
}: Props) => {
const [editingDatabase, setEditingDatabase] = useState<Database>();
@@ -50,6 +42,36 @@ export const EditDatabaseSpecificDataComponent = ({
const [isTestingConnection, setIsTestingConnection] = useState(false);
const [isConnectionFailed, setIsConnectionFailed] = useState(false);
const hasAdvancedValues = !!database.postgresql?.includeSchemas?.length;
const [isShowAdvanced, setShowAdvanced] = useState(hasAdvancedValues);
const [hasAutoAddedPublicSchema, setHasAutoAddedPublicSchema] = useState(false);
const autoAddPublicSchemaForSupabase = (updatedDatabase: Database): Database => {
if (hasAutoAddedPublicSchema) return updatedDatabase;
const host = updatedDatabase.postgresql?.host || '';
const username = updatedDatabase.postgresql?.username || '';
const isSupabase = host.includes('supabase') || username.includes('supabase');
if (isSupabase && updatedDatabase.postgresql) {
setHasAutoAddedPublicSchema(true);
const currentSchemas = updatedDatabase.postgresql.includeSchemas || [];
if (!currentSchemas.includes('public')) {
return {
...updatedDatabase,
postgresql: {
...updatedDatabase.postgresql,
includeSchemas: ['public', ...currentSchemas],
},
};
}
}
return updatedDatabase;
};
const testConnection = async () => {
if (!editingDatabase) return;
setIsTestingConnection(true);
@@ -100,7 +122,6 @@ export const EditDatabaseSpecificDataComponent = ({
if (!editingDatabase) return null;
let isAllFieldsFilled = true;
if (!editingDatabase.postgresql?.version) isAllFieldsFilled = false;
if (!editingDatabase.postgresql?.host) isAllFieldsFilled = false;
if (!editingDatabase.postgresql?.port) isAllFieldsFilled = false;
if (!editingDatabase.postgresql?.username) isAllFieldsFilled = false;
@@ -111,51 +132,14 @@ export const EditDatabaseSpecificDataComponent = ({
editingDatabase.postgresql?.host?.includes('localhost') ||
editingDatabase.postgresql?.host?.includes('127.0.0.1');
const isSupabaseDb =
editingDatabase.postgresql?.host?.includes('supabase') ||
editingDatabase.postgresql?.username?.includes('supabase');
return (
<div>
{editingDatabase.type === DatabaseType.POSTGRES && (
<>
<div className="mb-1 flex w-full items-center">
<div className="min-w-[150px]">PG version</div>
<Select
value={editingDatabase.postgresql?.version}
onChange={(v) => {
if (!editingDatabase.postgresql) return;
setEditingDatabase({
...editingDatabase,
postgresql: {
...editingDatabase.postgresql,
version: v as PostgresqlVersion,
},
});
setIsConnectionTested(false);
}}
size="small"
className="max-w-[200px] grow"
placeholder="Select PG version"
options={[
{ label: '12', value: PostgresqlVersion.PostgresqlVersion12 },
{ label: '13', value: PostgresqlVersion.PostgresqlVersion13 },
{ label: '14', value: PostgresqlVersion.PostgresqlVersion14 },
{ label: '15', value: PostgresqlVersion.PostgresqlVersion15 },
{ label: '16', value: PostgresqlVersion.PostgresqlVersion16 },
{ label: '17', value: PostgresqlVersion.PostgresqlVersion17 },
{ label: '18', value: PostgresqlVersion.PostgresqlVersion18 },
]}
/>
{isShowDbVersionHint && (
<Tooltip
className="cursor-pointer"
title="Please select the version of PostgreSQL you are backing up now. You will be able to restore backup to the same version or higher"
>
<InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />
</Tooltip>
)}
</div>
<div className="mb-1 flex w-full items-center">
<div className="min-w-[150px]">Host</div>
<Input
@@ -163,13 +147,14 @@ export const EditDatabaseSpecificDataComponent = ({
onChange={(e) => {
if (!editingDatabase.postgresql) return;
setEditingDatabase({
const updatedDatabase = {
...editingDatabase,
postgresql: {
...editingDatabase.postgresql,
host: e.target.value.trim().replace('https://', '').replace('http://', ''),
},
});
};
setEditingDatabase(autoAddPublicSchemaForSupabase(updatedDatabase));
setIsConnectionTested(false);
}}
size="small"
@@ -184,7 +169,7 @@ export const EditDatabaseSpecificDataComponent = ({
<div className="max-w-[200px] text-xs text-gray-500 dark:text-gray-400">
Please{' '}
<a
href="https://postgresus.com/faq#how-to-backup-localhost"
href="https://postgresus.com/faq/localhost"
target="_blank"
rel="noreferrer"
className="!text-blue-600 dark:!text-blue-400"
@@ -196,6 +181,24 @@ export const EditDatabaseSpecificDataComponent = ({
</div>
)}
{isSupabaseDb && (
<div className="mb-1 flex">
<div className="min-w-[150px]" />
<div className="max-w-[200px] text-xs text-gray-500 dark:text-gray-400">
Please{' '}
<a
href="https://postgresus.com/faq/supabase"
target="_blank"
rel="noreferrer"
className="!text-blue-600 dark:!text-blue-400"
>
read this document
</a>{' '}
to study how to backup Supabase database
</div>
</div>
)}
<div className="mb-1 flex w-full items-center">
<div className="min-w-[150px]">Port</div>
<InputNumber
@@ -223,10 +226,11 @@ export const EditDatabaseSpecificDataComponent = ({
onChange={(e) => {
if (!editingDatabase.postgresql) return;
setEditingDatabase({
const updatedDatabase = {
...editingDatabase,
postgresql: { ...editingDatabase.postgresql, username: e.target.value.trim() },
});
};
setEditingDatabase(autoAddPublicSchemaForSupabase(updatedDatabase));
setIsConnectionTested(false);
}}
size="small"
@@ -291,6 +295,43 @@ export const EditDatabaseSpecificDataComponent = ({
size="small"
/>
</div>
<div className="mt-4 mb-3 flex items-center">
<div
className="flex cursor-pointer items-center text-sm text-blue-600 hover:text-blue-800"
onClick={() => setShowAdvanced(!isShowAdvanced)}
>
<span className="mr-2">Advanced settings</span>
{isShowAdvanced ? (
<UpOutlined style={{ fontSize: '12px' }} />
) : (
<DownOutlined style={{ fontSize: '12px' }} />
)}
</div>
</div>
{isShowAdvanced && (
<div className="mb-1 flex w-full items-center">
<div className="min-w-[150px]">Include schemas</div>
<Select
mode="tags"
value={editingDatabase.postgresql?.includeSchemas || []}
onChange={(values) => {
if (!editingDatabase.postgresql) return;
setEditingDatabase({
...editingDatabase,
postgresql: { ...editingDatabase.postgresql, includeSchemas: values },
});
}}
size="small"
className="max-w-[200px] grow"
placeholder="All schemas (default)"
tokenSeparators={[',']}
/>
</div>
)}
</>
)}

View File

@@ -57,6 +57,13 @@ export const ShowDatabaseSpecificDataComponent = ({ database }: Props) => {
<div className="min-w-[150px]">Use HTTPS</div>
<div>{database.postgresql?.isHttps ? 'Yes' : 'No'}</div>
</div>
{!!database.postgresql?.includeSchemas?.length && (
<div className="mb-1 flex w-full items-center">
<div className="min-w-[150px]">Include schemas</div>
<div>{database.postgresql.includeSchemas.join(', ')}</div>
</div>
)}
</>
)}
</div>

View File

@@ -111,7 +111,6 @@ export const RestoresComponent = ({ database, backup }: Props) => {
setEditingDatabase({ ...database });
restore(database);
}}
isShowDbVersionHint={false}
/>
</>
);

View File

@@ -39,6 +39,7 @@ export function EditStorageComponent({
const [isTestingConnection, setIsTestingConnection] = useState(false);
const [isTestConnectionSuccess, setIsTestConnectionSuccess] = useState(false);
const [connectionError, setConnectionError] = useState<string | undefined>();
const save = async () => {
if (!storage) return;
@@ -60,6 +61,7 @@ export function EditStorageComponent({
if (!storage) return;
setIsTestingConnection(true);
setConnectionError(undefined);
try {
await storageApi.testStorageConnectionDirect(storage);
@@ -69,7 +71,9 @@ export function EditStorageComponent({
description: 'Storage connection tested successfully',
});
} catch (e) {
alert((e as Error).message);
const errorMessage = (e as Error).message;
setConnectionError(errorMessage);
alert(errorMessage);
}
setIsTestingConnection(false);
@@ -290,7 +294,9 @@ export function EditStorageComponent({
setUnsaved={() => {
setIsUnsaved(true);
setIsTestConnectionSuccess(false);
setConnectionError(undefined);
}}
connectionError={connectionError}
/>
)}

View File

@@ -1,6 +1,6 @@
import { DownOutlined, InfoCircleOutlined, UpOutlined } from '@ant-design/icons';
import { Checkbox, Input, Tooltip } from 'antd';
import { useState } from 'react';
import { useEffect, useState } from 'react';
import type { Storage } from '../../../../../entity/storages';
@@ -8,13 +8,27 @@ interface Props {
storage: Storage;
setStorage: (storage: Storage) => void;
setUnsaved: () => void;
connectionError?: string;
}
export function EditS3StorageComponent({ storage, setStorage, setUnsaved }: Props) {
export function EditS3StorageComponent({
storage,
setStorage,
setUnsaved,
connectionError,
}: Props) {
const hasAdvancedValues =
!!storage?.s3Storage?.s3Prefix || !!storage?.s3Storage?.s3UseVirtualHostedStyle;
!!storage?.s3Storage?.s3Prefix ||
!!storage?.s3Storage?.s3UseVirtualHostedStyle ||
!!storage?.s3Storage?.skipTLSVerify;
const [showAdvanced, setShowAdvanced] = useState(hasAdvancedValues);
useEffect(() => {
if (connectionError?.includes('failed to verify certificate')) {
setShowAdvanced(true);
}
}, [connectionError]);
return (
<>
<div className="mb-2 flex items-center">
@@ -226,6 +240,36 @@ export function EditS3StorageComponent({ storage, setStorage, setUnsaved }: Prop
</Tooltip>
</div>
</div>
<div className="mb-1 flex w-full flex-col items-start sm:flex-row sm:items-center">
<div className="mb-1 min-w-[110px] sm:mb-0">Skip TLS verify</div>
<div className="flex items-center">
<Checkbox
checked={storage?.s3Storage?.skipTLSVerify || false}
onChange={(e) => {
if (!storage?.s3Storage) return;
setStorage({
...storage,
s3Storage: {
...storage.s3Storage,
skipTLSVerify: e.target.checked,
},
});
setUnsaved();
}}
>
Skip TLS
</Checkbox>
<Tooltip
className="cursor-pointer"
title="Skip TLS certificate verification. Enable this if your S3-compatible storage uses a self-signed certificate. Warning: this reduces security."
>
<InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />
</Tooltip>
</div>
</div>
</>
)}

View File

@@ -45,6 +45,13 @@ export function ShowS3StorageComponent({ storage }: Props) {
Enabled
</div>
)}
{storage?.s3Storage?.skipTLSVerify && (
<div className="mb-1 flex items-center">
<div className="min-w-[110px]">Skip TLS</div>
Enabled
</div>
)}
</>
);
}