diff --git a/backend/internal/features/backups/backups/backuping/cleaner.go b/backend/internal/features/backups/backups/backuping/cleaner.go index 167b781..0eff38a 100644 --- a/backend/internal/features/backups/backups/backuping/cleaner.go +++ b/backend/internal/features/backups/backups/backuping/cleaner.go @@ -80,8 +80,7 @@ func (c *BackupCleaner) DeleteBackup(backup *backups_core.Backup) error { return err } - err = storage.DeleteFile(c.fieldEncryptor, backup.FileName) - if err != nil { + if err := storage.DeleteFile(c.fieldEncryptor, backup.FileName); err != nil { // we do not return error here, because sometimes clean up performed // before unavailable storage removal or change - therefore we should // proceed even in case of error. It's possible that some S3 or @@ -408,6 +407,10 @@ func buildGFSKeepSet( ) map[uuid.UUID]bool { keep := make(map[uuid.UUID]bool) + if len(backups) == 0 { + return keep + } + hoursSeen := make(map[string]bool) daysSeen := make(map[string]bool) weeksSeen := make(map[string]bool) @@ -416,6 +419,52 @@ func buildGFSKeepSet( hoursKept, daysKept, weeksKept, monthsKept, yearsKept := 0, 0, 0, 0, 0 + // Compute per-level time-window cutoffs so higher-frequency slots + // cannot absorb backups that belong to lower-frequency levels. + ref := backups[0].CreatedAt + + rawHourlyCutoff := ref.Add(-time.Duration(hours) * time.Hour) + rawDailyCutoff := ref.Add(-time.Duration(days) * 24 * time.Hour) + rawWeeklyCutoff := ref.Add(-time.Duration(weeks) * 7 * 24 * time.Hour) + rawMonthlyCutoff := ref.AddDate(0, -months, 0) + rawYearlyCutoff := ref.AddDate(-years, 0, 0) + + // Hierarchical capping: each level's window cannot extend further back + // than the nearest active lower-frequency level's window. + yearlyCutoff := rawYearlyCutoff + + monthlyCutoff := rawMonthlyCutoff + if years > 0 { + monthlyCutoff = laterOf(monthlyCutoff, yearlyCutoff) + } + + weeklyCutoff := rawWeeklyCutoff + if months > 0 { + weeklyCutoff = laterOf(weeklyCutoff, monthlyCutoff) + } else if years > 0 { + weeklyCutoff = laterOf(weeklyCutoff, yearlyCutoff) + } + + dailyCutoff := rawDailyCutoff + if weeks > 0 { + dailyCutoff = laterOf(dailyCutoff, weeklyCutoff) + } else if months > 0 { + dailyCutoff = laterOf(dailyCutoff, monthlyCutoff) + } else if years > 0 { + dailyCutoff = laterOf(dailyCutoff, yearlyCutoff) + } + + hourlyCutoff := rawHourlyCutoff + if days > 0 { + hourlyCutoff = laterOf(hourlyCutoff, dailyCutoff) + } else if weeks > 0 { + hourlyCutoff = laterOf(hourlyCutoff, weeklyCutoff) + } else if months > 0 { + hourlyCutoff = laterOf(hourlyCutoff, monthlyCutoff) + } else if years > 0 { + hourlyCutoff = laterOf(hourlyCutoff, yearlyCutoff) + } + for _, backup := range backups { t := backup.CreatedAt @@ -426,31 +475,31 @@ func buildGFSKeepSet( monthKey := t.Format("2006-01") yearKey := t.Format("2006") - if hours > 0 && hoursKept < hours && !hoursSeen[hourKey] { + if hours > 0 && hoursKept < hours && !hoursSeen[hourKey] && t.After(hourlyCutoff) { keep[backup.ID] = true hoursSeen[hourKey] = true hoursKept++ } - if days > 0 && daysKept < days && !daysSeen[dayKey] { + if days > 0 && daysKept < days && !daysSeen[dayKey] && t.After(dailyCutoff) { keep[backup.ID] = true daysSeen[dayKey] = true daysKept++ } - if weeks > 0 && weeksKept < weeks && !weeksSeen[weekKey] { + if weeks > 0 && weeksKept < weeks && !weeksSeen[weekKey] && t.After(weeklyCutoff) { keep[backup.ID] = true weeksSeen[weekKey] = true weeksKept++ } - if months > 0 && monthsKept < months && !monthsSeen[monthKey] { + if months > 0 && monthsKept < months && !monthsSeen[monthKey] && t.After(monthlyCutoff) { keep[backup.ID] = true monthsSeen[monthKey] = true monthsKept++ } - if years > 0 && yearsKept < years && !yearsSeen[yearKey] { + if years > 0 && yearsKept < years && !yearsSeen[yearKey] && t.After(yearlyCutoff) { keep[backup.ID] = true yearsSeen[yearKey] = true yearsKept++ @@ -459,3 +508,11 @@ func buildGFSKeepSet( return keep } + +func laterOf(a, b time.Time) time.Time { + if a.After(b) { + return a + } + + return b +} diff --git a/backend/internal/features/backups/backups/backuping/cleaner_gfs_test.go b/backend/internal/features/backups/backups/backuping/cleaner_gfs_test.go new file mode 100644 index 0000000..551db5c --- /dev/null +++ b/backend/internal/features/backups/backups/backuping/cleaner_gfs_test.go @@ -0,0 +1,1175 @@ +package backuping + +import ( + "fmt" + "testing" + "time" + + backups_core "databasus-backend/internal/features/backups/backups/core" + backups_config "databasus-backend/internal/features/backups/config" + "databasus-backend/internal/features/databases" + "databasus-backend/internal/features/notifiers" + "databasus-backend/internal/features/storages" + users_enums "databasus-backend/internal/features/users/enums" + users_testing "databasus-backend/internal/features/users/testing" + workspaces_testing "databasus-backend/internal/features/workspaces/testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" +) + +func Test_BuildGFSKeepSet(t *testing.T) { + // Fixed reference time: a Wednesday mid-month to avoid boundary edge cases in the default tests. + // Use time.Date for determinism across test runs. + ref := time.Date(2025, 6, 18, 12, 0, 0, 0, time.UTC) // Wednesday, 2025-06-18 + + day := 24 * time.Hour + week := 7 * day + + newBackup := func(createdAt time.Time) *backups_core.Backup { + return &backups_core.Backup{ID: uuid.New(), CreatedAt: createdAt} + } + + // backupsEveryDay returns n backups, newest-first, each 1 day apart. + backupsEveryDay := func(n int) []*backups_core.Backup { + bs := make([]*backups_core.Backup, n) + for i := 0; i < n; i++ { + bs[i] = newBackup(ref.Add(-time.Duration(i) * day)) + } + return bs + } + + // backupsEveryWeek returns n backups, newest-first, each 7 days apart. + backupsEveryWeek := func(n int) []*backups_core.Backup { + bs := make([]*backups_core.Backup, n) + for i := 0; i < n; i++ { + bs[i] = newBackup(ref.Add(-time.Duration(i) * week)) + } + return bs + } + + hour := time.Hour + + // backupsEveryHour returns n backups, newest-first, each 1 hour apart. + backupsEveryHour := func(n int) []*backups_core.Backup { + bs := make([]*backups_core.Backup, n) + for i := 0; i < n; i++ { + bs[i] = newBackup(ref.Add(-time.Duration(i) * hour)) + } + return bs + } + + // backupsEveryMonth returns n backups, newest-first, each ~1 month apart. + backupsEveryMonth := func(n int) []*backups_core.Backup { + bs := make([]*backups_core.Backup, n) + for i := 0; i < n; i++ { + bs[i] = newBackup(ref.AddDate(0, -i, 0)) + } + return bs + } + + // backupsEveryYear returns n backups, newest-first, each 1 year apart. + backupsEveryYear := func(n int) []*backups_core.Backup { + bs := make([]*backups_core.Backup, n) + for i := 0; i < n; i++ { + bs[i] = newBackup(ref.AddDate(-i, 0, 0)) + } + return bs + } + + tests := []struct { + name string + backups []*backups_core.Backup + hours int + days int + weeks int + months int + years int + keptIndices []int // which indices in backups should be kept + deletedRange *[2]int // optional: all indices in [from, to) must be deleted + }{ + { + name: "OnlyHourlySlots_KeepsNewest3Of5", + backups: backupsEveryHour(5), + hours: 3, + keptIndices: []int{0, 1, 2}, + }, + { + name: "SameHourDedup_OnlyNewestKeptForHourlySlot", + backups: []*backups_core.Backup{ + newBackup(ref.Truncate(hour).Add(45 * time.Minute)), + newBackup(ref.Truncate(hour).Add(10 * time.Minute)), + }, + hours: 1, + keptIndices: []int{0}, + }, + { + name: "OnlyDailySlots_KeepsNewest3Of5", + backups: backupsEveryDay(5), + days: 3, + keptIndices: []int{0, 1, 2}, + }, + { + name: "OnlyDailySlots_FewerBackupsThanSlots_KeepsAll", + backups: backupsEveryDay(2), + days: 5, + keptIndices: []int{0, 1}, + }, + { + name: "OnlyWeeklySlots_KeepsNewest2Weeks", + backups: backupsEveryWeek(4), + weeks: 2, + keptIndices: []int{0, 1}, + }, + { + name: "OnlyMonthlySlots_KeepsNewest2Months", + backups: []*backups_core.Backup{ + newBackup(time.Date(2025, 6, 1, 12, 0, 0, 0, time.UTC)), + newBackup(time.Date(2025, 5, 1, 12, 0, 0, 0, time.UTC)), + newBackup(time.Date(2025, 4, 1, 12, 0, 0, 0, time.UTC)), + }, + months: 2, + keptIndices: []int{0, 1}, + }, + { + name: "OnlyYearlySlots_KeepsNewest2Years", + backups: []*backups_core.Backup{ + newBackup(time.Date(2025, 6, 1, 12, 0, 0, 0, time.UTC)), + newBackup(time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC)), + newBackup(time.Date(2023, 6, 1, 12, 0, 0, 0, time.UTC)), + }, + years: 2, + keptIndices: []int{0, 1}, + }, + { + name: "SameDayDedup_OnlyNewestKeptForDailySlot", + backups: []*backups_core.Backup{ + // Two backups on the same day; newest-first order + newBackup(ref.Truncate(day).Add(10 * time.Hour)), + newBackup(ref.Truncate(day).Add(2 * time.Hour)), + }, + days: 1, + keptIndices: []int{0}, + }, + { + name: "SameWeekDedup_OnlyNewestKeptForWeeklySlot", + backups: []*backups_core.Backup{ + // ref is Wednesday; add Thursday of same week + newBackup(ref.Add(1 * day)), // Thursday same week + newBackup(ref), // Wednesday same week + }, + weeks: 1, + keptIndices: []int{0}, + }, + { + name: "AdditiveSlots_NewestFillsDailyAndWeeklyAndMonthly", + // Newest backup fills daily + weekly + monthly simultaneously + backups: []*backups_core.Backup{ + newBackup(time.Date(2025, 6, 18, 12, 0, 0, 0, time.UTC)), // newest + newBackup(time.Date(2025, 6, 11, 12, 0, 0, 0, time.UTC)), // 1 week ago + newBackup(time.Date(2025, 5, 18, 12, 0, 0, 0, time.UTC)), // 1 month ago + newBackup(time.Date(2025, 4, 18, 12, 0, 0, 0, time.UTC)), // 2 months ago + }, + days: 1, + weeks: 2, + months: 2, + keptIndices: []int{0, 1, 2}, + }, + { + name: "YearBoundary_CorrectlySplitsAcrossYears", + backups: []*backups_core.Backup{ + newBackup(time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC)), + newBackup(time.Date(2024, 12, 31, 12, 0, 0, 0, time.UTC)), + newBackup(time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC)), + newBackup(time.Date(2023, 6, 1, 12, 0, 0, 0, time.UTC)), + }, + years: 2, + keptIndices: []int{0, 1}, // 2025 and 2024 kept; 2024-06 and 2023 deleted + }, + { + name: "ISOWeekBoundary_Jan1UsesCorrectISOWeek", + // 2025-01-01 is ISO week 1 of 2025; 2024-12-28 is ISO week 52 of 2024 + backups: []*backups_core.Backup{ + newBackup(time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC)), // ISO week 2025-W01 + newBackup(time.Date(2024, 12, 28, 12, 0, 0, 0, time.UTC)), // ISO week 2024-W52 + }, + weeks: 2, + keptIndices: []int{0, 1}, // different ISO weeks → both kept + }, + { + name: "EmptyBackups_ReturnsEmptyKeepSet", + backups: []*backups_core.Backup{}, + hours: 3, + days: 3, + weeks: 2, + months: 1, + years: 1, + keptIndices: []int{}, + }, + { + name: "AllZeroSlots_KeepsNothing", + backups: backupsEveryDay(5), + hours: 0, + days: 0, + weeks: 0, + months: 0, + years: 0, + keptIndices: []int{}, + }, + { + name: "AllSlotsActive_FullCombination", + backups: backupsEveryWeek(12), + days: 2, + weeks: 3, + months: 2, + years: 1, + // 2 daily (indices 0,1) + 3rd weekly slot (index 2) + 2nd monthly slot (index 3 or later). + // Additive slots: newest fills daily+weekly+monthly+yearly; each subsequent week fills another weekly, + // and a backup ~4 weeks later fills the 2nd monthly slot. + keptIndices: []int{0, 1, 2, 3}, + }, + { + name: "RealisticGFS_20DailyBackups_7Days4Weeks1Month", + backups: backupsEveryDay(20), + days: 7, + weeks: 4, + months: 1, + // 7 daily: indices 0-6 (Jun 18-12). Index 0 also fills week 25 + month 2025-06. + // Index 3 (Jun 15, Sun) fills week 24. Indices 7-9 are week 24 (seen) → not kept. + // Index 10 (Jun 8, Sun) fills week 23. Indices 11-16 are week 23 (seen) → not kept. + // Index 17 (Jun 1, Sun) fills week 22. Index 18-19 are week 22 (seen) → not kept. + // Monthly slot 1 filled by index 0 (2025-06). Total kept: 9. + keptIndices: []int{0, 1, 2, 3, 4, 5, 6, 10, 17}, + }, + + // Cross-level absorption tests: when backup frequency is lower than slot granularity, + // each backup gets a unique key at the higher-frequency level, filling slots that + // should only cover recent time periods. This causes excess backups to be retained. + + // Adjacent-level absorption: + { + // Daily backups with hourly slots: each daily backup has a unique hour key + // (different date-hour combo), so 23 backups fill 23/24 hourly slots. + // Hourly slots should only cover the most recent 24 hours, not span weeks. + name: "HourlyAbsorbsDaily_DailyBackupsWithHourlySlots_KeepsTooMany", + backups: backupsEveryDay(23), + hours: 24, + days: 7, + weeks: 4, + months: 12, + years: 3, + // Correct behavior: 10 kept (7 daily + 2 extra weekly + 1 monthly for 2025-05). + // Index 18 (May 31) is the first backup in month "2025-05" within the 12-month window. + // Bug: all 23 kept because each daily backup fills a unique hourly slot. + keptIndices: []int{0, 1, 2, 3, 4, 5, 6, 10, 17, 18}, + }, + { + // Weekly backups with daily slots: each weekly backup has a unique day key, + // so 10 weekly backups fill 7/7 daily slots + 3 weekly slots = all 10 kept. + // Daily slots should only cover the most recent 7 days, not span months. + name: "DailyAbsorbsWeekly_WeeklyBackupsWithDailySlots_KeepsTooMany", + backups: backupsEveryWeek(10), + days: 7, + weeks: 4, + // Correct behavior: ~8 kept (4 weekly, some overlap with daily slots that + // should only cover recent 7 days). Extra daily slots shouldn't retain + // backups older than 7 days. + // Bug: all 10 kept because each weekly backup fills a unique day slot. + keptIndices: []int{0, 1, 2, 3}, + }, + { + // Monthly backups with many weekly slots: each monthly backup has a unique week key, + // so 8 monthly backups fill 8/52 weekly slots, all getting kept. + // Weekly slots should only cover the most recent weeks, not span years. + name: "WeeklyAbsorbsMonthly_MonthlyBackupsWithWeeklySlots_KeepsTooMany", + backups: backupsEveryMonth(8), + weeks: 52, + months: 3, + // Correct behavior: 3 kept (3 monthly slots, weekly should only cover recent 52 weeks + // but not artificially retain monthly backups). + // Bug: all 8 kept because each monthly backup fills a unique week slot. + keptIndices: []int{0, 1, 2}, + }, + { + // Yearly backups with monthly slots: each yearly backup (on different month of year) + // has a unique month key, so 5 yearly backups fill 5/12 monthly slots = all 5 kept. + // Monthly slots should only cover the most recent 12 months, not span decades. + name: "MonthlyAbsorbsYearly_YearlyBackupsWithMonthlySlots_KeepsTooMany", + backups: backupsEveryYear(5), + months: 12, + years: 3, + // Correct behavior: 3 kept (3 yearly slots). + // Bug: all 5 kept because each yearly backup fills a unique month slot. + keptIndices: []int{0, 1, 2}, + }, + + // Non-adjacent (skip-level) absorption: + { + // Weekly backups with hourly slots: each weekly backup has a unique hour key, + // so 8 weekly backups fill 8/24 hourly slots = all 8 kept. + name: "HourlyAbsorbsWeekly_WeeklyBackupsWithHourlySlots_KeepsTooMany", + backups: backupsEveryWeek(8), + hours: 24, + weeks: 4, + // Correct behavior: 4 kept (4 weekly slots, hourly should only cover recent 24h). + // Bug: all 8 kept because each weekly backup fills a unique hourly slot. + keptIndices: []int{0, 1, 2, 3}, + }, + { + // Monthly backups with daily slots: each monthly backup has a unique day key, + // so 6 monthly backups fill 6/7 daily slots = all 6 kept. + name: "DailyAbsorbsMonthly_MonthlyBackupsWithDailySlots_KeepsTooMany", + backups: backupsEveryMonth(6), + days: 7, + months: 3, + // Correct behavior: 3 kept (3 monthly slots, daily should only cover recent 7 days). + // Bug: all 6 kept because each monthly backup fills a unique day slot. + keptIndices: []int{0, 1, 2}, + }, + + // Full-stack (production-like config): + { + // Production config with all levels active. Daily backups span 30 days. + // Hourly slots absorb everything because each daily backup has a unique hour key. + name: "FullStack_AllLevelsAbsorb_DailyBackupsWithFullConfig_KeepsTooMany", + backups: backupsEveryDay(30), + hours: 24, + days: 7, + weeks: 4, + months: 1, + years: 1, + // Correct behavior: ~9 kept (7 daily + ~2 extra weekly, month+year overlap). + // Bug: 24+ kept because hourly absorbs 24 unique date-hour combos. + keptIndices: []int{0, 1, 2, 3, 4, 5, 6, 10, 17}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + keepSet := buildGFSKeepSet(tc.backups, tc.hours, tc.days, tc.weeks, tc.months, tc.years) + + keptIndexSet := make(map[int]bool, len(tc.keptIndices)) + for _, idx := range tc.keptIndices { + keptIndexSet[idx] = true + } + + for i, backup := range tc.backups { + if keptIndexSet[i] { + assert.True(t, keepSet[backup.ID], + "backup at index %d (date=%s) should be kept", + i, backup.CreatedAt.Format("2006-01-02 15:04")) + } else { + assert.False(t, keepSet[backup.ID], + "backup at index %d (date=%s) should be deleted", + i, backup.CreatedAt.Format("2006-01-02 15:04")) + } + } + }) + } +} + +func Test_CleanByGFS_KeepsCorrectBackupsPerSlot(t *testing.T) { + router := CreateTestRouter() + owner := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) + storage := storages.CreateTestStorage(workspace.ID) + notifier := notifiers.CreateTestNotifier(workspace.ID) + database := databases.CreateTestDatabase(workspace.ID, storage, notifier) + + defer func() { + backups, _ := backupRepository.FindByDatabaseID(database.ID) + for _, backup := range backups { + backupRepository.DeleteByID(backup.ID) + } + + databases.RemoveTestDatabase(database) + time.Sleep(50 * time.Millisecond) + notifiers.RemoveTestNotifier(notifier) + storages.RemoveTestStorage(storage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) + }() + + interval := createTestInterval() + + backupConfig := &backups_config.BackupConfig{ + DatabaseID: database.ID, + IsBackupsEnabled: true, + RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, + RetentionGfsDays: 3, + RetentionGfsWeeks: 0, + RetentionGfsMonths: 0, + RetentionGfsYears: 0, + StorageID: &storage.ID, + BackupIntervalID: interval.ID, + BackupInterval: interval, + } + _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) + assert.NoError(t, err) + + now := time.Now().UTC() + + // Create 5 backups on 5 different days; only the 3 newest days should be kept + var backupIDs []uuid.UUID + for i := 0; i < 5; i++ { + backup := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: storage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 10, + CreatedAt: now.Add(-time.Duration(4-i) * 24 * time.Hour).Truncate(24 * time.Hour), + } + err = backupRepository.Save(backup) + assert.NoError(t, err) + backupIDs = append(backupIDs, backup.ID) + } + + cleaner := GetBackupCleaner() + err = cleaner.cleanByRetentionPolicy() + assert.NoError(t, err) + + remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) + assert.NoError(t, err) + assert.Equal(t, 3, len(remainingBackups)) + + remainingIDs := make(map[uuid.UUID]bool) + for _, backup := range remainingBackups { + remainingIDs[backup.ID] = true + } + assert.False(t, remainingIDs[backupIDs[0]], "Oldest daily backup should be deleted") + assert.False(t, remainingIDs[backupIDs[1]], "2nd oldest daily backup should be deleted") + assert.True(t, remainingIDs[backupIDs[2]], "3rd backup should remain") + assert.True(t, remainingIDs[backupIDs[3]], "4th backup should remain") + assert.True(t, remainingIDs[backupIDs[4]], "Newest backup should remain") +} + +func Test_CleanByGFS_WithWeeklyAndMonthlySlots_KeepsWiderSpread(t *testing.T) { + router := CreateTestRouter() + owner := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) + storage := storages.CreateTestStorage(workspace.ID) + notifier := notifiers.CreateTestNotifier(workspace.ID) + database := databases.CreateTestDatabase(workspace.ID, storage, notifier) + + defer func() { + backups, _ := backupRepository.FindByDatabaseID(database.ID) + for _, backup := range backups { + backupRepository.DeleteByID(backup.ID) + } + + databases.RemoveTestDatabase(database) + time.Sleep(50 * time.Millisecond) + notifiers.RemoveTestNotifier(notifier) + storages.RemoveTestStorage(storage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) + }() + + interval := createTestInterval() + + backupConfig := &backups_config.BackupConfig{ + DatabaseID: database.ID, + IsBackupsEnabled: true, + RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, + RetentionGfsDays: 2, + RetentionGfsWeeks: 2, + RetentionGfsMonths: 1, + RetentionGfsYears: 0, + StorageID: &storage.ID, + BackupIntervalID: interval.ID, + BackupInterval: interval, + } + _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) + assert.NoError(t, err) + + now := time.Now().UTC() + + // Create one backup per week for 6 weeks (each on Monday of that week) + // GFS should keep: 2 daily (most recent 2 unique days) + 2 weekly + 1 monthly = up to 5 unique + var createdIDs []uuid.UUID + for i := 0; i < 6; i++ { + weekOffset := time.Duration(5-i) * 7 * 24 * time.Hour + backup := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: storage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 10, + CreatedAt: now.Add(-weekOffset).Truncate(24 * time.Hour), + } + err = backupRepository.Save(backup) + assert.NoError(t, err) + createdIDs = append(createdIDs, backup.ID) + } + + cleaner := GetBackupCleaner() + err = cleaner.cleanByRetentionPolicy() + assert.NoError(t, err) + + remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) + assert.NoError(t, err) + + // We should have at most 5 backups kept (2 daily + 2 weekly + 1 monthly, but with overlap possible) + // The exact count depends on how many unique periods are covered + assert.LessOrEqual(t, len(remainingBackups), 5) + assert.GreaterOrEqual(t, len(remainingBackups), 1) + + // The two most recent backups should always be retained (daily slots) + remainingIDs := make(map[uuid.UUID]bool) + for _, backup := range remainingBackups { + remainingIDs[backup.ID] = true + } + assert.True(t, remainingIDs[createdIDs[4]], "Second newest backup should be retained (daily)") + assert.True(t, remainingIDs[createdIDs[5]], "Newest backup should be retained (daily)") +} + +func Test_CleanByGFS_WithHourlySlots_KeepsCorrectBackups(t *testing.T) { + router := CreateTestRouter() + owner := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) + testStorage := storages.CreateTestStorage(workspace.ID) + notifier := notifiers.CreateTestNotifier(workspace.ID) + database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier) + + defer func() { + backups, _ := backupRepository.FindByDatabaseID(database.ID) + for _, backup := range backups { + backupRepository.DeleteByID(backup.ID) + } + + databases.RemoveTestDatabase(database) + time.Sleep(50 * time.Millisecond) + notifiers.RemoveTestNotifier(notifier) + storages.RemoveTestStorage(testStorage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) + }() + + interval := createTestInterval() + + backupConfig := &backups_config.BackupConfig{ + DatabaseID: database.ID, + IsBackupsEnabled: true, + RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, + RetentionGfsHours: 3, + StorageID: &testStorage.ID, + BackupIntervalID: interval.ID, + BackupInterval: interval, + } + _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) + assert.NoError(t, err) + + now := time.Now().UTC() + + // Create 5 backups spaced 1 hour apart; only the 3 newest hours should be kept + var backupIDs []uuid.UUID + for i := 0; i < 5; i++ { + backup := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: testStorage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 10, + CreatedAt: now.Add(-time.Duration(4-i) * time.Hour).Truncate(time.Hour), + } + err = backupRepository.Save(backup) + assert.NoError(t, err) + backupIDs = append(backupIDs, backup.ID) + } + + cleaner := GetBackupCleaner() + err = cleaner.cleanByRetentionPolicy() + assert.NoError(t, err) + + remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) + assert.NoError(t, err) + assert.Equal(t, 3, len(remainingBackups)) + + remainingIDs := make(map[uuid.UUID]bool) + for _, backup := range remainingBackups { + remainingIDs[backup.ID] = true + } + assert.False(t, remainingIDs[backupIDs[0]], "Oldest hourly backup should be deleted") + assert.False(t, remainingIDs[backupIDs[1]], "2nd oldest hourly backup should be deleted") + assert.True(t, remainingIDs[backupIDs[2]], "3rd backup should remain") + assert.True(t, remainingIDs[backupIDs[3]], "4th backup should remain") + assert.True(t, remainingIDs[backupIDs[4]], "Newest backup should remain") +} + +func Test_CleanByGFS_SkipsRecentBackup_WhenNotInKeepSet(t *testing.T) { + router := CreateTestRouter() + owner := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) + storage := storages.CreateTestStorage(workspace.ID) + notifier := notifiers.CreateTestNotifier(workspace.ID) + database := databases.CreateTestDatabase(workspace.ID, storage, notifier) + + defer func() { + backups, _ := backupRepository.FindByDatabaseID(database.ID) + for _, backup := range backups { + backupRepository.DeleteByID(backup.ID) + } + + databases.RemoveTestDatabase(database) + time.Sleep(50 * time.Millisecond) + notifiers.RemoveTestNotifier(notifier) + storages.RemoveTestStorage(storage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) + }() + + interval := createTestInterval() + + // Keep only 1 daily slot. We create 2 old backups plus two recent backups on today. + // Backups are ordered newest-first, so the 15-min-old backup fills the single daily slot. + // The 30-min-old backup is the same day → not in the GFS keep-set, but it is still recent + // (within grace period) and must be preserved. + backupConfig := &backups_config.BackupConfig{ + DatabaseID: database.ID, + IsBackupsEnabled: true, + RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, + RetentionGfsDays: 1, + StorageID: &storage.ID, + BackupIntervalID: interval.ID, + BackupInterval: interval, + } + _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) + assert.NoError(t, err) + + now := time.Now().UTC() + + oldBackup1 := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: storage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 10, + CreatedAt: now.Add(-3 * 24 * time.Hour).Truncate(24 * time.Hour), + } + oldBackup2 := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: storage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 10, + CreatedAt: now.Add(-2 * 24 * time.Hour).Truncate(24 * time.Hour), + } + // Newest backup today — will fill the single GFS daily slot. + newestTodayBackup := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: storage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 10, + CreatedAt: now.Add(-15 * time.Minute), + } + // Slightly older backup, also today — NOT in GFS keep-set (duplicate day), + // but within the 60-minute grace period so it must survive. + recentNotInKeepSet := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: storage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 10, + CreatedAt: now.Add(-30 * time.Minute), + } + + for _, b := range []*backups_core.Backup{oldBackup1, oldBackup2, newestTodayBackup, recentNotInKeepSet} { + err = backupRepository.Save(b) + assert.NoError(t, err) + } + + cleaner := GetBackupCleaner() + err = cleaner.cleanByRetentionPolicy() + assert.NoError(t, err) + + remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) + assert.NoError(t, err) + + remainingIDs := make(map[uuid.UUID]bool) + for _, backup := range remainingBackups { + remainingIDs[backup.ID] = true + } + + assert.False(t, remainingIDs[oldBackup1.ID], "Old backup 1 should be deleted by GFS") + assert.False(t, remainingIDs[oldBackup2.ID], "Old backup 2 should be deleted by GFS") + assert.True( + t, + remainingIDs[newestTodayBackup.ID], + "Newest backup fills GFS daily slot and must remain", + ) + assert.True( + t, + remainingIDs[recentNotInKeepSet.ID], + "Recent backup not in keep-set must be preserved by grace period", + ) +} + +func Test_CleanByGFS_With20DailyBackups_KeepsOnlyExpectedCount(t *testing.T) { + router := CreateTestRouter() + owner := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) + testStorage := storages.CreateTestStorage(workspace.ID) + notifier := notifiers.CreateTestNotifier(workspace.ID) + database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier) + + defer func() { + backups, _ := backupRepository.FindByDatabaseID(database.ID) + for _, backup := range backups { + backupRepository.DeleteByID(backup.ID) + } + + databases.RemoveTestDatabase(database) + time.Sleep(50 * time.Millisecond) + notifiers.RemoveTestNotifier(notifier) + storages.RemoveTestStorage(testStorage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) + }() + + interval := createTestInterval() + + backupConfig := &backups_config.BackupConfig{ + DatabaseID: database.ID, + IsBackupsEnabled: true, + RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, + RetentionGfsDays: 7, + RetentionGfsWeeks: 4, + RetentionGfsMonths: 1, + StorageID: &testStorage.ID, + BackupIntervalID: interval.ID, + BackupInterval: interval, + } + _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) + assert.NoError(t, err) + + now := time.Now().UTC() + + // Create 20 daily backups, all older than the 60-minute grace period. + // backupIDs[0] is oldest (19 days ago), backupIDs[19] is newest (~2h ago). + var backupIDs []uuid.UUID + for i := range 20 { + backup := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: testStorage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 10, + CreatedAt: now.Add(-time.Duration(19-i)*24*time.Hour - 2*time.Hour), + } + err = backupRepository.Save(backup) + assert.NoError(t, err) + backupIDs = append(backupIDs, backup.ID) + } + + cleaner := GetBackupCleaner() + err = cleaner.cleanByRetentionPolicy() + assert.NoError(t, err) + + remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) + assert.NoError(t, err) + + // GFS 7d/4w/1m: 7 daily slots + up to 4 weekly + 1 monthly (with overlap). + // Theoretical max unique kept: 12. Realistic: ~9 depending on ISO week layout. + assert.LessOrEqual(t, len(remainingBackups), 12, + "At most 12 backups should be retained (7d+4w+1m)") + assert.GreaterOrEqual(t, len(remainingBackups), 7, + "At least 7 backups should be retained for daily slots") + + remainingIDs := make(map[uuid.UUID]bool) + for _, backup := range remainingBackups { + remainingIDs[backup.ID] = true + } + + // The newest 7 backups (indices 13-19) must always be retained (daily slots) + for i := 13; i < 20; i++ { + assert.True(t, remainingIDs[backupIDs[i]], + "Backup at position %d (one of 7 newest) should be retained", i) + } +} + +func Test_CleanByGFS_WithMultipleBackupsPerDay_KeepsOnlyOnePerDailySlot(t *testing.T) { + router := CreateTestRouter() + owner := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) + testStorage := storages.CreateTestStorage(workspace.ID) + notifier := notifiers.CreateTestNotifier(workspace.ID) + database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier) + + defer func() { + backups, _ := backupRepository.FindByDatabaseID(database.ID) + for _, backup := range backups { + backupRepository.DeleteByID(backup.ID) + } + + databases.RemoveTestDatabase(database) + time.Sleep(50 * time.Millisecond) + notifiers.RemoveTestNotifier(notifier) + storages.RemoveTestStorage(testStorage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) + }() + + interval := createTestInterval() + + backupConfig := &backups_config.BackupConfig{ + DatabaseID: database.ID, + IsBackupsEnabled: true, + RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, + RetentionGfsDays: 7, + RetentionGfsWeeks: 4, + StorageID: &testStorage.ID, + BackupIntervalID: interval.ID, + BackupInterval: interval, + } + _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) + assert.NoError(t, err) + + now := time.Now().UTC() + + // Create 3 backups per day for 10 days = 30 total, all beyond grace period. + // Each day gets backups at base+0h, base+6h, base+12h. + for day := 0; day < 10; day++ { + for sub := 0; sub < 3; sub++ { + backup := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: testStorage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 10, + CreatedAt: now.Add( + -time.Duration(9-day)*24*time.Hour - + 2*time.Hour - + time.Duration(2-sub)*6*time.Hour, + ), + } + err = backupRepository.Save(backup) + assert.NoError(t, err) + } + } + + cleaner := GetBackupCleaner() + err = cleaner.cleanByRetentionPolicy() + assert.NoError(t, err) + + remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) + assert.NoError(t, err) + + // 10 days spans ~2 ISO weeks. GFS 7d/4w: weekly slots overlap with daily. + // Expected: ~7-9 backups (7 daily + possibly extra weekly for uncovered weeks). + assert.LessOrEqual(t, len(remainingBackups), 11, + "Should keep at most 11 backups (7d+4w theoretical max)") + assert.GreaterOrEqual(t, len(remainingBackups), 7, + "Should keep at least 7 backups for daily slots") + + // Only 1 backup per calendar day should be retained + dayCount := make(map[string]int) + for _, backup := range remainingBackups { + dayKey := backup.CreatedAt.Format("2006-01-02") + dayCount[dayKey]++ + } + + for dayKey, count := range dayCount { + assert.Equal(t, 1, count, + "Day %s should have exactly 1 retained backup, got %d", dayKey, count) + } +} + +// When backups are created once per day but hourly slots are configured (e.g. 24h), +// each daily backup lands in a unique hour key (different date-hour combo), filling +// hourly slots with old backups that shouldn't be retained by hourly rotation. +// Hourly slots should only cover the most recent hours, not span across weeks. +func Test_CleanByGFS_With24HourlySlotsAnd23DailyBackups_DeletesExcessBackups(t *testing.T) { + router := CreateTestRouter() + owner := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) + testStorage := storages.CreateTestStorage(workspace.ID) + notifier := notifiers.CreateTestNotifier(workspace.ID) + database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier) + + defer func() { + backups, _ := backupRepository.FindByDatabaseID(database.ID) + for _, backup := range backups { + backupRepository.DeleteByID(backup.ID) + } + + databases.RemoveTestDatabase(database) + time.Sleep(50 * time.Millisecond) + notifiers.RemoveTestNotifier(notifier) + storages.RemoveTestStorage(testStorage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) + }() + + interval := createTestInterval() + + backupConfig := &backups_config.BackupConfig{ + DatabaseID: database.ID, + IsBackupsEnabled: true, + RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, + RetentionGfsHours: 24, + RetentionGfsDays: 7, + RetentionGfsWeeks: 4, + RetentionGfsMonths: 12, + RetentionGfsYears: 3, + StorageID: &testStorage.ID, + BackupIntervalID: interval.ID, + BackupInterval: interval, + } + _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) + assert.NoError(t, err) + + now := time.Now().UTC() + + for i := 0; i < 23; i++ { + backup := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: testStorage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 0.02, + CreatedAt: now.Add(-time.Duration(22-i)*24*time.Hour - 2*time.Hour), + } + err = backupRepository.Save(backup) + assert.NoError(t, err) + } + + cleaner := GetBackupCleaner() + err = cleaner.cleanByRetentionPolicy() + assert.NoError(t, err) + + remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) + assert.NoError(t, err) + + assert.Less(t, len(remainingBackups), 20, + "GFS with 24h/7d/4w/12m/3y should not retain all 23 daily backups") + assert.GreaterOrEqual(t, len(remainingBackups), 7, + "At least 7 backups should be retained for daily slots") +} + +// Same scenario as above but with hourly slots disabled (0h). This verifies +// that the daily/weekly/monthly/yearly rotation correctly prunes excess backups +// when hourly slots are not involved. +func Test_CleanByGFS_WithDisabledHourlySlotsAnd23DailyBackups_DeletesExcessBackups(t *testing.T) { + router := CreateTestRouter() + owner := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) + testStorage := storages.CreateTestStorage(workspace.ID) + notifier := notifiers.CreateTestNotifier(workspace.ID) + database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier) + + defer func() { + backups, _ := backupRepository.FindByDatabaseID(database.ID) + for _, backup := range backups { + backupRepository.DeleteByID(backup.ID) + } + + databases.RemoveTestDatabase(database) + time.Sleep(50 * time.Millisecond) + notifiers.RemoveTestNotifier(notifier) + storages.RemoveTestStorage(testStorage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) + }() + + interval := createTestInterval() + + backupConfig := &backups_config.BackupConfig{ + DatabaseID: database.ID, + IsBackupsEnabled: true, + RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, + RetentionGfsHours: 0, + RetentionGfsDays: 7, + RetentionGfsWeeks: 4, + RetentionGfsMonths: 12, + RetentionGfsYears: 3, + StorageID: &testStorage.ID, + BackupIntervalID: interval.ID, + BackupInterval: interval, + } + _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) + assert.NoError(t, err) + + now := time.Now().UTC() + + for i := 0; i < 23; i++ { + backup := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: testStorage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 0.02, + CreatedAt: now.Add(-time.Duration(22-i)*24*time.Hour - 2*time.Hour), + } + err = backupRepository.Save(backup) + assert.NoError(t, err) + } + + cleaner := GetBackupCleaner() + err = cleaner.cleanByRetentionPolicy() + assert.NoError(t, err) + + remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) + assert.NoError(t, err) + + assert.Less(t, len(remainingBackups), 15, + "GFS with 0h/7d/4w/12m/3y should not retain all 23 daily backups") + assert.GreaterOrEqual(t, len(remainingBackups), 7, + "At least 7 backups should be retained for daily slots") +} + +// When weekly backups exist but daily slots are configured, each weekly backup +// has a unique day key, filling daily slots with backups that are weeks apart. +// Daily slots should only cover the most recent days, not span months. +func Test_CleanByGFS_WithDailySlotsAndWeeklyBackups_DeletesExcessBackups(t *testing.T) { + router := CreateTestRouter() + owner := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) + testStorage := storages.CreateTestStorage(workspace.ID) + notifier := notifiers.CreateTestNotifier(workspace.ID) + database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier) + + defer func() { + backups, _ := backupRepository.FindByDatabaseID(database.ID) + for _, backup := range backups { + backupRepository.DeleteByID(backup.ID) + } + + databases.RemoveTestDatabase(database) + time.Sleep(50 * time.Millisecond) + notifiers.RemoveTestNotifier(notifier) + storages.RemoveTestStorage(testStorage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) + }() + + interval := createTestInterval() + + backupConfig := &backups_config.BackupConfig{ + DatabaseID: database.ID, + IsBackupsEnabled: true, + RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, + RetentionGfsDays: 7, + RetentionGfsWeeks: 4, + StorageID: &testStorage.ID, + BackupIntervalID: interval.ID, + BackupInterval: interval, + } + _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) + assert.NoError(t, err) + + now := time.Now().UTC() + + // Create 10 weekly backups (1 per week, all >2h old past grace period). + // With 7d/4w config, correct behavior: ~8 kept (4 weekly + overlap with daily for recent ones). + // Daily slots should NOT absorb weekly backups that are older than 7 days. + for i := 0; i < 10; i++ { + backup := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: testStorage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 0.02, + CreatedAt: now.Add(-time.Duration(9-i)*7*24*time.Hour - 2*time.Hour), + } + err = backupRepository.Save(backup) + assert.NoError(t, err) + } + + cleaner := GetBackupCleaner() + err = cleaner.cleanByRetentionPolicy() + assert.NoError(t, err) + + remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) + assert.NoError(t, err) + + fmt.Printf( + "[TEST] WithDailySlotsAndWeeklyBackups: %d remaining out of 10\n", + len(remainingBackups), + ) + + // With weekly backups and 7d/4w config, daily slots should only cover the most recent + // 7 days. Since backups are 1 week apart, at most 1 backup falls within the daily window. + // Correct: ~4-5 kept (4 weekly + possibly 1 overlapping daily). Not 7+. + assert.LessOrEqual( + t, + len(remainingBackups), + 5, + "GFS with 7d/4w should keep at most ~5 weekly backups — daily slots should not absorb weekly backups older than 7 days", + ) + assert.GreaterOrEqual(t, len(remainingBackups), 4, + "At least 4 backups should be retained for weekly slots") +} + +// When monthly backups exist but weekly slots are configured, each monthly backup +// has a unique week key, filling weekly slots with backups that are months apart. +// Weekly slots should only cover the most recent weeks, not span years. +func Test_CleanByGFS_WithWeeklySlotsAndMonthlyBackups_DeletesExcessBackups(t *testing.T) { + router := CreateTestRouter() + owner := users_testing.CreateTestUser(users_enums.UserRoleMember) + workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) + testStorage := storages.CreateTestStorage(workspace.ID) + notifier := notifiers.CreateTestNotifier(workspace.ID) + database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier) + + defer func() { + backups, _ := backupRepository.FindByDatabaseID(database.ID) + for _, backup := range backups { + backupRepository.DeleteByID(backup.ID) + } + + databases.RemoveTestDatabase(database) + time.Sleep(50 * time.Millisecond) + notifiers.RemoveTestNotifier(notifier) + storages.RemoveTestStorage(testStorage.ID) + workspaces_testing.RemoveTestWorkspace(workspace, router) + }() + + interval := createTestInterval() + + backupConfig := &backups_config.BackupConfig{ + DatabaseID: database.ID, + IsBackupsEnabled: true, + RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, + RetentionGfsWeeks: 52, + RetentionGfsMonths: 3, + StorageID: &testStorage.ID, + BackupIntervalID: interval.ID, + BackupInterval: interval, + } + _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) + assert.NoError(t, err) + + now := time.Now().UTC() + + // Create 8 monthly backups (1 per month, all >2h old past grace period). + // With 52w/3m config, correct behavior: 3 kept (3 monthly slots; weekly should only + // cover recent 52 weeks but not artificially retain old monthly backups). + // Bug: all 8 kept because each monthly backup fills a unique weekly slot. + for i := 0; i < 8; i++ { + backup := &backups_core.Backup{ + ID: uuid.New(), + DatabaseID: database.ID, + StorageID: testStorage.ID, + Status: backups_core.BackupStatusCompleted, + BackupSizeMb: 0.02, + CreatedAt: now.AddDate(0, -(7 - i), 0).Add(-2 * time.Hour), + } + err = backupRepository.Save(backup) + assert.NoError(t, err) + } + + cleaner := GetBackupCleaner() + err = cleaner.cleanByRetentionPolicy() + assert.NoError(t, err) + + remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) + assert.NoError(t, err) + + fmt.Printf( + "[TEST] WithWeeklySlotsAndMonthlyBackups: %d remaining out of 8\n", + len(remainingBackups), + ) + + // Weekly slots should not absorb monthly backups. Correct: ~3 kept (3 monthly). + assert.LessOrEqual( + t, + len(remainingBackups), + 5, + "GFS with 52w/3m should keep at most ~5 backups — weekly slots should not absorb monthly backups", + ) + assert.GreaterOrEqual(t, len(remainingBackups), 3, + "At least 3 backups should be retained for monthly slots") +} diff --git a/backend/internal/features/backups/backups/backuping/cleaner_test.go b/backend/internal/features/backups/backups/backuping/cleaner_test.go index dc5d44f..bb8bc0e 100644 --- a/backend/internal/features/backups/backups/backuping/cleaner_test.go +++ b/backend/internal/features/backups/backups/backuping/cleaner_test.go @@ -697,160 +697,6 @@ func Test_CleanByCount_DoesNotDeleteInProgressBackups(t *testing.T) { assert.True(t, inProgressFound, "In-progress backup should not be deleted by count policy") } -func Test_CleanByGFS_KeepsCorrectBackupsPerSlot(t *testing.T) { - router := CreateTestRouter() - owner := users_testing.CreateTestUser(users_enums.UserRoleMember) - workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) - storage := storages.CreateTestStorage(workspace.ID) - notifier := notifiers.CreateTestNotifier(workspace.ID) - database := databases.CreateTestDatabase(workspace.ID, storage, notifier) - - defer func() { - backups, _ := backupRepository.FindByDatabaseID(database.ID) - for _, backup := range backups { - backupRepository.DeleteByID(backup.ID) - } - - databases.RemoveTestDatabase(database) - time.Sleep(50 * time.Millisecond) - notifiers.RemoveTestNotifier(notifier) - storages.RemoveTestStorage(storage.ID) - workspaces_testing.RemoveTestWorkspace(workspace, router) - }() - - interval := createTestInterval() - - backupConfig := &backups_config.BackupConfig{ - DatabaseID: database.ID, - IsBackupsEnabled: true, - RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, - RetentionGfsDays: 3, - RetentionGfsWeeks: 0, - RetentionGfsMonths: 0, - RetentionGfsYears: 0, - StorageID: &storage.ID, - BackupIntervalID: interval.ID, - BackupInterval: interval, - } - _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) - assert.NoError(t, err) - - now := time.Now().UTC() - - // Create 5 backups on 5 different days; only the 3 newest days should be kept - var backupIDs []uuid.UUID - for i := 0; i < 5; i++ { - backup := &backups_core.Backup{ - ID: uuid.New(), - DatabaseID: database.ID, - StorageID: storage.ID, - Status: backups_core.BackupStatusCompleted, - BackupSizeMb: 10, - CreatedAt: now.Add(-time.Duration(4-i) * 24 * time.Hour).Truncate(24 * time.Hour), - } - err = backupRepository.Save(backup) - assert.NoError(t, err) - backupIDs = append(backupIDs, backup.ID) - } - - cleaner := GetBackupCleaner() - err = cleaner.cleanByRetentionPolicy() - assert.NoError(t, err) - - remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) - assert.NoError(t, err) - assert.Equal(t, 3, len(remainingBackups)) - - remainingIDs := make(map[uuid.UUID]bool) - for _, backup := range remainingBackups { - remainingIDs[backup.ID] = true - } - assert.False(t, remainingIDs[backupIDs[0]], "Oldest daily backup should be deleted") - assert.False(t, remainingIDs[backupIDs[1]], "2nd oldest daily backup should be deleted") - assert.True(t, remainingIDs[backupIDs[2]], "3rd backup should remain") - assert.True(t, remainingIDs[backupIDs[3]], "4th backup should remain") - assert.True(t, remainingIDs[backupIDs[4]], "Newest backup should remain") -} - -func Test_CleanByGFS_WithWeeklyAndMonthlySlots_KeepsWiderSpread(t *testing.T) { - router := CreateTestRouter() - owner := users_testing.CreateTestUser(users_enums.UserRoleMember) - workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) - storage := storages.CreateTestStorage(workspace.ID) - notifier := notifiers.CreateTestNotifier(workspace.ID) - database := databases.CreateTestDatabase(workspace.ID, storage, notifier) - - defer func() { - backups, _ := backupRepository.FindByDatabaseID(database.ID) - for _, backup := range backups { - backupRepository.DeleteByID(backup.ID) - } - - databases.RemoveTestDatabase(database) - time.Sleep(50 * time.Millisecond) - notifiers.RemoveTestNotifier(notifier) - storages.RemoveTestStorage(storage.ID) - workspaces_testing.RemoveTestWorkspace(workspace, router) - }() - - interval := createTestInterval() - - backupConfig := &backups_config.BackupConfig{ - DatabaseID: database.ID, - IsBackupsEnabled: true, - RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, - RetentionGfsDays: 2, - RetentionGfsWeeks: 2, - RetentionGfsMonths: 1, - RetentionGfsYears: 0, - StorageID: &storage.ID, - BackupIntervalID: interval.ID, - BackupInterval: interval, - } - _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) - assert.NoError(t, err) - - now := time.Now().UTC() - - // Create one backup per week for 6 weeks (each on Monday of that week) - // GFS should keep: 2 daily (most recent 2 unique days) + 2 weekly + 1 monthly = up to 5 unique - var createdIDs []uuid.UUID - for i := 0; i < 6; i++ { - weekOffset := time.Duration(5-i) * 7 * 24 * time.Hour - backup := &backups_core.Backup{ - ID: uuid.New(), - DatabaseID: database.ID, - StorageID: storage.ID, - Status: backups_core.BackupStatusCompleted, - BackupSizeMb: 10, - CreatedAt: now.Add(-weekOffset).Truncate(24 * time.Hour), - } - err = backupRepository.Save(backup) - assert.NoError(t, err) - createdIDs = append(createdIDs, backup.ID) - } - - cleaner := GetBackupCleaner() - err = cleaner.cleanByRetentionPolicy() - assert.NoError(t, err) - - remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) - assert.NoError(t, err) - - // We should have at most 5 backups kept (2 daily + 2 weekly + 1 monthly, but with overlap possible) - // The exact count depends on how many unique periods are covered - assert.LessOrEqual(t, len(remainingBackups), 5) - assert.GreaterOrEqual(t, len(remainingBackups), 1) - - // The two most recent backups should always be retained (daily slots) - remainingIDs := make(map[uuid.UUID]bool) - for _, backup := range remainingBackups { - remainingIDs[backup.ID] = true - } - assert.True(t, remainingIDs[createdIDs[4]], "Second newest backup should be retained (daily)") - assert.True(t, remainingIDs[createdIDs[5]], "Newest backup should be retained (daily)") -} - // Test_DeleteBackup_WhenStorageDeleteFails_BackupStillRemovedFromDatabase verifies resilience // when storage becomes unavailable. Even if storage.DeleteFile fails (e.g., storage is offline, // credentials changed, or storage was deleted), the backup record should still be removed from @@ -897,292 +743,6 @@ func Test_DeleteBackup_WhenStorageDeleteFails_BackupStillRemovedFromDatabase(t * assert.Nil(t, deletedBackup) } -func Test_CleanByGFS_WithHourlySlots_KeepsCorrectBackups(t *testing.T) { - router := CreateTestRouter() - owner := users_testing.CreateTestUser(users_enums.UserRoleMember) - workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) - testStorage := storages.CreateTestStorage(workspace.ID) - notifier := notifiers.CreateTestNotifier(workspace.ID) - database := databases.CreateTestDatabase(workspace.ID, testStorage, notifier) - - defer func() { - backups, _ := backupRepository.FindByDatabaseID(database.ID) - for _, backup := range backups { - backupRepository.DeleteByID(backup.ID) - } - - databases.RemoveTestDatabase(database) - time.Sleep(50 * time.Millisecond) - notifiers.RemoveTestNotifier(notifier) - storages.RemoveTestStorage(testStorage.ID) - workspaces_testing.RemoveTestWorkspace(workspace, router) - }() - - interval := createTestInterval() - - backupConfig := &backups_config.BackupConfig{ - DatabaseID: database.ID, - IsBackupsEnabled: true, - RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, - RetentionGfsHours: 3, - StorageID: &testStorage.ID, - BackupIntervalID: interval.ID, - BackupInterval: interval, - } - _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) - assert.NoError(t, err) - - now := time.Now().UTC() - - // Create 5 backups spaced 1 hour apart; only the 3 newest hours should be kept - var backupIDs []uuid.UUID - for i := 0; i < 5; i++ { - backup := &backups_core.Backup{ - ID: uuid.New(), - DatabaseID: database.ID, - StorageID: testStorage.ID, - Status: backups_core.BackupStatusCompleted, - BackupSizeMb: 10, - CreatedAt: now.Add(-time.Duration(4-i) * time.Hour).Truncate(time.Hour), - } - err = backupRepository.Save(backup) - assert.NoError(t, err) - backupIDs = append(backupIDs, backup.ID) - } - - cleaner := GetBackupCleaner() - err = cleaner.cleanByRetentionPolicy() - assert.NoError(t, err) - - remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) - assert.NoError(t, err) - assert.Equal(t, 3, len(remainingBackups)) - - remainingIDs := make(map[uuid.UUID]bool) - for _, backup := range remainingBackups { - remainingIDs[backup.ID] = true - } - assert.False(t, remainingIDs[backupIDs[0]], "Oldest hourly backup should be deleted") - assert.False(t, remainingIDs[backupIDs[1]], "2nd oldest hourly backup should be deleted") - assert.True(t, remainingIDs[backupIDs[2]], "3rd backup should remain") - assert.True(t, remainingIDs[backupIDs[3]], "4th backup should remain") - assert.True(t, remainingIDs[backupIDs[4]], "Newest backup should remain") -} - -func Test_BuildGFSKeepSet(t *testing.T) { - // Fixed reference time: a Wednesday mid-month to avoid boundary edge cases in the default tests. - // Use time.Date for determinism across test runs. - ref := time.Date(2025, 6, 18, 12, 0, 0, 0, time.UTC) // Wednesday, 2025-06-18 - - day := 24 * time.Hour - week := 7 * day - - newBackup := func(createdAt time.Time) *backups_core.Backup { - return &backups_core.Backup{ID: uuid.New(), CreatedAt: createdAt} - } - - // backupsEveryDay returns n backups, newest-first, each 1 day apart. - backupsEveryDay := func(n int) []*backups_core.Backup { - bs := make([]*backups_core.Backup, n) - for i := 0; i < n; i++ { - bs[i] = newBackup(ref.Add(-time.Duration(i) * day)) - } - return bs - } - - // backupsEveryWeek returns n backups, newest-first, each 7 days apart. - backupsEveryWeek := func(n int) []*backups_core.Backup { - bs := make([]*backups_core.Backup, n) - for i := 0; i < n; i++ { - bs[i] = newBackup(ref.Add(-time.Duration(i) * week)) - } - return bs - } - - hour := time.Hour - - // backupsEveryHour returns n backups, newest-first, each 1 hour apart. - backupsEveryHour := func(n int) []*backups_core.Backup { - bs := make([]*backups_core.Backup, n) - for i := 0; i < n; i++ { - bs[i] = newBackup(ref.Add(-time.Duration(i) * hour)) - } - return bs - } - - tests := []struct { - name string - backups []*backups_core.Backup - hours int - days int - weeks int - months int - years int - keptIndices []int // which indices in backups should be kept - deletedRange *[2]int // optional: all indices in [from, to) must be deleted - }{ - { - name: "OnlyHourlySlots_KeepsNewest3Of5", - backups: backupsEveryHour(5), - hours: 3, - keptIndices: []int{0, 1, 2}, - }, - { - name: "SameHourDedup_OnlyNewestKeptForHourlySlot", - backups: []*backups_core.Backup{ - newBackup(ref.Truncate(hour).Add(45 * time.Minute)), - newBackup(ref.Truncate(hour).Add(10 * time.Minute)), - }, - hours: 1, - keptIndices: []int{0}, - }, - { - name: "OnlyDailySlots_KeepsNewest3Of5", - backups: backupsEveryDay(5), - days: 3, - keptIndices: []int{0, 1, 2}, - }, - { - name: "OnlyDailySlots_FewerBackupsThanSlots_KeepsAll", - backups: backupsEveryDay(2), - days: 5, - keptIndices: []int{0, 1}, - }, - { - name: "OnlyWeeklySlots_KeepsNewest2Weeks", - backups: backupsEveryWeek(4), - weeks: 2, - keptIndices: []int{0, 1}, - }, - { - name: "OnlyMonthlySlots_KeepsNewest2Months", - backups: []*backups_core.Backup{ - newBackup(time.Date(2025, 6, 1, 12, 0, 0, 0, time.UTC)), - newBackup(time.Date(2025, 5, 1, 12, 0, 0, 0, time.UTC)), - newBackup(time.Date(2025, 4, 1, 12, 0, 0, 0, time.UTC)), - }, - months: 2, - keptIndices: []int{0, 1}, - }, - { - name: "OnlyYearlySlots_KeepsNewest2Years", - backups: []*backups_core.Backup{ - newBackup(time.Date(2025, 6, 1, 12, 0, 0, 0, time.UTC)), - newBackup(time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC)), - newBackup(time.Date(2023, 6, 1, 12, 0, 0, 0, time.UTC)), - }, - years: 2, - keptIndices: []int{0, 1}, - }, - { - name: "SameDayDedup_OnlyNewestKeptForDailySlot", - backups: []*backups_core.Backup{ - // Two backups on the same day; newest-first order - newBackup(ref.Truncate(day).Add(10 * time.Hour)), - newBackup(ref.Truncate(day).Add(2 * time.Hour)), - }, - days: 1, - keptIndices: []int{0}, - }, - { - name: "SameWeekDedup_OnlyNewestKeptForWeeklySlot", - backups: []*backups_core.Backup{ - // ref is Wednesday; add Thursday of same week - newBackup(ref.Add(1 * day)), // Thursday same week - newBackup(ref), // Wednesday same week - }, - weeks: 1, - keptIndices: []int{0}, - }, - { - name: "AdditiveSlots_NewestFillsDailyAndWeeklyAndMonthly", - // Newest backup fills daily + weekly + monthly simultaneously - backups: []*backups_core.Backup{ - newBackup(time.Date(2025, 6, 18, 12, 0, 0, 0, time.UTC)), // newest - newBackup(time.Date(2025, 6, 11, 12, 0, 0, 0, time.UTC)), // 1 week ago - newBackup(time.Date(2025, 5, 18, 12, 0, 0, 0, time.UTC)), // 1 month ago - newBackup(time.Date(2025, 4, 18, 12, 0, 0, 0, time.UTC)), // 2 months ago - }, - days: 1, - weeks: 2, - months: 2, - keptIndices: []int{0, 1, 2}, - }, - { - name: "YearBoundary_CorrectlySplitsAcrossYears", - backups: []*backups_core.Backup{ - newBackup(time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC)), - newBackup(time.Date(2024, 12, 31, 12, 0, 0, 0, time.UTC)), - newBackup(time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC)), - newBackup(time.Date(2023, 6, 1, 12, 0, 0, 0, time.UTC)), - }, - years: 2, - keptIndices: []int{0, 1}, // 2025 and 2024 kept; 2024-06 and 2023 deleted - }, - { - name: "ISOWeekBoundary_Jan1UsesCorrectISOWeek", - // 2025-01-01 is ISO week 1 of 2025; 2024-12-28 is ISO week 52 of 2024 - backups: []*backups_core.Backup{ - newBackup(time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC)), // ISO week 2025-W01 - newBackup(time.Date(2024, 12, 28, 12, 0, 0, 0, time.UTC)), // ISO week 2024-W52 - }, - weeks: 2, - keptIndices: []int{0, 1}, // different ISO weeks → both kept - }, - { - name: "EmptyBackups_ReturnsEmptyKeepSet", - backups: []*backups_core.Backup{}, - hours: 3, - days: 3, - weeks: 2, - months: 1, - years: 1, - keptIndices: []int{}, - }, - { - name: "AllZeroSlots_KeepsNothing", - backups: backupsEveryDay(5), - hours: 0, - days: 0, - weeks: 0, - months: 0, - years: 0, - keptIndices: []int{}, - }, - { - name: "AllSlotsActive_FullCombination", - backups: backupsEveryWeek(12), - days: 2, - weeks: 3, - months: 2, - years: 1, - // 2 daily (indices 0,1) + 3rd weekly slot (index 2) + 2nd monthly slot (index 3 or later). - // Additive slots: newest fills daily+weekly+monthly+yearly; each subsequent week fills another weekly, - // and a backup ~4 weeks later fills the 2nd monthly slot. - keptIndices: []int{0, 1, 2, 3}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - keepSet := buildGFSKeepSet(tc.backups, tc.hours, tc.days, tc.weeks, tc.months, tc.years) - - keptIndexSet := make(map[int]bool, len(tc.keptIndices)) - for _, idx := range tc.keptIndices { - keptIndexSet[idx] = true - } - - for i, backup := range tc.backups { - if keptIndexSet[i] { - assert.True(t, keepSet[backup.ID], "backup at index %d should be kept", i) - } else { - assert.False(t, keepSet[backup.ID], "backup at index %d should be deleted", i) - } - } - }) - } -} - func Test_CleanByTimePeriod_SkipsRecentBackup_EvenIfOlderThanRetention(t *testing.T) { router := CreateTestRouter() owner := users_testing.CreateTestUser(users_enums.UserRoleMember) @@ -1354,114 +914,6 @@ func Test_CleanByCount_SkipsRecentBackup_EvenIfOverLimit(t *testing.T) { assert.True(t, remainingIDs[newestBackup.ID], "Newest backup should be preserved") } -func Test_CleanByGFS_SkipsRecentBackup_WhenNotInKeepSet(t *testing.T) { - router := CreateTestRouter() - owner := users_testing.CreateTestUser(users_enums.UserRoleMember) - workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", owner, router) - storage := storages.CreateTestStorage(workspace.ID) - notifier := notifiers.CreateTestNotifier(workspace.ID) - database := databases.CreateTestDatabase(workspace.ID, storage, notifier) - - defer func() { - backups, _ := backupRepository.FindByDatabaseID(database.ID) - for _, backup := range backups { - backupRepository.DeleteByID(backup.ID) - } - - databases.RemoveTestDatabase(database) - time.Sleep(50 * time.Millisecond) - notifiers.RemoveTestNotifier(notifier) - storages.RemoveTestStorage(storage.ID) - workspaces_testing.RemoveTestWorkspace(workspace, router) - }() - - interval := createTestInterval() - - // Keep only 1 daily slot. We create 2 old backups plus two recent backups on today. - // Backups are ordered newest-first, so the 15-min-old backup fills the single daily slot. - // The 30-min-old backup is the same day → not in the GFS keep-set, but it is still recent - // (within grace period) and must be preserved. - backupConfig := &backups_config.BackupConfig{ - DatabaseID: database.ID, - IsBackupsEnabled: true, - RetentionPolicyType: backups_config.RetentionPolicyTypeGFS, - RetentionGfsDays: 1, - StorageID: &storage.ID, - BackupIntervalID: interval.ID, - BackupInterval: interval, - } - _, err := backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig) - assert.NoError(t, err) - - now := time.Now().UTC() - - oldBackup1 := &backups_core.Backup{ - ID: uuid.New(), - DatabaseID: database.ID, - StorageID: storage.ID, - Status: backups_core.BackupStatusCompleted, - BackupSizeMb: 10, - CreatedAt: now.Add(-3 * 24 * time.Hour).Truncate(24 * time.Hour), - } - oldBackup2 := &backups_core.Backup{ - ID: uuid.New(), - DatabaseID: database.ID, - StorageID: storage.ID, - Status: backups_core.BackupStatusCompleted, - BackupSizeMb: 10, - CreatedAt: now.Add(-2 * 24 * time.Hour).Truncate(24 * time.Hour), - } - // Newest backup today — will fill the single GFS daily slot. - newestTodayBackup := &backups_core.Backup{ - ID: uuid.New(), - DatabaseID: database.ID, - StorageID: storage.ID, - Status: backups_core.BackupStatusCompleted, - BackupSizeMb: 10, - CreatedAt: now.Add(-15 * time.Minute), - } - // Slightly older backup, also today — NOT in GFS keep-set (duplicate day), - // but within the 60-minute grace period so it must survive. - recentNotInKeepSet := &backups_core.Backup{ - ID: uuid.New(), - DatabaseID: database.ID, - StorageID: storage.ID, - Status: backups_core.BackupStatusCompleted, - BackupSizeMb: 10, - CreatedAt: now.Add(-30 * time.Minute), - } - - for _, b := range []*backups_core.Backup{oldBackup1, oldBackup2, newestTodayBackup, recentNotInKeepSet} { - err = backupRepository.Save(b) - assert.NoError(t, err) - } - - cleaner := GetBackupCleaner() - err = cleaner.cleanByRetentionPolicy() - assert.NoError(t, err) - - remainingBackups, err := backupRepository.FindByDatabaseID(database.ID) - assert.NoError(t, err) - - remainingIDs := make(map[uuid.UUID]bool) - for _, backup := range remainingBackups { - remainingIDs[backup.ID] = true - } - - assert.False(t, remainingIDs[oldBackup1.ID], "Old backup 1 should be deleted by GFS") - assert.False(t, remainingIDs[oldBackup2.ID], "Old backup 2 should be deleted by GFS") - assert.True( - t, - remainingIDs[newestTodayBackup.ID], - "Newest backup fills GFS daily slot and must remain", - ) - assert.True( - t, - remainingIDs[recentNotInKeepSet.ID], - "Recent backup not in keep-set must be preserved by grace period", - ) -} - func Test_CleanExceededBackups_SkipsRecentBackup_WhenOverTotalSizeLimit(t *testing.T) { router := CreateTestRouter() owner := users_testing.CreateTestUser(users_enums.UserRoleMember)