RavenDB-23683 - fix how we determine if we run a backup or not

grisha-kotler · Feb 24, 2025 · f6f5488 · f6f5488
1 parent 368997b
commit f6f5488
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 17 deletions.
diff --git a/src/Raven.Server/Documents/DatabasesLandlord.cs b/src/Raven.Server/Documents/DatabasesLandlord.cs
@@ -1405,7 +1405,7 @@ private void NextScheduledActivityCallback(string databaseName, IdleDatabaseActi
                             break;
 
                         case IdleDatabaseActivityType.WakeUpDatabase:
-                            if (_serverStore.ConcurrentBackupsCounter.CanRunBackup == false)
+                            if (_serverStore.ConcurrentBackupsCounter.CanRunBackup(ShardHelper.ToDatabaseName(databaseName)) == false)
                             {
                                 // reached max concurrent backups
                                 var delayInMs = RescheduleDatabaseWakeup();

diff --git a/src/Raven.Server/Documents/PeriodicBackup/ConcurrentBackupsCounter.cs b/src/Raven.Server/Documents/PeriodicBackup/ConcurrentBackupsCounter.cs
@@ -32,17 +32,6 @@ public int CurrentNumberOfRunningBackups
             }
         }
 
-        public bool CanRunBackup
-        {
-            get
-            {
-                lock (_locker)
-                {
-                    return _runningBackupsPerDatabase.Count - 1 > 0;
-                }
-            }
-        }
-
         public ConcurrentBackupsCounter(BackupConfiguration backupConfiguration, LicenseManager licenseManager)
         {
             _licenseManager = licenseManager;
@@ -65,6 +54,20 @@ public ConcurrentBackupsCounter(BackupConfiguration backupConfiguration, License
             _skipModifications = skipModifications;
         }
 
+        public bool CanRunBackup(string databaseName)
+        {
+            lock (_locker)
+            {
+                if (_runningBackupsPerDatabase.TryGetValue(databaseName, out _))
+                {
+                    //  allow to backup all shards of the same database concurrently
+                    return true;
+                }
+
+                return MaxNumberOfConcurrentBackups - _runningBackupsPerDatabase.Count >= 1;
+            }
+        }
+
         public void StartBackup(string databaseName, string backupName, Logger logger)
         {
             lock (_locker)

diff --git a/test/SlowTests/Server/Documents/PeriodicBackup/PeriodicBackupSlowTests.cs b/test/SlowTests/Server/Documents/PeriodicBackup/PeriodicBackupSlowTests.cs
@@ -3482,38 +3482,55 @@ public async Task NumberOfCurrentlyRunningBackupsShouldBeCorrectAfterBackupTaskD
 
             var backupPath = NewDataPath(suffix: "BackupFolder");
 
-            using (var store = GetDocumentStore())
+            using (var server = GetNewServer(new ServerCreationOptions
+            {
+                CustomSettings = new Dictionary<string, string>
+                {
+                    [RavenConfiguration.GetKey(x => x.Backup.MaxNumberOfConcurrentBackups)] = 1.ToString()
+                }
+            }))
+            using (var store = GetDocumentStore(new Options
+            {
+                Server = server
+            }))
             {
                 using (var session = store.OpenAsyncSession())
                     await Backup.FillDatabaseWithRandomDataAsync(databaseSizeInMb: 1, session);
 
-                var database = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
+                var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
                 Assert.NotNull(database);
 
                 await Backup.HoldBackupExecutionIfNeededAndInvoke(database.PeriodicBackupRunner.ForTestingPurposesOnly(), async () =>
                 {
                     var config = Backup.CreateBackupConfiguration(backupPath, fullBackupFrequency: "* * * * *");
-                    var taskId = await Backup.UpdateConfigAndRunBackupAsync(Server, config, store, opStatus: OperationStatus.InProgress);
+                    var taskId = await Backup.UpdateConfigAndRunBackupAsync(server, config, store, opStatus: OperationStatus.InProgress);
 
                     // The backup task is running, and the next backup should be scheduled for the next minute (based on the backup configuration)
                     var taskBackupInfo = await store.Maintenance.SendAsync(new GetOngoingTaskInfoOperation(taskId, OngoingTaskType.Backup)) as OngoingTaskBackup;
                     Assert.NotNull(taskBackupInfo);
                     Assert.NotNull(taskBackupInfo.OnGoingBackup);
 
-                    using (Server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
+                    using (server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                     using (context.OpenReadTransaction())
                     {
                         AssertNumberOfConcurrentBackups(expectedNumber: 1);
 
+                        Assert.False(server.ServerStore.ConcurrentBackupsCounter.CanRunBackup(database.Name + "_Test"));
+
+                        // checking for the same database name (imitating a different shard for the same database).
+                        Assert.True(server.ServerStore.ConcurrentBackupsCounter.CanRunBackup(database.Name));
+
                         // Let's delay the backup task to 1 hour
                         var delayDuration = TimeSpan.FromHours(1);
                         await store.Maintenance.SendAsync(new DelayBackupOperation(taskBackupInfo.OnGoingBackup.RunningBackupTaskId, delayDuration));
 
                         AssertNumberOfConcurrentBackups(expectedNumber: 0);
 
+                        Assert.True(server.ServerStore.ConcurrentBackupsCounter.CanRunBackup(database.Name + "_Test"));
+
                         void AssertNumberOfConcurrentBackups(int expectedNumber)
                         {
-                            int concurrentBackups = WaitForValue(() => Server.ServerStore.ConcurrentBackupsCounter.CurrentNumberOfRunningBackups,
+                            int concurrentBackups = WaitForValue(() => server.ServerStore.ConcurrentBackupsCounter.CurrentNumberOfRunningBackups,
                                 expectedVal: expectedNumber,
                                 timeout: Convert.ToInt32(TimeSpan.FromMinutes(1).TotalMilliseconds),
                                 interval: Convert.ToInt32(TimeSpan.FromSeconds(1).TotalMilliseconds));