Feature/improve automated backup (#597)

* Add option to disable cleanup of backups

* Ensure the minimum TTL of backups to 1 day

* Schedule the automated backup on a specific time of the day

* Introduce scheduler that takes system hibernation time into account

In case the system was hibernating/suspended scheduled task that should have been executed during that time would not get triggered and thus, miss an execution.

To prevent this, this new scheduler periodically checks if the system was suspended and in case it was, triggers any task that missed its last execution

* Use new scheduler

gradle/libs.versions.toml

@@ -128,6 +128,12 @@ twelvemonkeys-imageio-webp = { module = "com.twelvemonkeys.imageio:imageio-webp"
 # Testing
 mockk = "io.mockk:mockk:1.13.2"
 
+# cron scheduler
+cron4j = "it.sauronsoftware.cron4j:cron4j:2.2.5"
+
+# cron-utils
+cronUtils = "com.cronutils:cron-utils:9.2.0"
+
 [plugins]
 # Kotlin
 kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin"}

server/build.gradle.kts

@@ -70,6 +70,10 @@ dependencies {
     implementation("com.graphql-java:graphql-java-extended-scalars:20.0")
 
     testImplementation(libs.mockk)
+
+    implementation(libs.cron4j)
+
+    implementation(libs.cronUtils)
 }
 
 application {

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt

@@ -36,13 +36,12 @@ import suwayomi.tachidesk.manga.model.table.SourceTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
 import suwayomi.tachidesk.server.ApplicationDirs
 import suwayomi.tachidesk.server.serverConfig
+import suwayomi.tachidesk.util.HAScheduler
 import java.io.ByteArrayOutputStream
 import java.io.File
 import java.io.InputStream
 import java.text.SimpleDateFormat
 import java.util.Date
-import java.util.Timer
-import java.util.TimerTask
 import java.util.concurrent.TimeUnit
 import java.util.prefs.Preferences
 import kotlin.time.Duration.Companion.days
@@ -50,40 +49,37 @@ import kotlin.time.Duration.Companion.days
 object ProtoBackupExport : ProtoBackupBase() {
     private val logger = KotlinLogging.logger { }
     private val applicationDirs by DI.global.instance<ApplicationDirs>()
+    private var backupSchedulerJobId: String = ""
     private const val lastAutomatedBackupKey = "lastAutomatedBackupKey"
     private val preferences = Preferences.userNodeForPackage(ProtoBackupExport::class.java)
 
-    private val backupTimer = Timer()
-    private var currentAutomatedBackupTask: TimerTask? = null
-
     fun scheduleAutomatedBackupTask() {
+        HAScheduler.deschedule(backupSchedulerJobId)
+
         if (!serverConfig.automatedBackups) {
-            currentAutomatedBackupTask?.cancel()
             return
         }
 
-        val minInterval = 1.days
-        val interval = serverConfig.backupInterval.days
-        val backupInterval = interval.coerceAtLeast(minInterval).inWholeMilliseconds
-
-        val lastAutomatedBackup = preferences.getLong(lastAutomatedBackupKey, 0)
-        val initialDelay =
-            backupInterval - (System.currentTimeMillis() - lastAutomatedBackup) % backupInterval
-
-        currentAutomatedBackupTask?.cancel()
-        currentAutomatedBackupTask = object : TimerTask() {
-            override fun run() {
-                cleanupAutomatedBackups()
-                createAutomatedBackup()
-                preferences.putLong(lastAutomatedBackupKey, System.currentTimeMillis())
-            }
+        val task = {
+            cleanupAutomatedBackups()
+            createAutomatedBackup()
+            preferences.putLong(lastAutomatedBackupKey, System.currentTimeMillis())
         }
 
-        backupTimer.scheduleAtFixedRate(
-            currentAutomatedBackupTask,
-            initialDelay,
-            backupInterval
-        )
+        val (hour, minute) = serverConfig.backupTime.split(":").map { it.toInt() }
+        val backupHour = hour.coerceAtLeast(0).coerceAtMost(23)
+        val backupMinute = minute.coerceAtLeast(0).coerceAtMost(59)
+        val backupInterval = serverConfig.backupInterval.days.coerceAtLeast(1.days)
+
+        // trigger last backup in case the server wasn't running on the scheduled time
+        val lastAutomatedBackup = preferences.getLong(lastAutomatedBackupKey, System.currentTimeMillis())
+        val wasPreviousBackupTriggered =
+            (System.currentTimeMillis() - lastAutomatedBackup) < backupInterval.inWholeMilliseconds
+        if (!wasPreviousBackupTriggered) {
+            task()
+        }
+
+        HAScheduler.schedule(task, "$backupMinute $backupHour */${backupInterval.inWholeDays} * *", "backup")
     }
 
     private fun createAutomatedBackup() {
@@ -105,11 +101,15 @@ object ProtoBackupExport : ProtoBackupBase() {
 
             backupFile.outputStream().use { output -> input.copyTo(output) }
         }
-
     }
 
     private fun cleanupAutomatedBackups() {
-        logger.debug { "Cleanup automated backups" }
+        logger.debug { "Cleanup automated backups (ttl= ${serverConfig.backupTTL})" }
+
+        val isCleanupDisabled = serverConfig.backupTTL == 0
+        if (isCleanupDisabled) {
+            return
+        }
 
         val automatedBackupDir = File(applicationDirs.automatedBackupRoot)
         if (!automatedBackupDir.isDirectory) {
@@ -132,7 +132,7 @@ object ProtoBackupExport : ProtoBackupBase() {
 
         val lastAccessTime = file.lastModified()
         val isTTLReached =
-            System.currentTimeMillis() - lastAccessTime >= serverConfig.backupTTL.days.inWholeMilliseconds
+            System.currentTimeMillis() - lastAccessTime >= serverConfig.backupTTL.days.coerceAtLeast(1.days).inWholeMilliseconds
         if (isTTLReached) {
             file.delete()
         }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt

@@ -1,6 +1,8 @@
 package suwayomi.tachidesk.manga.impl.update
 
 import eu.kanade.tachiyomi.source.model.UpdateStrategy
+import it.sauronsoftware.cron4j.Task
+import it.sauronsoftware.cron4j.TaskExecutionContext
 import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
@@ -24,11 +26,13 @@ import org.kodein.di.instance
 import suwayomi.tachidesk.manga.impl.Category
 import suwayomi.tachidesk.manga.impl.CategoryManga
 import suwayomi.tachidesk.manga.impl.Chapter
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport
 import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
 import suwayomi.tachidesk.manga.model.dataclass.IncludeInUpdate
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
 import suwayomi.tachidesk.manga.model.table.MangaStatus
 import suwayomi.tachidesk.server.serverConfig
+import suwayomi.tachidesk.util.HAScheduler
 import java.util.Date
 import java.util.Timer
 import java.util.TimerTask
@@ -51,41 +55,46 @@ class Updater : IUpdater {
     private val lastAutomatedUpdateKey = "lastAutomatedUpdateKey"
     private val preferences = Preferences.userNodeForPackage(Updater::class.java)
 
-    private val updateTimer = Timer()
-    private var currentUpdateTask: TimerTask? = null
+    private var currentUpdateTaskId = ""
 
     init {
         scheduleUpdateTask()
     }
 
+
+    private fun autoUpdateTask() {
+        val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0)
+        preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis())
+
+        if (status.value.running) {
+            logger.debug { "Global update is already in progress" }
+            return
+        }
+
+        logger.info { "Trigger global update (interval= ${serverConfig.globalUpdateInterval}h, lastAutomatedUpdate= ${Date(lastAutomatedUpdate)})" }
+        addCategoriesToUpdateQueue(Category.getCategoryList(), true)
+    }
+
     private fun scheduleUpdateTask() {
+        HAScheduler.deschedule(currentUpdateTaskId)
+
         if (!serverConfig.automaticallyTriggerGlobalUpdate) {
             return
         }
 
         val minInterval = 6.hours
         val interval = serverConfig.globalUpdateInterval.hours
-        val updateInterval = interval.coerceAtLeast(minInterval).inWholeMilliseconds
+        val updateInterval = interval.coerceAtLeast(minInterval)
+        val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, System.currentTimeMillis())
 
-        val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0)
-        val initialDelay = updateInterval - (System.currentTimeMillis() - lastAutomatedUpdate) % updateInterval
-
-        currentUpdateTask?.cancel()
-        currentUpdateTask = object : TimerTask() {
-            override fun run() {
-                preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis())
-
-                if (status.value.running) {
-                    logger.debug { "Global update is already in progress, do not trigger global update" }
-                    return
-                }
-
-                logger.info { "Trigger global update (interval= ${serverConfig.globalUpdateInterval}h, lastAutomatedUpdate= ${Date(lastAutomatedUpdate)})" }
-                addCategoriesToUpdateQueue(Category.getCategoryList(), true)
-            }
+        // trigger update in case the server wasn't running on the scheduled time
+        val wasPreviousUpdateTriggered =
+            (System.currentTimeMillis() - lastAutomatedUpdate) < updateInterval.inWholeMilliseconds
+        if (!wasPreviousUpdateTriggered) {
+            autoUpdateTask()
         }
 
-        updateTimer.scheduleAtFixedRate(currentUpdateTask, initialDelay, updateInterval)
+        HAScheduler.schedule(::autoUpdateTask, "* */${updateInterval.inWholeHours} * * *", "global-update")
     }
 
     private fun getOrCreateUpdateChannelFor(source: String): Channel<UpdateJob> {

server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt

@@ -52,6 +52,7 @@ class ServerConfig(getConfig: () -> Config, moduleName: String = MODULE_NAME) :
 
     // backup
     var backupPath: String by overridableConfig
+    var backupTime: String by overridableConfig
     var backupInterval: Int by overridableConfig
     var automatedBackups: Boolean by overridableConfig
     var backupTTL: Int by overridableConfig

server/src/main/kotlin/suwayomi/tachidesk/util/HAScheduler.kt

@@ -0,0 +1,131 @@
+
+package suwayomi.tachidesk.util
+
+import com.cronutils.model.CronType.CRON4J
+import com.cronutils.model.definition.CronDefinitionBuilder
+import com.cronutils.model.time.ExecutionTime
+import com.cronutils.parser.CronParser
+import it.sauronsoftware.cron4j.Scheduler
+import it.sauronsoftware.cron4j.Task
+import it.sauronsoftware.cron4j.TaskExecutionContext
+import mu.KotlinLogging
+import java.time.ZonedDateTime
+import java.util.PriorityQueue
+import java.util.Timer
+import java.util.TimerTask
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.minutes
+import kotlin.time.Duration.Companion.seconds
+
+val cronParser = CronParser(CronDefinitionBuilder.instanceDefinitionFor(CRON4J))
+
+class HATask(val id: String, val cronExpr: String, val execute: () -> Unit, val name: String?) : Comparable<HATask> {
+    private val executionTime = ExecutionTime.forCron(cronParser.parse(cronExpr))
+
+    fun getLastExecutionTime(): Long {
+        return executionTime.lastExecution(ZonedDateTime.now()).get().toEpochSecond().seconds.inWholeMilliseconds
+    }
+
+    fun getNextExecutionTime(): Long {
+        return executionTime.nextExecution(ZonedDateTime.now()).get().toEpochSecond().seconds.inWholeMilliseconds
+    }
+
+    fun getTimeToNextExecution(): Long {
+        return executionTime.timeToNextExecution(ZonedDateTime.now()).get().toMillis()
+    }
+
+    override fun compareTo(other: HATask): Int {
+        return getTimeToNextExecution().compareTo(other.getTimeToNextExecution())
+    }
+}
+
+/**
+ * The "HAScheduler" ("HibernateAwareScheduler") is a scheduler that recognizes when the system was hibernating/suspended
+ * and triggers tasks that have missed their execution points.
+ */
+object HAScheduler {
+    private val logger = KotlinLogging.logger { }
+
+    private val scheduledTasks = PriorityQueue<HATask>()
+    private val scheduler = Scheduler()
+
+    private val HIBERNATION_THRESHOLD = 10.seconds.inWholeMilliseconds
+    private const val TASK_THRESHOLD = 0.1
+
+    init {
+        scheduleHibernateCheckerTask(1.minutes)
+    }
+
+    private fun scheduleHibernateCheckerTask(interval: Duration) {
+        val timer = Timer()
+        timer.scheduleAtFixedRate(
+            object : TimerTask() {
+                var lastExecutionTime = System.currentTimeMillis()
+
+                override fun run() {
+                    val currentTime = System.currentTimeMillis()
+                    val elapsedTime = currentTime - lastExecutionTime
+                    lastExecutionTime = currentTime
+
+                    val systemWasInHibernation = elapsedTime > interval.inWholeMilliseconds + HIBERNATION_THRESHOLD
+                    if (systemWasInHibernation) {
+                        logger.debug { "System hibernation detected, task was delayed by ${elapsedTime - interval.inWholeMilliseconds}ms" }
+                        scheduledTasks.forEach {
+                            val missedExecution = currentTime - it.getLastExecutionTime() - elapsedTime < 0
+                            val taskInterval = it.getNextExecutionTime() - it.getLastExecutionTime()
+                            // in case the next task execution doesn't take long the missed execution can be ignored to prevent a double execution
+                            val taskThresholdMet = taskInterval * TASK_THRESHOLD > it.getTimeToNextExecution()
+
+                            val triggerTask = missedExecution && taskThresholdMet
+                            if (triggerTask) {
+                                logger.debug { "Task \"${it.name ?: it.id}\" missed its execution, executing now..." }
+                                reschedule(it.id, it.cronExpr)
+                                it.execute()
+                            }
+
+                            // queue is ordered by next execution time, thus, loop can be exited early
+                            if (!missedExecution) {
+                                return@forEach
+                            }
+                        }
+                    }
+                }
+            },
+            interval.inWholeMilliseconds,
+            interval.inWholeMilliseconds
+        )
+    }
+
+    fun schedule(execute: () -> Unit, cronExpr: String, name: String?): String {
+        if (!scheduler.isStarted) {
+            scheduler.start()
+        }
+
+        val taskId = scheduler.schedule(
+            cronExpr,
+            object : Task() {
+                override fun execute(context: TaskExecutionContext?) {
+                    execute()
+                }
+            }
+        )
+
+        scheduledTasks.add(HATask(taskId, cronExpr, execute, name))
+
+        return taskId
+    }
+
+    fun deschedule(taskId: String) {
+        scheduler.deschedule(taskId)
+        scheduledTasks.removeIf { it.id == taskId }
+    }
+
+    fun reschedule(taskId: String, cronExpr: String) {
+        val task = scheduledTasks.find { it.id == taskId } ?: return
+
+        scheduledTasks.remove(task)
+        scheduledTasks.add(HATask(taskId, cronExpr, task.execute, task.name))
+
+        scheduler.reschedule(taskId, cronExpr)
+    }
+}

server/src/main/resources/server-reference.conf

@@ -37,6 +37,7 @@ server.systemTrayEnabled = true
 
 # backup
 server.backupPath = ""
+server.backupTime = "00:00" # range: hour: 0-23, minute: 0-59 - default: "00:00" - time of day at which the automated backup should be triggered
 server.backupInterval = 1 # time in days - range: 1 <= n < ∞ - default: 1 day - interval in which the server will automatically create a backup
 server.automatedBackups = true
-server.backupTTL = 14 # time in days - range: 1 <= n < ∞ - default: 14 days - how long backup files will be kept before they will get deleted
+server.backupTTL = 14 # time in days - 0 to disable it - range: 1 <= n < ∞ - default: 14 days - how long backup files will be kept before they will get deleted

server/src/test/resources/server-reference.conf

@@ -30,6 +30,7 @@ server.electronPath = ""
 
 # backup
 server.backupPath = ""
+server.backupTime = "00:00"
 server.backupInterval = 1
 server.automatedBackups = true
 server.backupTTL = 14