From c1d702a51c69538f9bd785e515aa3d881d582370 Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Thu, 20 Jul 2023 23:47:30 +0200 Subject: [PATCH] Feature/improve automated backup (#597) * Add option to disable cleanup of backups * Ensure the minimum TTL of backups to 1 day * Schedule the automated backup on a specific time of the day * Introduce scheduler that takes system hibernation time into account In case the system was hibernating/suspended scheduled task that should have been executed during that time would not get triggered and thus, miss an execution. To prevent this, this new scheduler periodically checks if the system was suspended and in case it was, triggers any task that missed its last execution * Use new scheduler --- gradle/libs.versions.toml | 6 + server/build.gradle.kts | 4 + .../impl/backup/proto/ProtoBackupExport.kt | 58 ++++---- .../tachidesk/manga/impl/update/Updater.kt | 49 ++++--- .../suwayomi/tachidesk/server/ServerConfig.kt | 1 + .../suwayomi/tachidesk/util/HAScheduler.kt | 131 ++++++++++++++++++ .../src/main/resources/server-reference.conf | 3 +- .../src/test/resources/server-reference.conf | 1 + 8 files changed, 203 insertions(+), 50 deletions(-) create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/util/HAScheduler.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9d4383d7..ba911764 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -128,6 +128,12 @@ twelvemonkeys-imageio-webp = { module = "com.twelvemonkeys.imageio:imageio-webp" # Testing mockk = "io.mockk:mockk:1.13.2" +# cron scheduler +cron4j = "it.sauronsoftware.cron4j:cron4j:2.2.5" + +# cron-utils +cronUtils = "com.cronutils:cron-utils:9.2.0" + [plugins] # Kotlin kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin"} diff --git a/server/build.gradle.kts b/server/build.gradle.kts index c3a092ce..27e82808 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -70,6 +70,10 @@ dependencies { implementation("com.graphql-java:graphql-java-extended-scalars:20.0") testImplementation(libs.mockk) + + implementation(libs.cron4j) + + implementation(libs.cronUtils) } application { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt index 8895ae9b..a0f3236e 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt @@ -36,13 +36,12 @@ import suwayomi.tachidesk.manga.model.table.SourceTable import suwayomi.tachidesk.manga.model.table.toDataClass import suwayomi.tachidesk.server.ApplicationDirs import suwayomi.tachidesk.server.serverConfig +import suwayomi.tachidesk.util.HAScheduler import java.io.ByteArrayOutputStream import java.io.File import java.io.InputStream import java.text.SimpleDateFormat import java.util.Date -import java.util.Timer -import java.util.TimerTask import java.util.concurrent.TimeUnit import java.util.prefs.Preferences import kotlin.time.Duration.Companion.days @@ -50,40 +49,37 @@ import kotlin.time.Duration.Companion.days object ProtoBackupExport : ProtoBackupBase() { private val logger = KotlinLogging.logger { } private val applicationDirs by DI.global.instance() + private var backupSchedulerJobId: String = "" private const val lastAutomatedBackupKey = "lastAutomatedBackupKey" private val preferences = Preferences.userNodeForPackage(ProtoBackupExport::class.java) - private val backupTimer = Timer() - private var currentAutomatedBackupTask: TimerTask? = null - fun scheduleAutomatedBackupTask() { + HAScheduler.deschedule(backupSchedulerJobId) + if (!serverConfig.automatedBackups) { - currentAutomatedBackupTask?.cancel() return } - val minInterval = 1.days - val interval = serverConfig.backupInterval.days - val backupInterval = interval.coerceAtLeast(minInterval).inWholeMilliseconds - - val lastAutomatedBackup = preferences.getLong(lastAutomatedBackupKey, 0) - val initialDelay = - backupInterval - (System.currentTimeMillis() - lastAutomatedBackup) % backupInterval - - currentAutomatedBackupTask?.cancel() - currentAutomatedBackupTask = object : TimerTask() { - override fun run() { - cleanupAutomatedBackups() - createAutomatedBackup() - preferences.putLong(lastAutomatedBackupKey, System.currentTimeMillis()) - } + val task = { + cleanupAutomatedBackups() + createAutomatedBackup() + preferences.putLong(lastAutomatedBackupKey, System.currentTimeMillis()) } - backupTimer.scheduleAtFixedRate( - currentAutomatedBackupTask, - initialDelay, - backupInterval - ) + val (hour, minute) = serverConfig.backupTime.split(":").map { it.toInt() } + val backupHour = hour.coerceAtLeast(0).coerceAtMost(23) + val backupMinute = minute.coerceAtLeast(0).coerceAtMost(59) + val backupInterval = serverConfig.backupInterval.days.coerceAtLeast(1.days) + + // trigger last backup in case the server wasn't running on the scheduled time + val lastAutomatedBackup = preferences.getLong(lastAutomatedBackupKey, System.currentTimeMillis()) + val wasPreviousBackupTriggered = + (System.currentTimeMillis() - lastAutomatedBackup) < backupInterval.inWholeMilliseconds + if (!wasPreviousBackupTriggered) { + task() + } + + HAScheduler.schedule(task, "$backupMinute $backupHour */${backupInterval.inWholeDays} * *", "backup") } private fun createAutomatedBackup() { @@ -105,11 +101,15 @@ object ProtoBackupExport : ProtoBackupBase() { backupFile.outputStream().use { output -> input.copyTo(output) } } - } private fun cleanupAutomatedBackups() { - logger.debug { "Cleanup automated backups" } + logger.debug { "Cleanup automated backups (ttl= ${serverConfig.backupTTL})" } + + val isCleanupDisabled = serverConfig.backupTTL == 0 + if (isCleanupDisabled) { + return + } val automatedBackupDir = File(applicationDirs.automatedBackupRoot) if (!automatedBackupDir.isDirectory) { @@ -132,7 +132,7 @@ object ProtoBackupExport : ProtoBackupBase() { val lastAccessTime = file.lastModified() val isTTLReached = - System.currentTimeMillis() - lastAccessTime >= serverConfig.backupTTL.days.inWholeMilliseconds + System.currentTimeMillis() - lastAccessTime >= serverConfig.backupTTL.days.coerceAtLeast(1.days).inWholeMilliseconds if (isTTLReached) { file.delete() } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt index dea7e1ad..1a373a5b 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt @@ -1,6 +1,8 @@ package suwayomi.tachidesk.manga.impl.update import eu.kanade.tachiyomi.source.model.UpdateStrategy +import it.sauronsoftware.cron4j.Task +import it.sauronsoftware.cron4j.TaskExecutionContext import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -24,11 +26,13 @@ import org.kodein.di.instance import suwayomi.tachidesk.manga.impl.Category import suwayomi.tachidesk.manga.impl.CategoryManga import suwayomi.tachidesk.manga.impl.Chapter +import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass import suwayomi.tachidesk.manga.model.dataclass.IncludeInUpdate import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass import suwayomi.tachidesk.manga.model.table.MangaStatus import suwayomi.tachidesk.server.serverConfig +import suwayomi.tachidesk.util.HAScheduler import java.util.Date import java.util.Timer import java.util.TimerTask @@ -51,41 +55,46 @@ class Updater : IUpdater { private val lastAutomatedUpdateKey = "lastAutomatedUpdateKey" private val preferences = Preferences.userNodeForPackage(Updater::class.java) - private val updateTimer = Timer() - private var currentUpdateTask: TimerTask? = null + private var currentUpdateTaskId = "" init { scheduleUpdateTask() } + + private fun autoUpdateTask() { + val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0) + preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis()) + + if (status.value.running) { + logger.debug { "Global update is already in progress" } + return + } + + logger.info { "Trigger global update (interval= ${serverConfig.globalUpdateInterval}h, lastAutomatedUpdate= ${Date(lastAutomatedUpdate)})" } + addCategoriesToUpdateQueue(Category.getCategoryList(), true) + } + private fun scheduleUpdateTask() { + HAScheduler.deschedule(currentUpdateTaskId) + if (!serverConfig.automaticallyTriggerGlobalUpdate) { return } val minInterval = 6.hours val interval = serverConfig.globalUpdateInterval.hours - val updateInterval = interval.coerceAtLeast(minInterval).inWholeMilliseconds + val updateInterval = interval.coerceAtLeast(minInterval) + val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, System.currentTimeMillis()) - val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0) - val initialDelay = updateInterval - (System.currentTimeMillis() - lastAutomatedUpdate) % updateInterval - - currentUpdateTask?.cancel() - currentUpdateTask = object : TimerTask() { - override fun run() { - preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis()) - - if (status.value.running) { - logger.debug { "Global update is already in progress, do not trigger global update" } - return - } - - logger.info { "Trigger global update (interval= ${serverConfig.globalUpdateInterval}h, lastAutomatedUpdate= ${Date(lastAutomatedUpdate)})" } - addCategoriesToUpdateQueue(Category.getCategoryList(), true) - } + // trigger update in case the server wasn't running on the scheduled time + val wasPreviousUpdateTriggered = + (System.currentTimeMillis() - lastAutomatedUpdate) < updateInterval.inWholeMilliseconds + if (!wasPreviousUpdateTriggered) { + autoUpdateTask() } - updateTimer.scheduleAtFixedRate(currentUpdateTask, initialDelay, updateInterval) + HAScheduler.schedule(::autoUpdateTask, "* */${updateInterval.inWholeHours} * * *", "global-update") } private fun getOrCreateUpdateChannelFor(source: String): Channel { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt index 1b1eca02..caa22e8d 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt @@ -52,6 +52,7 @@ class ServerConfig(getConfig: () -> Config, moduleName: String = MODULE_NAME) : // backup var backupPath: String by overridableConfig + var backupTime: String by overridableConfig var backupInterval: Int by overridableConfig var automatedBackups: Boolean by overridableConfig var backupTTL: Int by overridableConfig diff --git a/server/src/main/kotlin/suwayomi/tachidesk/util/HAScheduler.kt b/server/src/main/kotlin/suwayomi/tachidesk/util/HAScheduler.kt new file mode 100644 index 00000000..fc78c691 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/util/HAScheduler.kt @@ -0,0 +1,131 @@ + +package suwayomi.tachidesk.util + +import com.cronutils.model.CronType.CRON4J +import com.cronutils.model.definition.CronDefinitionBuilder +import com.cronutils.model.time.ExecutionTime +import com.cronutils.parser.CronParser +import it.sauronsoftware.cron4j.Scheduler +import it.sauronsoftware.cron4j.Task +import it.sauronsoftware.cron4j.TaskExecutionContext +import mu.KotlinLogging +import java.time.ZonedDateTime +import java.util.PriorityQueue +import java.util.Timer +import java.util.TimerTask +import kotlin.time.Duration +import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds + +val cronParser = CronParser(CronDefinitionBuilder.instanceDefinitionFor(CRON4J)) + +class HATask(val id: String, val cronExpr: String, val execute: () -> Unit, val name: String?) : Comparable { + private val executionTime = ExecutionTime.forCron(cronParser.parse(cronExpr)) + + fun getLastExecutionTime(): Long { + return executionTime.lastExecution(ZonedDateTime.now()).get().toEpochSecond().seconds.inWholeMilliseconds + } + + fun getNextExecutionTime(): Long { + return executionTime.nextExecution(ZonedDateTime.now()).get().toEpochSecond().seconds.inWholeMilliseconds + } + + fun getTimeToNextExecution(): Long { + return executionTime.timeToNextExecution(ZonedDateTime.now()).get().toMillis() + } + + override fun compareTo(other: HATask): Int { + return getTimeToNextExecution().compareTo(other.getTimeToNextExecution()) + } +} + +/** + * The "HAScheduler" ("HibernateAwareScheduler") is a scheduler that recognizes when the system was hibernating/suspended + * and triggers tasks that have missed their execution points. + */ +object HAScheduler { + private val logger = KotlinLogging.logger { } + + private val scheduledTasks = PriorityQueue() + private val scheduler = Scheduler() + + private val HIBERNATION_THRESHOLD = 10.seconds.inWholeMilliseconds + private const val TASK_THRESHOLD = 0.1 + + init { + scheduleHibernateCheckerTask(1.minutes) + } + + private fun scheduleHibernateCheckerTask(interval: Duration) { + val timer = Timer() + timer.scheduleAtFixedRate( + object : TimerTask() { + var lastExecutionTime = System.currentTimeMillis() + + override fun run() { + val currentTime = System.currentTimeMillis() + val elapsedTime = currentTime - lastExecutionTime + lastExecutionTime = currentTime + + val systemWasInHibernation = elapsedTime > interval.inWholeMilliseconds + HIBERNATION_THRESHOLD + if (systemWasInHibernation) { + logger.debug { "System hibernation detected, task was delayed by ${elapsedTime - interval.inWholeMilliseconds}ms" } + scheduledTasks.forEach { + val missedExecution = currentTime - it.getLastExecutionTime() - elapsedTime < 0 + val taskInterval = it.getNextExecutionTime() - it.getLastExecutionTime() + // in case the next task execution doesn't take long the missed execution can be ignored to prevent a double execution + val taskThresholdMet = taskInterval * TASK_THRESHOLD > it.getTimeToNextExecution() + + val triggerTask = missedExecution && taskThresholdMet + if (triggerTask) { + logger.debug { "Task \"${it.name ?: it.id}\" missed its execution, executing now..." } + reschedule(it.id, it.cronExpr) + it.execute() + } + + // queue is ordered by next execution time, thus, loop can be exited early + if (!missedExecution) { + return@forEach + } + } + } + } + }, + interval.inWholeMilliseconds, + interval.inWholeMilliseconds + ) + } + + fun schedule(execute: () -> Unit, cronExpr: String, name: String?): String { + if (!scheduler.isStarted) { + scheduler.start() + } + + val taskId = scheduler.schedule( + cronExpr, + object : Task() { + override fun execute(context: TaskExecutionContext?) { + execute() + } + } + ) + + scheduledTasks.add(HATask(taskId, cronExpr, execute, name)) + + return taskId + } + + fun deschedule(taskId: String) { + scheduler.deschedule(taskId) + scheduledTasks.removeIf { it.id == taskId } + } + + fun reschedule(taskId: String, cronExpr: String) { + val task = scheduledTasks.find { it.id == taskId } ?: return + + scheduledTasks.remove(task) + scheduledTasks.add(HATask(taskId, cronExpr, task.execute, task.name)) + + scheduler.reschedule(taskId, cronExpr) + } +} diff --git a/server/src/main/resources/server-reference.conf b/server/src/main/resources/server-reference.conf index 007cd8b7..dba2eb06 100644 --- a/server/src/main/resources/server-reference.conf +++ b/server/src/main/resources/server-reference.conf @@ -37,6 +37,7 @@ server.systemTrayEnabled = true # backup server.backupPath = "" +server.backupTime = "00:00" # range: hour: 0-23, minute: 0-59 - default: "00:00" - time of day at which the automated backup should be triggered server.backupInterval = 1 # time in days - range: 1 <= n < ∞ - default: 1 day - interval in which the server will automatically create a backup server.automatedBackups = true -server.backupTTL = 14 # time in days - range: 1 <= n < ∞ - default: 14 days - how long backup files will be kept before they will get deleted +server.backupTTL = 14 # time in days - 0 to disable it - range: 1 <= n < ∞ - default: 14 days - how long backup files will be kept before they will get deleted diff --git a/server/src/test/resources/server-reference.conf b/server/src/test/resources/server-reference.conf index e8297386..a27efccf 100644 --- a/server/src/test/resources/server-reference.conf +++ b/server/src/test/resources/server-reference.conf @@ -30,6 +30,7 @@ server.electronPath = "" # backup server.backupPath = "" +server.backupTime = "00:00" server.backupInterval = 1 server.automatedBackups = true server.backupTTL = 14