From 49f2d8588ad8797ebb559c704fd01cbc2c76ae9e Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Mon, 10 Jul 2023 11:43:53 +0200 Subject: [PATCH] Feature/automated backups (#595) * Automatically create backups * Cleanup automated backups * Extract backup filename creation into function --- .../graphql/mutations/BackupMutation.kt | 5 +- .../manga/controller/BackupController.kt | 5 +- .../impl/backup/proto/ProtoBackupExport.kt | 108 ++++++++++++++++++ .../suwayomi/tachidesk/server/ServerConfig.kt | 6 + .../suwayomi/tachidesk/server/ServerSetup.kt | 5 + .../src/main/resources/server-reference.conf | 6 + .../src/test/resources/server-reference.conf | 6 + 7 files changed, 133 insertions(+), 8 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/BackupMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/BackupMutation.kt index 8a1b0176..156724ca 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/BackupMutation.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/BackupMutation.kt @@ -14,8 +14,6 @@ import suwayomi.tachidesk.manga.impl.backup.BackupFlags import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupImport import suwayomi.tachidesk.server.JavalinSetup.future -import java.text.SimpleDateFormat -import java.util.Date import java.util.concurrent.CompletableFuture import kotlin.time.Duration.Companion.seconds @@ -62,8 +60,7 @@ class BackupMutation { fun createBackup( input: CreateBackupInput? = null ): CreateBackupPayload { - val currentDate = SimpleDateFormat("yyyy-MM-dd_HH-mm").format(Date()) - val filename = "tachidesk_$currentDate.proto.gz" + val filename = ProtoBackupExport.getBackupFilename() val backup = ProtoBackupExport.createBackup( BackupFlags( diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/BackupController.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/BackupController.kt index 65022b1b..47157569 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/BackupController.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/BackupController.kt @@ -8,8 +8,6 @@ import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator import suwayomi.tachidesk.server.JavalinSetup.future import suwayomi.tachidesk.server.util.handler import suwayomi.tachidesk.server.util.withOperation -import java.text.SimpleDateFormat -import java.util.Date /* * Copyright (C) Contributors to the Suwayomi project @@ -105,9 +103,8 @@ object BackupController { }, behaviorOf = { ctx -> ctx.contentType("application/octet-stream") - val currentDate = SimpleDateFormat("yyyy-MM-dd_HH-mm").format(Date()) - ctx.header("Content-Disposition", """attachment; filename="tachidesk_$currentDate.proto.gz"""") + ctx.header("Content-Disposition", """attachment; filename="${ProtoBackupExport.getBackupFilename()}"""") ctx.future( future { ProtoBackupExport.createBackup( diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt index a311b32e..8895ae9b 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt @@ -8,6 +8,7 @@ package suwayomi.tachidesk.manga.impl.backup.proto * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ import eu.kanade.tachiyomi.source.model.UpdateStrategy +import mu.KotlinLogging import okio.buffer import okio.gzip import okio.sink @@ -16,6 +17,9 @@ import org.jetbrains.exposed.sql.SortOrder import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.selectAll import org.jetbrains.exposed.sql.transactions.transaction +import org.kodein.di.DI +import org.kodein.di.conf.global +import org.kodein.di.instance import suwayomi.tachidesk.manga.impl.CategoryManga import suwayomi.tachidesk.manga.impl.backup.BackupFlags import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup @@ -30,11 +34,115 @@ import suwayomi.tachidesk.manga.model.table.MangaStatus import suwayomi.tachidesk.manga.model.table.MangaTable import suwayomi.tachidesk.manga.model.table.SourceTable import suwayomi.tachidesk.manga.model.table.toDataClass +import suwayomi.tachidesk.server.ApplicationDirs +import suwayomi.tachidesk.server.serverConfig import java.io.ByteArrayOutputStream +import java.io.File import java.io.InputStream +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Timer +import java.util.TimerTask import java.util.concurrent.TimeUnit +import java.util.prefs.Preferences +import kotlin.time.Duration.Companion.days object ProtoBackupExport : ProtoBackupBase() { + private val logger = KotlinLogging.logger { } + private val applicationDirs by DI.global.instance() + private const val lastAutomatedBackupKey = "lastAutomatedBackupKey" + private val preferences = Preferences.userNodeForPackage(ProtoBackupExport::class.java) + + private val backupTimer = Timer() + private var currentAutomatedBackupTask: TimerTask? = null + + fun scheduleAutomatedBackupTask() { + if (!serverConfig.automatedBackups) { + currentAutomatedBackupTask?.cancel() + return + } + + val minInterval = 1.days + val interval = serverConfig.backupInterval.days + val backupInterval = interval.coerceAtLeast(minInterval).inWholeMilliseconds + + val lastAutomatedBackup = preferences.getLong(lastAutomatedBackupKey, 0) + val initialDelay = + backupInterval - (System.currentTimeMillis() - lastAutomatedBackup) % backupInterval + + currentAutomatedBackupTask?.cancel() + currentAutomatedBackupTask = object : TimerTask() { + override fun run() { + cleanupAutomatedBackups() + createAutomatedBackup() + preferences.putLong(lastAutomatedBackupKey, System.currentTimeMillis()) + } + } + + backupTimer.scheduleAtFixedRate( + currentAutomatedBackupTask, + initialDelay, + backupInterval + ) + } + + private fun createAutomatedBackup() { + logger.info { "Creating automated backup..." } + + createBackup( + BackupFlags( + includeManga = true, + includeCategories = true, + includeChapters = true, + includeTracking = true, + includeHistory = true + ) + ).use { input -> + val automatedBackupDir = File(applicationDirs.automatedBackupRoot) + automatedBackupDir.mkdirs() + + val backupFile = File(applicationDirs.automatedBackupRoot, getBackupFilename()) + + backupFile.outputStream().use { output -> input.copyTo(output) } + } + + } + + private fun cleanupAutomatedBackups() { + logger.debug { "Cleanup automated backups" } + + val automatedBackupDir = File(applicationDirs.automatedBackupRoot) + if (!automatedBackupDir.isDirectory) { + return + } + + automatedBackupDir.walkTopDown().forEach { file -> + try { + cleanupAutomatedBackupFile(file) + } catch (e: Exception) { + // ignore, will be retried on next cleanup + } + } + } + + private fun cleanupAutomatedBackupFile(file: File) { + if (!file.isFile) { + return + } + + val lastAccessTime = file.lastModified() + val isTTLReached = + System.currentTimeMillis() - lastAccessTime >= serverConfig.backupTTL.days.inWholeMilliseconds + if (isTTLReached) { + file.delete() + } + } + + fun getBackupFilename(): String { + val currentDate = SimpleDateFormat("yyyy-MM-dd_HH-mm").format(Date()) + return "tachidesk_$currentDate.proto.gz" + } + fun createBackup(flags: BackupFlags): InputStream { // Create root object diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt index 3a51645a..bc3b6110 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt @@ -48,6 +48,12 @@ class ServerConfig(getConfig: () -> Config, moduleName: String = MODULE_NAME) : var debugLogsEnabled: Boolean = debugLogsEnabled(GlobalConfigManager.config) var systemTrayEnabled: Boolean by overridableConfig + // backup + var backupPath: String by overridableConfig + var backupInterval: Int by overridableConfig + var automatedBackups: Boolean by overridableConfig + var backupTTL: Int by overridableConfig + companion object { fun register(getConfig: () -> Config) = ServerConfig({ getConfig().getConfig(MODULE_NAME) }) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt index 52065283..9a8adf3d 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt @@ -18,6 +18,7 @@ import org.kodein.di.DI import org.kodein.di.bind import org.kodein.di.conf.global import org.kodein.di.singleton +import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport import suwayomi.tachidesk.manga.impl.update.IUpdater import suwayomi.tachidesk.manga.impl.update.Updater import suwayomi.tachidesk.manga.impl.util.lang.renameTo @@ -45,6 +46,7 @@ class ApplicationDirs( val mangaDownloadsRoot = serverConfig.downloadsPath.ifBlank { "$dataRoot/downloads" } val localMangaRoot = "$dataRoot/local" val webUIRoot = "$dataRoot/webUI" + val automatedBackupRoot = serverConfig.backupPath.ifBlank { "$dataRoot/backups" } val tempMangaCacheRoot = "$tempRoot/manga-cache" } @@ -165,4 +167,7 @@ fun applicationSetup() { // AES/CBC/PKCS7Padding Cypher provider for zh.copymanga Security.addProvider(BouncyCastleProvider()) + + // start automated backups + ProtoBackupExport.scheduleAutomatedBackupTask() } diff --git a/server/src/main/resources/server-reference.conf b/server/src/main/resources/server-reference.conf index 4847a0ec..9e65dd75 100644 --- a/server/src/main/resources/server-reference.conf +++ b/server/src/main/resources/server-reference.conf @@ -32,3 +32,9 @@ server.basicAuthPassword = "" # misc server.debugLogsEnabled = false server.systemTrayEnabled = true + +# backup +server.backupPath = "" +server.backupInterval = 1 # time in days - range: 1 <= n < ∞ - default: 1 day - interval in which the server will automatically create a backup +server.automatedBackups = true +server.backupTTL = 14 # time in days - range: 1 <= n < ∞ - default: 14 days - how long backup files will be kept before they will get deleted diff --git a/server/src/test/resources/server-reference.conf b/server/src/test/resources/server-reference.conf index 324cde08..faf04303 100644 --- a/server/src/test/resources/server-reference.conf +++ b/server/src/test/resources/server-reference.conf @@ -25,3 +25,9 @@ server.webUIEnabled = true server.initialOpenInBrowserEnabled = true server.webUIInterface = "browser" # "browser" or "electron" server.electronPath = "" + +# backup +server.backupPath = "" +server.backupInterval = 1 +server.automatedBackups = true +server.backupTTL = 14