Feature/backup import add backup flags (#1697)

* Add backup flags to backup restore

* Cleanup default backup flags handling

* Optionally exclude manga from backup
This commit is contained in:
schroda
2025-10-06 00:52:45 +02:00
committed by GitHub
parent 3ce9f72e3f
commit 0d79ac68f8
7 changed files with 213 additions and 135 deletions

View File

@@ -1,11 +1,13 @@
package suwayomi.tachidesk.graphql.mutations
import com.expediagroup.graphql.generator.annotations.GraphQLDeprecated
import io.javalin.http.UploadedFile
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.withTimeout
import suwayomi.tachidesk.graphql.directives.RequireAuth
import suwayomi.tachidesk.graphql.server.TemporaryFileStorage
import suwayomi.tachidesk.graphql.types.BackupRestoreStatus
import suwayomi.tachidesk.graphql.types.PartialBackupFlags
import suwayomi.tachidesk.graphql.types.toStatus
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport
@@ -19,6 +21,7 @@ class BackupMutation {
data class RestoreBackupInput(
val clientMutationId: String? = null,
val backup: UploadedFile,
val flags: PartialBackupFlags? = null,
)
data class RestoreBackupPayload(
@@ -29,10 +32,14 @@ class BackupMutation {
@RequireAuth
fun restoreBackup(input: RestoreBackupInput): CompletableFuture<RestoreBackupPayload> {
val (clientMutationId, backup) = input
val (clientMutationId, backup, flags) = input
return future {
val restoreId = ProtoBackupImport.restore(backup.content())
val restoreId =
ProtoBackupImport.restore(
backup.content(),
BackupFlags.fromPartial(flags),
)
withTimeout(10.seconds) {
ProtoBackupImport.notifyFlow.first {
@@ -46,11 +53,18 @@ class BackupMutation {
data class CreateBackupInput(
val clientMutationId: String? = null,
val flags: PartialBackupFlags? = null,
@GraphQLDeprecated("Will get removed", replaceWith = ReplaceWith("flags"))
val includeChapters: Boolean? = null,
@GraphQLDeprecated("Will get removed", replaceWith = ReplaceWith("flags"))
val includeCategories: Boolean? = null,
@GraphQLDeprecated("Will get removed", replaceWith = ReplaceWith("flags"))
val includeTracking: Boolean? = null,
@GraphQLDeprecated("Will get removed", replaceWith = ReplaceWith("flags"))
val includeHistory: Boolean? = null,
@GraphQLDeprecated("Will get removed", replaceWith = ReplaceWith("flags"))
val includeClientData: Boolean? = null,
@GraphQLDeprecated("Will get removed", replaceWith = ReplaceWith("flags"))
val includeServerSettings: Boolean? = null,
)
@@ -65,15 +79,19 @@ class BackupMutation {
val backup =
ProtoBackupExport.createBackup(
BackupFlags(
includeManga = true,
includeCategories = input?.includeCategories ?: true,
includeChapters = input?.includeChapters ?: true,
includeTracking = input?.includeTracking ?: true,
includeHistory = input?.includeHistory ?: true,
includeClientData = input?.includeClientData ?: true,
includeServerSettings = input?.includeServerSettings ?: true,
),
if (input?.flags != null) {
BackupFlags.fromPartial(input.flags)
} else {
BackupFlags(
includeManga = BackupFlags.DEFAULT.includeManga,
includeCategories = input?.includeCategories ?: BackupFlags.DEFAULT.includeCategories,
includeChapters = input?.includeChapters ?: BackupFlags.DEFAULT.includeChapters,
includeTracking = input?.includeTracking ?: BackupFlags.DEFAULT.includeTracking,
includeHistory = input?.includeHistory ?: BackupFlags.DEFAULT.includeHistory,
includeClientData = input?.includeClientData ?: BackupFlags.DEFAULT.includeClientData,
includeServerSettings = input?.includeServerSettings ?: BackupFlags.DEFAULT.includeServerSettings,
)
},
)
TemporaryFileStorage.saveFile(filename, backup)

View File

@@ -1,7 +1,18 @@
package suwayomi.tachidesk.graphql.types
import suwayomi.tachidesk.manga.impl.backup.IBackupFlags
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupImport
data class PartialBackupFlags(
override val includeManga: Boolean?,
override val includeCategories: Boolean?,
override val includeChapters: Boolean?,
override val includeTracking: Boolean?,
override val includeHistory: Boolean?,
override val includeClientData: Boolean?,
override val includeServerSettings: Boolean?,
) : IBackupFlags
enum class BackupRestoreState {
IDLE,
SUCCESS,

View File

@@ -89,17 +89,7 @@ object BackupController {
ctx.contentType("application/octet-stream")
ctx.future {
future {
ProtoBackupExport.createBackup(
BackupFlags(
includeManga = true,
includeCategories = true,
includeChapters = true,
includeTracking = true,
includeHistory = true,
includeClientData = true,
includeServerSettings = true,
),
)
ProtoBackupExport.createBackup(BackupFlags.DEFAULT)
}.thenApply { ctx.result(it) }
}
},
@@ -124,17 +114,7 @@ object BackupController {
ctx.header("Content-Disposition", """attachment; filename="${Backup.getFilename()}"""")
ctx.future {
future {
ProtoBackupExport.createBackup(
BackupFlags(
includeManga = true,
includeCategories = true,
includeChapters = true,
includeTracking = true,
includeHistory = true,
includeClientData = true,
includeServerSettings = true,
),
)
ProtoBackupExport.createBackup(BackupFlags.DEFAULT)
}.thenApply { ctx.result(it) }
}
},

View File

@@ -1,5 +1,7 @@
package suwayomi.tachidesk.manga.impl.backup
import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
/*
* Copyright (C) Contributors to the Suwayomi project
*
@@ -7,12 +9,46 @@ package suwayomi.tachidesk.manga.impl.backup
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
interface IBackupFlags {
val includeManga: Boolean?
val includeCategories: Boolean?
val includeChapters: Boolean?
val includeTracking: Boolean?
val includeHistory: Boolean?
val includeClientData: Boolean?
val includeServerSettings: Boolean?
}
data class BackupFlags(
val includeManga: Boolean,
val includeCategories: Boolean,
val includeChapters: Boolean,
val includeTracking: Boolean,
val includeHistory: Boolean,
val includeClientData: Boolean,
val includeServerSettings: Boolean,
)
override val includeManga: Boolean,
override val includeCategories: Boolean,
override val includeChapters: Boolean,
override val includeTracking: Boolean,
override val includeHistory: Boolean,
override val includeClientData: Boolean,
override val includeServerSettings: Boolean,
) : IBackupFlags {
companion object {
val DEFAULT =
BackupFlags(
includeManga = true,
includeCategories = true,
includeChapters = true,
includeTracking = true,
includeHistory = true,
includeClientData = true,
includeServerSettings = true,
)
fun fromPartial(partialFlags: IBackupFlags?): BackupFlags =
BackupFlags(
includeManga = partialFlags?.includeManga ?: DEFAULT.includeManga,
includeCategories = partialFlags?.includeCategories ?: DEFAULT.includeCategories,
includeChapters = partialFlags?.includeChapters ?: DEFAULT.includeChapters,
includeTracking = partialFlags?.includeTracking ?: DEFAULT.includeTracking,
includeHistory = partialFlags?.includeHistory ?: DEFAULT.includeHistory,
includeClientData = partialFlags?.includeClientData ?: DEFAULT.includeClientData,
includeServerSettings = partialFlags?.includeServerSettings ?: DEFAULT.includeServerSettings,
)
}
}

View File

@@ -21,6 +21,7 @@ import okio.Sink
import okio.buffer
import okio.gzip
import org.jetbrains.exposed.sql.Query
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.transaction
@@ -118,17 +119,7 @@ object ProtoBackupExport : ProtoBackupBase() {
private fun createAutomatedBackup() {
logger.info { "Creating automated backup..." }
createBackup(
BackupFlags(
includeManga = true,
includeCategories = true,
includeChapters = true,
includeTracking = true,
includeHistory = true,
includeClientData = true,
includeServerSettings = true,
),
).use { input ->
createBackup(BackupFlags.DEFAULT).use { input ->
val automatedBackupDir = File(applicationDirs.automatedBackupRoot)
automatedBackupDir.mkdirs()
@@ -179,7 +170,12 @@ object ProtoBackupExport : ProtoBackupBase() {
fun createBackup(flags: BackupFlags): InputStream {
// Create root object
val databaseManga = transaction { MangaTable.selectAll().where { MangaTable.inLibrary eq true } }
val databaseManga =
if (flags.includeManga) {
transaction { MangaTable.selectAll().where { MangaTable.inLibrary eq true }.toList() }
} else {
emptyList()
}
val backup: Backup =
transaction {
@@ -204,7 +200,7 @@ object ProtoBackupExport : ProtoBackupBase() {
}
private fun backupManga(
databaseManga: Query,
databaseManga: List<ResultRow>,
flags: BackupFlags,
): List<BackupManga> =
databaseManga.map { mangaRow ->
@@ -336,7 +332,7 @@ object ProtoBackupExport : ProtoBackupBase() {
}
private fun backupExtensionInfo(
mangas: Query,
mangas: List<ResultRow>,
flags: BackupFlags,
): List<BackupSource> {
val inLibraryMangaSourceIds =

View File

@@ -39,6 +39,7 @@ import suwayomi.tachidesk.manga.impl.Chapter.modifyChaptersMetas
import suwayomi.tachidesk.manga.impl.Manga.clearThumbnail
import suwayomi.tachidesk.manga.impl.Manga.modifyMangasMetas
import suwayomi.tachidesk.manga.impl.Source.modifySourceMetas
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.validate
import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler
@@ -139,7 +140,10 @@ object ProtoBackupImport : ProtoBackupBase() {
}
@OptIn(DelicateCoroutinesApi::class)
fun restore(sourceStream: InputStream): String {
fun restore(
sourceStream: InputStream,
flags: BackupFlags,
): String {
val restoreId = System.currentTimeMillis().toString()
logger.info { "restore($restoreId): queued" }
@@ -147,7 +151,7 @@ object ProtoBackupImport : ProtoBackupBase() {
updateRestoreState(restoreId, BackupRestoreState.Idle)
GlobalScope.launch {
restoreLegacy(sourceStream, restoreId)
restoreLegacy(sourceStream, restoreId, flags)
}
return restoreId
@@ -156,11 +160,12 @@ object ProtoBackupImport : ProtoBackupBase() {
suspend fun restoreLegacy(
sourceStream: InputStream,
restoreId: String = "legacy",
flags: BackupFlags = BackupFlags.DEFAULT,
): ValidationResult =
backupMutex.withLock {
try {
logger.info { "restore($restoreId): restoring..." }
performRestore(restoreId, sourceStream)
performRestore(restoreId, sourceStream, flags)
} catch (e: Exception) {
logger.error(e) { "restore($restoreId): failed due to" }
@@ -180,6 +185,7 @@ object ProtoBackupImport : ProtoBackupBase() {
private fun performRestore(
id: String,
sourceStream: InputStream,
flags: BackupFlags,
): ValidationResult {
val backupString =
sourceStream
@@ -191,28 +197,36 @@ object ProtoBackupImport : ProtoBackupBase() {
val validationResult = validate(backup)
val restoreCategories = 1
val restoreMeta = 1
val restoreSettings = 1
val restoreCategories = if (flags.includeCategories) 1 else 0
val restoreMeta = if (flags.includeClientData) 1 else 0
val restoreSettings = if (flags.includeServerSettings) 1 else 0
val getRestoreAmount = { size: Int -> size + restoreCategories + restoreMeta + restoreSettings }
val restoreAmount = getRestoreAmount(backup.backupManga.size)
val restoreAmount = getRestoreAmount(if (flags.includeManga) backup.backupManga.size else 0)
updateRestoreState(
id,
BackupRestoreState.RestoringSettings(restoreSettings, restoreAmount),
)
if (flags.includeServerSettings) {
updateRestoreState(
id,
BackupRestoreState.RestoringSettings(restoreSettings, restoreAmount),
)
BackupSettingsHandler.restore(backup.serverSettings)
BackupSettingsHandler.restore(backup.serverSettings)
}
updateRestoreState(id, BackupRestoreState.RestoringCategories(restoreSettings + restoreCategories, restoreAmount))
val categoryMapping =
if (flags.includeCategories) {
updateRestoreState(id, BackupRestoreState.RestoringCategories(restoreSettings + restoreCategories, restoreAmount))
restoreCategories(backup.backupCategories)
} else {
emptyMap()
}
val categoryMapping = restoreCategories(backup.backupCategories)
if (flags.includeClientData) {
updateRestoreState(id, BackupRestoreState.RestoringMeta(restoreSettings + restoreCategories + restoreMeta, restoreAmount))
updateRestoreState(id, BackupRestoreState.RestoringMeta(restoreSettings + restoreCategories + restoreMeta, restoreAmount))
restoreGlobalMeta(backup.meta)
restoreGlobalMeta(backup.meta)
restoreSourceMeta(backup.backupSources)
restoreSourceMeta(backup.backupSources)
}
// Store source mapping for error messages
val sourceMapping = backup.getSourceMap()
@@ -220,22 +234,25 @@ object ProtoBackupImport : ProtoBackupBase() {
val errors = mutableListOf<Pair<Date, String>>()
// Restore individual manga
backup.backupManga.forEachIndexed { index, manga ->
updateRestoreState(
id,
BackupRestoreState.RestoringManga(
current = getRestoreAmount(index + 1),
totalManga = restoreAmount,
title = manga.title,
),
)
if (flags.includeManga) {
backup.backupManga.forEachIndexed { index, manga ->
updateRestoreState(
id,
BackupRestoreState.RestoringManga(
current = getRestoreAmount(index + 1),
totalManga = restoreAmount,
title = manga.title,
),
)
restoreManga(
backupManga = manga,
categoryMapping = categoryMapping,
sourceMapping = sourceMapping,
errors = errors,
)
restoreManga(
backupManga = manga,
categoryMapping = categoryMapping,
sourceMapping = sourceMapping,
errors = errors,
flags = flags,
)
}
}
logger.info {
@@ -279,15 +296,17 @@ object ProtoBackupImport : ProtoBackupBase() {
categoryMapping: Map<Int, Int>,
sourceMapping: Map<Long, String>,
errors: MutableList<Pair<Date, String>>,
flags: BackupFlags,
) {
val chapters = backupManga.chapters
val categories = backupManga.categories
val history = backupManga.history
val tracking = backupManga.tracking
val dbCategoryIds = categories.map { categoryMapping[it]!! }
val dbCategoryIds = categories.mapNotNull { categoryMapping[it] }
try {
restoreMangaData(backupManga, chapters, dbCategoryIds, history, backupManga.tracking)
restoreMangaData(backupManga, chapters, dbCategoryIds, history, tracking, flags)
} catch (e: Exception) {
val sourceName = sourceMapping[backupManga.source] ?: backupManga.source.toString()
errors.add(Date() to "${backupManga.title} [$sourceName]: ${e.message}")
@@ -300,6 +319,7 @@ object ProtoBackupImport : ProtoBackupBase() {
categoryIds: List<Int>,
history: List<BackupHistory>,
tracks: List<BackupTracking>,
flags: BackupFlags,
) {
val dbManga =
transaction {
@@ -362,20 +382,26 @@ object ProtoBackupImport : ProtoBackupBase() {
// delete thumbnail in case cached data still exists
clearThumbnail(mangaId)
if (manga.meta.isNotEmpty()) {
if (flags.includeClientData && manga.meta.isNotEmpty()) {
modifyMangasMetas(mapOf(mangaId to manga.meta))
}
// merge chapter data
restoreMangaChapterData(mangaId, restoreMode, chapters, history)
if (flags.includeChapters || flags.includeHistory) {
restoreMangaChapterData(mangaId, restoreMode, chapters, history, flags)
}
// merge categories
restoreMangaCategoryData(mangaId, categoryIds)
if (flags.includeCategories) {
restoreMangaCategoryData(mangaId, categoryIds)
}
mangaId
}
restoreMangaTrackerData(mangaId, tracks)
if (flags.includeTracking) {
restoreMangaTrackerData(mangaId, tracks)
}
// TODO: insert/merge history
}
@@ -404,64 +430,79 @@ object ProtoBackupImport : ProtoBackupBase() {
restoreMode: RestoreMode,
chapters: List<BackupChapter>,
history: List<BackupHistory>,
flags: BackupFlags,
) = dbTransaction {
val (chaptersToInsert, chaptersToUpdateToDbChapter) = getMangaChapterToRestoreInfo(mangaId, restoreMode, chapters)
val historyByChapter = history.groupBy({ it.url }, { it.lastRead })
val insertedChapterIds =
ChapterTable
.batchInsert(chaptersToInsert) { chapter ->
this[ChapterTable.url] = chapter.url
this[ChapterTable.name] = chapter.name
if (chapter.dateUpload == 0L) {
this[ChapterTable.date_upload] = chapter.dateFetch
} else {
this[ChapterTable.date_upload] = chapter.dateUpload
}
this[ChapterTable.chapter_number] = chapter.chapterNumber
this[ChapterTable.scanlator] = chapter.scanlator
if (flags.includeChapters) {
ChapterTable
.batchInsert(chaptersToInsert) { chapter ->
this[ChapterTable.url] = chapter.url
this[ChapterTable.name] = chapter.name
if (chapter.dateUpload == 0L) {
this[ChapterTable.date_upload] = chapter.dateFetch
} else {
this[ChapterTable.date_upload] = chapter.dateUpload
}
this[ChapterTable.chapter_number] = chapter.chapterNumber
this[ChapterTable.scanlator] = chapter.scanlator
this[ChapterTable.sourceOrder] = chaptersToInsert.size - chapter.sourceOrder
this[ChapterTable.manga] = mangaId
this[ChapterTable.sourceOrder] = chaptersToInsert.size - chapter.sourceOrder
this[ChapterTable.manga] = mangaId
this[ChapterTable.isRead] = chapter.read
this[ChapterTable.lastPageRead] = chapter.lastPageRead.coerceAtLeast(0)
this[ChapterTable.isBookmarked] = chapter.bookmark
this[ChapterTable.isRead] = chapter.read
this[ChapterTable.lastPageRead] = chapter.lastPageRead.coerceAtLeast(0)
this[ChapterTable.isBookmarked] = chapter.bookmark
this[ChapterTable.fetchedAt] = chapter.dateFetch.milliseconds.inWholeSeconds
this[ChapterTable.fetchedAt] = chapter.dateFetch.milliseconds.inWholeSeconds
this[ChapterTable.lastReadAt] = historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0
}.map { it[ChapterTable.id].value }
if (flags.includeHistory) {
this[ChapterTable.lastReadAt] =
historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0
}
}.map { it[ChapterTable.id].value }
} else {
emptyList()
}
if (chaptersToUpdateToDbChapter.isNotEmpty()) {
BatchUpdateStatement(ChapterTable).apply {
chaptersToUpdateToDbChapter.forEach { (backupChapter, dbChapter) ->
addBatch(EntityID(dbChapter[ChapterTable.id].value, ChapterTable))
this[ChapterTable.isRead] = backupChapter.read || dbChapter[ChapterTable.isRead]
this[ChapterTable.lastPageRead] =
max(backupChapter.lastPageRead, dbChapter[ChapterTable.lastPageRead]).coerceAtLeast(0)
this[ChapterTable.isBookmarked] = backupChapter.bookmark || dbChapter[ChapterTable.isBookmarked]
this[ChapterTable.lastReadAt] =
(historyByChapter[backupChapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0)
.coerceAtLeast(dbChapter[ChapterTable.lastReadAt])
if (flags.includeChapters) {
this[ChapterTable.isRead] = backupChapter.read || dbChapter[ChapterTable.isRead]
this[ChapterTable.lastPageRead] =
max(backupChapter.lastPageRead, dbChapter[ChapterTable.lastPageRead]).coerceAtLeast(0)
this[ChapterTable.isBookmarked] = backupChapter.bookmark || dbChapter[ChapterTable.isBookmarked]
}
if (flags.includeHistory) {
this[ChapterTable.lastReadAt] =
(historyByChapter[backupChapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0)
.coerceAtLeast(dbChapter[ChapterTable.lastReadAt])
}
}
execute(this@dbTransaction)
}
}
val chaptersToInsertByChapterId = insertedChapterIds.zip(chaptersToInsert)
val chapterToUpdateByChapterId =
chaptersToUpdateToDbChapter.map { (backupChapter, dbChapter) ->
dbChapter[ChapterTable.id].value to
backupChapter
}
val metaEntryByChapterId =
(chaptersToInsertByChapterId + chapterToUpdateByChapterId)
.associate { (chapterId, backupChapter) ->
chapterId to backupChapter.meta
if (flags.includeClientData) {
val chaptersToInsertByChapterId = insertedChapterIds.zip(chaptersToInsert)
val chapterToUpdateByChapterId =
chaptersToUpdateToDbChapter.map { (backupChapter, dbChapter) ->
dbChapter[ChapterTable.id].value to
backupChapter
}
val metaEntryByChapterId =
(chaptersToInsertByChapterId + chapterToUpdateByChapterId)
.associate { (chapterId, backupChapter) ->
chapterId to backupChapter.meta
}
modifyChaptersMetas(metaEntryByChapterId)
modifyChaptersMetas(metaEntryByChapterId)
}
}
private fun restoreMangaCategoryData(

View File

@@ -28,10 +28,6 @@ object ProtoBackupValidator {
)
fun validate(backup: Backup): ValidationResult {
if (backup.backupManga.isEmpty()) {
throw Exception("Backup does not contain any manga.")
}
val sources = backup.getSourceMap()
val missingSources =