diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt index d62c54b3..8d9679b4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt @@ -9,7 +9,6 @@ package suwayomi.tachidesk.manga.impl.backup.proto import android.app.Application import android.content.Context -import eu.kanade.tachiyomi.source.model.UpdateStrategy import io.github.oshai.kotlinlogging.KotlinLogging import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers @@ -20,33 +19,14 @@ import okio.Buffer import okio.Sink import okio.buffer import okio.gzip -import org.jetbrains.exposed.sql.Query -import org.jetbrains.exposed.sql.ResultRow -import org.jetbrains.exposed.sql.SortOrder -import org.jetbrains.exposed.sql.selectAll import org.jetbrains.exposed.sql.transactions.transaction -import suwayomi.tachidesk.global.impl.GlobalMeta -import suwayomi.tachidesk.manga.impl.Category -import suwayomi.tachidesk.manga.impl.CategoryManga -import suwayomi.tachidesk.manga.impl.Chapter -import suwayomi.tachidesk.manga.impl.Manga -import suwayomi.tachidesk.manga.impl.Source import suwayomi.tachidesk.manga.impl.backup.BackupFlags +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupCategoryHandler +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupGlobalMetaHandler +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup -import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory -import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter -import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupHistory -import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga -import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource -import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking -import suwayomi.tachidesk.manga.impl.track.Track -import suwayomi.tachidesk.manga.model.table.CategoryTable -import suwayomi.tachidesk.manga.model.table.ChapterTable -import suwayomi.tachidesk.manga.model.table.MangaStatus -import suwayomi.tachidesk.manga.model.table.MangaTable -import suwayomi.tachidesk.manga.model.table.SourceTable -import suwayomi.tachidesk.manga.model.table.toDataClass import suwayomi.tachidesk.server.ApplicationDirs import suwayomi.tachidesk.server.serverConfig import suwayomi.tachidesk.util.HAScheduler @@ -56,7 +36,6 @@ import uy.kohesive.injekt.injectLazy import java.io.File import java.io.InputStream import kotlin.time.Duration.Companion.days -import kotlin.time.Duration.Companion.seconds object ProtoBackupExport : ProtoBackupBase() { private val logger = KotlinLogging.logger { } @@ -170,20 +149,15 @@ object ProtoBackupExport : ProtoBackupBase() { fun createBackup(flags: BackupFlags): InputStream { // Create root object - val databaseManga = - if (flags.includeManga) { - transaction { MangaTable.selectAll().where { MangaTable.inLibrary eq true }.toList() } - } else { - emptyList() - } + val backupMangas = BackupMangaHandler.backup(flags) val backup: Backup = transaction { Backup( - backupManga(databaseManga, flags), - backupCategories(flags), - backupExtensionInfo(databaseManga, flags), - backupGlobalMeta(flags), + BackupMangaHandler.backup(flags), + BackupCategoryHandler.backup(flags), + BackupSourceHandler.backup(backupMangas, flags), + BackupGlobalMetaHandler.backup(flags), BackupSettingsHandler.backup(flags), ) } @@ -198,171 +172,4 @@ object ProtoBackupExport : ProtoBackupBase() { return byteStream.inputStream() } - - private fun backupManga( - databaseManga: List, - flags: BackupFlags, - ): List = - databaseManga.map { mangaRow -> - val backupManga = - BackupManga( - source = mangaRow[MangaTable.sourceReference], - url = mangaRow[MangaTable.url], - title = mangaRow[MangaTable.title], - artist = mangaRow[MangaTable.artist], - author = mangaRow[MangaTable.author], - description = mangaRow[MangaTable.description], - genre = mangaRow[MangaTable.genre]?.split(", ") ?: emptyList(), - status = MangaStatus.valueOf(mangaRow[MangaTable.status]).value, - thumbnailUrl = mangaRow[MangaTable.thumbnail_url], - dateAdded = mangaRow[MangaTable.inLibraryAt].seconds.inWholeMilliseconds, - viewer = 0, // not supported in Tachidesk - updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy]), - ) - - val mangaId = mangaRow[MangaTable.id].value - - if (flags.includeClientData) { - backupManga.meta = Manga.getMangaMetaMap(mangaId) - } - - if (flags.includeChapters || flags.includeHistory) { - val chapters = - transaction { - ChapterTable - .selectAll() - .where { ChapterTable.manga eq mangaId } - .orderBy(ChapterTable.sourceOrder to SortOrder.DESC) - .map { - ChapterTable.toDataClass(it) - } - } - if (flags.includeChapters) { - val chapterToMeta = Chapter.getChaptersMetaMaps(chapters.map { it.id }) - - backupManga.chapters = - chapters.map { - BackupChapter( - it.url, - it.name, - it.scanlator, - it.read, - it.bookmarked, - it.lastPageRead, - it.fetchedAt.seconds.inWholeMilliseconds, - it.uploadDate, - it.chapterNumber, - chapters.size - it.index, - ).apply { - if (flags.includeClientData) { - this.meta = chapterToMeta[it.id] ?: emptyMap() - } - } - } - } - if (flags.includeHistory) { - backupManga.history = - chapters.mapNotNull { - if (it.lastReadAt > 0) { - BackupHistory( - url = it.url, - lastRead = it.lastReadAt.seconds.inWholeMilliseconds, - ) - } else { - null - } - } - } - } - - if (flags.includeCategories) { - backupManga.categories = CategoryManga.getMangaCategories(mangaId).map { it.order } - } - - if (flags.includeTracking) { - val tracks = - Track.getTrackRecordsByMangaId(mangaRow[MangaTable.id].value).mapNotNull { - if (it.record == null) { - null - } else { - BackupTracking( - syncId = it.record.trackerId, - // forced not null so its compatible with 1.x backup system - libraryId = it.record.libraryId ?: 0, - mediaId = it.record.remoteId, - title = it.record.title, - lastChapterRead = it.record.lastChapterRead.toFloat(), - totalChapters = it.record.totalChapters, - score = it.record.score.toFloat(), - status = it.record.status, - startedReadingDate = it.record.startDate, - finishedReadingDate = it.record.finishDate, - trackingUrl = it.record.remoteUrl, - private = it.record.private, - ) - } - } - if (tracks.isNotEmpty()) { - backupManga.tracking = tracks - } - } - - backupManga - } - - private fun backupCategories(flags: BackupFlags): List { - val categories = - CategoryTable - .selectAll() - .orderBy(CategoryTable.order to SortOrder.ASC) - .map { CategoryTable.toDataClass(it) } - val categoryToMeta = Category.getCategoriesMetaMaps(categories.map { it.id }) - - return categories.map { - BackupCategory( - it.name, - it.order, - 0, // not supported in Tachidesk - ).apply { - if (flags.includeClientData) { - this.meta = categoryToMeta[it.id] ?: emptyMap() - } - } - } - } - - private fun backupExtensionInfo( - mangas: List, - flags: BackupFlags, - ): List { - val inLibraryMangaSourceIds = - mangas - .asSequence() - .map { it[MangaTable.sourceReference] } - .distinct() - .toList() - val sources = SourceTable.selectAll().where { SourceTable.id inList inLibraryMangaSourceIds } - val sourceToMeta = Source.getSourcesMetaMaps(sources.map { it[SourceTable.id].value }) - - return inLibraryMangaSourceIds - .map { mangaSourceId -> - val source = sources.firstOrNull { it[SourceTable.id].value == mangaSourceId } - BackupSource( - source?.get(SourceTable.name) ?: "", - mangaSourceId, - ).apply { - if (flags.includeClientData) { - this.meta = sourceToMeta[mangaSourceId] ?: emptyMap() - } - } - }.toList() - } - - private fun backupGlobalMeta(flags: BackupFlags): Map { - if (!flags.includeClientData) { - return emptyMap() - } - - return GlobalMeta.getMetaMap() - } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt index 6ebeaad2..a4f31f74 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt @@ -21,50 +21,21 @@ import kotlinx.coroutines.sync.withLock import okio.buffer import okio.gzip import okio.source -import org.jetbrains.exposed.dao.id.EntityID -import org.jetbrains.exposed.sql.ResultRow -import org.jetbrains.exposed.sql.and -import org.jetbrains.exposed.sql.batchInsert -import org.jetbrains.exposed.sql.insertAndGetId -import org.jetbrains.exposed.sql.selectAll -import org.jetbrains.exposed.sql.statements.BatchUpdateStatement -import org.jetbrains.exposed.sql.transactions.transaction -import org.jetbrains.exposed.sql.update -import suwayomi.tachidesk.global.impl.GlobalMeta import suwayomi.tachidesk.graphql.types.toStatus -import suwayomi.tachidesk.manga.impl.Category -import suwayomi.tachidesk.manga.impl.Category.modifyCategoriesMetas -import suwayomi.tachidesk.manga.impl.CategoryManga -import suwayomi.tachidesk.manga.impl.Chapter.modifyChaptersMetas -import suwayomi.tachidesk.manga.impl.Manga.clearThumbnail -import suwayomi.tachidesk.manga.impl.Manga.modifyMangasMetas -import suwayomi.tachidesk.manga.impl.Source.modifySourceMetas import suwayomi.tachidesk.manga.impl.backup.BackupFlags import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.validate +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupCategoryHandler +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupGlobalMetaHandler +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup -import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory -import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter -import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupHistory -import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga -import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource -import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking -import suwayomi.tachidesk.manga.impl.track.tracker.TrackerManager -import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrack -import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrackRecordDataClass -import suwayomi.tachidesk.manga.model.dataclass.TrackRecordDataClass -import suwayomi.tachidesk.manga.model.table.ChapterTable -import suwayomi.tachidesk.manga.model.table.MangaTable -import suwayomi.tachidesk.server.database.dbTransaction import java.io.InputStream import java.util.Date import java.util.Timer import java.util.TimerTask import java.util.concurrent.ConcurrentHashMap -import kotlin.math.max -import kotlin.time.Duration.Companion.milliseconds -import suwayomi.tachidesk.manga.impl.track.Track as Tracker object ProtoBackupImport : ProtoBackupBase() { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) @@ -73,11 +44,6 @@ object ProtoBackupImport : ProtoBackupBase() { private val backupMutex = Mutex() - enum class RestoreMode { - NEW, - EXISTING, - } - sealed class BackupRestoreState { data object Idle : BackupRestoreState() @@ -215,7 +181,7 @@ object ProtoBackupImport : ProtoBackupBase() { val categoryMapping = if (flags.includeCategories) { updateRestoreState(id, BackupRestoreState.RestoringCategories(restoreSettings + restoreCategories, restoreAmount)) - restoreCategories(backup.backupCategories) + BackupCategoryHandler.restore(backup.backupCategories) } else { emptyMap() } @@ -223,9 +189,9 @@ object ProtoBackupImport : ProtoBackupBase() { if (flags.includeClientData) { updateRestoreState(id, BackupRestoreState.RestoringMeta(restoreSettings + restoreCategories + restoreMeta, restoreAmount)) - restoreGlobalMeta(backup.meta) + BackupGlobalMetaHandler.restore(backup.meta) - restoreSourceMeta(backup.backupSources) + BackupSourceHandler.restore(backup.backupSources) } // Store source mapping for error messages @@ -245,7 +211,7 @@ object ProtoBackupImport : ProtoBackupBase() { ), ) - restoreManga( + BackupMangaHandler.restore( backupManga = manga, categoryMapping = categoryMapping, sourceMapping = sourceMapping, @@ -273,292 +239,4 @@ object ProtoBackupImport : ProtoBackupBase() { return validationResult } - - private fun restoreCategories(backupCategories: List): Map { - val categoryIds = Category.createCategories(backupCategories.map { it.name }) - - val metaEntryByCategoryId = - categoryIds - .zip(backupCategories) - .associate { (categoryId, backupCategory) -> - categoryId to backupCategory.meta - } - - modifyCategoriesMetas(metaEntryByCategoryId) - - return backupCategories.withIndex().associate { (index, backupCategory) -> - backupCategory.order to categoryIds[index] - } - } - - private fun restoreManga( - backupManga: BackupManga, - categoryMapping: Map, - sourceMapping: Map, - errors: MutableList>, - flags: BackupFlags, - ) { - val chapters = backupManga.chapters - val categories = backupManga.categories - val history = backupManga.history - val tracking = backupManga.tracking - - val dbCategoryIds = categories.mapNotNull { categoryMapping[it] } - - try { - restoreMangaData(backupManga, chapters, dbCategoryIds, history, tracking, flags) - } catch (e: Exception) { - val sourceName = sourceMapping[backupManga.source] ?: backupManga.source.toString() - errors.add(Date() to "${backupManga.title} [$sourceName]: ${e.message}") - } - } - - private fun restoreMangaData( - manga: BackupManga, - chapters: List, - categoryIds: List, - history: List, - tracks: List, - flags: BackupFlags, - ) { - val dbManga = - transaction { - MangaTable - .selectAll() - .where { (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq manga.source) } - .firstOrNull() - } - val restoreMode = if (dbManga != null) RestoreMode.EXISTING else RestoreMode.NEW - - val mangaId = - transaction { - val mangaId = - if (dbManga == null) { - // insert manga to database - MangaTable - .insertAndGetId { - it[url] = manga.url - it[title] = manga.title - - it[artist] = manga.artist - it[author] = manga.author - it[description] = manga.description - it[genre] = manga.genre.joinToString() - it[status] = manga.status - it[thumbnail_url] = manga.thumbnailUrl - it[updateStrategy] = manga.updateStrategy.name - - it[sourceReference] = manga.source - - it[initialized] = manga.description != null - - it[inLibrary] = manga.favorite - - it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds - }.value - } else { - val dbMangaId = dbManga[MangaTable.id].value - - // Merge manga data - MangaTable.update({ MangaTable.id eq dbMangaId }) { - it[artist] = manga.artist ?: dbManga[artist] - it[author] = manga.author ?: dbManga[author] - it[description] = manga.description ?: dbManga[description] - it[genre] = manga.genre.ifEmpty { null }?.joinToString() ?: dbManga[genre] - it[status] = manga.status - it[thumbnail_url] = manga.thumbnailUrl ?: dbManga[thumbnail_url] - it[updateStrategy] = manga.updateStrategy.name - - it[initialized] = dbManga[initialized] || manga.description != null - - it[inLibrary] = manga.favorite || dbManga[inLibrary] - - it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds - } - - dbMangaId - } - - // delete thumbnail in case cached data still exists - clearThumbnail(mangaId) - - if (flags.includeClientData && manga.meta.isNotEmpty()) { - modifyMangasMetas(mapOf(mangaId to manga.meta)) - } - - // merge chapter data - if (flags.includeChapters || flags.includeHistory) { - restoreMangaChapterData(mangaId, restoreMode, chapters, history, flags) - } - - // merge categories - if (flags.includeCategories) { - restoreMangaCategoryData(mangaId, categoryIds) - } - - mangaId - } - - if (flags.includeTracking) { - restoreMangaTrackerData(mangaId, tracks) - } - - // TODO: insert/merge history - } - - private fun getMangaChapterToRestoreInfo( - mangaId: Int, - restoreMode: RestoreMode, - chapters: List, - ): Pair, List>> { - val uniqueChapters = chapters.distinctBy { it.url } - - if (restoreMode == RestoreMode.NEW) { - return Pair(uniqueChapters, emptyList()) - } - - val dbChaptersByUrl = ChapterTable.selectAll().where { ChapterTable.manga eq mangaId }.associateBy { it[ChapterTable.url] } - - val (chaptersToUpdate, chaptersToInsert) = uniqueChapters.partition { dbChaptersByUrl.contains(it.url) } - val chaptersToUpdateToDbChapter = chaptersToUpdate.map { it to dbChaptersByUrl[it.url]!! } - - return chaptersToInsert to chaptersToUpdateToDbChapter - } - - private fun restoreMangaChapterData( - mangaId: Int, - restoreMode: RestoreMode, - chapters: List, - history: List, - flags: BackupFlags, - ) = dbTransaction { - val (chaptersToInsert, chaptersToUpdateToDbChapter) = getMangaChapterToRestoreInfo(mangaId, restoreMode, chapters) - val historyByChapter = history.groupBy({ it.url }, { it.lastRead }) - - val insertedChapterIds = - if (flags.includeChapters) { - ChapterTable - .batchInsert(chaptersToInsert) { chapter -> - this[ChapterTable.url] = chapter.url - this[ChapterTable.name] = chapter.name - if (chapter.dateUpload == 0L) { - this[ChapterTable.date_upload] = chapter.dateFetch - } else { - this[ChapterTable.date_upload] = chapter.dateUpload - } - this[ChapterTable.chapter_number] = chapter.chapterNumber - this[ChapterTable.scanlator] = chapter.scanlator - - this[ChapterTable.sourceOrder] = chaptersToInsert.size - chapter.sourceOrder - this[ChapterTable.manga] = mangaId - - this[ChapterTable.isRead] = chapter.read - this[ChapterTable.lastPageRead] = chapter.lastPageRead.coerceAtLeast(0) - this[ChapterTable.isBookmarked] = chapter.bookmark - - this[ChapterTable.fetchedAt] = chapter.dateFetch.milliseconds.inWholeSeconds - - if (flags.includeHistory) { - this[ChapterTable.lastReadAt] = - historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0 - } - }.map { it[ChapterTable.id].value } - } else { - emptyList() - } - - if (chaptersToUpdateToDbChapter.isNotEmpty()) { - BatchUpdateStatement(ChapterTable).apply { - chaptersToUpdateToDbChapter.forEach { (backupChapter, dbChapter) -> - addBatch(EntityID(dbChapter[ChapterTable.id].value, ChapterTable)) - if (flags.includeChapters) { - this[ChapterTable.isRead] = backupChapter.read || dbChapter[ChapterTable.isRead] - this[ChapterTable.lastPageRead] = - max(backupChapter.lastPageRead, dbChapter[ChapterTable.lastPageRead]).coerceAtLeast(0) - this[ChapterTable.isBookmarked] = backupChapter.bookmark || dbChapter[ChapterTable.isBookmarked] - } - - if (flags.includeHistory) { - this[ChapterTable.lastReadAt] = - (historyByChapter[backupChapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0) - .coerceAtLeast(dbChapter[ChapterTable.lastReadAt]) - } - } - execute(this@dbTransaction) - } - } - - if (flags.includeClientData) { - val chaptersToInsertByChapterId = insertedChapterIds.zip(chaptersToInsert) - val chapterToUpdateByChapterId = - chaptersToUpdateToDbChapter.map { (backupChapter, dbChapter) -> - dbChapter[ChapterTable.id].value to - backupChapter - } - val metaEntryByChapterId = - (chaptersToInsertByChapterId + chapterToUpdateByChapterId) - .associate { (chapterId, backupChapter) -> - chapterId to backupChapter.meta - } - - modifyChaptersMetas(metaEntryByChapterId) - } - } - - private fun restoreMangaCategoryData( - mangaId: Int, - categoryIds: List, - ) { - CategoryManga.addMangaToCategories(mangaId, categoryIds) - } - - private fun restoreMangaTrackerData( - mangaId: Int, - tracks: List, - ) { - val dbTrackRecordsByTrackerId = - Tracker - .getTrackRecordsByMangaId(mangaId) - .mapNotNull { it.record?.toTrack() } - .associateBy { it.tracker_id } - - val (existingTracks, newTracks) = - tracks - .mapNotNull { backupTrack -> - val track = backupTrack.toTrack(mangaId) - - val isUnsupportedTracker = TrackerManager.getTracker(track.tracker_id) == null - if (isUnsupportedTracker) { - return@mapNotNull null - } - - val dbTrack = - dbTrackRecordsByTrackerId[backupTrack.syncId] - ?: // new track - return@mapNotNull track - - if (track.toTrackRecordDataClass().forComparison() == dbTrack.toTrackRecordDataClass().forComparison()) { - return@mapNotNull null - } - - dbTrack.also { - it.remote_id = track.remote_id - it.library_id = track.library_id - it.last_chapter_read = max(dbTrack.last_chapter_read, track.last_chapter_read) - } - }.partition { (it.id ?: -1) > 0 } - - Tracker.updateTrackRecords(existingTracks) - Tracker.insertTrackRecords(newTracks) - } - - private fun restoreGlobalMeta(meta: Map) { - GlobalMeta.modifyMetas(meta) - } - - private fun restoreSourceMeta(backupSources: List) { - modifySourceMetas(backupSources.associateBy { it.sourceId }.mapValues { it.value.meta }) - } - - private fun TrackRecordDataClass.forComparison() = this.copy(id = 0, mangaId = 0) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupCategoryHandler.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupCategoryHandler.kt new file mode 100644 index 00000000..2603fe0c --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupCategoryHandler.kt @@ -0,0 +1,63 @@ +package suwayomi.tachidesk.manga.impl.backup.proto.handlers + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import org.jetbrains.exposed.sql.SortOrder +import org.jetbrains.exposed.sql.selectAll +import suwayomi.tachidesk.manga.impl.Category +import suwayomi.tachidesk.manga.impl.Category.modifyCategoriesMetas +import suwayomi.tachidesk.manga.impl.backup.BackupFlags +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory +import suwayomi.tachidesk.manga.model.table.CategoryTable +import suwayomi.tachidesk.manga.model.table.toDataClass +import suwayomi.tachidesk.server.database.dbTransaction + +object BackupCategoryHandler { + fun backup(flags: BackupFlags): List = + dbTransaction { + val categories = + CategoryTable + .selectAll() + .orderBy(CategoryTable.order to SortOrder.ASC) + .map { CategoryTable.toDataClass(it) } + + val categoryToMeta = + if (flags.includeClientData) { + Category.getCategoriesMetaMaps(categories.map { it.id }) + } else { + emptyMap() + } + + categories.map { + BackupCategory( + it.name, + it.order, + 0, // not supported in Tachidesk + ).apply { + this.meta = categoryToMeta[it.id] ?: emptyMap() + } + } + } + + fun restore(backupCategories: List): Map { + val categoryIds = Category.createCategories(backupCategories.map { it.name }) + + val metaEntryByCategoryId = + categoryIds + .zip(backupCategories) + .associate { (categoryId, backupCategory) -> + categoryId to backupCategory.meta + } + + modifyCategoriesMetas(metaEntryByCategoryId) + + return backupCategories.withIndex().associate { (index, backupCategory) -> + backupCategory.order to categoryIds[index] + } + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupGlobalMetaHandler.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupGlobalMetaHandler.kt new file mode 100644 index 00000000..43938fb0 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupGlobalMetaHandler.kt @@ -0,0 +1,25 @@ +package suwayomi.tachidesk.manga.impl.backup.proto.handlers + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import suwayomi.tachidesk.global.impl.GlobalMeta +import suwayomi.tachidesk.manga.impl.backup.BackupFlags + +object BackupGlobalMetaHandler { + fun backup(flags: BackupFlags): Map { + if (!flags.includeClientData) { + return emptyMap() + } + + return GlobalMeta.getMetaMap() + } + + fun restore(meta: Map) { + GlobalMeta.modifyMetas(meta) + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt new file mode 100644 index 00000000..4c0f2bbb --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt @@ -0,0 +1,431 @@ +package suwayomi.tachidesk.manga.impl.backup.proto.handlers + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import eu.kanade.tachiyomi.source.model.UpdateStrategy +import org.jetbrains.exposed.dao.id.EntityID +import org.jetbrains.exposed.sql.ResultRow +import org.jetbrains.exposed.sql.SortOrder +import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.batchInsert +import org.jetbrains.exposed.sql.insertAndGetId +import org.jetbrains.exposed.sql.selectAll +import org.jetbrains.exposed.sql.statements.BatchUpdateStatement +import org.jetbrains.exposed.sql.transactions.transaction +import org.jetbrains.exposed.sql.update +import suwayomi.tachidesk.manga.impl.CategoryManga +import suwayomi.tachidesk.manga.impl.Chapter +import suwayomi.tachidesk.manga.impl.Chapter.modifyChaptersMetas +import suwayomi.tachidesk.manga.impl.Manga +import suwayomi.tachidesk.manga.impl.Manga.clearThumbnail +import suwayomi.tachidesk.manga.impl.Manga.modifyMangasMetas +import suwayomi.tachidesk.manga.impl.backup.BackupFlags +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupHistory +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking +import suwayomi.tachidesk.manga.impl.track.tracker.TrackerManager +import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrack +import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrackRecordDataClass +import suwayomi.tachidesk.manga.model.dataclass.TrackRecordDataClass +import suwayomi.tachidesk.manga.model.table.ChapterTable +import suwayomi.tachidesk.manga.model.table.MangaStatus +import suwayomi.tachidesk.manga.model.table.MangaTable +import suwayomi.tachidesk.manga.model.table.toDataClass +import suwayomi.tachidesk.server.database.dbTransaction +import java.util.Date +import kotlin.math.max +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds +import suwayomi.tachidesk.manga.impl.track.Track as Tracker + +object BackupMangaHandler { + private enum class RestoreMode { + NEW, + EXISTING, + } + + fun backup(flags: BackupFlags): List = + dbTransaction { + if (!flags.includeManga) { + return@dbTransaction emptyList() + } + + val manga = MangaTable.selectAll().where { MangaTable.inLibrary eq true }.toList() + + manga.map { mangaRow -> + val backupManga = + BackupManga( + source = mangaRow[MangaTable.sourceReference], + url = mangaRow[MangaTable.url], + title = mangaRow[MangaTable.title], + artist = mangaRow[MangaTable.artist], + author = mangaRow[MangaTable.author], + description = mangaRow[MangaTable.description], + genre = mangaRow[MangaTable.genre]?.split(", ") ?: emptyList(), + status = MangaStatus.valueOf(mangaRow[MangaTable.status]).value, + thumbnailUrl = mangaRow[MangaTable.thumbnail_url], + dateAdded = mangaRow[MangaTable.inLibraryAt].seconds.inWholeMilliseconds, + viewer = 0, // not supported in Tachidesk + updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy]), + ) + + val mangaId = mangaRow[MangaTable.id].value + + if (flags.includeClientData) { + backupManga.meta = Manga.getMangaMetaMap(mangaId) + } + + if (flags.includeChapters || flags.includeHistory) { + val chapters = + transaction { + ChapterTable + .selectAll() + .where { ChapterTable.manga eq mangaId } + .orderBy(ChapterTable.sourceOrder to SortOrder.DESC) + .map { + ChapterTable.toDataClass(it) + } + } + if (flags.includeChapters) { + val chapterToMeta = Chapter.getChaptersMetaMaps(chapters.map { it.id }) + + backupManga.chapters = + chapters.map { + BackupChapter( + it.url, + it.name, + it.scanlator, + it.read, + it.bookmarked, + it.lastPageRead, + it.fetchedAt.seconds.inWholeMilliseconds, + it.uploadDate, + it.chapterNumber, + chapters.size - it.index, + ).apply { + if (flags.includeClientData) { + this.meta = chapterToMeta[it.id] ?: emptyMap() + } + } + } + } + if (flags.includeHistory) { + backupManga.history = + chapters.mapNotNull { + if (it.lastReadAt > 0) { + BackupHistory( + url = it.url, + lastRead = it.lastReadAt.seconds.inWholeMilliseconds, + ) + } else { + null + } + } + } + } + + if (flags.includeCategories) { + backupManga.categories = CategoryManga.getMangaCategories(mangaId).map { it.order } + } + + if (flags.includeTracking) { + val tracks = + Tracker.getTrackRecordsByMangaId(mangaRow[MangaTable.id].value).mapNotNull { + if (it.record == null) { + null + } else { + BackupTracking( + syncId = it.record.trackerId, + // forced not null so its compatible with 1.x backup system + libraryId = it.record.libraryId ?: 0, + mediaId = it.record.remoteId, + title = it.record.title, + lastChapterRead = it.record.lastChapterRead.toFloat(), + totalChapters = it.record.totalChapters, + score = it.record.score.toFloat(), + status = it.record.status, + startedReadingDate = it.record.startDate, + finishedReadingDate = it.record.finishDate, + trackingUrl = it.record.remoteUrl, + private = it.record.private, + ) + } + } + if (tracks.isNotEmpty()) { + backupManga.tracking = tracks + } + } + + backupManga + } + } + + fun restore( + backupManga: BackupManga, + categoryMapping: Map, + sourceMapping: Map, + errors: MutableList>, + flags: BackupFlags, + ) { + val chapters = backupManga.chapters + val categories = backupManga.categories + val history = backupManga.history + val tracking = backupManga.tracking + + val dbCategoryIds = categories.mapNotNull { categoryMapping[it] } + + try { + restoreMangaData(backupManga, chapters, dbCategoryIds, history, tracking, flags) + } catch (e: Exception) { + val sourceName = sourceMapping[backupManga.source] ?: backupManga.source.toString() + errors.add(Date() to "${backupManga.title} [$sourceName]: ${e.message}") + } + } + + private fun restoreMangaData( + manga: BackupManga, + chapters: List, + categoryIds: List, + history: List, + tracks: List, + flags: BackupFlags, + ) { + val dbManga = + transaction { + MangaTable + .selectAll() + .where { (MangaTable.url eq manga.url) and (MangaTable.sourceReference eq manga.source) } + .firstOrNull() + } + val restoreMode = if (dbManga != null) RestoreMode.EXISTING else RestoreMode.NEW + + val mangaId = + transaction { + val mangaId = + if (dbManga == null) { + // insert manga to database + MangaTable + .insertAndGetId { + it[url] = manga.url + it[title] = manga.title + + it[artist] = manga.artist + it[author] = manga.author + it[description] = manga.description + it[genre] = manga.genre.joinToString() + it[status] = manga.status + it[thumbnail_url] = manga.thumbnailUrl + it[updateStrategy] = manga.updateStrategy.name + + it[sourceReference] = manga.source + + it[initialized] = manga.description != null + + it[inLibrary] = manga.favorite + + it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds + }.value + } else { + val dbMangaId = dbManga[MangaTable.id].value + + // Merge manga data + MangaTable.update({ MangaTable.id eq dbMangaId }) { + it[artist] = manga.artist ?: dbManga[artist] + it[author] = manga.author ?: dbManga[author] + it[description] = manga.description ?: dbManga[description] + it[genre] = manga.genre.ifEmpty { null }?.joinToString() ?: dbManga[genre] + it[status] = manga.status + it[thumbnail_url] = manga.thumbnailUrl ?: dbManga[thumbnail_url] + it[updateStrategy] = manga.updateStrategy.name + + it[initialized] = dbManga[initialized] || manga.description != null + + it[inLibrary] = manga.favorite || dbManga[inLibrary] + + it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds + } + + dbMangaId + } + + // delete thumbnail in case cached data still exists + clearThumbnail(mangaId) + + if (flags.includeClientData && manga.meta.isNotEmpty()) { + modifyMangasMetas(mapOf(mangaId to manga.meta)) + } + + // merge chapter data + if (flags.includeChapters || flags.includeHistory) { + restoreMangaChapterData(mangaId, restoreMode, chapters, history, flags) + } + + // merge categories + if (flags.includeCategories) { + restoreMangaCategoryData(mangaId, categoryIds) + } + + mangaId + } + + if (flags.includeTracking) { + restoreMangaTrackerData(mangaId, tracks) + } + + // TODO: insert/merge history + } + + private fun getMangaChapterToRestoreInfo( + mangaId: Int, + restoreMode: RestoreMode, + chapters: List, + ): Pair, List>> { + val uniqueChapters = chapters.distinctBy { it.url } + + if (restoreMode == RestoreMode.NEW) { + return Pair(uniqueChapters, emptyList()) + } + + val dbChaptersByUrl = ChapterTable.selectAll().where { ChapterTable.manga eq mangaId }.associateBy { it[ChapterTable.url] } + + val (chaptersToUpdate, chaptersToInsert) = uniqueChapters.partition { dbChaptersByUrl.contains(it.url) } + val chaptersToUpdateToDbChapter = chaptersToUpdate.map { it to dbChaptersByUrl[it.url]!! } + + return chaptersToInsert to chaptersToUpdateToDbChapter + } + + private fun restoreMangaChapterData( + mangaId: Int, + restoreMode: RestoreMode, + chapters: List, + history: List, + flags: BackupFlags, + ) = dbTransaction { + val (chaptersToInsert, chaptersToUpdateToDbChapter) = getMangaChapterToRestoreInfo(mangaId, restoreMode, chapters) + val historyByChapter = history.groupBy({ it.url }, { it.lastRead }) + + val insertedChapterIds = + if (flags.includeChapters) { + ChapterTable + .batchInsert(chaptersToInsert) { chapter -> + this[ChapterTable.url] = chapter.url + this[ChapterTable.name] = chapter.name + if (chapter.dateUpload == 0L) { + this[ChapterTable.date_upload] = chapter.dateFetch + } else { + this[ChapterTable.date_upload] = chapter.dateUpload + } + this[ChapterTable.chapter_number] = chapter.chapterNumber + this[ChapterTable.scanlator] = chapter.scanlator + + this[ChapterTable.sourceOrder] = chaptersToInsert.size - chapter.sourceOrder + this[ChapterTable.manga] = mangaId + + this[ChapterTable.isRead] = chapter.read + this[ChapterTable.lastPageRead] = chapter.lastPageRead.coerceAtLeast(0) + this[ChapterTable.isBookmarked] = chapter.bookmark + + this[ChapterTable.fetchedAt] = chapter.dateFetch.milliseconds.inWholeSeconds + + if (flags.includeHistory) { + this[ChapterTable.lastReadAt] = + historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0 + } + }.map { it[ChapterTable.id].value } + } else { + emptyList() + } + + if (chaptersToUpdateToDbChapter.isNotEmpty()) { + BatchUpdateStatement(ChapterTable).apply { + chaptersToUpdateToDbChapter.forEach { (backupChapter, dbChapter) -> + addBatch(EntityID(dbChapter[ChapterTable.id].value, ChapterTable)) + if (flags.includeChapters) { + this[ChapterTable.isRead] = backupChapter.read || dbChapter[ChapterTable.isRead] + this[ChapterTable.lastPageRead] = + max(backupChapter.lastPageRead, dbChapter[ChapterTable.lastPageRead]).coerceAtLeast(0) + this[ChapterTable.isBookmarked] = backupChapter.bookmark || dbChapter[ChapterTable.isBookmarked] + } + + if (flags.includeHistory) { + this[ChapterTable.lastReadAt] = + (historyByChapter[backupChapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0) + .coerceAtLeast(dbChapter[ChapterTable.lastReadAt]) + } + } + execute(this@dbTransaction) + } + } + + if (flags.includeClientData) { + val chaptersToInsertByChapterId = insertedChapterIds.zip(chaptersToInsert) + val chapterToUpdateByChapterId = + chaptersToUpdateToDbChapter.map { (backupChapter, dbChapter) -> + dbChapter[ChapterTable.id].value to + backupChapter + } + val metaEntryByChapterId = + (chaptersToInsertByChapterId + chapterToUpdateByChapterId) + .associate { (chapterId, backupChapter) -> + chapterId to backupChapter.meta + } + + modifyChaptersMetas(metaEntryByChapterId) + } + } + + private fun restoreMangaCategoryData( + mangaId: Int, + categoryIds: List, + ) { + CategoryManga.addMangaToCategories(mangaId, categoryIds) + } + + private fun restoreMangaTrackerData( + mangaId: Int, + tracks: List, + ) { + val dbTrackRecordsByTrackerId = + Tracker + .getTrackRecordsByMangaId(mangaId) + .mapNotNull { it.record?.toTrack() } + .associateBy { it.tracker_id } + + val (existingTracks, newTracks) = + tracks + .mapNotNull { backupTrack -> + val track = backupTrack.toTrack(mangaId) + + val isUnsupportedTracker = TrackerManager.getTracker(track.tracker_id) == null + if (isUnsupportedTracker) { + return@mapNotNull null + } + + val dbTrack = + dbTrackRecordsByTrackerId[backupTrack.syncId] + ?: // new track + return@mapNotNull track + + if (track.toTrackRecordDataClass().forComparison() == dbTrack.toTrackRecordDataClass().forComparison()) { + return@mapNotNull null + } + + dbTrack.also { + it.remote_id = track.remote_id + it.library_id = track.library_id + it.last_chapter_read = max(dbTrack.last_chapter_read, track.last_chapter_read) + } + }.partition { (it.id ?: -1) > 0 } + + Tracker.updateTrackRecords(existingTracks) + Tracker.insertTrackRecords(newTracks) + } + + private fun TrackRecordDataClass.forComparison() = this.copy(id = 0, mangaId = 0) +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupSourceHandler.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupSourceHandler.kt new file mode 100644 index 00000000..f940217a --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupSourceHandler.kt @@ -0,0 +1,56 @@ +package suwayomi.tachidesk.manga.impl.backup.proto.handlers + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +import org.jetbrains.exposed.sql.selectAll +import suwayomi.tachidesk.manga.impl.Source +import suwayomi.tachidesk.manga.impl.Source.modifySourceMetas +import suwayomi.tachidesk.manga.impl.backup.BackupFlags +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource +import suwayomi.tachidesk.manga.model.table.SourceTable +import suwayomi.tachidesk.server.database.dbTransaction + +object BackupSourceHandler { + fun backup( + backupMangas: List, + flags: BackupFlags, + ): List = + dbTransaction { + val inLibraryMangaSourceIds = + backupMangas + .asSequence() + .map { it.source } + .distinct() + .toList() + val sources = SourceTable.selectAll().where { SourceTable.id inList inLibraryMangaSourceIds } + val sourceToMeta = + if (flags.includeClientData) { + Source.getSourcesMetaMaps(sources.map { it[SourceTable.id].value }) + } else { + emptyMap() + } + + inLibraryMangaSourceIds + .map { mangaSourceId -> + val source = sources.firstOrNull { it[SourceTable.id].value == mangaSourceId } + BackupSource( + source?.get(SourceTable.name) ?: "", + mangaSourceId, + ).apply { + if (flags.includeClientData) { + this.meta = sourceToMeta[mangaSourceId] ?: emptyMap() + } + } + }.toList() + } + + fun restore(backupSources: List) { + modifySourceMetas(backupSources.associateBy { it.sourceId }.mapValues { it.value.meta }) + } +}