From d3d53d1a4e276d741930d02822e3d37c66dc6ec1 Mon Sep 17 00:00:00 2001 From: Aria Moradi Date: Thu, 19 Aug 2021 00:46:45 +0430 Subject: [PATCH] initial support for portobuf backup --- .../kanade/tachiyomi/source/model/SManga.kt | 28 +++++++++ .../legacy/serializer/MangaTypeAdapter.kt | 4 +- .../manga/impl/backup/models/MangaImpl.kt | 8 ++- .../impl/backup/proto/ProtoBackupBase.kt | 16 +++++ .../impl/backup/proto/ProtoBackupExport.kt | 35 +++++++++-- .../impl/backup/proto/ProtoBackupImport.kt | 59 +++++++++++++++++-- .../impl/backup/proto/ProtoBackupValidator.kt | 34 ++++++++++- 7 files changed, 166 insertions(+), 18 deletions(-) create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupBase.kt diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SManga.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SManga.kt index 63911e10..4c0f1ce4 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SManga.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/SManga.kt @@ -61,3 +61,31 @@ interface SManga : Serializable { } } } + +//fun SManga.toMangaInfo(): MangaInfo { +// return MangaInfo( +// key = this.url, +// title = this.title, +// artist = this.artist ?: "", +// author = this.author ?: "", +// description = this.description ?: "", +// genres = this.genre?.split(", ") ?: emptyList(), +// status = this.status, +// cover = this.thumbnail_url ?: "" +// ) +//} +// +//fun MangaInfo.toSManga(): SManga { +// val mangaInfo = this +// return SManga.create().apply { +// url = mangaInfo.key +// title = mangaInfo.title +// artist = mangaInfo.artist +// author = mangaInfo.author +// description = mangaInfo.description +// genre = mangaInfo.genres.joinToString(", ") +// status = mangaInfo.status +// thumbnail_url = mangaInfo.cover +// } +//} + diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/serializer/MangaTypeAdapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/serializer/MangaTypeAdapter.kt index dee97338..c54b7621 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/serializer/MangaTypeAdapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/legacy/serializer/MangaTypeAdapter.kt @@ -16,7 +16,7 @@ object MangaTypeAdapter { value(it.url) value(it.title) value(it.source) - value(it.viewer) + value(it.viewer_flags) value(it.chapter_flags) endArray() } @@ -27,7 +27,7 @@ object MangaTypeAdapter { manga.url = nextString() manga.title = nextString() manga.source = nextLong() - manga.viewer = nextInt() + manga.viewer_flags = nextInt() manga.chapter_flags = nextInt() endArray() manga diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/models/MangaImpl.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/models/MangaImpl.kt index 8124cdda..a0f6f8fe 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/models/MangaImpl.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/models/MangaImpl.kt @@ -5,7 +5,7 @@ import suwayomi.tachidesk.manga.model.table.MangaTable open class MangaImpl : Manga { - override var id: Long? = 0 + override var id: Long? = null override var source: Long = -1 @@ -29,6 +29,8 @@ open class MangaImpl : Manga { override var last_update: Long = 0 + override var next_update: Long = 0 + override var date_added: Long = 0 override var initialized: Boolean = false @@ -42,7 +44,7 @@ open class MangaImpl : Manga { * 4 -> Webtoon * 5 -> Continues Vertical */ - override var viewer: Int = 0 + override var viewer_flags: Int = 0 /** Contains some useful info about */ @@ -70,7 +72,7 @@ open class MangaImpl : Manga { url = mangaRecord[MangaTable.url] title = mangaRecord[MangaTable.title] source = mangaRecord[MangaTable.sourceReference] - viewer = 0 // TODO: implement + viewer_flags = 0 // TODO: implement chapter_flags = 0 // TODO: implement } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupBase.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupBase.kt new file mode 100644 index 00000000..66c15327 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupBase.kt @@ -0,0 +1,16 @@ +package suwayomi.tachidesk.manga.impl.backup.proto + +import kotlinx.serialization.protobuf.ProtoBuf + +/* + * Copyright (C) Contributors to the Suwayomi project + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +open class ProtoBackupBase { + var sourceMapping: Map = emptyMap() + + val parser = ProtoBuf +} \ No newline at end of file diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt index 2bfbc688..84d574d1 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupExport.kt @@ -1,7 +1,5 @@ package suwayomi.tachidesk.manga.impl.backup.proto -import suwayomi.tachidesk.manga.impl.backup.BackupFlags - /* * Copyright (C) Contributors to the Suwayomi project * @@ -9,8 +7,35 @@ import suwayomi.tachidesk.manga.impl.backup.BackupFlags * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -object ProtoBackupExport { - suspend fun createBackup(flags: BackupFlags): String? { - TODO() +import okio.buffer +import okio.gzip +import okio.sink +import suwayomi.tachidesk.manga.impl.backup.BackupFlags +import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSerializer +import java.io.ByteArrayOutputStream + +object ProtoBackupExport: ProtoBackupBase() { + suspend fun createBackup(flags: BackupFlags): ByteArray { + // Create root object + var backup: Backup? = null + +// databaseHelper.inTransaction { +// val databaseManga = getFavoriteManga() +// +// backup = Backup( +// backupManga(databaseManga, flags), +// backupCategories(), +// backupExtensionInfo(databaseManga) +// ) +// } + + val byteArray = parser.encodeToByteArray(BackupSerializer, backup!!) + byteArray.inputStream() + + val byteStream = ByteArrayOutputStream() + byteStream.sink().gzip().buffer().use { it.write(byteArray) } + + return byteStream.toByteArray() } } \ No newline at end of file diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt index 88f497fa..1c88e0ea 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt @@ -7,25 +7,74 @@ package suwayomi.tachidesk.manga.impl.backup.proto * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import kotlinx.serialization.protobuf.ProtoBuf import mu.KotlinLogging import okio.buffer import okio.gzip import okio.source import suwayomi.tachidesk.manga.impl.backup.AbstractBackupValidator.ValidationResult +import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.validate +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSerializer import java.io.InputStream private val logger = KotlinLogging.logger {} -object ProtoBackupImport { +object ProtoBackupImport: ProtoBackupBase() { + var restoreAmount = 0 + suspend fun performRestore(sourceStream: InputStream): ValidationResult { val backupString = sourceStream.source().gzip().buffer().use { it.readByteArray() } - val backup = ProtoBuf.decodeFromByteArray(BackupSerializer, backupString) + val backup = parser.decodeFromByteArray(BackupSerializer, backupString) + + val validationResult = validate(backup) + + restoreAmount = backup.backupManga.size + 1 // +1 for categories + + // Restore categories + if (backup.backupCategories.isNotEmpty()) { + restoreCategories(backup.backupCategories) + } + + // Store source mapping for error messages + sourceMapping = backup.backupSources.map { it.sourceId to it.name }.toMap() + + // Restore individual manga + backup.backupManga.forEach { + restoreManga(it, backup.backupCategories) + } + + // TODO: optionally trigger online library + tracker update + + return validationResult + } + + private fun restoreCategories(backupCategories: List) { // TODO +// db.inTransaction { +// backupManager.restoreCategories(backupCategories) +// } +// +// restoreProgress += 1 +// showRestoreProgress(restoreProgress, restoreAmount, context.getString(R.string.categories)) + } - - TODO() + private fun restoreManga(backupManga: BackupManga, backupCategories: List) { // TODO +// val manga = backupManga.getMangaImpl() +// val chapters = backupManga.getChaptersImpl() +// val categories = backupManga.categories +// val history = backupManga.history +// val tracks = backupManga.getTrackingImpl() +// +// try { +// restoreMangaData(manga, chapters, categories, history, tracks, backupCategories) +// } catch (e: Exception) { +// val sourceName = sourceMapping[manga.source] ?: manga.source.toString() +// errors.add(Date() to "${manga.title} [$sourceName]: ${e.message}") +// } +// +// restoreProgress += 1 +// showRestoreProgress(restoreProgress, restoreAmount, manga.title) } } \ No newline at end of file diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupValidator.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupValidator.kt index fe463555..2f4deae2 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupValidator.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupValidator.kt @@ -7,11 +7,39 @@ package suwayomi.tachidesk.manga.impl.backup.proto * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import com.google.gson.JsonObject +import org.jetbrains.exposed.sql.select +import org.jetbrains.exposed.sql.transactions.transaction import suwayomi.tachidesk.manga.impl.backup.AbstractBackupValidator +import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup +import suwayomi.tachidesk.manga.model.table.SourceTable object ProtoBackupValidator: AbstractBackupValidator() { - fun validate(json: JsonObject): ValidationResult { - TODO() + fun validate(backup: Backup): ValidationResult { + if (backup.backupManga.isEmpty()) { + throw Exception("Backup does not contain any manga.") + } + + val sources = backup.backupSources.map { it.sourceId to it.name }.toMap() + + val missingSources = transaction { + sources + .filter { SourceTable.select { SourceTable.id eq it.key }.firstOrNull() == null } + .map { "${it.value} (${it.key})" } + .sorted() + } + +// val trackers = backup.backupManga +// .flatMap { it.tracking } +// .map { it.syncId } +// .distinct() + + val missingTrackers = listOf("") +// val missingTrackers = trackers +// .mapNotNull { trackManager.getService(it) } +// .filter { !it.isLogged } +// .map { context.getString(it.nameRes()) } +// .sorted() + + return ValidationResult(missingSources, missingTrackers) } } \ No newline at end of file