Feature/backup tracking (#940)

* Include tracking in validation of backup

* Always return track records

Not clear why an empty list should be returned in case no trackers are logged in

* Include tracking in backup creation

* Restore tracking from backup
This commit is contained in:
schroda
2024-05-05 19:24:16 +02:00
committed by GitHub
parent cf1ede9cf7
commit 7df5f1c4c4
9 changed files with 294 additions and 158 deletions

View File

@@ -16,14 +16,20 @@ class BackupQuery {
val name: String,
)
data class ValidateBackupTracker(
val name: String,
)
data class ValidateBackupResult(
val missingSources: List<ValidateBackupSource>,
val missingTrackers: List<ValidateBackupTracker>,
)
fun validateBackup(input: ValidateBackupInput): ValidateBackupResult {
val result = ProtoBackupValidator.validate(input.backup.content)
return ValidateBackupResult(
result.missingSourceIds.map { ValidateBackupSource(it.first, it.second) },
result.missingTrackers.map { ValidateBackupTracker(it) },
)
}

View File

@@ -11,7 +11,7 @@ interface Track : Serializable {
var sync_id: Int
var media_id: Int
var media_id: Long
var library_id: Long?

View File

@@ -9,7 +9,7 @@ class TrackImpl : Track {
override var sync_id: Int = 0
override var media_id: Int = 0
override var media_id: Long = 0L
override var library_id: Long? = null
@@ -43,7 +43,7 @@ class TrackImpl : Track {
override fun hashCode(): Int {
var result = (manga_id xor manga_id.ushr(32)).toInt()
result = 31 * result + sync_id
result = 31 * result + media_id
result = (31 * result + media_id).toInt()
return result
}
}

View File

@@ -31,6 +31,8 @@ import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSerializer
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking
import suwayomi.tachidesk.manga.impl.track.Track
import suwayomi.tachidesk.manga.model.table.CategoryTable
import suwayomi.tachidesk.manga.model.table.ChapterTable
import suwayomi.tachidesk.manga.model.table.MangaStatus
@@ -230,9 +232,32 @@ object ProtoBackupExport : ProtoBackupBase() {
backupManga.categories = CategoryManga.getMangaCategories(mangaId).map { it.order }
}
// if(flags.includeTracking) {
// backupManga.tracking = TODO()
// }
if (flags.includeTracking) {
val tracks =
Track.getTrackRecordsByMangaId(mangaRow[MangaTable.id].value).mapNotNull {
if (it.record == null) {
null
} else {
BackupTracking(
syncId = it.record.trackerId,
// forced not null so its compatible with 1.x backup system
libraryId = it.record.libraryId ?: 0,
mediaId = it.record.remoteId,
title = it.record.title,
lastChapterRead = it.record.lastChapterRead.toFloat(),
totalChapters = it.record.totalChapters,
score = it.record.score.toFloat(),
status = it.record.status,
startedReadingDate = it.record.startDate,
finishedReadingDate = it.record.finishDate,
trackingUrl = it.record.remoteUrl,
)
}
}
if (tracks.isNotEmpty()) {
backupManga.tracking = tracks
}
}
// if (flags.includeHistory) {
// backupManga.history = TODO()

View File

@@ -34,22 +34,26 @@ import suwayomi.tachidesk.manga.impl.CategoryManga
import suwayomi.tachidesk.manga.impl.Manga.clearThumbnail
import suwayomi.tachidesk.manga.impl.backup.models.Chapter
import suwayomi.tachidesk.manga.impl.backup.models.Manga
import suwayomi.tachidesk.manga.impl.backup.models.Track
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult
import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.validate
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupHistory
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSerializer
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking
import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrack
import suwayomi.tachidesk.manga.impl.track.tracker.model.toTrackRecordDataClass
import suwayomi.tachidesk.manga.model.dataclass.TrackRecordDataClass
import suwayomi.tachidesk.manga.model.table.CategoryTable
import suwayomi.tachidesk.manga.model.table.ChapterTable
import suwayomi.tachidesk.manga.model.table.MangaTable
import java.io.InputStream
import java.lang.Integer.max
import java.util.Date
import java.util.Timer
import java.util.TimerTask
import java.util.concurrent.TimeUnit
import kotlin.math.max
import suwayomi.tachidesk.manga.impl.track.Track as Tracker
object ProtoBackupImport : ProtoBackupBase() {
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
@@ -239,10 +243,9 @@ object ProtoBackupImport : ProtoBackupBase() {
val chapters = backupManga.getChaptersImpl()
val categories = backupManga.categories
val history = backupManga.brokenHistory.map { BackupHistory(it.url, it.lastRead) } + backupManga.history
val tracks = backupManga.getTrackingImpl()
try {
restoreMangaData(manga, chapters, categories, history, tracks, backupCategories, categoryMapping)
restoreMangaData(manga, chapters, categories, history, backupManga.tracking, backupCategories, categoryMapping)
} catch (e: Exception) {
val sourceName = sourceMapping[manga.source] ?: manga.source.toString()
errors.add(Date() to "${manga.title} [$sourceName]: ${e.message}")
@@ -255,7 +258,7 @@ object ProtoBackupImport : ProtoBackupBase() {
chapters: List<Chapter>,
categories: List<Int>,
history: List<BackupHistory>,
tracks: List<Track>,
tracks: List<BackupTracking>,
backupCategories: List<BackupCategory>,
categoryMapping: Map<Int, Int>,
) {
@@ -265,127 +268,159 @@ object ProtoBackupImport : ProtoBackupBase() {
.firstOrNull()
}
if (dbManga == null) { // Manga not in database
transaction {
// insert manga to database
val mangaId =
MangaTable.insertAndGetId {
it[url] = manga.url
it[title] = manga.title
val mangaId =
if (dbManga == null) { // Manga not in database
transaction {
// insert manga to database
val mangaId =
MangaTable.insertAndGetId {
it[url] = manga.url
it[title] = manga.title
it[artist] = manga.artist
it[author] = manga.author
it[description] = manga.description
it[genre] = manga.genre
it[artist] = manga.artist
it[author] = manga.author
it[description] = manga.description
it[genre] = manga.genre
it[status] = manga.status
it[thumbnail_url] = manga.thumbnail_url
it[updateStrategy] = manga.update_strategy.name
it[sourceReference] = manga.source
it[initialized] = manga.description != null
it[inLibrary] = manga.favorite
it[inLibraryAt] = TimeUnit.MILLISECONDS.toSeconds(manga.date_added)
}.value
// delete thumbnail in case cached data still exists
clearThumbnail(mangaId)
// insert chapter data
val chaptersLength = chapters.size
ChapterTable.batchInsert(chapters) { chapter ->
this[ChapterTable.url] = chapter.url
this[ChapterTable.name] = chapter.name
if (chapter.date_upload == 0L) {
this[ChapterTable.date_upload] = chapter.date_fetch
} else {
this[ChapterTable.date_upload] = chapter.date_upload
}
this[ChapterTable.chapter_number] = chapter.chapter_number
this[ChapterTable.scanlator] = chapter.scanlator
this[ChapterTable.sourceOrder] = chaptersLength - chapter.source_order
this[ChapterTable.manga] = mangaId
this[ChapterTable.isRead] = chapter.read
this[ChapterTable.lastPageRead] = chapter.last_page_read
this[ChapterTable.isBookmarked] = chapter.bookmark
this[ChapterTable.fetchedAt] = TimeUnit.MILLISECONDS.toSeconds(chapter.date_fetch)
}
// insert categories
categories.forEach { backupCategoryOrder ->
CategoryManga.addMangaToCategory(mangaId, categoryMapping[backupCategoryOrder]!!)
}
mangaId
}
} else { // Manga in database
transaction {
val mangaId = dbManga[MangaTable.id].value
// Merge manga data
MangaTable.update({ MangaTable.id eq mangaId }) {
it[artist] = manga.artist ?: dbManga[artist]
it[author] = manga.author ?: dbManga[author]
it[description] = manga.description ?: dbManga[description]
it[genre] = manga.genre ?: dbManga[genre]
it[status] = manga.status
it[thumbnail_url] = manga.thumbnail_url
it[thumbnail_url] = manga.thumbnail_url ?: dbManga[thumbnail_url]
it[updateStrategy] = manga.update_strategy.name
it[sourceReference] = manga.source
it[initialized] = dbManga[initialized] || manga.description != null
it[initialized] = manga.description != null
it[inLibrary] = manga.favorite
it[inLibrary] = manga.favorite || dbManga[inLibrary]
it[inLibraryAt] = TimeUnit.MILLISECONDS.toSeconds(manga.date_added)
}.value
// delete thumbnail in case cached data still exists
clearThumbnail(mangaId)
// insert chapter data
val chaptersLength = chapters.size
ChapterTable.batchInsert(chapters) { chapter ->
this[ChapterTable.url] = chapter.url
this[ChapterTable.name] = chapter.name
if (chapter.date_upload == 0L) {
this[ChapterTable.date_upload] = chapter.date_fetch
} else {
this[ChapterTable.date_upload] = chapter.date_upload
}
this[ChapterTable.chapter_number] = chapter.chapter_number
this[ChapterTable.scanlator] = chapter.scanlator
this[ChapterTable.sourceOrder] = chaptersLength - chapter.source_order
this[ChapterTable.manga] = mangaId
// merge chapter data
val chaptersLength = chapters.size
val dbChapters = ChapterTable.select { ChapterTable.manga eq mangaId }
this[ChapterTable.isRead] = chapter.read
this[ChapterTable.lastPageRead] = chapter.last_page_read
this[ChapterTable.isBookmarked] = chapter.bookmark
chapters.forEach { chapter ->
val dbChapter = dbChapters.find { it[ChapterTable.url] == chapter.url }
this[ChapterTable.fetchedAt] = TimeUnit.MILLISECONDS.toSeconds(chapter.date_fetch)
}
if (dbChapter == null) {
ChapterTable.insert {
it[url] = chapter.url
it[name] = chapter.name
if (chapter.date_upload == 0L) {
it[date_upload] = chapter.date_fetch
} else {
it[date_upload] = chapter.date_upload
}
it[chapter_number] = chapter.chapter_number
it[scanlator] = chapter.scanlator
// insert categories
categories.forEach { backupCategoryOrder ->
CategoryManga.addMangaToCategory(mangaId, categoryMapping[backupCategoryOrder]!!)
}
}
} else { // Manga in database
transaction {
val mangaId = dbManga[MangaTable.id].value
it[sourceOrder] = chaptersLength - chapter.source_order
it[ChapterTable.manga] = mangaId
// Merge manga data
MangaTable.update({ MangaTable.id eq mangaId }) {
it[artist] = manga.artist ?: dbManga[artist]
it[author] = manga.author ?: dbManga[author]
it[description] = manga.description ?: dbManga[description]
it[genre] = manga.genre ?: dbManga[genre]
it[status] = manga.status
it[thumbnail_url] = manga.thumbnail_url ?: dbManga[thumbnail_url]
it[updateStrategy] = manga.update_strategy.name
it[initialized] = dbManga[initialized] || manga.description != null
it[inLibrary] = manga.favorite || dbManga[inLibrary]
it[inLibraryAt] = TimeUnit.MILLISECONDS.toSeconds(manga.date_added)
}
// merge chapter data
val chaptersLength = chapters.size
val dbChapters = ChapterTable.select { ChapterTable.manga eq mangaId }
chapters.forEach { chapter ->
val dbChapter = dbChapters.find { it[ChapterTable.url] == chapter.url }
if (dbChapter == null) {
ChapterTable.insert {
it[url] = chapter.url
it[name] = chapter.name
if (chapter.date_upload == 0L) {
it[date_upload] = chapter.date_fetch
} else {
it[date_upload] = chapter.date_upload
it[isRead] = chapter.read
it[lastPageRead] = chapter.last_page_read
it[isBookmarked] = chapter.bookmark
}
} else {
ChapterTable.update({ (ChapterTable.url eq dbChapter[ChapterTable.url]) and (ChapterTable.manga eq mangaId) }) {
it[isRead] = chapter.read || dbChapter[isRead]
it[lastPageRead] = max(chapter.last_page_read, dbChapter[lastPageRead])
it[isBookmarked] = chapter.bookmark || dbChapter[isBookmarked]
}
it[chapter_number] = chapter.chapter_number
it[scanlator] = chapter.scanlator
it[sourceOrder] = chaptersLength - chapter.source_order
it[ChapterTable.manga] = mangaId
it[isRead] = chapter.read
it[lastPageRead] = chapter.last_page_read
it[isBookmarked] = chapter.bookmark
}
} else {
ChapterTable.update({ (ChapterTable.url eq dbChapter[ChapterTable.url]) and (ChapterTable.manga eq mangaId) }) {
it[isRead] = chapter.read || dbChapter[isRead]
it[lastPageRead] = max(chapter.last_page_read, dbChapter[lastPageRead])
it[isBookmarked] = chapter.bookmark || dbChapter[isBookmarked]
}
}
}
// merge categories
categories.forEach { backupCategoryOrder ->
CategoryManga.addMangaToCategory(mangaId, categoryMapping[backupCategoryOrder]!!)
// merge categories
categories.forEach { backupCategoryOrder ->
CategoryManga.addMangaToCategory(mangaId, categoryMapping[backupCategoryOrder]!!)
}
mangaId
}
}
}
val dbTrackRecordsByTrackerId =
Tracker.getTrackRecordsByMangaId(mangaId)
.mapNotNull { it.record?.toTrack() }
.associateBy { it.sync_id }
val (existingTracks, newTracks) =
tracks.mapNotNull { backupTrack ->
val track = backupTrack.toTrack(mangaId)
val dbTrack =
dbTrackRecordsByTrackerId[backupTrack.syncId]
?: // new track
return@mapNotNull track
if (track.toTrackRecordDataClass().forComparison() == dbTrack.toTrackRecordDataClass().forComparison()) {
return@mapNotNull null
}
dbTrack.also {
it.media_id = track.media_id
it.library_id = track.library_id
it.last_chapter_read = max(dbTrack.last_chapter_read, track.last_chapter_read)
}
}.partition { (it.id ?: -1) > 0 }
existingTracks.forEach(Tracker::updateTrackRecord)
newTracks.forEach(Tracker::insertTrackRecord)
// TODO: insert/merge history
// TODO: insert/merge tracking
}
private fun TrackRecordDataClass.forComparison() = this.copy(id = 0, mangaId = 0)
}

View File

@@ -15,6 +15,7 @@ import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSerializer
import suwayomi.tachidesk.manga.impl.track.tracker.TrackerManager
import suwayomi.tachidesk.manga.model.table.SourceTable
import java.io.InputStream
@@ -39,17 +40,18 @@ object ProtoBackupValidator {
sources.filter { SourceTable.select { SourceTable.id eq it.key }.firstOrNull() == null }
}
// val trackers = backup.backupManga
// .flatMap { it.tracking }
// .map { it.syncId }
// .distinct()
val trackers =
backup.backupManga
.flatMap { it.tracking }
.map { it.syncId }
.distinct()
val missingTrackers = listOf("")
// val missingTrackers = trackers
// .mapNotNull { trackManager.getService(it) }
// .filter { !it.isLogged }
// .map { context.getString(it.nameRes()) }
// .sorted()
val missingTrackers =
trackers
.mapNotNull { TrackerManager.getTracker(it) }
.filter { !it.isLoggedIn }
.map { it.name }
.sorted()
return ValidationResult(
missingSources

View File

@@ -8,11 +8,12 @@ import suwayomi.tachidesk.manga.impl.backup.models.TrackImpl
@Serializable
data class BackupTracking(
// in 1.x some of these values have different types or names
// syncId is called siteId in 1,x
@ProtoNumber(1) var syncId: Int,
// LibraryId is not null in 1.x
@ProtoNumber(2) var libraryId: Long,
@ProtoNumber(3) var mediaId: Int = 0,
@Deprecated("Use mediaId instead", level = DeprecationLevel.WARNING)
@ProtoNumber(3)
var mediaIdInt: Int = 0,
// trackingUrl is called mediaUrl in 1.x
@ProtoNumber(4) var trackingUrl: String = "",
@ProtoNumber(5) var title: String = "",
@@ -25,11 +26,17 @@ data class BackupTracking(
@ProtoNumber(10) var startedReadingDate: Long = 0,
// finishedReadingDate is called endReadTime in 1.x
@ProtoNumber(11) var finishedReadingDate: Long = 0,
@ProtoNumber(100) var mediaId: Long = 0,
) {
fun getTrackingImpl(): TrackImpl {
return TrackImpl().apply {
sync_id = this@BackupTracking.syncId
media_id = this@BackupTracking.mediaId
media_id =
if (this@BackupTracking.mediaIdInt != 0) {
this@BackupTracking.mediaIdInt.toLong()
} else {
this@BackupTracking.mediaId
}
library_id = this@BackupTracking.libraryId
title = this@BackupTracking.title
// convert from float to int because of 1.x types

View File

@@ -74,9 +74,6 @@ object Track {
}
fun getTrackRecordsByMangaId(mangaId: Int): List<MangaTrackerDataClass> {
if (!TrackerManager.hasLoggedTracker()) {
return emptyList()
}
val recordMap =
transaction {
TrackRecordTable.select { TrackRecordTable.mangaId eq mangaId }
@@ -342,7 +339,7 @@ object Track {
}
}
private fun upsertTrackRecord(track: Track): Int {
fun upsertTrackRecord(track: Track): Int {
return transaction {
val existingRecord =
TrackRecordTable.select {
@@ -352,41 +349,53 @@ object Track {
.singleOrNull()
if (existingRecord != null) {
TrackRecordTable.update({
(TrackRecordTable.mangaId eq track.manga_id) and
(TrackRecordTable.trackerId eq track.sync_id)
}) {
it[remoteId] = track.media_id
it[libraryId] = track.library_id
it[title] = track.title
it[lastChapterRead] = track.last_chapter_read.toDouble()
it[totalChapters] = track.total_chapters
it[status] = track.status
it[score] = track.score.toDouble()
it[remoteUrl] = track.tracking_url
it[startDate] = track.started_reading_date
it[finishDate] = track.finished_reading_date
}
updateTrackRecord(track)
existingRecord[TrackRecordTable.id].value
} else {
TrackRecordTable.insertAndGetId {
it[mangaId] = track.manga_id
it[trackerId] = track.sync_id
it[remoteId] = track.media_id
it[libraryId] = track.library_id
it[title] = track.title
it[lastChapterRead] = track.last_chapter_read.toDouble()
it[totalChapters] = track.total_chapters
it[status] = track.status
it[score] = track.score.toDouble()
it[remoteUrl] = track.tracking_url
it[startDate] = track.started_reading_date
it[finishDate] = track.finished_reading_date
}.value
insertTrackRecord(track)
}
}
}
fun updateTrackRecord(track: Track): Int =
transaction {
TrackRecordTable.update(
{
(TrackRecordTable.mangaId eq track.manga_id) and
(TrackRecordTable.trackerId eq track.sync_id)
},
) {
it[remoteId] = track.media_id
it[libraryId] = track.library_id
it[title] = track.title
it[lastChapterRead] = track.last_chapter_read.toDouble()
it[totalChapters] = track.total_chapters
it[status] = track.status
it[score] = track.score.toDouble()
it[remoteUrl] = track.tracking_url
it[startDate] = track.started_reading_date
it[finishDate] = track.finished_reading_date
}
}
fun insertTrackRecord(track: Track): Int =
transaction {
TrackRecordTable.insertAndGetId {
it[mangaId] = track.manga_id
it[trackerId] = track.sync_id
it[remoteId] = track.media_id
it[libraryId] = track.library_id
it[title] = track.title
it[lastChapterRead] = track.last_chapter_read.toDouble()
it[totalChapters] = track.total_chapters
it[status] = track.status
it[score] = track.score.toDouble()
it[remoteUrl] = track.tracking_url
it[startDate] = track.started_reading_date
it[finishDate] = track.finished_reading_date
}.value
}
@Serializable
data class LoginInput(
val trackerId: Int,

View File

@@ -1,8 +1,11 @@
package suwayomi.tachidesk.manga.impl.track.tracker.model
import org.jetbrains.exposed.sql.ResultRow
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupTracking
import suwayomi.tachidesk.manga.model.dataclass.TrackRecordDataClass
import suwayomi.tachidesk.manga.model.table.TrackRecordTable
import suwayomi.tachidesk.manga.model.table.TrackRecordTable.lastChapterRead
import suwayomi.tachidesk.manga.model.table.TrackRecordTable.remoteUrl
fun ResultRow.toTrackRecordDataClass(): TrackRecordDataClass =
TrackRecordDataClass(
@@ -36,3 +39,52 @@ fun ResultRow.toTrack(): Track =
it.started_reading_date = this[TrackRecordTable.startDate]
it.finished_reading_date = this[TrackRecordTable.finishDate]
}
fun BackupTracking.toTrack(mangaId: Int): Track =
Track.create(syncId).also {
it.id = -1
it.manga_id = mangaId
it.media_id = mediaId
it.library_id = libraryId
it.title = title
it.last_chapter_read = lastChapterRead
it.total_chapters = totalChapters
it.status = status
it.score = score
it.tracking_url = trackingUrl
it.started_reading_date = startedReadingDate
it.finished_reading_date = finishedReadingDate
}
fun TrackRecordDataClass.toTrack(): Track =
Track.create(trackerId).also {
it.id = id
it.manga_id = mangaId
it.media_id = remoteId
it.library_id = libraryId
it.title = title
it.last_chapter_read = lastChapterRead.toFloat()
it.total_chapters = totalChapters
it.status = status
it.score = score.toFloat()
it.tracking_url = remoteUrl
it.started_reading_date = startDate
it.finished_reading_date = finishDate
}
fun Track.toTrackRecordDataClass(): TrackRecordDataClass =
TrackRecordDataClass(
id = id ?: -1,
mangaId = manga_id,
trackerId = sync_id,
remoteId = media_id,
libraryId = library_id,
title = title,
lastChapterRead = last_chapter_read.toDouble(),
totalChapters = total_chapters,
status = status,
score = score.toDouble(),
remoteUrl = tracking_url,
startDate = started_reading_date,
finishDate = finished_reading_date,
)