From 48e19f7914fee1ea1789b217d5df9b05acb49203 Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sun, 7 Apr 2024 04:53:36 +0200 Subject: [PATCH] Feature/auto download of new chapters improve handling of unhandable reuploads (#921) * Update test/server-reference file * Properly handle re-uploaded chapters in auto download of new chapters In case of unhandable re-uploaded chapters (different chapter numbers) they potentially would have prevented auto downloads due being considered as unread. Additionally, they would not have been considered to get downloaded due to not having a higher chapter number than the previous latest existing chapter before the chapter list fetch. * Add option to ignore re-uploads for auto downloads * Extract check for manga category download inclusion * Extract logic to get new chapter ids to download * Simplify manga category download inclusion check In case the DEFAULT category does not exist, someone messed with the database and it is basically corrupted --- .../graphql/mutations/SettingsMutation.kt | 1 + .../tachidesk/graphql/types/SettingsType.kt | 4 + .../suwayomi/tachidesk/manga/impl/Chapter.kt | 118 +++++++----------- .../suwayomi/tachidesk/manga/impl/Manga.kt | 55 ++++++++ .../suwayomi/tachidesk/server/ServerConfig.kt | 1 + .../src/main/resources/server-reference.conf | 1 + .../src/test/resources/server-reference.conf | 60 ++++++--- 7 files changed, 144 insertions(+), 96 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt index c88b1e85..fc083ca1 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt @@ -58,6 +58,7 @@ class SettingsMutation { updateSetting(settings.excludeEntryWithUnreadChapters, serverConfig.excludeEntryWithUnreadChapters) updateSetting(settings.autoDownloadAheadLimit, serverConfig.autoDownloadNewChaptersLimit) // deprecated updateSetting(settings.autoDownloadNewChaptersLimit, serverConfig.autoDownloadNewChaptersLimit) + updateSetting(settings.autoDownloadIgnoreReUploads, serverConfig.autoDownloadIgnoreReUploads) // extension updateSetting(settings.extensionRepos, serverConfig.extensionRepos) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt index 5a28e513..a980d5f4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt @@ -49,6 +49,7 @@ interface Settings : Node { ) val autoDownloadAheadLimit: Int? val autoDownloadNewChaptersLimit: Int? + val autoDownloadIgnoreReUploads: Boolean? // extension val extensionRepos: List? @@ -118,6 +119,7 @@ data class PartialSettingsType( ) override val autoDownloadAheadLimit: Int?, override val autoDownloadNewChaptersLimit: Int?, + override val autoDownloadIgnoreReUploads: Boolean?, // extension override val extensionRepos: List?, // requests @@ -179,6 +181,7 @@ class SettingsType( ) override val autoDownloadAheadLimit: Int, override val autoDownloadNewChaptersLimit: Int, + override val autoDownloadIgnoreReUploads: Boolean?, // extension override val extensionRepos: List, // requests @@ -235,6 +238,7 @@ class SettingsType( config.excludeEntryWithUnreadChapters.value, config.autoDownloadNewChaptersLimit.value, // deprecated config.autoDownloadNewChaptersLimit.value, + config.autoDownloadIgnoreReUploads.value, // extension config.extensionRepos.value, // requests diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt index ce998814..5de631d6 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt @@ -20,7 +20,6 @@ import kotlinx.serialization.Serializable import mu.KotlinLogging import org.jetbrains.exposed.dao.id.EntityID import org.jetbrains.exposed.sql.Op -import org.jetbrains.exposed.sql.ResultRow import org.jetbrains.exposed.sql.SortOrder import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList import org.jetbrains.exposed.sql.and @@ -37,7 +36,6 @@ import suwayomi.tachidesk.manga.impl.download.DownloadManager.EnqueueInput import suwayomi.tachidesk.manga.impl.track.Track import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass -import suwayomi.tachidesk.manga.model.dataclass.IncludeOrExclude import suwayomi.tachidesk.manga.model.dataclass.MangaChapterDataClass import suwayomi.tachidesk.manga.model.dataclass.PaginatedList import suwayomi.tachidesk.manga.model.dataclass.paginatedFrom @@ -136,6 +134,7 @@ object Chapter { url = manga.url } + val currentLatestChapterNumber = Manga.getLatestChapter(mangaId)?.chapterNumber ?: 0f val numberOfCurrentChapters = getCountOfMangaChapters(mangaId) val chapterList = source.getChapterList(sManga) @@ -164,7 +163,10 @@ object Chapter { .toList() } - val chaptersToInsert = mutableListOf() + // new chapters after they have been added to the database for auto downloads + val insertedChapters = mutableListOf() + + val chaptersToInsert = mutableListOf() // do not yet have an ID from the database val chaptersToUpdate = mutableListOf() chapterList.reversed().forEachIndexed { index, fetchedChapter -> @@ -260,7 +262,7 @@ object Chapter { this[ChapterTable.fetchedAt] = it } } - } + }.forEach { insertedChapters.add(ChapterTable.toDataClass(it)) } } if (chaptersToUpdate.isNotEmpty()) { @@ -283,14 +285,8 @@ object Chapter { } } - val newChapters = - transaction { - ChapterTable.select { ChapterTable.manga eq mangaId } - .orderBy(ChapterTable.sourceOrder to SortOrder.DESC).toList() - } - if (manga.inLibrary) { - downloadNewChapters(mangaId, numberOfCurrentChapters, newChapters) + downloadNewChapters(mangaId, currentLatestChapterNumber, numberOfCurrentChapters, insertedChapters) } chapterList @@ -301,16 +297,19 @@ object Chapter { private fun downloadNewChapters( mangaId: Int, + prevLatestChapterNumber: Float, prevNumberOfChapters: Int, - updatedChapterList: List, + newChapters: List, ) { val log = KotlinLogging.logger( "${logger.name}::downloadNewChapters(" + "mangaId= $mangaId, " + + "prevLatestChapterNumber= $prevLatestChapterNumber, " + "prevNumberOfChapters= $prevNumberOfChapters, " + - "updatedChapterList= ${updatedChapterList.size}, " + - "autoDownloadNewChaptersLimit= ${serverConfig.autoDownloadNewChaptersLimit.value}" + + "newChapters= ${newChapters.size}, " + + "autoDownloadNewChaptersLimit= ${serverConfig.autoDownloadNewChaptersLimit.value}, " + + "autoDownloadIgnoreReUploads= ${serverConfig.autoDownloadIgnoreReUploads.value}" + ")", ) @@ -319,68 +318,22 @@ object Chapter { return } - // Only download if there are new chapters, or if this is the first fetch - val newNumberOfChapters = updatedChapterList.size - val numberOfNewChapters = newNumberOfChapters - prevNumberOfChapters - - val areNewChaptersAvailable = numberOfNewChapters > 0 - val wasInitialFetch = prevNumberOfChapters == 0 - - if (!areNewChaptersAvailable) { + if (newChapters.isEmpty()) { log.debug { "no new chapters available" } return } + val wasInitialFetch = prevNumberOfChapters == 0 if (wasInitialFetch) { log.debug { "skipping download on initial fetch" } return } - // Verify the manga is configured to be downloaded based on it's categories. - var mangaCategories = CategoryManga.getMangaCategories(mangaId).toSet() - // if the manga has no categories, then it's implicitly in the default category - if (mangaCategories.isEmpty()) { - val defaultCategory = Category.getCategoryById(Category.DEFAULT_CATEGORY_ID) - if (defaultCategory != null) { - mangaCategories = setOf(defaultCategory) - } else { - log.warn { "missing default category" } - } + if (!Manga.isInIncludedDownloadCategory(log, mangaId)) { + return } - if (mangaCategories.isNotEmpty()) { - val downloadCategoriesMap = Category.getCategoryList().groupBy { it.includeInDownload } - val unsetCategories = downloadCategoriesMap[IncludeOrExclude.UNSET].orEmpty() - // We only download if it's in the include list, and not in the exclude list. - // Use the unset categories as the included categories if the included categories is - // empty - val includedCategories = downloadCategoriesMap[IncludeOrExclude.INCLUDE].orEmpty().ifEmpty { unsetCategories } - val excludedCategories = downloadCategoriesMap[IncludeOrExclude.EXCLUDE].orEmpty() - // Only download manga that aren't in any excluded categories - val mangaExcludeCategories = mangaCategories.intersect(excludedCategories.toSet()) - if (mangaExcludeCategories.isNotEmpty()) { - log.debug { "download excluded by categories: '${mangaExcludeCategories.joinToString("', '") { it.name }}'" } - return - } - val mangaDownloadCategories = mangaCategories.intersect(includedCategories.toSet()) - if (mangaDownloadCategories.isNotEmpty()) { - log.debug { "download inluded by categories: '${mangaDownloadCategories.joinToString("', '") { it.name }}'" } - } else { - log.debug { "skipping download due to download categories configuration" } - return - } - } else { - log.debug { "no categories configured, skipping check for category download include/excludes" } - } - - val newChapters = updatedChapterList.subList(0, numberOfNewChapters) - - // make sure to only consider the latest chapters. e.g. old unread chapters should be ignored - val latestReadChapterIndex = - updatedChapterList.indexOfFirst { it[ChapterTable.isRead] }.takeIf { it > -1 } ?: (updatedChapterList.size) - val unreadChapters = - updatedChapterList.subList(numberOfNewChapters, latestReadChapterIndex) - .filter { !it[ChapterTable.isRead] } + val unreadChapters = Manga.getUnreadChapters(mangaId).subtract(newChapters.toSet()) val skipDueToUnreadChapters = serverConfig.excludeEntryWithUnreadChapters.value && unreadChapters.isNotEmpty() if (skipDueToUnreadChapters) { @@ -388,17 +341,7 @@ object Chapter { return } - val firstChapterToDownloadIndex = - if (serverConfig.autoDownloadNewChaptersLimit.value > 0) { - (numberOfNewChapters - serverConfig.autoDownloadNewChaptersLimit.value).coerceAtLeast(0) - } else { - 0 - } - - val chapterIdsToDownload = - newChapters.subList(firstChapterToDownloadIndex, numberOfNewChapters) - .filter { !it[ChapterTable.isRead] && !it[ChapterTable.isDownloaded] } - .map { it[ChapterTable.id].value } + val chapterIdsToDownload = getNewChapterIdsToDownload(newChapters, prevLatestChapterNumber) if (chapterIdsToDownload.isEmpty()) { log.debug { "no chapters available for download" } @@ -410,6 +353,29 @@ object Chapter { DownloadManager.enqueue(EnqueueInput(chapterIdsToDownload)) } + private fun getNewChapterIdsToDownload( + newChapters: List, + prevLatestChapterNumber: Float, + ): List { + val reUploadedChapters = newChapters.filter { it.chapterNumber < prevLatestChapterNumber } + val actualNewChapters = newChapters.subtract(reUploadedChapters.toSet()) + val chaptersToConsiderForDownloadLimit = + if (serverConfig.autoDownloadIgnoreReUploads.value) { + actualNewChapters + } else { + newChapters + }.sortedBy { it.index } + + val latestChapterToDownloadIndex = + if (serverConfig.autoDownloadNewChaptersLimit.value == 0) { + chaptersToConsiderForDownloadLimit.size + } else { + serverConfig.autoDownloadNewChaptersLimit.value.coerceAtMost(chaptersToConsiderForDownloadLimit.size) + } + + return chaptersToConsiderForDownloadLimit.subList(0, latestChapterToDownloadIndex).map { it.id } + } + fun modifyChapter( mangaId: Int, chapterIndex: Int, diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt index d15af084..e28e95d3 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt @@ -16,6 +16,7 @@ import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.model.UpdateStrategy import eu.kanade.tachiyomi.source.online.HttpSource import io.javalin.http.HttpCode +import mu.KLogger import mu.KotlinLogging import okhttp3.CacheControl import okhttp3.Response @@ -41,6 +42,8 @@ import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.clearCachedImage import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil import suwayomi.tachidesk.manga.impl.util.updateMangaDownloadDir +import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass +import suwayomi.tachidesk.manga.model.dataclass.IncludeOrExclude import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass import suwayomi.tachidesk.manga.model.dataclass.toGenreList import suwayomi.tachidesk.manga.model.table.ChapterTable @@ -366,4 +369,56 @@ object Manga { clearCachedImage(applicationDirs.tempThumbnailCacheRoot, fileName) clearCachedImage(applicationDirs.thumbnailDownloadsRoot, fileName) } + + fun getLatestChapter(mangaId: Int): ChapterDataClass? { + return transaction { + ChapterTable.select { ChapterTable.manga eq mangaId }.maxByOrNull { it[ChapterTable.sourceOrder] } + }?.let { ChapterTable.toDataClass(it) } + } + + fun getUnreadChapters(mangaId: Int): List { + return transaction { + ChapterTable.select { (ChapterTable.manga eq mangaId) and (ChapterTable.isRead eq false) } + .orderBy(ChapterTable.sourceOrder to SortOrder.DESC) + .map { ChapterTable.toDataClass(it) } + } + } + + fun isInIncludedDownloadCategory( + logContext: KLogger = logger, + mangaId: Int, + ): Boolean { + val log = KotlinLogging.logger { "${logContext.name}::isInExcludedDownloadCategory($mangaId)" } + + // Verify the manga is configured to be downloaded based on it's categories. + var mangaCategories = CategoryManga.getMangaCategories(mangaId).toSet() + // if the manga has no categories, then it's implicitly in the default category + if (mangaCategories.isEmpty()) { + val defaultCategory = Category.getCategoryById(Category.DEFAULT_CATEGORY_ID)!! + mangaCategories = setOf(defaultCategory) + } + + val downloadCategoriesMap = Category.getCategoryList().groupBy { it.includeInDownload } + val unsetCategories = downloadCategoriesMap[IncludeOrExclude.UNSET].orEmpty() + // We only download if it's in the include list, and not in the exclude list. + // Use the unset categories as the included categories if the included categories is + // empty + val includedCategories = downloadCategoriesMap[IncludeOrExclude.INCLUDE].orEmpty().ifEmpty { unsetCategories } + val excludedCategories = downloadCategoriesMap[IncludeOrExclude.EXCLUDE].orEmpty() + // Only download manga that aren't in any excluded categories + val mangaExcludeCategories = mangaCategories.intersect(excludedCategories.toSet()) + if (mangaExcludeCategories.isNotEmpty()) { + log.debug { "download excluded by categories: '${mangaExcludeCategories.joinToString("', '") { it.name }}'" } + return false + } + val mangaDownloadCategories = mangaCategories.intersect(includedCategories.toSet()) + if (mangaDownloadCategories.isNotEmpty()) { + log.debug { "download inluded by categories: '${mangaDownloadCategories.joinToString("', '") { it.name }}'" } + } else { + log.debug { "skipping download due to download categories configuration" } + return false + } + + return true + } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt index afab0831..21b7c7f3 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt @@ -101,6 +101,7 @@ class ServerConfig(getConfig: () -> Config, val moduleName: String = SERVER_CONF val autoDownloadNewChapters: MutableStateFlow by OverrideConfigValue(BooleanConfigAdapter) val excludeEntryWithUnreadChapters: MutableStateFlow by OverrideConfigValue(BooleanConfigAdapter) val autoDownloadNewChaptersLimit: MutableStateFlow by OverrideConfigValue(IntConfigAdapter) + val autoDownloadIgnoreReUploads: MutableStateFlow by OverrideConfigValue(BooleanConfigAdapter) // extensions val extensionRepos: MutableStateFlow> by OverrideConfigValues(StringConfigAdapter) diff --git a/server/src/main/resources/server-reference.conf b/server/src/main/resources/server-reference.conf index 5b4bd174..0cf00c4f 100644 --- a/server/src/main/resources/server-reference.conf +++ b/server/src/main/resources/server-reference.conf @@ -25,6 +25,7 @@ server.downloadsPath = "" server.autoDownloadNewChapters = false # if new chapters that have been retrieved should get automatically downloaded server.excludeEntryWithUnreadChapters = true # ignore automatic chapter downloads of entries with unread chapters server.autoDownloadNewChaptersLimit = 0 # 0 to disable it - how many unread downloaded chapters should be available - if the limit is reached, new chapters won't be downloaded automatically. this limit will also be applied to the auto download of new chapters on an update +server.autoDownloadIgnoreReUploads = false # decides if re-uploads should be ignored during auto download of new chapters # extension repos server.extensionRepos = [ diff --git a/server/src/test/resources/server-reference.conf b/server/src/test/resources/server-reference.conf index 7670bb57..0cf00c4f 100644 --- a/server/src/test/resources/server-reference.conf +++ b/server/src/test/resources/server-reference.conf @@ -10,40 +10,60 @@ server.socksProxyPort = "" server.socksProxyUsername = "" server.socksProxyPassword = "" +# webUI +server.webUIEnabled = true +server.webUIFlavor = "WebUI" # "WebUI", "VUI" or "Custom" +server.initialOpenInBrowserEnabled = true +server.webUIInterface = "browser" # "browser" or "electron" +server.electronPath = "" +server.webUIChannel = "stable" # "bundled" (the version bundled with the server release), "stable" or "preview" - the webUI version that should be used +server.webUIUpdateCheckInterval = 23 # time in hours - 0 to disable auto update - range: 1 <= n < 24 - default 23 hours - how often the server should check for webUI updates + # downloader server.downloadAsCbz = false -server.autoDownloadNewChapters = false -server.excludeEntryWithUnreadChapters = true -server.autoDownloadNewChaptersLimit = 0 +server.downloadsPath = "" +server.autoDownloadNewChapters = false # if new chapters that have been retrieved should get automatically downloaded +server.excludeEntryWithUnreadChapters = true # ignore automatic chapter downloads of entries with unread chapters +server.autoDownloadNewChaptersLimit = 0 # 0 to disable it - how many unread downloaded chapters should be available - if the limit is reached, new chapters won't be downloaded automatically. this limit will also be applied to the auto download of new chapters on an update +server.autoDownloadIgnoreReUploads = false # decides if re-uploads should be ignored during auto download of new chapters + +# extension repos +server.extensionRepos = [ + # an example: https://github.com/MY_ACCOUNT/MY_REPO/tree/repo +] # requests -server.maxSourcesInParallel = 10 +server.maxSourcesInParallel = 6 # range: 1 <= n <= 20 - default: 6 - sets how many sources can do requests (updates, downloads) in parallel. updates/downloads are grouped by source and all mangas of a source are updated/downloaded synchronously # updater server.excludeUnreadChapters = true server.excludeNotStarted = true server.excludeCompleted = true -server.globalUpdateInterval = 12 -server.updateMangas = false +server.globalUpdateInterval = 12 # time in hours - 0 to disable it - (doesn't have to be full hours e.g. 12.5) - range: 6 <= n < ∞ - default: 12 hours - interval in which the global update will be automatically triggered +server.updateMangas = false # if the mangas should be updated along with the chapter list during a library/category update + +# Authentication +server.basicAuthEnabled = false +server.basicAuthUsername = "" +server.basicAuthPassword = "" # misc -server.debugLogsEnabled = true -server.gqlDebugLogsEnabled = false -server.systemTrayEnabled = false - -# webUI -server.webUIEnabled = true -server.initialOpenInBrowserEnabled = true -server.webUIInterface = "browser" # "browser" or "electron" -server.electronPath = "" -server.webUIChannel = "stable" -server.webUIUpdateCheckInterval = 24 +server.debugLogsEnabled = false +server.gqlDebugLogsEnabled = false # this includes logs with non privacy safe information +server.systemTrayEnabled = true # backup server.backupPath = "" -server.backupTime = "00:00" -server.backupInterval = 1 -server.backupTTL = 14 +server.backupTime = "00:00" # range: hour: 0-23, minute: 0-59 - default: "00:00" - time of day at which the automated backup should be triggered +server.backupInterval = 1 # time in days - 0 to disable it - range: 1 <= n < ∞ - default: 1 day - interval in which the server will automatically create a backup +server.backupTTL = 14 # time in days - 0 to disable it - range: 1 <= n < ∞ - default: 14 days - how long backup files will be kept before they will get deleted # local source server.localSourcePath = "" + +# Cloudflare bypass +server.flareSolverrEnabled = false +server.flareSolverrUrl = "http://localhost:8191" +server.flareSolverrTimeout = 60 # time in seconds +server.flareSolverrSessionName = "suwayomi" +server.flareSolverrSessionTtl = 15 # time in minutes