diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt index c88b1e85..fc083ca1 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt @@ -58,6 +58,7 @@ class SettingsMutation { updateSetting(settings.excludeEntryWithUnreadChapters, serverConfig.excludeEntryWithUnreadChapters) updateSetting(settings.autoDownloadAheadLimit, serverConfig.autoDownloadNewChaptersLimit) // deprecated updateSetting(settings.autoDownloadNewChaptersLimit, serverConfig.autoDownloadNewChaptersLimit) + updateSetting(settings.autoDownloadIgnoreReUploads, serverConfig.autoDownloadIgnoreReUploads) // extension updateSetting(settings.extensionRepos, serverConfig.extensionRepos) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt index 5a28e513..a980d5f4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt @@ -49,6 +49,7 @@ interface Settings : Node { ) val autoDownloadAheadLimit: Int? val autoDownloadNewChaptersLimit: Int? + val autoDownloadIgnoreReUploads: Boolean? // extension val extensionRepos: List? @@ -118,6 +119,7 @@ data class PartialSettingsType( ) override val autoDownloadAheadLimit: Int?, override val autoDownloadNewChaptersLimit: Int?, + override val autoDownloadIgnoreReUploads: Boolean?, // extension override val extensionRepos: List?, // requests @@ -179,6 +181,7 @@ class SettingsType( ) override val autoDownloadAheadLimit: Int, override val autoDownloadNewChaptersLimit: Int, + override val autoDownloadIgnoreReUploads: Boolean?, // extension override val extensionRepos: List, // requests @@ -235,6 +238,7 @@ class SettingsType( config.excludeEntryWithUnreadChapters.value, config.autoDownloadNewChaptersLimit.value, // deprecated config.autoDownloadNewChaptersLimit.value, + config.autoDownloadIgnoreReUploads.value, // extension config.extensionRepos.value, // requests diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt index ce998814..5de631d6 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt @@ -20,7 +20,6 @@ import kotlinx.serialization.Serializable import mu.KotlinLogging import org.jetbrains.exposed.dao.id.EntityID import org.jetbrains.exposed.sql.Op -import org.jetbrains.exposed.sql.ResultRow import org.jetbrains.exposed.sql.SortOrder import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList import org.jetbrains.exposed.sql.and @@ -37,7 +36,6 @@ import suwayomi.tachidesk.manga.impl.download.DownloadManager.EnqueueInput import suwayomi.tachidesk.manga.impl.track.Track import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass -import suwayomi.tachidesk.manga.model.dataclass.IncludeOrExclude import suwayomi.tachidesk.manga.model.dataclass.MangaChapterDataClass import suwayomi.tachidesk.manga.model.dataclass.PaginatedList import suwayomi.tachidesk.manga.model.dataclass.paginatedFrom @@ -136,6 +134,7 @@ object Chapter { url = manga.url } + val currentLatestChapterNumber = Manga.getLatestChapter(mangaId)?.chapterNumber ?: 0f val numberOfCurrentChapters = getCountOfMangaChapters(mangaId) val chapterList = source.getChapterList(sManga) @@ -164,7 +163,10 @@ object Chapter { .toList() } - val chaptersToInsert = mutableListOf() + // new chapters after they have been added to the database for auto downloads + val insertedChapters = mutableListOf() + + val chaptersToInsert = mutableListOf() // do not yet have an ID from the database val chaptersToUpdate = mutableListOf() chapterList.reversed().forEachIndexed { index, fetchedChapter -> @@ -260,7 +262,7 @@ object Chapter { this[ChapterTable.fetchedAt] = it } } - } + }.forEach { insertedChapters.add(ChapterTable.toDataClass(it)) } } if (chaptersToUpdate.isNotEmpty()) { @@ -283,14 +285,8 @@ object Chapter { } } - val newChapters = - transaction { - ChapterTable.select { ChapterTable.manga eq mangaId } - .orderBy(ChapterTable.sourceOrder to SortOrder.DESC).toList() - } - if (manga.inLibrary) { - downloadNewChapters(mangaId, numberOfCurrentChapters, newChapters) + downloadNewChapters(mangaId, currentLatestChapterNumber, numberOfCurrentChapters, insertedChapters) } chapterList @@ -301,16 +297,19 @@ object Chapter { private fun downloadNewChapters( mangaId: Int, + prevLatestChapterNumber: Float, prevNumberOfChapters: Int, - updatedChapterList: List, + newChapters: List, ) { val log = KotlinLogging.logger( "${logger.name}::downloadNewChapters(" + "mangaId= $mangaId, " + + "prevLatestChapterNumber= $prevLatestChapterNumber, " + "prevNumberOfChapters= $prevNumberOfChapters, " + - "updatedChapterList= ${updatedChapterList.size}, " + - "autoDownloadNewChaptersLimit= ${serverConfig.autoDownloadNewChaptersLimit.value}" + + "newChapters= ${newChapters.size}, " + + "autoDownloadNewChaptersLimit= ${serverConfig.autoDownloadNewChaptersLimit.value}, " + + "autoDownloadIgnoreReUploads= ${serverConfig.autoDownloadIgnoreReUploads.value}" + ")", ) @@ -319,68 +318,22 @@ object Chapter { return } - // Only download if there are new chapters, or if this is the first fetch - val newNumberOfChapters = updatedChapterList.size - val numberOfNewChapters = newNumberOfChapters - prevNumberOfChapters - - val areNewChaptersAvailable = numberOfNewChapters > 0 - val wasInitialFetch = prevNumberOfChapters == 0 - - if (!areNewChaptersAvailable) { + if (newChapters.isEmpty()) { log.debug { "no new chapters available" } return } + val wasInitialFetch = prevNumberOfChapters == 0 if (wasInitialFetch) { log.debug { "skipping download on initial fetch" } return } - // Verify the manga is configured to be downloaded based on it's categories. - var mangaCategories = CategoryManga.getMangaCategories(mangaId).toSet() - // if the manga has no categories, then it's implicitly in the default category - if (mangaCategories.isEmpty()) { - val defaultCategory = Category.getCategoryById(Category.DEFAULT_CATEGORY_ID) - if (defaultCategory != null) { - mangaCategories = setOf(defaultCategory) - } else { - log.warn { "missing default category" } - } + if (!Manga.isInIncludedDownloadCategory(log, mangaId)) { + return } - if (mangaCategories.isNotEmpty()) { - val downloadCategoriesMap = Category.getCategoryList().groupBy { it.includeInDownload } - val unsetCategories = downloadCategoriesMap[IncludeOrExclude.UNSET].orEmpty() - // We only download if it's in the include list, and not in the exclude list. - // Use the unset categories as the included categories if the included categories is - // empty - val includedCategories = downloadCategoriesMap[IncludeOrExclude.INCLUDE].orEmpty().ifEmpty { unsetCategories } - val excludedCategories = downloadCategoriesMap[IncludeOrExclude.EXCLUDE].orEmpty() - // Only download manga that aren't in any excluded categories - val mangaExcludeCategories = mangaCategories.intersect(excludedCategories.toSet()) - if (mangaExcludeCategories.isNotEmpty()) { - log.debug { "download excluded by categories: '${mangaExcludeCategories.joinToString("', '") { it.name }}'" } - return - } - val mangaDownloadCategories = mangaCategories.intersect(includedCategories.toSet()) - if (mangaDownloadCategories.isNotEmpty()) { - log.debug { "download inluded by categories: '${mangaDownloadCategories.joinToString("', '") { it.name }}'" } - } else { - log.debug { "skipping download due to download categories configuration" } - return - } - } else { - log.debug { "no categories configured, skipping check for category download include/excludes" } - } - - val newChapters = updatedChapterList.subList(0, numberOfNewChapters) - - // make sure to only consider the latest chapters. e.g. old unread chapters should be ignored - val latestReadChapterIndex = - updatedChapterList.indexOfFirst { it[ChapterTable.isRead] }.takeIf { it > -1 } ?: (updatedChapterList.size) - val unreadChapters = - updatedChapterList.subList(numberOfNewChapters, latestReadChapterIndex) - .filter { !it[ChapterTable.isRead] } + val unreadChapters = Manga.getUnreadChapters(mangaId).subtract(newChapters.toSet()) val skipDueToUnreadChapters = serverConfig.excludeEntryWithUnreadChapters.value && unreadChapters.isNotEmpty() if (skipDueToUnreadChapters) { @@ -388,17 +341,7 @@ object Chapter { return } - val firstChapterToDownloadIndex = - if (serverConfig.autoDownloadNewChaptersLimit.value > 0) { - (numberOfNewChapters - serverConfig.autoDownloadNewChaptersLimit.value).coerceAtLeast(0) - } else { - 0 - } - - val chapterIdsToDownload = - newChapters.subList(firstChapterToDownloadIndex, numberOfNewChapters) - .filter { !it[ChapterTable.isRead] && !it[ChapterTable.isDownloaded] } - .map { it[ChapterTable.id].value } + val chapterIdsToDownload = getNewChapterIdsToDownload(newChapters, prevLatestChapterNumber) if (chapterIdsToDownload.isEmpty()) { log.debug { "no chapters available for download" } @@ -410,6 +353,29 @@ object Chapter { DownloadManager.enqueue(EnqueueInput(chapterIdsToDownload)) } + private fun getNewChapterIdsToDownload( + newChapters: List, + prevLatestChapterNumber: Float, + ): List { + val reUploadedChapters = newChapters.filter { it.chapterNumber < prevLatestChapterNumber } + val actualNewChapters = newChapters.subtract(reUploadedChapters.toSet()) + val chaptersToConsiderForDownloadLimit = + if (serverConfig.autoDownloadIgnoreReUploads.value) { + actualNewChapters + } else { + newChapters + }.sortedBy { it.index } + + val latestChapterToDownloadIndex = + if (serverConfig.autoDownloadNewChaptersLimit.value == 0) { + chaptersToConsiderForDownloadLimit.size + } else { + serverConfig.autoDownloadNewChaptersLimit.value.coerceAtMost(chaptersToConsiderForDownloadLimit.size) + } + + return chaptersToConsiderForDownloadLimit.subList(0, latestChapterToDownloadIndex).map { it.id } + } + fun modifyChapter( mangaId: Int, chapterIndex: Int, diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt index d15af084..e28e95d3 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt @@ -16,6 +16,7 @@ import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.model.UpdateStrategy import eu.kanade.tachiyomi.source.online.HttpSource import io.javalin.http.HttpCode +import mu.KLogger import mu.KotlinLogging import okhttp3.CacheControl import okhttp3.Response @@ -41,6 +42,8 @@ import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.clearCachedImage import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil import suwayomi.tachidesk.manga.impl.util.updateMangaDownloadDir +import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass +import suwayomi.tachidesk.manga.model.dataclass.IncludeOrExclude import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass import suwayomi.tachidesk.manga.model.dataclass.toGenreList import suwayomi.tachidesk.manga.model.table.ChapterTable @@ -366,4 +369,56 @@ object Manga { clearCachedImage(applicationDirs.tempThumbnailCacheRoot, fileName) clearCachedImage(applicationDirs.thumbnailDownloadsRoot, fileName) } + + fun getLatestChapter(mangaId: Int): ChapterDataClass? { + return transaction { + ChapterTable.select { ChapterTable.manga eq mangaId }.maxByOrNull { it[ChapterTable.sourceOrder] } + }?.let { ChapterTable.toDataClass(it) } + } + + fun getUnreadChapters(mangaId: Int): List { + return transaction { + ChapterTable.select { (ChapterTable.manga eq mangaId) and (ChapterTable.isRead eq false) } + .orderBy(ChapterTable.sourceOrder to SortOrder.DESC) + .map { ChapterTable.toDataClass(it) } + } + } + + fun isInIncludedDownloadCategory( + logContext: KLogger = logger, + mangaId: Int, + ): Boolean { + val log = KotlinLogging.logger { "${logContext.name}::isInExcludedDownloadCategory($mangaId)" } + + // Verify the manga is configured to be downloaded based on it's categories. + var mangaCategories = CategoryManga.getMangaCategories(mangaId).toSet() + // if the manga has no categories, then it's implicitly in the default category + if (mangaCategories.isEmpty()) { + val defaultCategory = Category.getCategoryById(Category.DEFAULT_CATEGORY_ID)!! + mangaCategories = setOf(defaultCategory) + } + + val downloadCategoriesMap = Category.getCategoryList().groupBy { it.includeInDownload } + val unsetCategories = downloadCategoriesMap[IncludeOrExclude.UNSET].orEmpty() + // We only download if it's in the include list, and not in the exclude list. + // Use the unset categories as the included categories if the included categories is + // empty + val includedCategories = downloadCategoriesMap[IncludeOrExclude.INCLUDE].orEmpty().ifEmpty { unsetCategories } + val excludedCategories = downloadCategoriesMap[IncludeOrExclude.EXCLUDE].orEmpty() + // Only download manga that aren't in any excluded categories + val mangaExcludeCategories = mangaCategories.intersect(excludedCategories.toSet()) + if (mangaExcludeCategories.isNotEmpty()) { + log.debug { "download excluded by categories: '${mangaExcludeCategories.joinToString("', '") { it.name }}'" } + return false + } + val mangaDownloadCategories = mangaCategories.intersect(includedCategories.toSet()) + if (mangaDownloadCategories.isNotEmpty()) { + log.debug { "download inluded by categories: '${mangaDownloadCategories.joinToString("', '") { it.name }}'" } + } else { + log.debug { "skipping download due to download categories configuration" } + return false + } + + return true + } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt index afab0831..21b7c7f3 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt @@ -101,6 +101,7 @@ class ServerConfig(getConfig: () -> Config, val moduleName: String = SERVER_CONF val autoDownloadNewChapters: MutableStateFlow by OverrideConfigValue(BooleanConfigAdapter) val excludeEntryWithUnreadChapters: MutableStateFlow by OverrideConfigValue(BooleanConfigAdapter) val autoDownloadNewChaptersLimit: MutableStateFlow by OverrideConfigValue(IntConfigAdapter) + val autoDownloadIgnoreReUploads: MutableStateFlow by OverrideConfigValue(BooleanConfigAdapter) // extensions val extensionRepos: MutableStateFlow> by OverrideConfigValues(StringConfigAdapter) diff --git a/server/src/main/resources/server-reference.conf b/server/src/main/resources/server-reference.conf index 5b4bd174..0cf00c4f 100644 --- a/server/src/main/resources/server-reference.conf +++ b/server/src/main/resources/server-reference.conf @@ -25,6 +25,7 @@ server.downloadsPath = "" server.autoDownloadNewChapters = false # if new chapters that have been retrieved should get automatically downloaded server.excludeEntryWithUnreadChapters = true # ignore automatic chapter downloads of entries with unread chapters server.autoDownloadNewChaptersLimit = 0 # 0 to disable it - how many unread downloaded chapters should be available - if the limit is reached, new chapters won't be downloaded automatically. this limit will also be applied to the auto download of new chapters on an update +server.autoDownloadIgnoreReUploads = false # decides if re-uploads should be ignored during auto download of new chapters # extension repos server.extensionRepos = [ diff --git a/server/src/test/resources/server-reference.conf b/server/src/test/resources/server-reference.conf index 7670bb57..0cf00c4f 100644 --- a/server/src/test/resources/server-reference.conf +++ b/server/src/test/resources/server-reference.conf @@ -10,40 +10,60 @@ server.socksProxyPort = "" server.socksProxyUsername = "" server.socksProxyPassword = "" +# webUI +server.webUIEnabled = true +server.webUIFlavor = "WebUI" # "WebUI", "VUI" or "Custom" +server.initialOpenInBrowserEnabled = true +server.webUIInterface = "browser" # "browser" or "electron" +server.electronPath = "" +server.webUIChannel = "stable" # "bundled" (the version bundled with the server release), "stable" or "preview" - the webUI version that should be used +server.webUIUpdateCheckInterval = 23 # time in hours - 0 to disable auto update - range: 1 <= n < 24 - default 23 hours - how often the server should check for webUI updates + # downloader server.downloadAsCbz = false -server.autoDownloadNewChapters = false -server.excludeEntryWithUnreadChapters = true -server.autoDownloadNewChaptersLimit = 0 +server.downloadsPath = "" +server.autoDownloadNewChapters = false # if new chapters that have been retrieved should get automatically downloaded +server.excludeEntryWithUnreadChapters = true # ignore automatic chapter downloads of entries with unread chapters +server.autoDownloadNewChaptersLimit = 0 # 0 to disable it - how many unread downloaded chapters should be available - if the limit is reached, new chapters won't be downloaded automatically. this limit will also be applied to the auto download of new chapters on an update +server.autoDownloadIgnoreReUploads = false # decides if re-uploads should be ignored during auto download of new chapters + +# extension repos +server.extensionRepos = [ + # an example: https://github.com/MY_ACCOUNT/MY_REPO/tree/repo +] # requests -server.maxSourcesInParallel = 10 +server.maxSourcesInParallel = 6 # range: 1 <= n <= 20 - default: 6 - sets how many sources can do requests (updates, downloads) in parallel. updates/downloads are grouped by source and all mangas of a source are updated/downloaded synchronously # updater server.excludeUnreadChapters = true server.excludeNotStarted = true server.excludeCompleted = true -server.globalUpdateInterval = 12 -server.updateMangas = false +server.globalUpdateInterval = 12 # time in hours - 0 to disable it - (doesn't have to be full hours e.g. 12.5) - range: 6 <= n < ∞ - default: 12 hours - interval in which the global update will be automatically triggered +server.updateMangas = false # if the mangas should be updated along with the chapter list during a library/category update + +# Authentication +server.basicAuthEnabled = false +server.basicAuthUsername = "" +server.basicAuthPassword = "" # misc -server.debugLogsEnabled = true -server.gqlDebugLogsEnabled = false -server.systemTrayEnabled = false - -# webUI -server.webUIEnabled = true -server.initialOpenInBrowserEnabled = true -server.webUIInterface = "browser" # "browser" or "electron" -server.electronPath = "" -server.webUIChannel = "stable" -server.webUIUpdateCheckInterval = 24 +server.debugLogsEnabled = false +server.gqlDebugLogsEnabled = false # this includes logs with non privacy safe information +server.systemTrayEnabled = true # backup server.backupPath = "" -server.backupTime = "00:00" -server.backupInterval = 1 -server.backupTTL = 14 +server.backupTime = "00:00" # range: hour: 0-23, minute: 0-59 - default: "00:00" - time of day at which the automated backup should be triggered +server.backupInterval = 1 # time in days - 0 to disable it - range: 1 <= n < ∞ - default: 1 day - interval in which the server will automatically create a backup +server.backupTTL = 14 # time in days - 0 to disable it - range: 1 <= n < ∞ - default: 14 days - how long backup files will be kept before they will get deleted # local source server.localSourcePath = "" + +# Cloudflare bypass +server.flareSolverrEnabled = false +server.flareSolverrUrl = "http://localhost:8191" +server.flareSolverrTimeout = 60 # time in seconds +server.flareSolverrSessionName = "suwayomi" +server.flareSolverrSessionTtl = 15 # time in minutes