Feature/auto download of new chapters improve handling of unhandable reuploads (#921)

* Update test/server-reference file

* Properly handle re-uploaded chapters in auto download of new chapters

In case of unhandable re-uploaded chapters (different chapter numbers) they potentially would have prevented auto downloads due being considered as unread.

Additionally, they would not have been considered to get downloaded due to not having a higher chapter number than the previous latest existing chapter before the chapter list fetch.

* Add option to ignore re-uploads for auto downloads

* Extract check for manga category download inclusion

* Extract logic to get new chapter ids to download

* Simplify manga category download inclusion check

In case the DEFAULT category does not exist, someone messed with the database and it is basically corrupted

server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SettingsMutation.kt

@@ -58,6 +58,7 @@ class SettingsMutation {
         updateSetting(settings.excludeEntryWithUnreadChapters, serverConfig.excludeEntryWithUnreadChapters)
         updateSetting(settings.autoDownloadAheadLimit, serverConfig.autoDownloadNewChaptersLimit) // deprecated
         updateSetting(settings.autoDownloadNewChaptersLimit, serverConfig.autoDownloadNewChaptersLimit)
+        updateSetting(settings.autoDownloadIgnoreReUploads, serverConfig.autoDownloadIgnoreReUploads)
 
         // extension
         updateSetting(settings.extensionRepos, serverConfig.extensionRepos)

server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SettingsType.kt

@@ -49,6 +49,7 @@ interface Settings : Node {
     )
     val autoDownloadAheadLimit: Int?
     val autoDownloadNewChaptersLimit: Int?
+    val autoDownloadIgnoreReUploads: Boolean?
 
     // extension
     val extensionRepos: List<String>?
@@ -118,6 +119,7 @@ data class PartialSettingsType(
     )
     override val autoDownloadAheadLimit: Int?,
     override val autoDownloadNewChaptersLimit: Int?,
+    override val autoDownloadIgnoreReUploads: Boolean?,
     // extension
     override val extensionRepos: List<String>?,
     // requests
@@ -179,6 +181,7 @@ class SettingsType(
     )
     override val autoDownloadAheadLimit: Int,
     override val autoDownloadNewChaptersLimit: Int,
+    override val autoDownloadIgnoreReUploads: Boolean?,
     // extension
     override val extensionRepos: List<String>,
     // requests
@@ -235,6 +238,7 @@ class SettingsType(
         config.excludeEntryWithUnreadChapters.value,
         config.autoDownloadNewChaptersLimit.value, // deprecated
         config.autoDownloadNewChaptersLimit.value,
+        config.autoDownloadIgnoreReUploads.value,
         // extension
         config.extensionRepos.value,
         // requests

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -20,7 +20,6 @@ import kotlinx.serialization.Serializable
 import mu.KotlinLogging
 import org.jetbrains.exposed.dao.id.EntityID
 import org.jetbrains.exposed.sql.Op
-import org.jetbrains.exposed.sql.ResultRow
 import org.jetbrains.exposed.sql.SortOrder
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList
 import org.jetbrains.exposed.sql.and
@@ -37,7 +36,6 @@ import suwayomi.tachidesk.manga.impl.download.DownloadManager.EnqueueInput
 import suwayomi.tachidesk.manga.impl.track.Track
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
 import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
-import suwayomi.tachidesk.manga.model.dataclass.IncludeOrExclude
 import suwayomi.tachidesk.manga.model.dataclass.MangaChapterDataClass
 import suwayomi.tachidesk.manga.model.dataclass.PaginatedList
 import suwayomi.tachidesk.manga.model.dataclass.paginatedFrom
@@ -136,6 +134,7 @@ object Chapter {
                         url = manga.url
                     }
 
+                val currentLatestChapterNumber = Manga.getLatestChapter(mangaId)?.chapterNumber ?: 0f
                 val numberOfCurrentChapters = getCountOfMangaChapters(mangaId)
                 val chapterList = source.getChapterList(sManga)
 
@@ -164,7 +163,10 @@ object Chapter {
                             .toList()
                     }
 
-                val chaptersToInsert = mutableListOf<ChapterDataClass>()
+                // new chapters after they have been added to the database for auto downloads
+                val insertedChapters = mutableListOf<ChapterDataClass>()
+
+                val chaptersToInsert = mutableListOf<ChapterDataClass>() // do not yet have an ID from the database
                 val chaptersToUpdate = mutableListOf<ChapterDataClass>()
 
                 chapterList.reversed().forEachIndexed { index, fetchedChapter ->
@@ -260,7 +262,7 @@ object Chapter {
                                     this[ChapterTable.fetchedAt] = it
                                 }
                             }
-                        }
+                        }.forEach { insertedChapters.add(ChapterTable.toDataClass(it)) }
                     }
 
                     if (chaptersToUpdate.isNotEmpty()) {
@@ -283,14 +285,8 @@ object Chapter {
                     }
                 }
 
-                val newChapters =
-                    transaction {
-                        ChapterTable.select { ChapterTable.manga eq mangaId }
-                            .orderBy(ChapterTable.sourceOrder to SortOrder.DESC).toList()
-                    }
-
                 if (manga.inLibrary) {
-                    downloadNewChapters(mangaId, numberOfCurrentChapters, newChapters)
+                    downloadNewChapters(mangaId, currentLatestChapterNumber, numberOfCurrentChapters, insertedChapters)
                 }
 
                 chapterList
@@ -301,16 +297,19 @@ object Chapter {
 
     private fun downloadNewChapters(
         mangaId: Int,
+        prevLatestChapterNumber: Float,
         prevNumberOfChapters: Int,
-        updatedChapterList: List<ResultRow>,
+        newChapters: List<ChapterDataClass>,
     ) {
         val log =
             KotlinLogging.logger(
                 "${logger.name}::downloadNewChapters(" +
                     "mangaId= $mangaId, " +
+                    "prevLatestChapterNumber= $prevLatestChapterNumber, " +
                     "prevNumberOfChapters= $prevNumberOfChapters, " +
-                    "updatedChapterList= ${updatedChapterList.size}, " +
+                    "newChapters= ${newChapters.size}, " +
-                    "autoDownloadNewChaptersLimit= ${serverConfig.autoDownloadNewChaptersLimit.value}" +
+                    "autoDownloadNewChaptersLimit= ${serverConfig.autoDownloadNewChaptersLimit.value}, " +
+                    "autoDownloadIgnoreReUploads= ${serverConfig.autoDownloadIgnoreReUploads.value}" +
                     ")",
             )
 
@@ -319,68 +318,22 @@ object Chapter {
             return
         }
 
-        // Only download if there are new chapters, or if this is the first fetch
+        if (newChapters.isEmpty()) {
-        val newNumberOfChapters = updatedChapterList.size
-        val numberOfNewChapters = newNumberOfChapters - prevNumberOfChapters
-
-        val areNewChaptersAvailable = numberOfNewChapters > 0
-        val wasInitialFetch = prevNumberOfChapters == 0
-
-        if (!areNewChaptersAvailable) {
             log.debug { "no new chapters available" }
             return
         }
 
+        val wasInitialFetch = prevNumberOfChapters == 0
         if (wasInitialFetch) {
             log.debug { "skipping download on initial fetch" }
             return
         }
 
-        // Verify the manga is configured to be downloaded based on it's categories.
+        if (!Manga.isInIncludedDownloadCategory(log, mangaId)) {
-        var mangaCategories = CategoryManga.getMangaCategories(mangaId).toSet()
+            return
-        // if the manga has no categories, then it's implicitly in the default category
-        if (mangaCategories.isEmpty()) {
-            val defaultCategory = Category.getCategoryById(Category.DEFAULT_CATEGORY_ID)
-            if (defaultCategory != null) {
-                mangaCategories = setOf(defaultCategory)
-            } else {
-                log.warn { "missing default category" }
-            }
         }
 
-        if (mangaCategories.isNotEmpty()) {
+        val unreadChapters = Manga.getUnreadChapters(mangaId).subtract(newChapters.toSet())
-            val downloadCategoriesMap = Category.getCategoryList().groupBy { it.includeInDownload }
-            val unsetCategories = downloadCategoriesMap[IncludeOrExclude.UNSET].orEmpty()
-            // We only download if it's in the include list, and not in the exclude list.
-            // Use the unset categories as the included categories if the included categories is
-            // empty
-            val includedCategories = downloadCategoriesMap[IncludeOrExclude.INCLUDE].orEmpty().ifEmpty { unsetCategories }
-            val excludedCategories = downloadCategoriesMap[IncludeOrExclude.EXCLUDE].orEmpty()
-            // Only download manga that aren't in any excluded categories
-            val mangaExcludeCategories = mangaCategories.intersect(excludedCategories.toSet())
-            if (mangaExcludeCategories.isNotEmpty()) {
-                log.debug { "download excluded by categories: '${mangaExcludeCategories.joinToString("', '") { it.name }}'" }
-                return
-            }
-            val mangaDownloadCategories = mangaCategories.intersect(includedCategories.toSet())
-            if (mangaDownloadCategories.isNotEmpty()) {
-                log.debug { "download inluded by categories: '${mangaDownloadCategories.joinToString("', '") { it.name }}'" }
-            } else {
-                log.debug { "skipping download due to download categories configuration" }
-                return
-            }
-        } else {
-            log.debug { "no categories configured, skipping check for category download include/excludes" }
-        }
-
-        val newChapters = updatedChapterList.subList(0, numberOfNewChapters)
-
-        // make sure to only consider the latest chapters. e.g. old unread chapters should be ignored
-        val latestReadChapterIndex =
-            updatedChapterList.indexOfFirst { it[ChapterTable.isRead] }.takeIf { it > -1 } ?: (updatedChapterList.size)
-        val unreadChapters =
-            updatedChapterList.subList(numberOfNewChapters, latestReadChapterIndex)
-                .filter { !it[ChapterTable.isRead] }
 
         val skipDueToUnreadChapters = serverConfig.excludeEntryWithUnreadChapters.value && unreadChapters.isNotEmpty()
         if (skipDueToUnreadChapters) {
@@ -388,17 +341,7 @@ object Chapter {
             return
         }
 
-        val firstChapterToDownloadIndex =
+        val chapterIdsToDownload = getNewChapterIdsToDownload(newChapters, prevLatestChapterNumber)
-            if (serverConfig.autoDownloadNewChaptersLimit.value > 0) {
-                (numberOfNewChapters - serverConfig.autoDownloadNewChaptersLimit.value).coerceAtLeast(0)
-            } else {
-                0
-            }
-
-        val chapterIdsToDownload =
-            newChapters.subList(firstChapterToDownloadIndex, numberOfNewChapters)
-                .filter { !it[ChapterTable.isRead] && !it[ChapterTable.isDownloaded] }
-                .map { it[ChapterTable.id].value }
 
         if (chapterIdsToDownload.isEmpty()) {
             log.debug { "no chapters available for download" }
@@ -410,6 +353,29 @@ object Chapter {
         DownloadManager.enqueue(EnqueueInput(chapterIdsToDownload))
     }
 
+    private fun getNewChapterIdsToDownload(
+        newChapters: List<ChapterDataClass>,
+        prevLatestChapterNumber: Float,
+    ): List<Int> {
+        val reUploadedChapters = newChapters.filter { it.chapterNumber < prevLatestChapterNumber }
+        val actualNewChapters = newChapters.subtract(reUploadedChapters.toSet())
+        val chaptersToConsiderForDownloadLimit =
+            if (serverConfig.autoDownloadIgnoreReUploads.value) {
+                actualNewChapters
+            } else {
+                newChapters
+            }.sortedBy { it.index }
+
+        val latestChapterToDownloadIndex =
+            if (serverConfig.autoDownloadNewChaptersLimit.value == 0) {
+                chaptersToConsiderForDownloadLimit.size
+            } else {
+                serverConfig.autoDownloadNewChaptersLimit.value.coerceAtMost(chaptersToConsiderForDownloadLimit.size)
+            }
+
+        return chaptersToConsiderForDownloadLimit.subList(0, latestChapterToDownloadIndex).map { it.id }
+    }
+
     fun modifyChapter(
         mangaId: Int,
         chapterIndex: Int,

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt

@@ -16,6 +16,7 @@ import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import eu.kanade.tachiyomi.source.online.HttpSource
 import io.javalin.http.HttpCode
+import mu.KLogger
 import mu.KotlinLogging
 import okhttp3.CacheControl
 import okhttp3.Response
@@ -41,6 +42,8 @@ import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.clearCachedImage
 import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
 import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
 import suwayomi.tachidesk.manga.impl.util.updateMangaDownloadDir
+import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
+import suwayomi.tachidesk.manga.model.dataclass.IncludeOrExclude
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
 import suwayomi.tachidesk.manga.model.dataclass.toGenreList
 import suwayomi.tachidesk.manga.model.table.ChapterTable
@@ -366,4 +369,56 @@ object Manga {
         clearCachedImage(applicationDirs.tempThumbnailCacheRoot, fileName)
         clearCachedImage(applicationDirs.thumbnailDownloadsRoot, fileName)
     }
+
+    fun getLatestChapter(mangaId: Int): ChapterDataClass? {
+        return transaction {
+            ChapterTable.select { ChapterTable.manga eq mangaId }.maxByOrNull { it[ChapterTable.sourceOrder] }
+        }?.let { ChapterTable.toDataClass(it) }
+    }
+
+    fun getUnreadChapters(mangaId: Int): List<ChapterDataClass> {
+        return transaction {
+            ChapterTable.select { (ChapterTable.manga eq mangaId) and (ChapterTable.isRead eq false) }
+                .orderBy(ChapterTable.sourceOrder to SortOrder.DESC)
+                .map { ChapterTable.toDataClass(it) }
+        }
+    }
+
+    fun isInIncludedDownloadCategory(
+        logContext: KLogger = logger,
+        mangaId: Int,
+    ): Boolean {
+        val log = KotlinLogging.logger { "${logContext.name}::isInExcludedDownloadCategory($mangaId)" }
+
+        // Verify the manga is configured to be downloaded based on it's categories.
+        var mangaCategories = CategoryManga.getMangaCategories(mangaId).toSet()
+        // if the manga has no categories, then it's implicitly in the default category
+        if (mangaCategories.isEmpty()) {
+            val defaultCategory = Category.getCategoryById(Category.DEFAULT_CATEGORY_ID)!!
+            mangaCategories = setOf(defaultCategory)
+        }
+
+        val downloadCategoriesMap = Category.getCategoryList().groupBy { it.includeInDownload }
+        val unsetCategories = downloadCategoriesMap[IncludeOrExclude.UNSET].orEmpty()
+        // We only download if it's in the include list, and not in the exclude list.
+        // Use the unset categories as the included categories if the included categories is
+        // empty
+        val includedCategories = downloadCategoriesMap[IncludeOrExclude.INCLUDE].orEmpty().ifEmpty { unsetCategories }
+        val excludedCategories = downloadCategoriesMap[IncludeOrExclude.EXCLUDE].orEmpty()
+        // Only download manga that aren't in any excluded categories
+        val mangaExcludeCategories = mangaCategories.intersect(excludedCategories.toSet())
+        if (mangaExcludeCategories.isNotEmpty()) {
+            log.debug { "download excluded by categories: '${mangaExcludeCategories.joinToString("', '") { it.name }}'" }
+            return false
+        }
+        val mangaDownloadCategories = mangaCategories.intersect(includedCategories.toSet())
+        if (mangaDownloadCategories.isNotEmpty()) {
+            log.debug { "download inluded by categories: '${mangaDownloadCategories.joinToString("', '") { it.name }}'" }
+        } else {
+            log.debug { "skipping download due to download categories configuration" }
+            return false
+        }
+
+        return true
+    }
 }

server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt

@@ -101,6 +101,7 @@ class ServerConfig(getConfig: () -> Config, val moduleName: String = SERVER_CONF
     val autoDownloadNewChapters: MutableStateFlow<Boolean> by OverrideConfigValue(BooleanConfigAdapter)
     val excludeEntryWithUnreadChapters: MutableStateFlow<Boolean> by OverrideConfigValue(BooleanConfigAdapter)
     val autoDownloadNewChaptersLimit: MutableStateFlow<Int> by OverrideConfigValue(IntConfigAdapter)
+    val autoDownloadIgnoreReUploads: MutableStateFlow<Boolean> by OverrideConfigValue(BooleanConfigAdapter)
 
     // extensions
     val extensionRepos: MutableStateFlow<List<String>> by OverrideConfigValues(StringConfigAdapter)

server/src/main/resources/server-reference.conf

@@ -25,6 +25,7 @@ server.downloadsPath = ""
 server.autoDownloadNewChapters = false # if new chapters that have been retrieved should get automatically downloaded
 server.excludeEntryWithUnreadChapters = true # ignore automatic chapter downloads of entries with unread chapters
 server.autoDownloadNewChaptersLimit = 0 # 0 to disable it - how many unread downloaded chapters should be available - if the limit is reached, new chapters won't be downloaded automatically. this limit will also be applied to the auto download of new chapters on an update
+server.autoDownloadIgnoreReUploads = false # decides if re-uploads should be ignored during auto download of new chapters
 
 # extension repos
 server.extensionRepos = [

server/src/test/resources/server-reference.conf

@@ -10,40 +10,60 @@ server.socksProxyPort = ""
 server.socksProxyUsername = ""
 server.socksProxyPassword = ""
 
+# webUI
+server.webUIEnabled = true
+server.webUIFlavor = "WebUI" # "WebUI", "VUI" or "Custom"
+server.initialOpenInBrowserEnabled = true
+server.webUIInterface = "browser" # "browser" or "electron"
+server.electronPath = ""
+server.webUIChannel = "stable" # "bundled" (the version bundled with the server release), "stable" or "preview" - the webUI version that should be used
+server.webUIUpdateCheckInterval = 23 # time in hours - 0 to disable auto update - range: 1 <= n < 24 - default 23 hours - how often the server should check for webUI updates
+
 # downloader
 server.downloadAsCbz = false
-server.autoDownloadNewChapters = false
+server.downloadsPath = ""
-server.excludeEntryWithUnreadChapters = true
+server.autoDownloadNewChapters = false # if new chapters that have been retrieved should get automatically downloaded
-server.autoDownloadNewChaptersLimit = 0
+server.excludeEntryWithUnreadChapters = true # ignore automatic chapter downloads of entries with unread chapters
+server.autoDownloadNewChaptersLimit = 0 # 0 to disable it - how many unread downloaded chapters should be available - if the limit is reached, new chapters won't be downloaded automatically. this limit will also be applied to the auto download of new chapters on an update
+server.autoDownloadIgnoreReUploads = false # decides if re-uploads should be ignored during auto download of new chapters
+
+# extension repos
+server.extensionRepos = [
+  # an example: https://github.com/MY_ACCOUNT/MY_REPO/tree/repo
+]
 
 # requests
-server.maxSourcesInParallel = 10
+server.maxSourcesInParallel = 6 # range: 1 <= n <= 20 - default: 6 - sets how many sources can do requests (updates, downloads) in parallel. updates/downloads are grouped by source and all mangas of a source are updated/downloaded synchronously
 
 # updater
 server.excludeUnreadChapters = true
 server.excludeNotStarted = true
 server.excludeCompleted = true
-server.globalUpdateInterval = 12
+server.globalUpdateInterval = 12 # time in hours - 0 to disable it - (doesn't have to be full hours e.g. 12.5) - range: 6 <= n < ∞ - default: 12 hours - interval in which the global update will be automatically triggered
-server.updateMangas = false
+server.updateMangas = false # if the mangas should be updated along with the chapter list during a library/category update
+
+# Authentication
+server.basicAuthEnabled = false
+server.basicAuthUsername = ""
+server.basicAuthPassword = ""
 
 # misc
-server.debugLogsEnabled = true
+server.debugLogsEnabled = false
-server.gqlDebugLogsEnabled = false
+server.gqlDebugLogsEnabled = false # this includes logs with non privacy safe information
-server.systemTrayEnabled = false
+server.systemTrayEnabled = true
-
-# webUI
-server.webUIEnabled = true
-server.initialOpenInBrowserEnabled = true
-server.webUIInterface = "browser" # "browser" or "electron"
-server.electronPath = ""
-server.webUIChannel = "stable"
-server.webUIUpdateCheckInterval = 24
 
 # backup
 server.backupPath = ""
-server.backupTime = "00:00"
+server.backupTime = "00:00" # range: hour: 0-23, minute: 0-59 - default: "00:00" - time of day at which the automated backup should be triggered
-server.backupInterval = 1
+server.backupInterval = 1 # time in days - 0 to disable it - range: 1 <= n < ∞ - default: 1 day - interval in which the server will automatically create a backup
-server.backupTTL = 14
+server.backupTTL = 14 # time in days - 0 to disable it - range: 1 <= n < ∞ - default: 14 days - how long backup files will be kept before they will get deleted
 
 # local source
 server.localSourcePath = ""
+
+# Cloudflare bypass
+server.flareSolverrEnabled = false
+server.flareSolverrUrl = "http://localhost:8191"
+server.flareSolverrTimeout = 60 # time in seconds
+server.flareSolverrSessionName = "suwayomi"
+server.flareSolverrSessionTtl = 15 # time in minutes