Feature/auto download ahead (#681)

* Add "download ahead" mutation

Checks if the specified number of unread chapters, that should be downloaded, are available.
In case not enough chapters are downloaded, the number of missing unread chapters will get downloaded

* Optionally pass the latest read chapter id of a manga

In case a chapter will get marked as read, which also triggered the download ahead call, it's possible, that by the time the download ahead logic gets triggered, the chapter hasn't been marked as read yet.
This could then cause this chapter to be included in the chapters to get downloaded.
By providing the chapter id, this chapter will be used as the latest read chapter instead, and thus, not be included inn the chapters to download.

server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/DownloadMutation.kt

@@ -7,6 +7,7 @@ import org.jetbrains.exposed.sql.transactions.transaction
 import suwayomi.tachidesk.graphql.types.ChapterType
 import suwayomi.tachidesk.graphql.types.DownloadStatus
 import suwayomi.tachidesk.manga.impl.Chapter
+import suwayomi.tachidesk.manga.impl.Manga
 import suwayomi.tachidesk.manga.impl.download.DownloadManager
 import suwayomi.tachidesk.manga.impl.download.model.Status
 import suwayomi.tachidesk.manga.model.table.ChapterTable
@@ -257,4 +258,20 @@ class DownloadMutation {
             )
         }
     }
+
+    data class DownloadAheadInput(
+        val clientMutationId: String? = null,
+        val mangaIds: List<Int> = emptyList(),
+        val latestReadChapterIds: List<Int>? = null
+    )
+
+    data class DownloadAheadPayload(val clientMutationId: String?)
+
+    fun downloadAhead(input: DownloadAheadInput): DownloadAheadPayload {
+        val (clientMutationId, mangaIds, latestReadChapterIds) = input
+
+        Manga.downloadAhead(mangaIds, latestReadChapterIds ?: emptyList())
+
+        return DownloadAheadPayload(clientMutationId)
+    }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt

@@ -13,6 +13,7 @@ import eu.kanade.tachiyomi.source.local.LocalSource
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import eu.kanade.tachiyomi.source.online.HttpSource
+import mu.KotlinLogging
 import org.jetbrains.exposed.sql.ResultRow
 import org.jetbrains.exposed.sql.SortOrder
 import org.jetbrains.exposed.sql.and
@@ -25,6 +26,8 @@ import org.kodein.di.conf.global
 import org.kodein.di.instance
 import suwayomi.tachidesk.manga.impl.MangaList.proxyThumbnailUrl
 import suwayomi.tachidesk.manga.impl.Source.getSource
+import suwayomi.tachidesk.manga.impl.download.DownloadManager
+import suwayomi.tachidesk.manga.impl.download.DownloadManager.EnqueueInput
 import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.MissingThumbnailException
 import suwayomi.tachidesk.manga.impl.util.network.await
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrNull
@@ -42,11 +45,17 @@ import suwayomi.tachidesk.manga.model.table.MangaStatus
 import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
 import suwayomi.tachidesk.server.ApplicationDirs
+import suwayomi.tachidesk.server.serverConfig
 import uy.kohesive.injekt.injectLazy
 import java.io.File
 import java.io.IOException
 import java.io.InputStream
 import java.time.Instant
+import java.util.Timer
+import java.util.TimerTask
+import java.util.concurrent.ConcurrentHashMap
+
+private val logger = KotlinLogging.logger { }
 
 object Manga {
     private fun truncate(text: String?, maxLength: Int): String? {
@@ -307,4 +316,87 @@ object Manga {
         clearCachedImage(applicationDirs.tempThumbnailCacheRoot, fileName)
         clearCachedImage(applicationDirs.thumbnailDownloadsRoot, fileName)
     }
+
+    private val downloadAheadQueue = ConcurrentHashMap<String, ConcurrentHashMap.KeySetView<Int, Boolean>>()
+    private var downloadAheadTimer: Timer? = null
+    fun downloadAhead(mangaIds: List<Int>, latestReadChapterIds: List<Int> = emptyList()) {
+        if (serverConfig.autoDownloadAheadLimit.value == 0) {
+            return
+        }
+
+        val MANGAS_KEY = "mangaIds"
+        val CHAPTERS_KEY = "chapterIds"
+
+        val updateDownloadAheadQueue = { key: String, ids: List<Int> ->
+            val idSet = downloadAheadQueue[key] ?: ConcurrentHashMap.newKeySet()
+            idSet.addAll(ids)
+            downloadAheadQueue[key] = idSet
+        }
+
+        updateDownloadAheadQueue(MANGAS_KEY, mangaIds)
+        updateDownloadAheadQueue(CHAPTERS_KEY, latestReadChapterIds)
+
+        // handle cases where this function gets called multiple times in quick succession.
+        // this could happen in case e.g. multiple chapters get marked as read without batching the operation
+        downloadAheadTimer?.cancel()
+        downloadAheadTimer = Timer().apply {
+            schedule(
+                object : TimerTask() {
+                    override fun run() {
+                        downloadAheadChapters(downloadAheadQueue[MANGAS_KEY]!!.toList(), downloadAheadQueue[CHAPTERS_KEY]!!.toList())
+                        downloadAheadQueue.clear()
+                    }
+                },
+                5000
+            )
+        }
+    }
+
+    /**
+     * Downloads the latest unread and not downloaded chapters for each passed manga id.
+     *
+     * To pass a specific chapter as the latest read chapter for a manga, it can be provided in the "latestReadChapterIds" list.
+     * This makes it possible to handle cases, where the actual latest read chapter isn't marked as read yet.
+     * E.g. the client marks a chapter as read and at the same time sends the "downloadAhead" mutation.
+     * In this case, the latest read chapter could potentially be the one, that just got send to get marked as read by the client.
+     * Without providing it in "latestReadChapterIds" it could be incorrectly included in the chapters, that will get downloaded.
+     *
+     * The latest read chapter will be considered the starting point.
+     * E.g.:
+     * - 20 chapters
+     * - chapter 15 marked as read
+     * - 16 - 20 marked as unread
+     * - 10 - 14 marked as unread
+     *
+     * will download the unread chapters starting from chapter 15
+     */
+    private fun downloadAheadChapters(mangaIds: List<Int>, latestReadChapterIds: List<Int>) {
+        val mangaToLatestReadChapterIndex = transaction {
+            ChapterTable.select { (ChapterTable.manga inList mangaIds) and (ChapterTable.isRead eq true) }
+                .orderBy(ChapterTable.sourceOrder to SortOrder.DESC).groupBy { it[ChapterTable.manga].value }
+        }.mapValues { (_, chapters) -> chapters.firstOrNull()?.let { it[ChapterTable.sourceOrder] } ?: 0 }
+
+        val mangaToUnreadChaptersMap = transaction {
+            ChapterTable.select { (ChapterTable.manga inList mangaIds) and (ChapterTable.isRead eq false) }
+                .orderBy(ChapterTable.sourceOrder to SortOrder.DESC)
+                .groupBy { it[ChapterTable.manga].value }
+        }
+
+        val chapterIdsToDownload = mangaToUnreadChaptersMap.map { (mangaId, unreadChapters) ->
+            val latestReadChapterIndex = mangaToLatestReadChapterIndex[mangaId] ?: 0
+            val lastChapterToDownloadIndex =
+                unreadChapters.indexOfLast { it[ChapterTable.sourceOrder] > latestReadChapterIndex && it[ChapterTable.id].value !in latestReadChapterIds }
+            val unreadChaptersToConsider = unreadChapters.subList(0, lastChapterToDownloadIndex + 1)
+            val firstChapterToDownloadIndex =
+                (unreadChaptersToConsider.size - serverConfig.autoDownloadAheadLimit.value).coerceAtLeast(0)
+            unreadChaptersToConsider.subList(firstChapterToDownloadIndex, lastChapterToDownloadIndex + 1)
+                .filter { !it[ChapterTable.isDownloaded] }
+                .map { it[ChapterTable.id].value }
+        }.flatten()
+
+        logger.info { "downloadAheadChapters: download chapters [${chapterIdsToDownload.joinToString(", ")}]" }
+
+        DownloadManager.dequeue(mangaIds, chapterIdsToDownload)
+        DownloadManager.enqueue(EnqueueInput(chapterIdsToDownload))
+    }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadManager.kt

@@ -329,6 +329,10 @@ object DownloadManager {
         dequeue(downloadQueue.filter { it.mangaId == mangaId && it.chapterIndex == chapterIndex }.toSet())
     }
 
+    fun dequeue(mangaIds: List<Int>, chaptersToIgnore: List<Int> = emptyList()) {
+        dequeue(downloadQueue.filter { it.mangaId in mangaIds && it.chapter.id !in chaptersToIgnore }.toSet())
+    }
+
     private fun dequeue(chapterDownloads: Set<DownloadChapter>) {
         logger.debug { "dequeue ${chapterDownloads.size} chapters [${chapterDownloads.joinToString(separator = ", ") { "$it" }}]" }