Feature/automatically download new chapters (#596)

* Automatically download new chapters

* Log queued downloads

* Add function to get number of manga chapters

server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/ChapterMutation.kt

@@ -122,17 +122,21 @@ class ChapterMutation {
         val (clientMutationId, mangaId) = input
 
         return future {
+            val numberOfCurrentChapters = Chapter.getCountOfMangaChapters(mangaId)
             Chapter.fetchChapterList(mangaId)
-        }.thenApply {
+            numberOfCurrentChapters
+        }.thenApply { numberOfCurrentChapters ->
             val chapters = transaction {
                 ChapterTable.select { ChapterTable.manga eq mangaId }
                     .orderBy(ChapterTable.sourceOrder)
-                    .map { ChapterType(it) }
             }
 
+            // download new chapters if settings flag is enabled
+            Chapter.downloadNewChapters(mangaId, numberOfCurrentChapters, chapters.toList())
+
             FetchChaptersPayload(
                 clientMutationId = clientMutationId,
-                chapters = chapters
+                chapters = chapters.map { ChapterType(it) }
             )
         }
     }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -12,8 +12,10 @@ import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.util.chapter.ChapterRecognition
 import kotlinx.serialization.Serializable
+import mu.KotlinLogging
 import org.jetbrains.exposed.dao.id.EntityID
 import org.jetbrains.exposed.sql.Op
+import org.jetbrains.exposed.sql.ResultRow
 import org.jetbrains.exposed.sql.SortOrder
 import org.jetbrains.exposed.sql.SortOrder.ASC
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
@@ -24,6 +26,8 @@ import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.Manga.getManga
+import suwayomi.tachidesk.manga.impl.download.DownloadManager
+import suwayomi.tachidesk.manga.impl.download.DownloadManager.EnqueueInput
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
 import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
@@ -35,9 +39,12 @@ import suwayomi.tachidesk.manga.model.table.ChapterTable
 import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.PageTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
+import suwayomi.tachidesk.server.serverConfig
 import java.time.Instant
 
 object Chapter {
+    private val logger = KotlinLogging.logger { }
+
     /** get chapter list when showing a manga */
     suspend fun getChapterList(mangaId: Int, onlineFetch: Boolean = false): List<ChapterDataClass> {
         return if (onlineFetch) {
@@ -55,6 +62,10 @@ object Chapter {
         }
     }
 
+    fun getCountOfMangaChapters(mangaId: Int): Int {
+        return transaction { ChapterTable.select { ChapterTable.manga eq mangaId }.count().toInt() }
+    }
+
     private suspend fun getSourceChapters(mangaId: Int): List<ChapterDataClass> {
         val chapterList = fetchChapterList(mangaId)
 
@@ -106,6 +117,7 @@ object Chapter {
             url = manga.url
         }
 
+        val numberOfCurrentChapters = getCountOfMangaChapters(mangaId)
         val chapterList = source.fetchChapterList(sManga).awaitSingle()
 
         // Recognize number for new chapters.
@@ -157,8 +169,13 @@ object Chapter {
             }
         }
 
+        val newChapters = transaction {
+            ChapterTable.select { ChapterTable.manga eq mangaId }
+                .orderBy(ChapterTable.sourceOrder to SortOrder.DESC).toList()
+        }
+
         // clear any orphaned/duplicate chapters that are in the db but not in `chapterList`
-        val dbChapterCount = transaction { ChapterTable.select { ChapterTable.manga eq mangaId }.count() }
+        val dbChapterCount = newChapters.count()
         if (dbChapterCount > chapterList.size) { // we got some clean up due
             val dbChapterList = transaction {
                 ChapterTable.select { ChapterTable.manga eq mangaId }
@@ -179,9 +196,39 @@ object Chapter {
             }
         }
 
+        downloadNewChapters(mangaId, numberOfCurrentChapters, newChapters)
+
         return chapterList
     }
 
+    fun downloadNewChapters(mangaId: Int, prevNumberOfChapters: Int, newChapters: List<ResultRow>) {
+        // convert numbers to be index based
+        val currentNumberOfChapters = (prevNumberOfChapters - 1).coerceAtLeast(0)
+        val updatedNumberOfChapters = (newChapters.size - 1).coerceAtLeast(0)
+
+        val areNewChaptersAvailable = currentNumberOfChapters < updatedNumberOfChapters
+        val wasInitialFetch = currentNumberOfChapters == 0
+
+        // make sure to ignore initial fetch
+        val downloadNewChapters = serverConfig.autoDownloadNewChapters && !wasInitialFetch && areNewChaptersAvailable
+        if (!downloadNewChapters) {
+            return
+        }
+
+        val numberOfNewChapters = updatedNumberOfChapters - currentNumberOfChapters
+        val chapterIdsToDownload = newChapters.subList(0, numberOfNewChapters)
+            .filter { !it[ChapterTable.isRead] && !it[ChapterTable.isDownloaded] }.map { it[ChapterTable.id].value }
+
+        if (chapterIdsToDownload.isEmpty()) {
+            return
+        }
+
+        logger.info { "downloadNewChapters($mangaId): Downloading \"${chapterIdsToDownload.size}\" new chapter(s)..." }
+
+        DownloadManager.enqueue(EnqueueInput(chapterIdsToDownload))
+        DownloadManager.start()
+    }
+
     fun modifyChapter(
         mangaId: Int,
         chapterIndex: Int,

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadManager.kt

@@ -122,7 +122,7 @@ object DownloadManager {
         scope.launch {
             downloaderWatch.sample(1.seconds).collect {
                 val runningDownloaders = downloaders.values.filter { it.isActive }
-                logger.info { "Running: ${runningDownloaders.size}" }
+                logger.info { "Running: ${runningDownloaders.size}, Queued: ${downloadQueue.size}" }
                 if (runningDownloaders.size < MAX_SOURCES_IN_PARAllEL) {
                     downloadQueue.asSequence()
                         .map { it.manga.sourceId.toLong() }

server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt

@@ -32,6 +32,7 @@ class ServerConfig(getConfig: () -> Config, moduleName: String = MODULE_NAME) :
     // downloader
     var downloadAsCbz: Boolean by overridableConfig
     var downloadsPath: String by overridableConfig
+    var autoDownloadNewChapters: Boolean by overridableConfig
 
     // updater
     var maxParallelUpdateRequests: Int by overridableConfig

server/src/main/resources/server-reference.conf

@@ -17,6 +17,7 @@ server.electronPath = ""
 # downloader
 server.downloadAsCbz = false
 server.downloadsPath = ""
+server.autoDownloadNewChapters = false # if new chapters that have been retrieved should get automatically downloaded
 
 # updater
 server.maxParallelUpdateRequests = 10 # sets how many sources can be updated in parallel. updates are grouped by source and all mangas of a source are updated synchronously

server/src/test/resources/server-reference.conf

@@ -9,6 +9,7 @@ server.socksProxyPort = ""
 
 # downloader
 server.downloadAsCbz = false
+server.autoDownloadNewChapters = false
 
 # updater
 server.maxParallelUpdateRequests = 10