Feature/global update trigger automatically (#593)

* Move "addCategoriesToUpdateQueue" to "Updater"

* Automatically trigger the global update

AndroidCompat/Config/src/main/java/xyz/nulldev/ts/config/ConfigModule.kt

@@ -41,6 +41,7 @@ class SystemPropertyOverrideDelegate(val getConfig: () -> Config, val moduleName
         return when (T::class.simpleName) {
             "Int" -> combined.toInt()
             "Boolean" -> combined.toBoolean()
+            "Double" -> combined.toDouble()
             // add more types as needed
             else -> combined // covers String
         } as T

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt

@@ -1,6 +1,5 @@
 package suwayomi.tachidesk.manga.controller
 
-import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import io.javalin.http.HttpCode
 import io.javalin.websocket.WsConfig
 import mu.KotlinLogging
@@ -8,19 +7,13 @@ import org.kodein.di.DI
 import org.kodein.di.conf.global
 import org.kodein.di.instance
 import suwayomi.tachidesk.manga.impl.Category
-import suwayomi.tachidesk.manga.impl.CategoryManga
 import suwayomi.tachidesk.manga.impl.Chapter
 import suwayomi.tachidesk.manga.impl.update.IUpdater
 import suwayomi.tachidesk.manga.impl.update.UpdateStatus
 import suwayomi.tachidesk.manga.impl.update.UpdaterSocket
-import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
-import suwayomi.tachidesk.manga.model.dataclass.IncludeInUpdate
 import suwayomi.tachidesk.manga.model.dataclass.MangaChapterDataClass
-import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
 import suwayomi.tachidesk.manga.model.dataclass.PaginatedList
-import suwayomi.tachidesk.manga.model.table.MangaStatus
 import suwayomi.tachidesk.server.JavalinSetup.future
-import suwayomi.tachidesk.server.serverConfig
 import suwayomi.tachidesk.server.util.formParam
 import suwayomi.tachidesk.server.util.handler
 import suwayomi.tachidesk.server.util.pathParam
@@ -72,13 +65,14 @@ object UpdateController {
             }
         },
         behaviorOf = { ctx, categoryId ->
+            val updater by DI.global.instance<IUpdater>()
             if (categoryId == null) {
                 logger.info { "Adding Library to Update Queue" }
-                addCategoriesToUpdateQueue(Category.getCategoryList(), true)
+                updater.addCategoriesToUpdateQueue(Category.getCategoryList(), true)
             } else {
                 val category = Category.getCategoryById(categoryId)
                 if (category != null) {
-                    addCategoriesToUpdateQueue(listOf(category), true)
+                    updater.addCategoriesToUpdateQueue(listOf(category), true)
                 } else {
                     logger.info { "No Category found" }
                     ctx.status(HttpCode.BAD_REQUEST)
@@ -91,45 +85,6 @@ object UpdateController {
         }
     )
 
-    private fun addCategoriesToUpdateQueue(categories: List<CategoryDataClass>, clear: Boolean = false) {
-        val updater by DI.global.instance<IUpdater>()
-        if (clear) {
-            updater.reset()
-        }
-
-        val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate }
-        val excludedCategories = includeInUpdateStatusToCategoryMap[IncludeInUpdate.EXCLUDE].orEmpty()
-        val includedCategories = includeInUpdateStatusToCategoryMap[IncludeInUpdate.INCLUDE].orEmpty()
-        val unsetCategories = includeInUpdateStatusToCategoryMap[IncludeInUpdate.UNSET].orEmpty()
-        val categoriesToUpdate = includedCategories.ifEmpty { unsetCategories }
-
-        logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" }
-
-        val categoriesToUpdateMangas = categoriesToUpdate
-            .flatMap { CategoryManga.getCategoryMangaList(it.id) }
-            .distinctBy { it.id }
-        val mangasToCategoriesMap = CategoryManga.getMangasCategories(categoriesToUpdateMangas.map { it.id })
-        val mangasToUpdate = categoriesToUpdateMangas
-            .asSequence()
-            .filter { it.updateStrategy == UpdateStrategy.ALWAYS_UPDATE }
-            .filter { if (serverConfig.excludeUnreadChapters) { (it.unreadCount ?: 0L) == 0L } else true }
-            .filter { if (serverConfig.excludeNotStarted) { it.lastReadAt != null } else true }
-            .filter { if (serverConfig.excludeCompleted) { it.status != MangaStatus.COMPLETED.name } else true }
-            .filter { !excludedCategories.any { category -> mangasToCategoriesMap[it.id]?.contains(category) == true } }
-            .toList()
-
-        // In case no manga gets updated and no update job was running before, the client would never receive an info about its update request
-        if (mangasToUpdate.isEmpty()) {
-            UpdaterSocket.notifyAllClients(UpdateStatus())
-            return
-        }
-
-        updater.addMangasToQueue(
-            mangasToUpdate
-                .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title))
-        )
-    }
-
     fun categoryUpdateWS(ws: WsConfig) {
         ws.onConnect { ctx ->
             UpdaterSocket.addClient(ctx)

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt

@@ -1,10 +1,10 @@
 package suwayomi.tachidesk.manga.impl.update
 
 import kotlinx.coroutines.flow.StateFlow
-import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
+import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
 
 interface IUpdater {
-    fun addMangasToQueue(mangas: List<MangaDataClass>)
+    fun addCategoriesToUpdateQueue(categories: List<CategoryDataClass>, clear: Boolean?)
     val status: StateFlow<UpdateStatus>
     fun reset()
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt

@@ -1,5 +1,6 @@
 package suwayomi.tachidesk.manga.impl.update
 
+import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
@@ -17,10 +18,23 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.sync.Semaphore
 import kotlinx.coroutines.sync.withPermit
 import mu.KotlinLogging
+import org.kodein.di.DI
+import org.kodein.di.conf.global
+import org.kodein.di.instance
+import suwayomi.tachidesk.manga.impl.Category
+import suwayomi.tachidesk.manga.impl.CategoryManga
 import suwayomi.tachidesk.manga.impl.Chapter
+import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
+import suwayomi.tachidesk.manga.model.dataclass.IncludeInUpdate
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
+import suwayomi.tachidesk.manga.model.table.MangaStatus
 import suwayomi.tachidesk.server.serverConfig
+import java.util.Date
+import java.util.Timer
+import java.util.TimerTask
 import java.util.concurrent.ConcurrentHashMap
+import java.util.prefs.Preferences
+import kotlin.time.Duration.Companion.hours
 
 class Updater : IUpdater {
     private val logger = KotlinLogging.logger {}
@@ -34,6 +48,46 @@ class Updater : IUpdater {
 
     private val semaphore = Semaphore(serverConfig.maxParallelUpdateRequests)
 
+    private val lastAutomatedUpdateKey = "lastAutomatedUpdateKey"
+    private val preferences = Preferences.userNodeForPackage(Updater::class.java)
+
+    private val updateTimer = Timer()
+    private var currentUpdateTask: TimerTask? = null
+
+    init {
+        scheduleUpdateTask()
+    }
+
+    private fun scheduleUpdateTask() {
+        if (!serverConfig.automaticallyTriggerGlobalUpdate) {
+            return
+        }
+
+        val minInterval = 6.hours
+        val interval = serverConfig.globalUpdateInterval.hours
+        val updateInterval = interval.coerceAtLeast(minInterval).inWholeMilliseconds
+
+        val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0)
+        val initialDelay = updateInterval - (System.currentTimeMillis() - lastAutomatedUpdate) % updateInterval
+
+        currentUpdateTask?.cancel()
+        currentUpdateTask = object : TimerTask() {
+            override fun run() {
+                preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis())
+
+                if (status.value.running) {
+                    logger.debug { "Global update is already in progress, do not trigger global update" }
+                    return
+                }
+
+                logger.info { "Trigger global update (interval= ${serverConfig.globalUpdateInterval}h, lastAutomatedUpdate= ${Date(lastAutomatedUpdate)})" }
+                addCategoriesToUpdateQueue(Category.getCategoryList(), true)
+            }
+        }
+
+        updateTimer.scheduleAtFixedRate(currentUpdateTask, initialDelay, updateInterval)
+    }
+
     private fun getOrCreateUpdateChannelFor(source: String): Channel<UpdateJob> {
         return updateChannels.getOrPut(source) {
             logger.debug { "getOrCreateUpdateChannelFor: created channel for $source - channels: ${updateChannels.size + 1}" }
@@ -74,7 +128,46 @@ class Updater : IUpdater {
         return tracker.values.toList()
     }
 
-    override fun addMangasToQueue(mangas: List<MangaDataClass>) {
+    override fun addCategoriesToUpdateQueue(categories: List<CategoryDataClass>, clear: Boolean?) {
+        val updater by DI.global.instance<IUpdater>()
+        if (clear == true) {
+            updater.reset()
+        }
+
+        val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate }
+        val excludedCategories = includeInUpdateStatusToCategoryMap[IncludeInUpdate.EXCLUDE].orEmpty()
+        val includedCategories = includeInUpdateStatusToCategoryMap[IncludeInUpdate.INCLUDE].orEmpty()
+        val unsetCategories = includeInUpdateStatusToCategoryMap[IncludeInUpdate.UNSET].orEmpty()
+        val categoriesToUpdate = includedCategories.ifEmpty { unsetCategories }
+
+        logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" }
+
+        val categoriesToUpdateMangas = categoriesToUpdate
+            .flatMap { CategoryManga.getCategoryMangaList(it.id) }
+            .distinctBy { it.id }
+        val mangasToCategoriesMap = CategoryManga.getMangasCategories(categoriesToUpdateMangas.map { it.id })
+        val mangasToUpdate = categoriesToUpdateMangas
+            .asSequence()
+            .filter { it.updateStrategy == UpdateStrategy.ALWAYS_UPDATE }
+            .filter { if (serverConfig.excludeUnreadChapters) { (it.unreadCount ?: 0L) == 0L } else true }
+            .filter { if (serverConfig.excludeNotStarted) { it.lastReadAt != null } else true }
+            .filter { if (serverConfig.excludeCompleted) { it.status != MangaStatus.COMPLETED.name } else true }
+            .filter { !excludedCategories.any { category -> mangasToCategoriesMap[it.id]?.contains(category) == true } }
+            .toList()
+
+        // In case no manga gets updated and no update job was running before, the client would never receive an info about its update request
+        if (mangasToUpdate.isEmpty()) {
+            UpdaterSocket.notifyAllClients(UpdateStatus())
+            return
+        }
+
+        addMangasToQueue(
+            mangasToUpdate
+                .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title))
+        )
+    }
+
+    private fun addMangasToQueue(mangas: List<MangaDataClass>) {
         mangas.forEach { tracker[it.id] = UpdateJob(it) }
         _status.update { UpdateStatus(tracker.values.toList(), mangas.isNotEmpty()) }
         mangas.forEach { addMangaToQueue(it) }

server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt

@@ -38,6 +38,8 @@ class ServerConfig(getConfig: () -> Config, moduleName: String = MODULE_NAME) :
     var excludeUnreadChapters: Boolean by overridableConfig
     var excludeNotStarted: Boolean by overridableConfig
     var excludeCompleted: Boolean by overridableConfig
+    var automaticallyTriggerGlobalUpdate: Boolean by overridableConfig
+    var globalUpdateInterval: Double by overridableConfig
 
     // Authentication
     var basicAuthEnabled: Boolean by overridableConfig

server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt

@@ -68,10 +68,12 @@ fun applicationSetup() {
     // Application dirs
     val applicationDirs = ApplicationDirs()
 
+    val updater = Updater()
+
     DI.global.addImport(
         DI.Module("Server") {
             bind<ApplicationDirs>() with singleton { applicationDirs }
-            bind<IUpdater>() with singleton { Updater() }
+            bind<IUpdater>() with singleton { updater }
             bind<JsonMapper>() with singleton { JavalinJackson() }
             bind<Json>() with singleton { Json { ignoreUnknownKeys = true } }
         }

server/src/main/resources/server-reference.conf

@@ -23,6 +23,8 @@ server.maxParallelUpdateRequests = 10 # sets how many sources can be updated in
 server.excludeUnreadChapters = true
 server.excludeNotStarted = true
 server.excludeCompleted = true
+server.automaticallyTriggerGlobalUpdate = false
+server.globalUpdateInterval = 12 # time in hours (doesn't have to be full hours e.g. 12.5) - range: 6 <= n < ∞ - default: 12 hours - interval in which the global update will be automatically triggered
 
 # Authentication
 server.basicAuthEnabled = false

server/src/test/resources/server-reference.conf

@@ -15,6 +15,8 @@ server.maxParallelUpdateRequests = 10
 server.excludeUnreadChapters = true
 server.excludeNotStarted = true
 server.excludeCompleted = true
+server.automaticallyTriggerGlobalUpdate = false
+server.globalUpdateInterval = 12
 
 # misc
 server.debugLogsEnabled = true