rethink image cache (#498)

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/ExtensionController.kt

@@ -15,7 +15,6 @@ import suwayomi.tachidesk.manga.model.dataclass.ExtensionDataClass
 import suwayomi.tachidesk.server.JavalinSetup.future
 import suwayomi.tachidesk.server.util.handler
 import suwayomi.tachidesk.server.util.pathParam
-import suwayomi.tachidesk.server.util.queryParam
 import suwayomi.tachidesk.server.util.withOperation
 
 object ExtensionController {
@@ -141,16 +140,15 @@ object ExtensionController {
     /** icon for extension named `apkName` */
     val icon = handler(
         pathParam<String>("apkName"),
-        queryParam("useCache", true),
         documentWith = {
             withOperation {
                 summary("Extension icon")
                 description("Icon for extension named `apkName`")
             }
         },
-        behaviorOf = { ctx, apkName, useCache ->
+        behaviorOf = { ctx, apkName ->
             ctx.future(
-                future { Extension.getExtensionIcon(apkName, useCache) }
+                future { Extension.getExtensionIcon(apkName) }
                     .thenApply {
                         ctx.header("content-type", it.second)
                         it.first

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/MangaController.kt

@@ -81,16 +81,15 @@ object MangaController {
     /** manga thumbnail */
     val thumbnail = handler(
         pathParam<Int>("mangaId"),
-        queryParam("useCache", true),
         documentWith = {
             withOperation {
                 summary("Get a manga thumbnail")
                 description("Get a manga thumbnail from the source or the cache.")
             }
         },
-        behaviorOf = { ctx, mangaId, useCache ->
+        behaviorOf = { ctx, mangaId ->
             ctx.future(
-                future { Manga.getMangaThumbnail(mangaId, useCache) }
+                future { Manga.getMangaThumbnail(mangaId) }
                     .thenApply {
                         ctx.header("content-type", it.second)
                         val httpCacheSeconds = 1.days.inWholeSeconds
@@ -375,16 +374,15 @@ object MangaController {
         pathParam<Int>("mangaId"),
         pathParam<Int>("chapterIndex"),
         pathParam<Int>("index"),
-        queryParam("useCache", true),
         documentWith = {
             withOperation {
                 summary("Get a chapter page")
                 description("Get a chapter page for a given index. Cache use can be disabled so it only retrieves it directly from the source.")
             }
         },
-        behaviorOf = { ctx, mangaId, chapterIndex, index, useCache ->
+        behaviorOf = { ctx, mangaId, chapterIndex, index ->
             ctx.future(
-                future { Page.getPageImage(mangaId, chapterIndex, index, useCache) }
+                future { Page.getPageImage(mangaId, chapterIndex, index) }
                     .thenApply {
                         ctx.header("content-type", it.second)
                         val httpCacheSeconds = 1.days.inWholeSeconds

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -23,7 +23,6 @@ import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.Manga.getManga
-import suwayomi.tachidesk.manga.impl.util.getChapterDir
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
 import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
@@ -36,7 +35,6 @@ import suwayomi.tachidesk.manga.model.table.ChapterTable.scanlator
 import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.PageTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
-import java.io.File
 import java.time.Instant
 
 object Chapter {
@@ -345,8 +343,7 @@ object Chapter {
                     .forEach { row ->
                         val chapterMangaId = row[ChapterTable.manga].value
                         val chapterId = row[ChapterTable.id].value
-                        val chapterDir = getChapterDir(chapterMangaId, chapterId)
-                        File(chapterDir).deleteRecursively()
+                        ChapterDownloadHelper.delete(chapterMangaId, chapterId)
                     }
 
                 ChapterTable.update({ ChapterTable.id inList chapterIds }) {
@@ -359,8 +356,8 @@ object Chapter {
                     .select { (ChapterTable.sourceOrder inList input.chapterIndexes) and (ChapterTable.manga eq mangaId) }
                     .map { row ->
                         val chapterId = row[ChapterTable.id].value
-                        val chapterDir = getChapterDir(mangaId, chapterId)
-                        File(chapterDir).deleteRecursively()
+                        ChapterDownloadHelper.delete(mangaId, chapterId)
+
                         chapterId
                     }
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt

@@ -31,7 +31,7 @@ import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogue
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
 import suwayomi.tachidesk.manga.impl.util.source.StubSource
 import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.clearCachedImage
-import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse
 import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
 import suwayomi.tachidesk.manga.impl.util.updateMangaDownloadDir
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
@@ -91,7 +91,7 @@ object Manga {
                     if (!sManga.thumbnail_url.isNullOrEmpty() && sManga.thumbnail_url != mangaEntry[MangaTable.thumbnail_url]) {
                         it[MangaTable.thumbnail_url] = sManga.thumbnail_url
                         it[MangaTable.thumbnailUrlLastFetched] = Instant.now().epochSecond
-                        clearMangaThumbnail(mangaId)
+                        clearMangaThumbnailCache(mangaId)
                     }
 
                     it[MangaTable.realUrl] = runCatching {
@@ -225,15 +225,15 @@ object Manga {
 
     private val applicationDirs by DI.global.instance<ApplicationDirs>()
     private val network: NetworkHelper by injectLazy()
-    suspend fun getMangaThumbnail(mangaId: Int, useCache: Boolean): Pair<InputStream, String> {
-        val saveDir = applicationDirs.thumbnailsRoot
+    suspend fun getMangaThumbnail(mangaId: Int): Pair<InputStream, String> {
+        val cacheSaveDir = applicationDirs.thumbnailsRoot
         val fileName = mangaId.toString()
 
         val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
         val sourceId = mangaEntry[MangaTable.sourceReference]
 
         return when (val source = getCatalogueSourceOrStub(sourceId)) {
-            is HttpSource -> getImageResponse(saveDir, fileName, useCache) {
+            is HttpSource -> getCachedImageResponse(cacheSaveDir, fileName) {
                 val thumbnailUrl = mangaEntry[MangaTable.thumbnail_url]
                     ?: if (!mangaEntry[MangaTable.initialized]) {
                         // initialize then try again
@@ -265,7 +265,7 @@ object Manga {
                 imageFile.inputStream() to contentType
             }
 
-            is StubSource -> getImageResponse(saveDir, fileName, useCache) {
+            is StubSource -> getCachedImageResponse(cacheSaveDir, fileName) {
                 val thumbnailUrl = mangaEntry[MangaTable.thumbnail_url]
                     ?: throw NullPointerException("No thumbnail found")
                 network.client.newCall(
@@ -277,7 +277,7 @@ object Manga {
         }
     }
 
-    private fun clearMangaThumbnail(mangaId: Int) {
+    private fun clearMangaThumbnailCache(mangaId: Int) {
         val saveDir = applicationDirs.thumbnailsRoot
         val fileName = mangaId.toString()
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt

@@ -15,9 +15,10 @@ import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
+import suwayomi.tachidesk.manga.impl.util.getChapterCachePath
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
-import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse
 import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
 import suwayomi.tachidesk.manga.model.table.ChapterTable
 import suwayomi.tachidesk.manga.model.table.MangaTable
@@ -37,7 +38,7 @@ object Page {
         return page.imageUrl!!
     }
 
-    suspend fun getPageImage(mangaId: Int, chapterIndex: Int, index: Int, useCache: Boolean = true, progressFlow: ((StateFlow<Int>) -> Unit)? = null): Pair<InputStream, String> {
+    suspend fun getPageImage(mangaId: Int, chapterIndex: Int, index: Int, progressFlow: ((StateFlow<Int>) -> Unit)? = null): Pair<InputStream, String> {
         val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
         val source = getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference])
         val chapterEntry = transaction {
@@ -87,7 +88,10 @@ object Page {
             return ChapterDownloadHelper.getImage(mangaId, chapterId, index)
         }
 
-        return getImageResponse(mangaId, chapterId, fileName, useCache) {
+        val cacheSaveDir = getChapterCachePath(mangaId, chapterId)
+
+        // Note: don't care about invalidating cache because OS cache is not permanent
+        return getCachedImageResponse(cacheSaveDir, fileName) {
             source.fetchImage(tachiyomiPage).awaitSingle()
         }
     }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt

@@ -16,7 +16,7 @@ import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.Page.getPageName
-import suwayomi.tachidesk.manga.impl.util.getChapterDir
+import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath
 import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
 import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse
@@ -133,7 +133,7 @@ private class ChapterForDownload(
     private fun firstPageExists(): Boolean {
         val chapterId = chapterEntry[ChapterTable.id].value
 
-        val chapterDir = getChapterDir(mangaId, chapterId)
+        val chapterDir = getChapterDownloadPath(mangaId, chapterId)
 
         println(chapterDir)
         println(getPageName(0))

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt

@@ -10,7 +10,7 @@ import kotlinx.coroutines.flow.sample
 import suwayomi.tachidesk.manga.impl.Page
 import suwayomi.tachidesk.manga.impl.Page.getPageName
 import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter
-import suwayomi.tachidesk.manga.impl.util.getChapterDir
+import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath
 import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse
 import java.io.File
 import java.io.FileInputStream
@@ -21,7 +21,7 @@ import java.io.InputStream
 * */
 class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(mangaId, chapterId) {
     override fun getImage(index: Int): Pair<InputStream, String> {
-        val chapterDir = getChapterDir(mangaId, chapterId)
+        val chapterDir = getChapterDownloadPath(mangaId, chapterId)
         val folder = File(chapterDir)
         folder.mkdirs()
         val file = folder.listFiles()?.get(index)
@@ -36,7 +36,7 @@ class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(man
         step: suspend (DownloadChapter?, Boolean) -> Unit
     ): Boolean {
         val pageCount = download.chapter.pageCount
-        val chapterDir = getChapterDir(mangaId, chapterId)
+        val chapterDir = getChapterDownloadPath(mangaId, chapterId)
         val folder = File(chapterDir)
         folder.mkdirs()
 
@@ -74,7 +74,7 @@ class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(man
     }
 
     override fun delete(): Boolean {
-        val chapterDir = getChapterDir(mangaId, chapterId)
+        val chapterDir = getChapterDownloadPath(mangaId, chapterId)
         return File(chapterDir).deleteRecursively()
     }
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/Extension.kt

@@ -40,7 +40,7 @@ import suwayomi.tachidesk.manga.impl.util.PackageTools.getPackageInfo
 import suwayomi.tachidesk.manga.impl.util.PackageTools.loadExtensionSources
 import suwayomi.tachidesk.manga.impl.util.network.await
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource
-import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
+import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getCachedImageResponse
 import suwayomi.tachidesk.manga.model.table.ExtensionTable
 import suwayomi.tachidesk.manga.model.table.SourceTable
 import suwayomi.tachidesk.server.ApplicationDirs
@@ -266,16 +266,16 @@ object Extension {
         return installExtension(pkgName)
     }
 
-    suspend fun getExtensionIcon(apkName: String, useCache: Boolean): Pair<InputStream, String> {
+    suspend fun getExtensionIcon(apkName: String): Pair<InputStream, String> {
         val iconUrl = if (apkName == "localSource") {
             ""
         } else {
             transaction { ExtensionTable.select { ExtensionTable.apkName eq apkName }.first() }[ExtensionTable.iconUrl]
         }
 
-        val saveDir = "${applicationDirs.extensionsRoot}/icon"
+        val cacheSaveDir = "${applicationDirs.extensionsRoot}/icon"
 
-        return getImageResponse(saveDir, apkName, useCache) {
+        return getCachedImageResponse(cacheSaveDir, apkName) {
             network.client.newCall(
                 GET(iconUrl)
             ).await()

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt

@@ -22,16 +22,16 @@ import java.io.File
 
 private val applicationDirs by DI.global.instance<ApplicationDirs>()
 
-fun getMangaDir(mangaId: Int, cache: Boolean = false): String {
+private fun getMangaDir(mangaId: Int): String {
     val mangaEntry = getMangaEntry(mangaId)
     val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference])
 
-    val sourceDir = source.toString()
+    val sourceDir = SafePath.buildValidFilename(source.toString())
     val mangaDir = SafePath.buildValidFilename(mangaEntry[MangaTable.title])
-    return (if (cache) applicationDirs.cacheRoot else applicationDirs.mangaDownloadsRoot) + "/$sourceDir/$mangaDir"
+    return "$sourceDir/$mangaDir"
 }
 
-fun getChapterDir(mangaId: Int, chapterId: Int, cache: Boolean = false): String {
+private fun getChapterDir(mangaId: Int, chapterId: Int): String {
     val chapterEntry = transaction { ChapterTable.select { ChapterTable.id eq chapterId }.first() }
 
     val chapterDir = SafePath.buildValidFilename(
@@ -41,9 +41,18 @@ fun getChapterDir(mangaId: Int, chapterId: Int, cache: Boolean = false): String
         }
     )
 
-    return getMangaDir(mangaId, cache) + "/$chapterDir"
+    return getMangaDir(mangaId) + "/$chapterDir"
 }
 
+fun getChapterDownloadPath(mangaId: Int, chapterId: Int): String {
+    return applicationDirs.mangaDownloadsRoot + "/" + getChapterDir(mangaId, chapterId)
+}
+fun getChapterCachePath(mangaId: Int, chapterId: Int): String {
+    return applicationDirs.tempMangaCacheRoot + "/" + getChapterDir(mangaId, chapterId)
+}
+
+// (if (useTempCache) applicationDirs.tempCacheRoot else )
+
 /** return value says if rename/move was successful */
 fun updateMangaDownloadDir(mangaId: Int, newTitle: String): Boolean {
     val mangaEntry = getMangaEntry(mangaId)

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt

@@ -9,7 +9,6 @@ package suwayomi.tachidesk.manga.impl.util.storage
 
 import okhttp3.Response
 import okhttp3.internal.closeQuietly
-import suwayomi.tachidesk.manga.impl.util.getChapterDir
 import java.io.File
 import java.io.FileInputStream
 import java.io.InputStream
@@ -19,6 +18,7 @@ object ImageResponse {
         return FileInputStream(path).buffered()
     }
 
+    /** find file with name when file extension is not known */
     fun findFileNameStartingWith(directoryPath: String, fileName: String): String? {
         val target = "$fileName."
         File(directoryPath).listFiles().orEmpty().forEach { file ->
@@ -29,8 +29,17 @@ object ImageResponse {
         return null
     }
 
-    /** fetch a cached image response, calls `fetcher` if cache fails */
-    private suspend fun getCachedImageResponse(saveDir: String, fileName: String, fetcher: suspend () -> Response): Pair<InputStream, String> {
+    /**
+     * Get a cached image response
+     *
+     * Note: The caller should also call [clearCachedImage] when appropriate
+     *
+     * @param cacheSavePath where to save the cached image. Caller should decide to use perma cache or temp cache (OS temp dir)
+     * @param fileName what the saved cache file should be named
+     */
+    suspend fun getCachedImageResponse(saveDir: String, fileName: String, fetcher: suspend () -> Response): Pair<InputStream, String> {
+        File(saveDir).mkdirs()
+
         val cachedFile = findFileNameStartingWith(saveDir, fileName)
         val filePath = "$saveDir/$fileName"
         if (cachedFile != null) {
@@ -52,6 +61,7 @@ object ImageResponse {
         }
     }
 
+    /** Save image safely */
     fun saveImage(filePath: String, image: InputStream): Pair<String, String> {
         val tmpSavePath = "$filePath.tmp"
         val tmpSaveFile = File(tmpSavePath)
@@ -73,39 +83,4 @@ object ImageResponse {
             File(it).delete()
         }
     }
-
-    private suspend fun getNoCacheImageResponse(fetcher: suspend () -> Response): Pair<InputStream, String> {
-        val response = fetcher()
-
-        if (response.code == 200) {
-            val responseBytes = response.body!!.bytes()
-
-            // find image type
-            val imageType = response.headers["content-type"]
-                ?: ImageUtil.findImageType { responseBytes.inputStream() }?.mime
-                ?: "image/jpeg"
-
-            return responseBytes.inputStream() to imageType
-        } else {
-            response.closeQuietly()
-            throw Exception("request error! ${response.code}")
-        }
-    }
-
-    suspend fun getImageResponse(saveDir: String, fileName: String, useCache: Boolean, fetcher: suspend () -> Response): Pair<InputStream, String> {
-        return if (useCache) {
-            getCachedImageResponse(saveDir, fileName, fetcher)
-        } else {
-            getNoCacheImageResponse(fetcher)
-        }
-    }
-
-    suspend fun getImageResponse(mangaId: Int, chapterId: Int, fileName: String, useCache: Boolean, fetcher: suspend () -> Response): Pair<InputStream, String> {
-        var saveDir = ""
-        if (useCache) {
-            saveDir = getChapterDir(mangaId, chapterId, true)
-            File(saveDir).mkdirs()
-        }
-        return getImageResponse(saveDir, fileName, useCache, fetcher)
-    }
 }

server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt

@@ -36,7 +36,8 @@ import java.util.Locale
 private val logger = KotlinLogging.logger {}
 
 class ApplicationDirs(
-    val dataRoot: String = ApplicationRootDir
+    val dataRoot: String = ApplicationRootDir,
+    val tempRoot: String = "${System.getProperty("java.io.tmpdir")}/Tachidesk"
 ) {
     val cacheRoot = System.getProperty("java.io.tmpdir") + "/tachidesk"
     val extensionsRoot = "$dataRoot/extensions"
@@ -44,6 +45,8 @@ class ApplicationDirs(
     val mangaDownloadsRoot = serverConfig.downloadsPath.ifBlank { "$dataRoot/downloads" }
     val localMangaRoot = "$dataRoot/local"
     val webUIRoot = "$dataRoot/webUI"
+
+    val tempMangaCacheRoot = "$tempRoot/manga-cache"
 }
 
 val serverConfig: ServerConfig by lazy { GlobalConfigManager.module() }