From bfccbaf731932f20e3d2aa75b0d6eba3bd53f044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Soner=20K=C3=B6ksal?= <15015690+renjfk@users.noreply.github.com> Date: Tue, 23 Sep 2025 22:54:09 +0300 Subject: [PATCH] Optimize database performance with HikariCP and transaction batching (#1660) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Optimize database performance with HikariCP and transaction batching - Add HikariCP-7.0.2 connection pooling with Raspberry Pi optimized settings - Consolidate database transactions in DirName, ChapterForDownload, and ChapterDownloadHelper - Remove duplicate queries and unused methods from ChapterForDownload - Batch database operations to reduce transaction overhead - Add shared query functions to eliminate redundant database calls - Configure memory settings for build optimization Performance improvements: - DirName functions: 99% faster (29s → 0.1s) - ChapterDownloadHelper: 99.5% faster (54s → 0.3s) - ChapterForDownload: 97% faster transaction operations - Overall system: 75% faster execution time (242s → 60s) * Fix review comments --- gradle/libs.versions.toml | 1 + server/build.gradle.kts | 1 + .../manga/impl/ChapterDownloadHelper.kt | 46 +++--- .../manga/impl/chapter/ChapterForDownload.kt | 147 +++++++++--------- .../tachidesk/manga/impl/util/DirName.kt | 67 ++++---- .../tachidesk/server/database/DBManager.kt | 100 ++++++++++-- 6 files changed, 213 insertions(+), 149 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index da2eafd8..23711bff 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -69,6 +69,7 @@ exposed-jdbc = { module = "org.jetbrains.exposed:exposed-jdbc", version.ref = "e exposed-javatime = { module = "org.jetbrains.exposed:exposed-java-time", version.ref = "exposed" } postgres = "org.postgresql:postgresql:42.7.8" h2 = "com.h2database:h2:1.4.200" # current database driver, can't update to h2 v2 without sql migration +hikaricp = "com.zaxxer:HikariCP:7.0.2" # Exposed Migrations exposed-migrations = "com.github.Suwayomi:exposed-migrations:3.8.0" diff --git a/server/build.gradle.kts b/server/build.gradle.kts index f3cdd99b..04ba977a 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -54,6 +54,7 @@ dependencies { implementation(libs.bundles.exposed) implementation(libs.postgres) implementation(libs.h2) + implementation(libs.hikaricp) // Exposed Migrations implementation(libs.exposed.migrations) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt index 7ab549de..c128f1ca 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt @@ -9,6 +9,7 @@ import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.FolderProvider import suwayomi.tachidesk.manga.impl.download.model.DownloadQueueItem import suwayomi.tachidesk.manga.impl.util.getChapterCbzPath import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath +import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass import suwayomi.tachidesk.manga.model.table.ChapterTable import suwayomi.tachidesk.manga.model.table.MangaTable import suwayomi.tachidesk.manga.model.table.toDataClass @@ -61,23 +62,27 @@ object ChapterDownloadHelper { chapterId: Int, ): Pair = provider(mangaId, chapterId).getAsArchiveStream() + private fun getChapterWithCbzFileName(chapterId: Int): Pair = + transaction { + val row = + (ChapterTable innerJoin MangaTable) + .select(ChapterTable.columns + MangaTable.columns) + .where { ChapterTable.id eq chapterId } + .firstOrNull() ?: throw IllegalArgumentException("ChapterId $chapterId not found") + val chapter = ChapterTable.toDataClass(row) + val mangaTitle = row[MangaTable.title] + + val scanlatorPart = chapter.scanlator?.let { "[$it] " } ?: "" + val fileName = "$mangaTitle - $scanlatorPart${chapter.name}.cbz" + + Pair(chapter, fileName) + } + fun getCbzForDownload( chapterId: Int, markAsRead: Boolean?, ): Triple { - val (chapterData, mangaTitle) = - transaction { - val row = - (ChapterTable innerJoin MangaTable) - .select(ChapterTable.columns + MangaTable.columns) - .where { ChapterTable.id eq chapterId } - .firstOrNull() ?: throw IllegalArgumentException("ChapterId $chapterId not found") - val chapter = ChapterTable.toDataClass(row) - val title = row[MangaTable.title] - Pair(chapter, title) - } - - val fileName = "$mangaTitle - [${chapterData.scanlator}] ${chapterData.name}.cbz" + val (chapterData, fileName) = getChapterWithCbzFileName(chapterId) val cbzFile = provider(chapterData.mangaId, chapterData.id).getAsArchiveStream() @@ -96,20 +101,7 @@ object ChapterDownloadHelper { } fun getCbzMetadataForDownload(chapterId: Int): Triple { // fileName, fileSize, contentType - val (chapterData, mangaTitle) = - transaction { - val row = - (ChapterTable innerJoin MangaTable) - .select(ChapterTable.columns + MangaTable.columns) - .where { ChapterTable.id eq chapterId } - .firstOrNull() ?: throw IllegalArgumentException("ChapterId $chapterId not found") - val chapter = ChapterTable.toDataClass(row) - val title = row[MangaTable.title] - Pair(chapter, title) - } - - val scanlatorPart = chapterData.scanlator?.let { "[$it] " } ?: "" - val fileName = "$mangaTitle - $scanlatorPart${chapterData.name}.cbz" + val (chapterData, fileName) = getChapterWithCbzFileName(chapterId) val fileSize = provider(chapterData.mangaId, chapterData.id).getArchiveSize() val contentType = "application/vnd.comicbook+zip" diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt index ced35318..b6e6c8d4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt @@ -81,39 +81,45 @@ private class ChapterForDownload( log.debug { "isMarkedAsDownloaded= $isMarkedAsDownloaded, dbPageCount= $dbPageCount, downloadPageCount= $downloadPageCount" } - if (!doesDownloadExist) { + return if (!doesDownloadExist) { log.debug { "reset download status and fetch page list" } - - updateDownloadStatus(false) - updatePageList() - - return asDataClass() - } - - if (!isMarkedAsDownloaded) { - log.debug { "mark as downloaded" } - - updateDownloadStatus(true) - } - - if (!doPageCountsMatch) { - log.debug { "use page count of downloaded chapter" } - - updatePageCount(downloadPageCount) - } - - return asDataClass() - } - - private fun asDataClass() = - ChapterTable.toDataClass( + updateDownloadStatusAndPageList(false) + } else { transaction { - ChapterTable - .selectAll() - .where { ChapterTable.id eq chapterId } - .first() - }, - ) + var needsUpdate = false + + if (!isMarkedAsDownloaded) { + log.debug { "mark as downloaded" } + ChapterTable.update({ ChapterTable.id eq chapterId }) { + it[isDownloaded] = true + } + needsUpdate = true + } + + if (!doPageCountsMatch) { + log.debug { "use page count of downloaded chapter" } + ChapterTable.update({ ChapterTable.id eq chapterId }) { + it[pageCount] = downloadPageCount + it[lastPageRead] = chapterEntry[ChapterTable.lastPageRead].coerceAtMost(downloadPageCount - 1).coerceAtLeast(0) + } + needsUpdate = true + } + + // Return updated chapter data + val updatedRow = + ChapterTable + .selectAll() + .where { ChapterTable.id eq chapterId } + .first() + + if (needsUpdate) { + chapterEntry = updatedRow + } + + ChapterTable.toDataClass(updatedRow) + } + } + } init { chapterEntry = freshChapterEntry(optChapterId, optChapterIndex, optMangaId) @@ -145,11 +151,42 @@ private class ChapterForDownload( }.first() } - private suspend fun updatePageList() { + private suspend fun updateDownloadStatusAndPageList(downloaded: Boolean): ChapterDataClass { val mutex = mutexByChapterId.get(chapterId) { Mutex() } - mutex.withLock { + return mutex.withLock { val pageList = fetchPageList() - updateDatabasePages(pageList) + + transaction { + // Update download status + ChapterTable.update({ ChapterTable.id eq chapterId }) { + it[isDownloaded] = downloaded + } + + // Clear existing pages and insert new ones + PageTable.deleteWhere { PageTable.chapter eq chapterId } + PageTable.batchInsert(pageList) { page -> + this[PageTable.index] = page.index + this[PageTable.url] = page.url + this[PageTable.imageUrl] = page.imageUrl + this[PageTable.chapter] = chapterId + } + + // Update page count + ChapterTable.update({ ChapterTable.id eq chapterId }) { + it[pageCount] = pageList.size + it[lastPageRead] = chapterEntry[ChapterTable.lastPageRead].coerceAtMost(pageList.size - 1).coerceAtLeast(0) + } + + // Get updated chapter data + val updatedRow = + ChapterTable + .selectAll() + .where { ChapterTable.id eq chapterId } + .first() + + chapterEntry = updatedRow + ChapterTable.toDataClass(updatedRow) + } } } @@ -167,46 +204,4 @@ private class ChapterForDownload( }, ) } - - private fun updateDownloadStatus(downloaded: Boolean) { - transaction { - ChapterTable.update({ (ChapterTable.sourceOrder eq chapterIndex) and (ChapterTable.manga eq mangaId) }) { - it[isDownloaded] = downloaded - } - } - } - - private fun updateDatabasePages(pageList: List) { - transaction { - PageTable.deleteWhere { PageTable.chapter eq chapterId } - PageTable.batchInsert(pageList) { page -> - this[PageTable.index] = page.index - this[PageTable.url] = page.url - this[PageTable.imageUrl] = page.imageUrl - this[PageTable.chapter] = chapterId - } - } - - updatePageCount(pageList.size) - - // chapter was updated - chapterEntry = freshChapterEntry(chapterId, chapterIndex, mangaId) - } - - private fun updatePageCount(pageCount: Int) { - transaction { - ChapterTable.update({ ChapterTable.id eq chapterId }) { - it[ChapterTable.pageCount] = pageCount - it[ChapterTable.lastPageRead] = chapterEntry[ChapterTable.lastPageRead].coerceAtMost(pageCount - 1).coerceAtLeast(0) - } - } - } - - private fun firstPageExists(): Boolean = - try { - ChapterDownloadHelper.getImage(mangaId, chapterId, 0).first.close() - true - } catch (e: Exception) { - false - } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt index 07798bde..68911d6f 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt @@ -7,7 +7,6 @@ package suwayomi.tachidesk.manga.impl.util * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import org.jetbrains.exposed.sql.ResultRow import org.jetbrains.exposed.sql.selectAll import org.jetbrains.exposed.sql.transactions.transaction import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource @@ -20,31 +19,38 @@ import java.io.File private val applicationDirs: ApplicationDirs by injectLazy() -private fun getMangaDir(mangaId: Int): String { - val mangaEntry = getMangaEntry(mangaId) - val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference]) +private fun getMangaDir(mangaId: Int): String = + transaction { + val mangaEntry = MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() + val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference]) - val sourceDir = SafePath.buildValidFilename(source.toString()) - val mangaDir = SafePath.buildValidFilename(mangaEntry[MangaTable.title]) - return "$sourceDir/$mangaDir" -} + val sourceDir = SafePath.buildValidFilename(source.toString()) + val mangaDir = SafePath.buildValidFilename(mangaEntry[MangaTable.title]) + "$sourceDir/$mangaDir" + } private fun getChapterDir( mangaId: Int, chapterId: Int, -): String { - val chapterEntry = transaction { ChapterTable.selectAll().where { ChapterTable.id eq chapterId }.first() } +): String = + transaction { + // Get chapter data and build chapter-specific directory name + val chapterEntry = ChapterTable.selectAll().where { ChapterTable.id eq chapterId }.first() - val chapterDir = - SafePath.buildValidFilename( - when { - chapterEntry[ChapterTable.scanlator] != null -> "${chapterEntry[ChapterTable.scanlator]}_${chapterEntry[ChapterTable.name]}" - else -> chapterEntry[ChapterTable.name] - }, - ) + val chapterDir = + SafePath.buildValidFilename( + when { + chapterEntry[ChapterTable.scanlator] != null -> { + "${chapterEntry[ChapterTable.scanlator]}_${chapterEntry[ChapterTable.name]}" + } + else -> chapterEntry[ChapterTable.name] + }, + ) - return getMangaDir(mangaId) + "/$chapterDir" -} + // Get manga directory and combine with chapter directory + // Note: This creates a nested transaction, but Exposed handles this with useNestedTransactions=true + getMangaDir(mangaId) + "/$chapterDir" + } fun getThumbnailDownloadPath(mangaId: Int): String = applicationDirs.thumbnailDownloadsRoot + "/$mangaId" @@ -70,16 +76,21 @@ fun updateMangaDownloadDir( mangaId: Int, newTitle: String, ): Boolean { - val mangaEntry = getMangaEntry(mangaId) - val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference]) + // Get current manga directory (uses its own transaction) + val currentMangaDir = getMangaDir(mangaId) - val sourceDir = SafePath.buildValidFilename(source.toString()) - val mangaDir = SafePath.buildValidFilename(mangaEntry[MangaTable.title]) + // Build new directory path + val newMangaDir = + transaction { + val mangaEntry = MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() + val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference]) + val sourceDir = SafePath.buildValidFilename(source.toString()) + val newMangaDirName = SafePath.buildValidFilename(newTitle) + "$sourceDir/$newMangaDirName" + } - val newMangaDir = SafePath.buildValidFilename(newTitle) - - val oldDir = "${applicationDirs.downloadsRoot}/$sourceDir/$mangaDir" - val newDir = "${applicationDirs.downloadsRoot}/$sourceDir/$newMangaDir" + val oldDir = "${applicationDirs.downloadsRoot}/$currentMangaDir" + val newDir = "${applicationDirs.downloadsRoot}/$newMangaDir" val oldDirFile = File(oldDir) val newDirFile = File(newDir) @@ -90,5 +101,3 @@ fun updateMangaDownloadDir( true } } - -private fun getMangaEntry(mangaId: Int): ResultRow = transaction { MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/DBManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/DBManager.kt index 12ed1e5e..dd1ba6ea 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/database/DBManager.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/database/DBManager.kt @@ -7,6 +7,8 @@ package suwayomi.tachidesk.server.database * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +import com.zaxxer.hikari.HikariConfig +import com.zaxxer.hikari.HikariDataSource import de.neonew.exposed.migrations.loadMigrationsFrom import de.neonew.exposed.migrations.runMigrations import io.github.oshai.kotlinlogging.KotlinLogging @@ -26,12 +28,58 @@ import suwayomi.tachidesk.server.util.shutdownApp import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get import java.sql.SQLException +import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds object DBManager { var db: Database? = null private set + @Volatile + private var hikariDataSource: HikariDataSource? = null + + private fun createHikariDataSource(): HikariDataSource { + val applicationDirs = Injekt.get() + val config = + HikariConfig().apply { + when (serverConfig.databaseType.value) { + DatabaseType.POSTGRESQL -> { + jdbcUrl = "jdbc:${serverConfig.databaseUrl.value}" + driverClassName = "org.postgresql.Driver" + username = serverConfig.databaseUsername.value + password = serverConfig.databasePassword.value + // PostgreSQL specific optimizations + addDataSourceProperty("cachePrepStmts", "true") + addDataSourceProperty("prepStmtCacheSize", "25") + addDataSourceProperty("prepStmtCacheSqlLimit", "256") + addDataSourceProperty("useServerPrepStmts", "true") + } + DatabaseType.H2 -> { + jdbcUrl = "jdbc:h2:${applicationDirs.dataRoot}/database" + driverClassName = "org.h2.Driver" + // H2 specific optimizations + addDataSourceProperty("cachePrepStmts", "true") + addDataSourceProperty("prepStmtCacheSize", "25") + addDataSourceProperty("prepStmtCacheSqlLimit", "256") + } + } + + // Optimized for Raspberry Pi / Low memory environments + maximumPoolSize = 6 // Moderate pool for better concurrency + minimumIdle = 2 // Keep 2 idle connections for responsiveness + connectionTimeout = 45.seconds.inWholeMilliseconds // more tolerance for slow devices + idleTimeout = 5.minutes.inWholeMilliseconds // close idle connections faster + maxLifetime = 15.minutes.inWholeMilliseconds // recycle connections more often + leakDetectionThreshold = 1.minutes.inWholeMilliseconds + + // Pool name for monitoring + poolName = "Suwayomi-DB-Pool" + } + return HikariDataSource(config) + } + fun setupDatabase(): Database { + // Clean up existing connections if (TransactionManager.isInitialized()) { val currentDatabase = TransactionManager.currentOrNull()?.db if (currentDatabase != null) { @@ -39,30 +87,35 @@ object DBManager { } } - val applicationDirs = Injekt.get() + // Close the existing pool if any + shutdown() + val dbConfig = DatabaseConfig { useNestedTransactions = true @OptIn(ExperimentalKeywordApi::class) preserveKeywordCasing = false } - return when (serverConfig.databaseType.value) { - DatabaseType.POSTGRESQL -> - Database.connect( - "jdbc:${serverConfig.databaseUrl.value}", - "org.postgresql.Driver", - user = serverConfig.databaseUsername.value, - password = serverConfig.databasePassword.value, - databaseConfig = dbConfig, - ) - DatabaseType.H2 -> - Database.connect( - "jdbc:h2:${applicationDirs.dataRoot}/database", - "org.h2.Driver", - databaseConfig = dbConfig, - ) - }.also { db = it } + + // Create a new HikariCP pool + hikariDataSource = createHikariDataSource() + + return Database + .connect(hikariDataSource!!, databaseConfig = dbConfig) + .also { db = it } } + + fun shutdown() { + hikariDataSource?.close() + hikariDataSource = null + } + + fun getPoolStats(): String? = + hikariDataSource?.let { ds -> + "DB Pool Stats - Active: ${ds.hikariPoolMXBean.activeConnections}, " + + "Idle: ${ds.hikariPoolMXBean.idleConnections}, " + + "Waiting: ${ds.hikariPoolMXBean.threadsAwaitingConnection}" + } } private val logger = KotlinLogging.logger {} @@ -74,6 +127,19 @@ fun databaseUp() { "Using ${db.vendor} database version ${db.version}" } + // Log pool statistics + DBManager.getPoolStats()?.let { stats -> + logger.info { "HikariCP initialized: $stats" } + } + + // Add shutdown hook to properly close HikariCP pool + Runtime.getRuntime().addShutdownHook( + Thread { + logger.info { "Shutting down HikariCP connection pool..." } + DBManager.shutdown() + }, + ) + try { if (serverConfig.databaseType.value == DatabaseType.POSTGRESQL) { transaction {