From d8876cf96a9fb5fb4c5ea67e2722a85a58009bf6 Mon Sep 17 00:00:00 2001 From: schroda <50052685+schroda@users.noreply.github.com> Date: Sun, 21 Jan 2024 01:42:01 +0100 Subject: [PATCH] Add mutex to "updateExtensionDatabase" (#829) If called in quick succession it is possible that duplicated extensions get inserted to the database, because it has not yet been updated by the first call --- .../manga/impl/extension/ExtensionsList.kt | 210 +++++++++--------- 1 file changed, 108 insertions(+), 102 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt index 40662fa1..7d3f4b7b 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt @@ -8,6 +8,8 @@ package suwayomi.tachidesk.manga.impl.extension * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ import eu.kanade.tachiyomi.source.local.LocalSource +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import mu.KotlinLogging import org.jetbrains.exposed.dao.id.EntityID import org.jetbrains.exposed.sql.ResultRow @@ -86,117 +88,121 @@ object ExtensionsList { } } - private fun updateExtensionDatabase(foundExtensions: List) { - transaction { - val uniqueExtensions = - foundExtensions.groupBy { it.pkgName }.mapValues { - (_, extension) -> - extension.maxBy { it.versionCode } - }.values - val installedExtensions = - ExtensionTable.selectAll().toList() - .associateBy { it[ExtensionTable.pkgName] } - val extensionsToUpdate = mutableListOf>() - val extensionsToInsert = mutableListOf() - val extensionsToDelete = - installedExtensions.filter { it.value[ExtensionTable.repo] != null }.mapNotNull { (pkgName, extension) -> - extension.takeUnless { uniqueExtensions.any { it.pkgName == pkgName } } - } - uniqueExtensions.forEach { - val extension = installedExtensions[it.pkgName] - if (extension != null) { - extensionsToUpdate.add(it to extension) - } else { - extensionsToInsert.add(it) - } - } - if (extensionsToUpdate.isNotEmpty()) { - val extensionsInstalled = - extensionsToUpdate - .groupBy { it.second[ExtensionTable.isInstalled] } - val installedExtensionsToUpdate = extensionsInstalled[true].orEmpty() - if (installedExtensionsToUpdate.isNotEmpty()) { - BatchUpdateStatement(ExtensionTable).apply { - installedExtensionsToUpdate.forEach { (foundExtension, extensionRecord) -> - addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable)) - // Always update icon url and repo - this[ExtensionTable.iconUrl] = foundExtension.iconUrl - this[ExtensionTable.repo] = foundExtension.repo + private val updateExtensionDatabaseMutex = Mutex() - // add these because batch updates need matching columns - this[ExtensionTable.hasUpdate] = extensionRecord[ExtensionTable.hasUpdate] - this[ExtensionTable.isObsolete] = extensionRecord[ExtensionTable.isObsolete] - - // a previously removed extension is now available again - if (extensionRecord[ExtensionTable.isObsolete] && - foundExtension.versionCode >= extensionRecord[ExtensionTable.versionCode] - ) { - this[ExtensionTable.isObsolete] = false - } - - when { - foundExtension.versionCode > extensionRecord[ExtensionTable.versionCode] -> { - // there is an update - this[ExtensionTable.hasUpdate] = true - updateMap.putIfAbsent(foundExtension.pkgName, foundExtension) - } - foundExtension.versionCode < extensionRecord[ExtensionTable.versionCode] -> { - // somehow the user installed an invalid version - this[ExtensionTable.isObsolete] = true - } - } - } - execute(this@transaction) + private suspend fun updateExtensionDatabase(foundExtensions: List) { + updateExtensionDatabaseMutex.withLock { + transaction { + val uniqueExtensions = + foundExtensions.groupBy { it.pkgName }.mapValues { + (_, extension) -> + extension.maxBy { it.versionCode } + }.values + val installedExtensions = + ExtensionTable.selectAll().toList() + .associateBy { it[ExtensionTable.pkgName] } + val extensionsToUpdate = mutableListOf>() + val extensionsToInsert = mutableListOf() + val extensionsToDelete = + installedExtensions.filter { it.value[ExtensionTable.repo] != null }.mapNotNull { (pkgName, extension) -> + extension.takeUnless { uniqueExtensions.any { it.pkgName == pkgName } } + } + uniqueExtensions.forEach { + val extension = installedExtensions[it.pkgName] + if (extension != null) { + extensionsToUpdate.add(it to extension) + } else { + extensionsToInsert.add(it) } } - val extensionsToFullyUpdate = extensionsInstalled[false].orEmpty() - if (extensionsToFullyUpdate.isNotEmpty()) { - BatchUpdateStatement(ExtensionTable).apply { - extensionsToFullyUpdate.forEach { (foundExtension, extensionRecord) -> - addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable)) - // extension is not installed, so we can overwrite the data without a care - this[ExtensionTable.repo] = foundExtension.repo - this[ExtensionTable.name] = foundExtension.name - this[ExtensionTable.versionName] = foundExtension.versionName - this[ExtensionTable.versionCode] = foundExtension.versionCode - this[ExtensionTable.lang] = foundExtension.lang - this[ExtensionTable.isNsfw] = foundExtension.isNsfw - this[ExtensionTable.apkName] = foundExtension.apkName - this[ExtensionTable.iconUrl] = foundExtension.iconUrl + if (extensionsToUpdate.isNotEmpty()) { + val extensionsInstalled = + extensionsToUpdate + .groupBy { it.second[ExtensionTable.isInstalled] } + val installedExtensionsToUpdate = extensionsInstalled[true].orEmpty() + if (installedExtensionsToUpdate.isNotEmpty()) { + BatchUpdateStatement(ExtensionTable).apply { + installedExtensionsToUpdate.forEach { (foundExtension, extensionRecord) -> + addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable)) + // Always update icon url and repo + this[ExtensionTable.iconUrl] = foundExtension.iconUrl + this[ExtensionTable.repo] = foundExtension.repo + + // add these because batch updates need matching columns + this[ExtensionTable.hasUpdate] = extensionRecord[ExtensionTable.hasUpdate] + this[ExtensionTable.isObsolete] = extensionRecord[ExtensionTable.isObsolete] + + // a previously removed extension is now available again + if (extensionRecord[ExtensionTable.isObsolete] && + foundExtension.versionCode >= extensionRecord[ExtensionTable.versionCode] + ) { + this[ExtensionTable.isObsolete] = false + } + + when { + foundExtension.versionCode > extensionRecord[ExtensionTable.versionCode] -> { + // there is an update + this[ExtensionTable.hasUpdate] = true + updateMap.putIfAbsent(foundExtension.pkgName, foundExtension) + } + foundExtension.versionCode < extensionRecord[ExtensionTable.versionCode] -> { + // somehow the user installed an invalid version + this[ExtensionTable.isObsolete] = true + } + } + } + execute(this@transaction) + } + } + val extensionsToFullyUpdate = extensionsInstalled[false].orEmpty() + if (extensionsToFullyUpdate.isNotEmpty()) { + BatchUpdateStatement(ExtensionTable).apply { + extensionsToFullyUpdate.forEach { (foundExtension, extensionRecord) -> + addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable)) + // extension is not installed, so we can overwrite the data without a care + this[ExtensionTable.repo] = foundExtension.repo + this[ExtensionTable.name] = foundExtension.name + this[ExtensionTable.versionName] = foundExtension.versionName + this[ExtensionTable.versionCode] = foundExtension.versionCode + this[ExtensionTable.lang] = foundExtension.lang + this[ExtensionTable.isNsfw] = foundExtension.isNsfw + this[ExtensionTable.apkName] = foundExtension.apkName + this[ExtensionTable.iconUrl] = foundExtension.iconUrl + } + execute(this@transaction) } - execute(this@transaction) } } - } - if (extensionsToInsert.isNotEmpty()) { - ExtensionTable.batchInsert(extensionsToInsert) { foundExtension -> - this[ExtensionTable.repo] = foundExtension.repo - this[ExtensionTable.name] = foundExtension.name - this[ExtensionTable.pkgName] = foundExtension.pkgName - this[ExtensionTable.versionName] = foundExtension.versionName - this[ExtensionTable.versionCode] = foundExtension.versionCode - this[ExtensionTable.lang] = foundExtension.lang - this[ExtensionTable.isNsfw] = foundExtension.isNsfw - this[ExtensionTable.apkName] = foundExtension.apkName - this[ExtensionTable.iconUrl] = foundExtension.iconUrl + if (extensionsToInsert.isNotEmpty()) { + ExtensionTable.batchInsert(extensionsToInsert) { foundExtension -> + this[ExtensionTable.repo] = foundExtension.repo + this[ExtensionTable.name] = foundExtension.name + this[ExtensionTable.pkgName] = foundExtension.pkgName + this[ExtensionTable.versionName] = foundExtension.versionName + this[ExtensionTable.versionCode] = foundExtension.versionCode + this[ExtensionTable.lang] = foundExtension.lang + this[ExtensionTable.isNsfw] = foundExtension.isNsfw + this[ExtensionTable.apkName] = foundExtension.apkName + this[ExtensionTable.iconUrl] = foundExtension.iconUrl + } } - } - // deal with obsolete extensions - val extensionsToRemove = - extensionsToDelete.groupBy { it[ExtensionTable.isInstalled] } - .mapValues { (_, extensions) -> extensions.map { it[ExtensionTable.pkgName] } } - // not in the repo, so these extensions are obsolete - val obsoleteExtensions = extensionsToRemove[true].orEmpty() - if (obsoleteExtensions.isNotEmpty()) { - ExtensionTable.update({ ExtensionTable.pkgName inList obsoleteExtensions }) { - it[isObsolete] = true + // deal with obsolete extensions + val extensionsToRemove = + extensionsToDelete.groupBy { it[ExtensionTable.isInstalled] } + .mapValues { (_, extensions) -> extensions.map { it[ExtensionTable.pkgName] } } + // not in the repo, so these extensions are obsolete + val obsoleteExtensions = extensionsToRemove[true].orEmpty() + if (obsoleteExtensions.isNotEmpty()) { + ExtensionTable.update({ ExtensionTable.pkgName inList obsoleteExtensions }) { + it[isObsolete] = true + } + } + // is not installed, so we can remove the record without a care + val removeExtensions = extensionsToRemove[false].orEmpty() + if (removeExtensions.isNotEmpty()) { + ExtensionTable.deleteWhere { ExtensionTable.pkgName inList removeExtensions } } - } - // is not installed, so we can remove the record without a care - val removeExtensions = extensionsToRemove[false].orEmpty() - if (removeExtensions.isNotEmpty()) { - ExtensionTable.deleteWhere { ExtensionTable.pkgName inList removeExtensions } } } }