Remote Image Processing (#1684)

* Update ServerConfig.kt

* Update ConversionUtil.kt

* Update Page.kt

* Update ServerConfig.kt

fixed deletions caused by ide

* Update ServerConfig.kt

* Update ServerConfig.kt

* Cleanup

* Post-processing terminology

* More comments

* Lint

* Add known image mimes

* Fix weird mime set/get

* Implement different downloadConversions and serveConversions

* Lint

* Improve Post-Processing massivly

* Fix thumbnail build

* Use Array for headers

* Actually fix headers

* Actually fix headers 2

* Manually parse DownloadConversion

* Cleanup parse

* Fix write

* Update TypeName

* Optimize imports

* Remove header type

* Fix build

---------

Co-authored-by: Syer10 <syer10@users.noreply.github.com>
This commit is contained in:
FadedSociety
2025-11-12 14:23:34 -07:00
committed by GitHub
parent 3e47859d88
commit 0a7e6cce87
16 changed files with 527 additions and 188 deletions

View File

@@ -14,6 +14,8 @@
* limitations under the License.
*/
package libcore.net;
import android.annotation.Nullable;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
@@ -249,10 +251,12 @@ public final class MimeUtils {
add("audio/x-scpls", "pls");
add("audio/x-sd2", "sd2");
add("audio/x-wav", "wav");
add("image/avif", "avif");
// image/bmp isn't IANA, so image/x-ms-bmp should come first.
add("image/x-ms-bmp", "bmp");
add("image/bmp", "bmp");
add("image/gif", "gif");
add("image/heif", "heif");
// image/ico isn't IANA, so image/x-icon should come first.
add("image/x-icon", "ico");
add("image/ico", "cur");
@@ -262,6 +266,7 @@ public final class MimeUtils {
add("image/jpeg", "jpg");
add("image/jpeg", "jpeg");
add("image/jpeg", "jpe");
add("image/jxl", "jxl");
add("image/pcx", "pcx");
add("image/png", "png");
add("image/svg+xml", "svg");
@@ -438,6 +443,7 @@ public final class MimeUtils {
* @return The extension has been registered for
* the given case insensitive MIME type or null if there is none.
*/
@Nullable
public static String guessExtensionFromMimeType(String mimeType) {
if (mimeType == null || mimeType.isEmpty()) {
return null;

View File

@@ -23,7 +23,7 @@ The configuration file is written in HOCON. Google is your friend if you want to
server.ip = "0.0.0.0"
server.port = 4567
```
- `server.ip` can be a IP or domain name.
- `server.ip` can be an IP or domain name.
### Socks5 proxy
```
@@ -78,8 +78,8 @@ server.downloadConversions = {}
- `server.excludeEntryWithUnreadChapters = true` controls if Suwayomi will download new chapters for titles with unread chapters (requires `server.autoDownloadNewChapters`).
- `server.autoDownloadNewChaptersLimit = 0` sets how many chapters should be downloaded at most, `0` to disable the limit; if the limit is reached, new chapters will not be downloaded (requires `server.autoDownloadNewChapters`).
- `server.autoDownloadIgnoreReUploads = false` controls if Suwayomi will re-download re-uploads on update (requires `server.autoDownloadNewChapters`).
- `server.downloadConversions = {}` configures optional image conversions for all downloads. This is an [JSON object](https://en.wikipedia.org/wiki/JSON#Syntax), with the source image [mime type](https://en.wikipedia.org/wiki/Media_type) as the key and an object with the target mime type and options as value.
The following options are both valid:
- `server.downloadConversions = {}` configures optional image conversions for all downloads. This is an [JSON object](https://en.wikipedia.org/wiki/JSON#Syntax), with the source image [mime type](https://en.wikipedia.org/wiki/Media_type) as the key and an object with the target mime type or url and options as value.
The following options are all valid:
```
server.downloadConversions = { "image/webp" : { target : "image/jpeg", compressionLevel = 0.8 }}
# -- or --
@@ -87,9 +87,26 @@ server.downloadConversions = {}
target = "image/jpeg" # image type to convert to
compressionLevel = 0.8 # quality in range [0,1], leave away to use default compression
}
# -- a url example --
server.downloadConversions = { "default" : { target : "http://localhost:9999/convert" }}
# -- a url with all parameters example --
server.downloadConversions = {
"default" : {
target : "http://localhost:9999/convert",
callTimeout : 10m,
connectTimeout : 10s,
headers : {
"authorization" : "MyPassword"
}
}
}
```
A source mime type `default` can be used as fallback to convert all images; a target mime type of `none` can be used to disable conversion for a particular format.
This is an example curl command for what Suwayomi-Server will send to the conversion url: `curl -X POST "http://localhost:9999/convert" -F "image=@cat.png;type=image/png"`
- `server.serveConversions = {}` configures optional image conversions before serving the image to the client. It follows the same format as `server.downloadConversions`.
### Updater
```
server.excludeUnreadChapters = true

View File

@@ -1,5 +1,5 @@
1. **Install the latest version(or preview):** https://github.com/Suwayomi/Suwayomi-Server/releases/latest
2. Find an extensions repo, there is now no default extensions and you have to use google to find a Tachiyomi extension repo.
2. Find an extensions repo, there is now no default extensions, and you have to use Google to find a Tachiyomi extension repo.
- Note: The repo should look like `https://raw.githubusercontent.com/user/reponame` or `https://www.github.com/user/reponame`
3. Configure it using one of the following:
- Suwayomi-Server option 1: With the **new launcher**, go to the `Extensions` tab and add the extensions repo.

View File

@@ -5,7 +5,7 @@ Follow the steps below to create local manga.
If you add more chapters then you'll have to manually refresh the chapter list.
Supported chapter formats are folder with pictures inside (such as `.jpg`, `.png`, etc), `ZIP`/`CBZ`, `RAR`/`CBR` and `EPUB`. But expect better performance with directories and `ZIP`/`CBZ`.
Supported chapter formats are folder with pictures inside (such as `.jpg`, `.png`, etc.), `ZIP`/`CBZ`, `RAR`/`CBR` and `EPUB`. But expect better performance with directories and `ZIP`/`CBZ`.
**Note:** While Suwayomi does support chapters compressed as **RAR** or **CBR**, note that **RAR** or **CBR** files using the **RAR5** format are not supported yet.
@@ -111,7 +111,7 @@ Archive files such as `ZIP`/`CBZ` are supported but the folder structure inside
It is possible to add details to local manga. Like manga from other catalogs, you add information about the manga such as the author, artist, description, and genre tags.
To import details along with your local manga, you have to create a json file. It can be named anything but it must be placed within the **Manga** folder. A standard file name is `details.json`. This file will contain the extended details about the manga in the `JSON` format. You can see the example below on how to build the file. Once the file is there, the app should load the data when you first open the manga or you can pull down to refresh the details.
To import details along with your local manga, you have to create a json file. It can be named anything but it must be placed within the **Manga** folder. A standard file name is `details.json`. This file will contain the extended details about the manga in the `JSON` format. You can see the example below on how to build the file. Once the file is there, the app should load the data when you first open the manga, or you can pull down to refresh the details.
You can copy the following example and edit the details as needed:
``` json

View File

@@ -1,5 +1,5 @@
## General Troubleshooting
This guide will try to fix Suwayomi by reseting it to a clean install state.
This guide will try to fix Suwayomi by reseting it to a clean installation state.
- Make sure you have a recent backup of your library or create one in the app (if possible) because we **are going to wipe all Suwayomi data**.
- Make sure Suwayomi is not running (right click on tray icon and quit or kill it through the way your Operating System provides)

View File

@@ -1,21 +1,42 @@
package suwayomi.tachidesk.graphql.types
import kotlin.time.Duration
// These types belong to SettingsType.kt. However, since that file is auto-generated, these types need to be placed in
// a "static" file.
data class DownloadConversion(
class DownloadConversion(
val target: String,
val compressionLevel: Double? = null,
val callTimeout: Duration? = null,
val connectTimeout: Duration? = null,
val headers: Map<String, String>? = null,
)
interface SettingsDownloadConversion {
val mimeType: String
val target: String
val compressionLevel: Double?
val callTimeout: Duration?
val connectTimeout: Duration?
val headers: List<SettingsDownloadConversionHeader>?
}
class SettingsDownloadConversionType(
override val mimeType: String,
override val target: String,
override val compressionLevel: Double?,
override val callTimeout: Duration?,
override val connectTimeout: Duration?,
override val headers: List<SettingsDownloadConversionHeaderType>?
) : SettingsDownloadConversion
interface SettingsDownloadConversionHeader {
val name: String
val value: String
}
class SettingsDownloadConversionHeaderType(
override val name: String,
override val value: String,
) : SettingsDownloadConversionHeader

View File

@@ -4,6 +4,8 @@ import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.Serializable
import kotlinx.serialization.protobuf.ProtoNumber
import suwayomi.tachidesk.graphql.types.SettingsDownloadConversion
import suwayomi.tachidesk.graphql.types.SettingsDownloadConversionHeader
import kotlin.time.Duration
@OptIn(ExperimentalSerializationApi::class)
@Serializable
@@ -11,4 +13,15 @@ class BackupSettingsDownloadConversionType(
@ProtoNumber(1) override val mimeType: String,
@ProtoNumber(2) override val target: String,
@ProtoNumber(3) override val compressionLevel: Double?,
@ProtoNumber(4) override val callTimeout: Duration?,
@ProtoNumber(5) override val connectTimeout: Duration?,
@ProtoNumber(6) override val headers: List<BackupSettingsDownloadConversionHeaderType>?
) : SettingsDownloadConversion
@OptIn(ExperimentalSerializationApi::class)
@Serializable
class BackupSettingsDownloadConversionHeaderType(
@ProtoNumber(1) override val name: String,
@ProtoNumber(2) override val value: String
): SettingsDownloadConversionHeader

View File

@@ -33,11 +33,13 @@ import suwayomi.tachidesk.graphql.types.DownloadConversion
import suwayomi.tachidesk.graphql.types.KoreaderSyncChecksumMethod
import suwayomi.tachidesk.graphql.types.KoreaderSyncConflictStrategy
import suwayomi.tachidesk.graphql.types.KoreaderSyncLegacyStrategy
import suwayomi.tachidesk.graphql.types.SettingsDownloadConversionHeaderType
import suwayomi.tachidesk.graphql.types.SettingsDownloadConversionType
import suwayomi.tachidesk.graphql.types.WebUIChannel
import suwayomi.tachidesk.graphql.types.WebUIFlavor
import suwayomi.tachidesk.graphql.types.WebUIInterface
import suwayomi.tachidesk.manga.impl.backup.BackupFlags
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSettingsDownloadConversionHeaderType
import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSettingsDownloadConversionType
import suwayomi.tachidesk.manga.impl.extension.repoMatchRegex
import suwayomi.tachidesk.server.settings.BooleanSetting
@@ -537,8 +539,8 @@ class ServerConfig(
excludeFromBackup = true,
)
val downloadConversions: MutableStateFlow<Map<String, DownloadConversion>> by MapSetting<String, DownloadConversion>(
protoNumber = 57,
fun createDownloadConversionsMap(protoNumber: Int, key: String) = MapSetting<String, DownloadConversion>(
protoNumber = protoNumber,
defaultValue = emptyMap(),
group = SettingGroup.DOWNLOADER,
typeInfo =
@@ -559,6 +561,14 @@ class ServerConfig(
it.key,
it.value.target,
it.value.compressionLevel,
it.value.callTimeout,
it.value.connectTimeout,
it.value.headers?.map { header ->
SettingsDownloadConversionHeaderType(
header.key,
header.value,
)
},
)
}
},
@@ -571,6 +581,11 @@ class ServerConfig(
DownloadConversion(
target = it.target,
compressionLevel = it.compressionLevel,
callTimeout = it.callTimeout,
connectTimeout = it.connectTimeout,
headers = it.headers?.associate { header ->
header.name to header.value
},
)
}
},
@@ -583,6 +598,14 @@ class ServerConfig(
it.key,
it.value.target,
it.value.compressionLevel,
it.value.callTimeout,
it.value.connectTimeout,
it.value.headers?.map { header ->
BackupSettingsDownloadConversionHeaderType(
header.key,
header.value,
)
},
)
}
},
@@ -590,12 +613,16 @@ class ServerConfig(
description =
"""
map input mime type to conversion information, or "default" for others
server.downloadConversions."image/webp" = {
target = "image/jpeg" # image type to convert to
server.$key."image/webp" = {
target = "image/jpeg" # image type to convert to, can also be a url to an external server
compressionLevel = 0.8 # quality in range [0,1], leave away to use default compression
}
""".trimIndent(),
)
val downloadConversions: MutableStateFlow<Map<String, DownloadConversion>> by createDownloadConversionsMap(
protoNumber = 57,
key = "downloadConversions"
)
val jwtAudience: MutableStateFlow<String> by StringSetting(
protoNumber = 58,
@@ -669,6 +696,7 @@ class ServerConfig(
typeInfo = SettingsRegistry.PartialTypeInfo(imports = listOf("suwayomi.tachidesk.graphql.types.KoreaderSyncChecksumMethod")),
)
@Suppress("DEPRECATION")
@Deprecated("Use koreaderSyncStrategyForward and koreaderSyncStrategyBackward instead")
val koreaderSyncStrategy: MutableStateFlow<KoreaderSyncLegacyStrategy> by MigratedConfigValue(
protoNumber = 64,
@@ -707,15 +735,11 @@ class ServerConfig(
),
readMigrated = {
// This is a best-effort reverse mapping. It's not perfect but covers common cases.
when {
koreaderSyncStrategyForward.value == KoreaderSyncConflictStrategy.PROMPT &&
koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.PROMPT -> KoreaderSyncLegacyStrategy.PROMPT
koreaderSyncStrategyForward.value == KoreaderSyncConflictStrategy.KEEP_REMOTE &&
koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.KEEP_LOCAL -> KoreaderSyncLegacyStrategy.SILENT
koreaderSyncStrategyForward.value == KoreaderSyncConflictStrategy.KEEP_LOCAL &&
koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.KEEP_LOCAL -> KoreaderSyncLegacyStrategy.SEND
koreaderSyncStrategyForward.value == KoreaderSyncConflictStrategy.KEEP_REMOTE &&
koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.KEEP_REMOTE -> KoreaderSyncLegacyStrategy.RECEIVE
when (koreaderSyncStrategyForward.value) {
KoreaderSyncConflictStrategy.PROMPT if koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.PROMPT -> KoreaderSyncLegacyStrategy.PROMPT
KoreaderSyncConflictStrategy.KEEP_REMOTE if koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.KEEP_LOCAL -> KoreaderSyncLegacyStrategy.SILENT
KoreaderSyncConflictStrategy.KEEP_LOCAL if koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.KEEP_LOCAL -> KoreaderSyncLegacyStrategy.SEND
KoreaderSyncConflictStrategy.KEEP_REMOTE if koreaderSyncStrategyBackward.value == KoreaderSyncConflictStrategy.KEEP_REMOTE -> KoreaderSyncLegacyStrategy.RECEIVE
else -> KoreaderSyncLegacyStrategy.DISABLED
}
},
@@ -885,6 +909,11 @@ class ServerConfig(
description = "Controls the MimeType that Suwayomi sends in OPDS entries for CBZ archives. Also affects global CBZ download. Modern follows recent IANA standard (2017), while LEGACY (deprecated mimetype for .cbz) and COMPATIBLE (deprecated mimetype for all comic archives) might be more compatible with older clients.",
)
val serveConversions: MutableStateFlow<Map<String, DownloadConversion>> by createDownloadConversionsMap(
protoNumber = 84,
key = "serveConversions"
)
/** ****************************************************************** **/

View File

@@ -14,6 +14,7 @@ object ConfigTypeRegistration {
registerCustomType(MutableStateFlowType())
registerCustomType(DurationType())
registerCustomType(DownloadConversionType())
registered = true
}

View File

@@ -0,0 +1,73 @@
package suwayomi.tachidesk.server.util
import com.typesafe.config.Config
import com.typesafe.config.ConfigFactory
import com.typesafe.config.ConfigValue
import io.github.config4k.ClassContainer
import io.github.config4k.CustomType
import io.github.config4k.extract
import io.github.config4k.toConfig
import suwayomi.tachidesk.graphql.types.DownloadConversion
import kotlin.time.Duration
class DownloadConversionType : CustomType {
override fun parse(
clazz: ClassContainer,
config: Config,
name: String,
): Any? {
val target = config.extract<String>("$name.target")
val compressionLevel = config.extract<Double?>("$name.compressionLevel")
val callTimeout = config.extract<Duration?>("$name.callTimeout")
val connectTimeout = config.extract<Duration?>("$name.connectTimeout")
val headers = config.extract<Map<String, String>?>("$name.headers")
return DownloadConversion(
target = target,
compressionLevel = compressionLevel,
callTimeout = callTimeout,
connectTimeout = connectTimeout,
headers = headers,
)
}
override fun testParse(clazz: ClassContainer): Boolean =
clazz.mapperClass.qualifiedName == "suwayomi.tachidesk.graphql.types.DownloadConversion"
override fun testToConfig(obj: Any): Boolean = obj is DownloadConversion
override fun toConfig(
obj: Any,
name: String,
): Config {
val conversion = obj as DownloadConversion
val builder = ConfigFactory.empty()
var config =
builder
.withValue("$name.target", conversion.target.asConfigValue())
.withValueIfPresent("$name.compressionLevel", conversion.compressionLevel)
.withValueIfPresent("$name.callTimeout", conversion.callTimeout?.toString())
.withValueIfPresent("$name.connectTimeout", conversion.connectTimeout?.toString())
if (conversion.headers != null) {
config =
config
.withValue("$name.headers", conversion.headers.asConfigValue())
}
return config
}
private fun Config.withValueIfPresent(
key: String,
value: Any?,
): Config =
if (value != null) {
withValue(key, value.asConfigValue())
} else {
this
}
private fun Any.asConfigValue(): ConfigValue = toConfig("internal").getValue("internal")
}

View File

@@ -475,7 +475,7 @@ object MangaController {
ctx.future {
future {
Page.getPageImage(
Page.getPageImageServe(
mangaId = mangaId,
chapterIndex = chapterIndex,
index = index,

View File

@@ -10,6 +10,7 @@ package suwayomi.tachidesk.manga.impl
import eu.kanade.tachiyomi.source.local.LocalSource
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.online.HttpSource
import io.github.oshai.kotlinlogging.KotlinLogging
import kotlinx.coroutines.flow.StateFlow
import libcore.net.MimeUtils
import org.jetbrains.exposed.sql.SortOrder
@@ -17,6 +18,7 @@ import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import suwayomi.tachidesk.graphql.types.DownloadConversion
import suwayomi.tachidesk.manga.impl.util.getChapterCachePath
import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse
@@ -24,14 +26,20 @@ import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
import suwayomi.tachidesk.manga.model.table.ChapterTable
import suwayomi.tachidesk.manga.model.table.MangaTable
import suwayomi.tachidesk.manga.model.table.PageTable
import suwayomi.tachidesk.server.serverConfig
import suwayomi.tachidesk.util.ConversionUtil
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.InputStream
import javax.imageio.IIOImage
import javax.imageio.ImageIO
import javax.imageio.ImageWriteParam
import javax.imageio.ImageWriter
object Page {
private val logger = KotlinLogging.logger {}
/**
* A page might have a imageUrl ready from the get go, or we might need to
* go an extra step and call fetchImageUrl to get it.
@@ -51,7 +59,6 @@ object Page {
chapterId: Int? = null,
chapterIndex: Int? = null,
index: Int,
format: String? = null,
progressFlow: ((StateFlow<Int>) -> Unit)? = null,
): Pair<InputStream, String> {
val mangaEntry = transaction { MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() }
@@ -73,7 +80,7 @@ object Page {
try {
if (chapterEntry[ChapterTable.isDownloaded]) {
return convertImageResponse(ChapterDownloadHelper.getImage(mangaId, chapterId, index), format)
return ChapterDownloadHelper.getImage(mangaId, chapterId, index)
}
} catch (_: Exception) {
// ignore and fetch again
@@ -102,15 +109,12 @@ object Page {
// is of archive format
if (LocalSource.pageCache.containsKey(chapterEntry[ChapterTable.url])) {
val pageStream = LocalSource.pageCache[chapterEntry[ChapterTable.url]]!![index]
return convertImageResponse(pageStream() to (ImageUtil.findImageType { pageStream() }?.mime ?: "image/jpeg"), format)
return pageStream() to (ImageUtil.findImageType { pageStream() }?.mime ?: "image/jpeg")
}
// is of directory format
val imageFile = File(tachiyomiPage.imageUrl!!)
return convertImageResponse(
imageFile.inputStream() to (ImageUtil.findImageType { imageFile.inputStream() }?.mime ?: "image/jpeg"),
format,
)
return imageFile.inputStream() to (ImageUtil.findImageType { imageFile.inputStream() }?.mime ?: "image/jpeg")
}
val source = getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference])
@@ -130,38 +134,210 @@ object Page {
val cacheSaveDir = getChapterCachePath(mangaId, chapterId)
// Note: don't care about invalidating cache because OS cache is not permanent
return convertImageResponse(
getImageResponse(cacheSaveDir, fileName) {
source.getImage(tachiyomiPage)
},
format,
)
return getImageResponse(cacheSaveDir, fileName) {
source.getImage(tachiyomiPage)
}
}
suspend fun getPageImageServe(
mangaId: Int,
chapterIndex: Int,
index: Int,
format: String? = null,
): Pair<InputStream, String> {
val (inputStream, mime) =
getPageImage(
mangaId = mangaId,
chapterIndex = chapterIndex,
index = index,
)
val conversions = serverConfig.serveConversions.value
val defaultConversion = conversions["default"]
val formatConversion = format?.let { DownloadConversion(target = it) }
val conversion =
formatConversion
?: conversions[mime]
?: defaultConversion
?: return inputStream to mime
val converted =
try {
convertImageResponse(
image = inputStream,
mime = mime,
conversion = conversion,
)
} catch (e: Exception) {
logger.error(e) { "Error while post-processing image" }
null
}
return converted?.also { inputStream.close() } ?: (inputStream to mime)
}
suspend fun getPageImageDownload(
mangaId: Int,
chapterId: Int,
index: Int,
downloadCacheFolder: File,
fileName: String,
progressFlow: (StateFlow<Int>) -> Unit,
) {
val (inputStream, mime) =
getPageImage(
mangaId = mangaId,
chapterId = chapterId,
index = index,
progressFlow = progressFlow,
)
val conversions = serverConfig.downloadConversions.value
if (conversions.isEmpty() || !downloadCacheFolder.exists()) {
inputStream.close()
return
}
val defaultConversion = conversions["default"]
val conversion =
conversions[mime]
?: defaultConversion
if (conversion == null) {
inputStream.close()
return
}
try {
val converted =
try {
convertImageResponse(
image = inputStream,
mime = mime,
conversion = conversion,
)
} catch (e: Exception) {
throw e
} finally {
inputStream.close()
}
if (converted != null) {
val (convertedStream, convertedMime) = converted
val convertedExtension =
MimeUtils.guessExtensionFromMimeType(convertedMime)
?: convertedMime.substringAfter('/')
val convertedPage =
File(
downloadCacheFolder,
"$fileName.$convertedExtension",
)
convertedPage.outputStream().use { outputStream ->
convertedStream.use { it.copyTo(outputStream) }
}
val extension =
MimeUtils.guessExtensionFromMimeType(mime)
?: mime.substringAfter('/')
if (extension != convertedExtension) {
File(
downloadCacheFolder,
"$fileName.$extension",
).delete()
}
}
} catch (e: Exception) {
logger.warn(e) { "Error while post-processing image" }
}
}
private suspend fun convertImageResponse(
image: Pair<InputStream, String>,
format: String? = null,
): Pair<InputStream, String> {
val imageExtension = MimeUtils.guessExtensionFromMimeType(image.second) ?: image.second.removePrefix("image/")
image: InputStream,
mime: String,
conversion: DownloadConversion,
): Pair<InputStream, String>? {
// Apply HTTP post-process if configured (complementary with format conversion)
if (ConversionUtil.isHttpPostProcess(conversion)) {
try {
val processedStream =
ConversionUtil
.imageHttpPostProcess(
inputStream = image,
mimeType = mime,
conversion = conversion,
)?.buffered()
if (processedStream != null) {
val mime =
ImageUtil.findImageType(processedStream)?.mime
?: "image/jpeg"
val targetExtension =
(if (format != imageExtension) format else null)
?: return image
return processedStream to mime
}
} catch (e: Exception) {
// HTTP post-processing failed, continue with original image
logger.warn(e) { "Error while post-processing image" }
}
return null
} else {
if (mime == conversion.target) {
return null
}
val outStream = ByteArrayOutputStream()
val writers = ImageIO.getImageWritersBySuffix(targetExtension)
val writer = writers.next()
ImageIO.createImageOutputStream(outStream).use { o ->
writer.setOutput(o)
val inImage =
ConversionUtil.readImage(image.first, image.second)
?: throw NoSuchElementException("No conversion to $targetExtension possible")
writer.write(inImage)
return convertToFormat(image, mime, conversion)
}
}
private fun convertToFormat(
inputStream: InputStream,
sourceMimeType: String,
target: DownloadConversion,
): Pair<InputStream, String>? {
val outStream = ByteArrayOutputStream()
val conversionWriter =
getConversionWriter(
target.target,
target.compressionLevel,
)
if (conversionWriter == null) {
logger.warn { "Conversion aborted: No reader for target format ${target.target}" }
return inputStream to sourceMimeType
}
val (writer, writerParams) = conversionWriter
try {
ImageIO.createImageOutputStream(outStream).use { o ->
writer.setOutput(o)
val inImage =
ConversionUtil.readImage(inputStream, sourceMimeType)
?: throw NoSuchElementException("No conversion to ${target.target} possible")
writer.write(null, IIOImage(inImage, null, null), writerParams)
}
} catch (e: Exception) {
logger.warn(e) { "Conversion aborted" }
return null
} finally {
writer.dispose()
}
writer.dispose()
val inStream = ByteArrayInputStream(outStream.toByteArray())
return Pair(inStream.buffered(), MimeUtils.guessMimeTypeFromExtension(targetExtension) ?: "image/$targetExtension")
return Pair(inStream.buffered(), target.target)
}
private fun getConversionWriter(
targetMime: String,
compressionLevel: Double?,
): Pair<ImageWriter, ImageWriteParam>? {
val writers = ImageIO.getImageWritersByMIMEType(targetMime)
val writer =
try {
writers.next()
} catch (_: NoSuchElementException) {
return null
}
val writerParams = writer.defaultWriteParam
compressionLevel?.let {
writerParams.compressionMode = ImageWriteParam.MODE_EXPLICIT
writerParams.compressionQuality = it.toFloat()
}
return writer to writerParams
}
/** converts 0 to "001" */

View File

@@ -151,10 +151,12 @@ abstract class ChaptersFilesProvider<Type : FileType>(
try {
Page
.getPageImage(
.getPageImageDownload(
mangaId = download.mangaId,
chapterId = download.chapterId,
index = pageNum,
downloadCacheFolder,
fileName,
) { flow ->
pageProgressJob =
flow
@@ -167,8 +169,7 @@ abstract class ChaptersFilesProvider<Type : FileType>(
false,
) // don't throw on canceled download here since we can't do anything
}.launchIn(scope)
}.first
.close()
}
} finally {
// always cancel the page progress job even if it throws an exception to avoid memory leaks
pageProgressJob?.cancel()
@@ -188,8 +189,6 @@ abstract class ChaptersFilesProvider<Type : FileType>(
},
)
maybeConvertPages(downloadCacheFolder)
handleSuccessfulDownload()
// Calculate and save Koreader hash for CBZ files
@@ -221,104 +220,4 @@ abstract class ChaptersFilesProvider<Type : FileType>(
abstract fun getAsArchiveStream(): Pair<InputStream, Long>
abstract fun getArchiveSize(): Long
private fun maybeConvertPages(chapterCacheFolder: File) {
val conversions = serverConfig.downloadConversions.value
if (!chapterCacheFolder.isDirectory || conversions.isEmpty()) {
return
}
val pages =
chapterCacheFolder
.listFiles()
.orEmpty()
.filter { it.name != COMIC_INFO_FILE }
val pagesByMimeType =
pages
.groupBy { MimeUtils.guessMimeTypeFromExtension(it.extension) }
.mapValues { it.value.map { it.nameWithoutExtension } }
logger.debug { "maybeConvertPages: pagesByMimeType= $pagesByMimeType; conversions= $conversions" }
pages.forEach { page ->
val imageType = MimeUtils.guessMimeTypeFromExtension(page.extension) ?: return@forEach
val defaultConversion = conversions["default"]
val conversion = conversions[imageType]
val targetConversion = conversion ?: defaultConversion ?: return@forEach
val (targetMime) = targetConversion
val requiresConversion = imageType != targetMime && targetMime != "none"
if (!requiresConversion) {
return@forEach
}
convertPage(page, targetConversion)
}
}
private fun convertPage(
page: File,
conversion: DownloadConversion,
) {
val (targetMime, compressionLevel) = conversion
val targetExtension =
MimeUtils.guessExtensionFromMimeType(targetMime) ?: targetMime.removePrefix("image/")
val convertedPage = File(page.parentFile, page.nameWithoutExtension + "." + targetExtension)
val conversionWriter = getConversionWriter(targetMime, compressionLevel)
if (conversionWriter == null) {
logger.warn { "Conversion aborted: No reader for target format $targetMime" }
return
}
val (writer, writerParams) = conversionWriter
val success =
try {
ImageIO.createImageOutputStream(convertedPage).use { outStream ->
writer.setOutput(outStream)
val inImage = ConversionUtil.readImage(page) ?: return@use false
writer.write(null, IIOImage(inImage, null, null), writerParams)
true
}
} catch (e: Exception) {
logger.warn(e) { "Conversion aborted: for image $page" }
false
}
writer.dispose()
if (success) {
page.delete()
} else {
convertedPage.delete()
}
}
private fun getConversionWriter(
targetMime: String,
compressionLevel: Double?,
): Pair<ImageWriter, ImageWriteParam>? {
val writers = ImageIO.getImageWritersByMIMEType(targetMime)
val writer =
try {
writers.next()
} catch (_: NoSuchElementException) {
return null
}
val writerParams = writer.defaultWriteParam
compressionLevel?.let {
writerParams.compressionMode = ImageWriteParam.MODE_EXPLICIT
writerParams.compressionQuality = it.toFloat()
}
return writer to writerParams
}
}

View File

@@ -44,10 +44,11 @@ class ThumbnailFileProvider(
return true
}
Manga.fetchMangaThumbnail(mangaId).first.use { image ->
val (inputStream, mime) = Manga.fetchMangaThumbnail(mangaId)
inputStream.use { image ->
makeSureDownloadDirExists()
val filePath = getThumbnailDownloadPath(mangaId)
ImageResponse.saveImage(filePath, image)
ImageResponse.saveImage(filePath, image, mime)
}
return true

View File

@@ -7,6 +7,7 @@ package suwayomi.tachidesk.manga.impl.util.storage
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
import libcore.net.MimeUtils
import okhttp3.Response
import okhttp3.internal.closeQuietly
import java.io.File
@@ -69,7 +70,12 @@ object ImageResponse {
try {
if (response.code == 200) {
val (actualSavePath, imageType) = saveImage(filePath, response.body.byteStream())
val (actualSavePath, imageType) =
saveImage(
filePath,
response.body.byteStream(),
response.header("Content-Type"),
)
return pathToInputStream(actualSavePath) to imageType
} else {
throw Exception("request error! ${response.code}")
@@ -87,20 +93,28 @@ object ImageResponse {
fun saveImage(
filePath: String,
image: InputStream,
mimeType: String?,
): Pair<String, String> {
val mimeType = mimeType?.takeIf { it.startsWith("image/") }?.lowercase()
val tmpSavePath = "$filePath.tmp"
val tmpSaveFile = File(tmpSavePath)
image.use { input -> tmpSaveFile.outputStream().use { output -> input.copyTo(output) } }
// find image type
val imageType =
ImageUtil.findImageType { tmpSaveFile.inputStream() }?.mime
?: "image/jpeg"
ImageUtil.findImageType { tmpSaveFile.inputStream() }
?: ImageUtil.ImageType.entries.find {
it.mime == mimeType
}
val extension =
imageType?.extension ?: mimeType?.let {
MimeUtils.guessExtensionFromMimeType(it)
} ?: "jpg"
val actualSavePath = "$filePath.${imageType.substringAfter("/")}"
val actualSavePath = "$filePath.$extension"
tmpSaveFile.renameTo(File(actualSavePath))
return Pair(actualSavePath, imageType)
return Pair(actualSavePath, imageType?.mime ?: mimeType ?: "image/jpeg")
}
fun clearCachedImage(

View File

@@ -1,35 +1,28 @@
package suwayomi.tachidesk.util
import eu.kanade.tachiyomi.network.NetworkHelper
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.network.await
import io.github.oshai.kotlinlogging.KotlinLogging
import libcore.net.MimeUtils
import okhttp3.Headers
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.MultipartBody
import okhttp3.RequestBody.Companion.asRequestBody
import suwayomi.tachidesk.graphql.types.DownloadConversion
import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
import uy.kohesive.injekt.injectLazy
import java.awt.image.BufferedImage
import java.io.File
import java.io.InputStream
import java.nio.file.Files
import javax.imageio.ImageIO
import kotlin.getValue
object ConversionUtil {
val logger = KotlinLogging.logger {}
public fun readImage(image: File): BufferedImage? {
val readers = ImageIO.getImageReadersBySuffix(image.extension)
image.inputStream().use {
ImageIO.createImageInputStream(it).use { inputStream ->
for (reader in readers) {
try {
reader.setInput(inputStream)
return reader.read(0)
} catch (e: Throwable) {
logger.debug(e) { "Reader ${reader.javaClass.name} not suitable" }
} finally {
reader.dispose()
}
}
}
}
logger.info { "No suitable image converter found for ${image.name}" }
return null
}
public fun readImage(
fun readImage(
image: InputStream,
mimeType: String,
): BufferedImage? {
@@ -49,4 +42,100 @@ object ConversionUtil {
logger.info { "No suitable image converter found for $mimeType" }
return null
}
private val networkService: NetworkHelper by injectLazy()
/**
* Send image to external HTTP service for post-processing
* Returns the processed image stream or null if failed
*/
suspend fun imageHttpPostProcess(
imageFile: File,
conversion: DownloadConversion,
mimeType: String,
): InputStream? =
try {
logger.debug { "Sending ${imageFile.name} to HTTP converter: ${conversion.target}" }
val requestBody =
MultipartBody
.Builder()
.setType(MultipartBody.FORM)
.addFormDataPart(
"image",
imageFile.name,
imageFile.asRequestBody(mimeType.toMediaType()),
).build()
val client =
networkService.client
.newBuilder()
.apply {
if (conversion.callTimeout != null) {
callTimeout(conversion.callTimeout!!)
}
if (conversion.connectTimeout != null) {
connectTimeout(conversion.connectTimeout!!)
}
}.build()
val response =
client
.newCall(
POST(
conversion.target,
body = requestBody,
headers =
Headers
.Builder()
.apply {
conversion.headers?.forEach {
set(it.key, it.value)
}
}.build(),
),
).await()
logger.debug { "HTTP conversion successful for ${imageFile.name}" }
response.body.byteStream()
} catch (e: Exception) {
logger.warn(e) { "HTTP conversion failed for ${imageFile.name}" }
null
}
/**
* Overload that takes InputStream and mimeType, creates temp file for HTTP upload
*/
suspend fun imageHttpPostProcess(
inputStream: InputStream,
mimeType: String,
conversion: DownloadConversion,
): InputStream? =
try {
// Create temporary file from input stream
val extension =
MimeUtils.guessExtensionFromMimeType(mimeType)
?: mimeType.substringAfter('/')
val tempFile = Files.createTempFile("conversion", ".$extension").toFile()
tempFile.outputStream().use { output ->
inputStream.copyTo(output)
}
// Convert using file method
val result = imageHttpPostProcess(tempFile, conversion, mimeType)
// Clean up temp file
tempFile.delete()
result
} catch (e: Exception) {
logger.warn(e) { "Failed to create temp file for HTTP converter" }
null
}
/**
* Check if a DownloadConversion target is an HTTP URL
*/
fun isHttpPostProcess(conversion: DownloadConversion): Boolean =
conversion.target.startsWith("http://") || conversion.target.startsWith("https://")
}