diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 69b28f85..48008bbd 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -94,9 +94,6 @@ appdirs = "net.harawata:appdirs:1.2.1" zip4j = "net.lingala.zip4j:zip4j:2.11.2" junrar = "com.github.junrar:junrar:7.5.3" -# CloudflareInterceptor WebView -jep = "black.ninia:jep:4.1.1" - # AES/CBC/PKCS7Padding Cypher provider bouncycastle = "org.bouncycastle:bcprov-jdk18on:1.72" diff --git a/server/build.gradle.kts b/server/build.gradle.kts index e2678cf7..2521bab5 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -50,9 +50,6 @@ dependencies { implementation(libs.zip4j) implementation(libs.junrar) - // CloudflareInterceptor - implementation(libs.jep) - // AES/CBC/PKCS7Padding Cypher provider for zh.copymanga implementation(libs.bouncycastle) diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt index a9e8db7a..2769e51d 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt @@ -1,8 +1,7 @@ package eu.kanade.tachiyomi.network.interceptor import eu.kanade.tachiyomi.network.NetworkHelper -import eu.kanade.tachiyomi.network.interceptor.CFClearance.resolveWithWebView -import jep.SharedInterpreter +import eu.kanade.tachiyomi.network.interceptor.CloudflareBypasser.resolveWithWebView import kotlinx.serialization.Serializable import kotlinx.serialization.decodeFromString import kotlinx.serialization.json.Json @@ -17,7 +16,12 @@ import org.kodein.di.conf.global import org.kodein.di.instance import suwayomi.tachidesk.server.serverConfig import uy.kohesive.injekt.injectLazy +import java.io.BufferedReader +import java.io.Closeable +import java.io.File import java.io.IOException +import java.io.InputStreamReader +import java.io.PrintWriter import java.util.concurrent.TimeUnit class CloudflareInterceptor : Interceptor { @@ -65,48 +69,28 @@ class CloudflareInterceptor : Interceptor { } } -/* - * This class is ported from https://github.com/vvanglro/cf-clearance - * The original code is licensed under Apache 2.0 -*/ -object CFClearance { +object CloudflareBypasser { private val logger = KotlinLogging.logger {} private val network: NetworkHelper by injectLazy() - private var undetectedChromeInitialized = false - - fun initializeUndetectedChrome() { - if (undetectedChromeInitialized) { - return - } - SharedInterpreter().use { jep -> - val uc = "/home/armor/programming/github-clones/undetected-chromedriver" - - jep.exec("import sys") - jep.exec("sys.path.insert(0,'$uc')") - - jep.exec("import undetected_chromedriver") // Cache import - } - undetectedChromeInitialized = true - } - fun resolveWithWebView(originalRequest: Request): Request { val url = originalRequest.url.toString() logger.debug { "resolveWithWebView($url)" } - initializeUndetectedChrome() - val cookies = SharedInterpreter().use { jep -> + val cookies = PythonInterpreter.create().use { py -> try { - jep.exec("import undetected_chromedriver as uc") + py.exec("import undetected_chromedriver as uc") - jep.exec("options = uc.ChromeOptions()") + py.exec("options = uc.ChromeOptions()") if (serverConfig.socksProxyEnabled) { val proxy = "socks5://${serverConfig.socksProxyHost}:${serverConfig.socksProxyPort}" - jep.exec("options.add_argument('--proxy-server=$proxy')") + py.exec("options.add_argument('--proxy-server=$proxy')") } - jep.exec("driver = uc.Chrome(options=options)") + py.exec("driver = uc.Chrome(options=options)") + + // TODO: handle custom userAgent // val userAgent = originalRequest.header("User-Agent") // if (userAgent != null) { @@ -116,11 +100,11 @@ object CFClearance { // } else { // browser.newPage().use { getCookies(it, url) } // } - jep.exec("driver.get('$url')") + py.exec("driver.get('$url')") - getCookies(jep) + getCookies(py) } finally { - jep.exec("driver.quit()") + py.exec("driver.quit()") } } @@ -164,18 +148,17 @@ object CFClearance { throw CloudflareBypassException("Webview is disabled, enable it in server config") } - initializeUndetectedChrome() - SharedInterpreter().use { jep -> - jep.exec("import undetected_chromedriver as uc") + PythonInterpreter.create().use { py -> + py.exec("import undetected_chromedriver as uc") - jep.exec("options = uc.ChromeOptions()") -// jep.exec("options.add_argument('--headless')") -// jep.exec("options.add_argument('--disable-gpu')") - jep.exec("driver = uc.Chrome(options=options)") + py.exec("options = uc.ChromeOptions()") + py.exec("options.add_argument('--headless')") + py.exec("options.add_argument('--disable-gpu')") + py.exec("driver = uc.Chrome(options=options)") - jep.exec("userAgent = driver.execute_script('return navigator.userAgent')") - val userAgent = jep.getValue("userAgent", java.lang.String::class.java).toString() - jep.exec("driver.quit()") + py.exec("userAgent = driver.execute_script('return navigator.userAgent')") + val userAgent = py.getValue("userAgent") + py.exec("driver.quit()") userAgent } @@ -198,18 +181,18 @@ object CFClearance { ) private val json by DI.global.instance() - private fun getCookies(jep: SharedInterpreter): List { - val challengeResolved = waitForChallengeResolve(jep) + private fun getCookies(py: PythonInterpreter): List { + val challengeResolved = waitForChallengeResolve(py) return if (challengeResolved) { - jep.exec("import json") - jep.exec("cookies = json.dumps(driver.get_cookies())") - val cookiesJson = jep.getValue("cookies", java.lang.String::class.java).toString() + py.exec("import json") + py.exec("cookies = json.dumps(driver.get_cookies())") + val cookiesJson = py.getValue("cookies") val cookies = json.decodeFromString>(cookiesJson) logger.debug { - jep.exec("userAgent = driver.execute_script('return navigator.userAgent')") - val userAgent = jep.getValue("userAgent", java.lang.String::class.java).toString() + py.exec("userAgent = driver.execute_script('return navigator.userAgent')") + val userAgent = py.getValue("userAgent") "Webview User-Agent is $userAgent" } @@ -230,14 +213,14 @@ object CFClearance { } } - private fun waitForChallengeResolve(jep: SharedInterpreter): Boolean { + private fun waitForChallengeResolve(py: PythonInterpreter): Boolean { // sometimes the user has to solve the captcha challenge manually and multiple times, potentially wait a long time val timeoutSeconds = 120 repeat(timeoutSeconds) { TimeUnit.SECONDS.sleep(1) val success = try { - jep.exec("r = driver.execute_script('return document.querySelector(\"#challenge-form\") == null')") - jep.getValue("r", java.lang.Boolean::class.java).toString().toBoolean() + py.exec("r = driver.execute_script('return document.querySelector(\"#challenge-form\") == null')") + py.getValue("r").lowercase().toBoolean() } catch (e: Exception) { logger.debug(e) { "query Error" } false @@ -248,3 +231,99 @@ object CFClearance { } } private class CloudflareBypassException(message: String?) : Exception(message) + +class PythonInterpreter private constructor(private val process: Process) : Closeable { + private val logger = KotlinLogging.logger {} + + private val stdin = process.outputStream + private val stdout = process.inputStream + private val stderr = process.errorStream + + private val stdinWriter = PrintWriter(stdin) + private val stdoutReader = BufferedReader(InputStreamReader(stdout)) + private val stderrReader = BufferedReader(InputStreamReader(stderr)) + + private fun rawExec(command: String) { + stdinWriter.println(command) + stdinWriter.flush() + } + + val BUFF_SIZE = 102400 + fun exec(command: String) { + logger.debug { "Python Command: $command" } + rawExec(command) + makeSureExecDone() + } + + private val commandOutputs = mutableListOf() + + fun makeSureExecDone() { + val makeSureString = "PYTHON_IS_READY" + + rawExec("print('$makeSureString')") + var line: String? + do { + line = stdoutReader.readLine() + if (line != makeSureString) { + commandOutputs.add(line) + } + } while (line != makeSureString) + + val pyError = buildString { + while (stderrReader.ready()) + append(stderr.read().toChar()) + } + if (pyError.isNotEmpty()) { + println("Python STDERR: $pyError") + } + } + + fun getValue(variableName: String): String { + exec("print($variableName)") + return commandOutputs.last() + } + + private fun flushStdoutReader() { + var line: String? + while (stdoutReader.ready()) { + val line = stdoutReader.readLine() + } + } + + fun destroy() { + stdinWriter.close() + stdoutReader.close() + stderr.close() + process.destroy() + } + override fun close() { + destroy() + } + companion object { + fun create(pythonPath: String, workingDir: String, pythonStartupFile: String? = null): PythonInterpreter { + val processBuilder = ProcessBuilder() + .command(pythonPath, "-i", "-q") + processBuilder.directory(File(workingDir)) + + if (pythonStartupFile != null) { + val environment = processBuilder.environment() + environment["PYTHONSTARTUP"] = pythonStartupFile + } + + val process = processBuilder.start() + + val pythonInterpreter = PythonInterpreter(process) + + return pythonInterpreter + } + + fun create(): PythonInterpreter { + val uc = serverConfig.undetectedChromePath + return create( + "$uc/venv/bin/python", + uc, + "$uc/console.py" + ) + } + } +} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt index 69d88d86..7324c012 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt @@ -3,7 +3,7 @@ package eu.kanade.tachiyomi.source.online import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.NetworkHelper import eu.kanade.tachiyomi.network.asObservableSuccess -import eu.kanade.tachiyomi.network.interceptor.CFClearance.getWebViewUserAgent +import eu.kanade.tachiyomi.network.interceptor.CloudflareBypasser.getWebViewUserAgent import eu.kanade.tachiyomi.network.newCallWithProgress import eu.kanade.tachiyomi.source.CatalogueSource import eu.kanade.tachiyomi.source.model.FilterList diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt index 3e86c76c..74101473 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt @@ -44,7 +44,10 @@ class ServerConfig(config: Config, moduleName: String = MODULE_NAME) : SystemPro // misc val debugLogsEnabled: Boolean = debugLogsEnabled(GlobalConfigManager.config) val systemTrayEnabled: Boolean by overridableConfig + + // Webview val webviewEnabled: Boolean by overridableConfig + val undetectedChromePath: String by overridableConfig companion object { fun register(config: Config) = ServerConfig(config.getConfig(MODULE_NAME)) diff --git a/server/src/main/resources/server-reference.conf b/server/src/main/resources/server-reference.conf index 6b4bc6fb..6572aa5f 100644 --- a/server/src/main/resources/server-reference.conf +++ b/server/src/main/resources/server-reference.conf @@ -29,4 +29,7 @@ server.basicAuthPassword = "" # misc server.debugLogsEnabled = false server.systemTrayEnabled = true + +# Webview server.webviewEnabled = false +server.undetectedChromePath = ""