From 3272b9dec5fe0da170f7106dc865bc438fe200a2 Mon Sep 17 00:00:00 2001 From: Aria Moradi Date: Mon, 23 Aug 2021 03:45:10 +0430 Subject: [PATCH] add CloudflareInterceptor from TachiWeb-Server --- server/build.gradle.kts | 3 + .../network/CloudflareInterceptor.kt | 177 ------------------ .../kanade/tachiyomi/network/NetworkHelper.kt | 74 ++++---- .../tachiyomi/network/PersistentCookieJar.kt | 20 ++ .../network/PersistentCookieStore.kt | 80 ++++++++ .../interceptor/CloudflareInterceptor.kt | 95 ++++++++++ .../interceptor/UserAgentInterceptor.kt | 22 +++ .../tachiyomi/source/online/HttpSource.kt | 2 +- 8 files changed, 255 insertions(+), 218 deletions(-) delete mode 100644 server/src/main/kotlin/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt create mode 100644 server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieJar.kt create mode 100644 server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieStore.kt create mode 100644 server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt create mode 100644 server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/UserAgentInterceptor.kt diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 7de1464d..bec9ba46 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -69,6 +69,9 @@ dependencies { // extracting zip files implementation("net.lingala.zip4j:zip4j:2.9.0") + // CloudflareInterceptor + implementation("net.sourceforge.htmlunit:htmlunit:2.52.0") + // Source models and interfaces from Tachiyomi 1.x // using source class from tachiyomi commit 9493577de27c40ce8b2b6122cc447d025e34c477 to not depend on tachiyomi.sourceapi // implementation("tachiyomi.sourceapi:source-api:1.1") diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt deleted file mode 100644 index beb97611..00000000 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt +++ /dev/null @@ -1,177 +0,0 @@ -package eu.kanade.tachiyomi.network - -/* - * Copyright (C) Contributors to the Suwayomi project - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ - -// import android.annotation.SuppressLint -// import android.content.Context -// import android.os.Build -// import android.os.Handler -// import android.os.Looper -// import android.webkit.WebSettings -// import android.webkit.WebView -// import android.widget.Toast -// import eu.kanade.tachiyomi.R -// import eu.kanade.tachiyomi.util.lang.launchUI -// import eu.kanade.tachiyomi.util.system.WebViewClientCompat -// import eu.kanade.tachiyomi.util.system.WebViewUtil -// import eu.kanade.tachiyomi.util.system.isOutdated -// import eu.kanade.tachiyomi.util.system.setDefaultSettings -// import eu.kanade.tachiyomi.util.system.toast -import okhttp3.Interceptor -import okhttp3.Response -// import uy.kohesive.injekt.injectLazy - -class CloudflareInterceptor() : Interceptor { - -// private val handler = Handler(Looper.getMainLooper()) - -// private val networkHelper = NetworkHelper() - - /** - * When this is called, it initializes the WebView if it wasn't already. We use this to avoid - * blocking the main thread too much. If used too often we could consider moving it to the - * Application class. - */ -// private val initWebView by lazy { -// WebSettings.getDefaultUserAgent(context) -// } - - @Synchronized - override fun intercept(chain: Interceptor.Chain): Response { - val originalRequest = chain.request() - return chain.proceed(originalRequest) - -// if (!WebViewUtil.supportsWebView(context)) { -// launchUI { -// context.toast(R.string.information_webview_required, Toast.LENGTH_LONG) -// } -// return chain.proceed(originalRequest) -// } -// -// initWebView -// -// val response = chain.proceed(originalRequest) -// -// // Check if Cloudflare anti-bot is on -// if (response.code != 503 || response.header("Server") !in SERVER_CHECK) { -// return response -// } -// -// try { -// response.close() -// networkHelper.cookieManager.remove(originalRequest.url, COOKIE_NAMES, 0) -// val oldCookie = networkHelper.cookieManager.get(originalRequest.url) -// .firstOrNull { it.name == "cf_clearance" } -// resolveWithWebView(originalRequest, oldCookie) -// -// return chain.proceed(originalRequest) -// } catch (e: Exception) { -// // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that -// // we don't crash the entire app -// throw IOException(e) -// } - } -// -// // @SuppressLint("SetJavaScriptEnabled") -// private fun resolveWithWebView(request: Request, oldCookie: Cookie?) { -// // We need to lock this thread until the WebView finds the challenge solution url, because -// // OkHttp doesn't support asynchronous interceptors. -// val latch = CountDownLatch(1) -// -// var webView: WebView? = null -// -// var challengeFound = false -// var cloudflareBypassed = false -// var isWebViewOutdated = false -// -// val origRequestUrl = request.url.toString() -// val headers = request.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap() -// headers["X-Requested-With"] = WebViewUtil.REQUESTED_WITH -// -// handler.post { -// val webview = WebView(context) -// webView = webview -// webview.setDefaultSettings() -// -// // Avoid sending empty User-Agent, Chromium WebView will reset to default if empty -// webview.settings.userAgentString = request.header("User-Agent") -// ?: HttpSource.DEFAULT_USERAGENT -// -// webview.webViewClient = object : WebViewClientCompat() { -// override fun onPageFinished(view: WebView, url: String) { -// fun isCloudFlareBypassed(): Boolean { -// return networkHelper.cookieManager.get(origRequestUrl.toHttpUrl()) -// .firstOrNull { it.name == "cf_clearance" } -// .let { it != null && it != oldCookie } -// } -// -// if (isCloudFlareBypassed()) { -// cloudflareBypassed = true -// latch.countDown() -// } -// -// // HTTP error codes are only received since M -// if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && -// url == origRequestUrl && !challengeFound -// ) { -// // The first request didn't return the challenge, abort. -// latch.countDown() -// } -// } -// -// override fun onReceivedErrorCompat( -// view: WebView, -// errorCode: Int, -// description: String?, -// failingUrl: String, -// isMainFrame: Boolean -// ) { -// if (isMainFrame) { -// if (errorCode == 503) { -// // Found the Cloudflare challenge page. -// challengeFound = true -// } else { -// // Unlock thread, the challenge wasn't found. -// latch.countDown() -// } -// } -// } -// } -// -// webView?.loadUrl(origRequestUrl, headers) -// } -// -// // Wait a reasonable amount of time to retrieve the solution. The minimum should be -// // around 4 seconds but it can take more due to slow networks or server issues. -// latch.await(12, TimeUnit.SECONDS) -// -// handler.post { -// if (!cloudflareBypassed) { -// isWebViewOutdated = webView?.isOutdated() == true -// } -// -// webView?.stopLoading() -// webView?.destroy() -// } -// -// // Throw exception if we failed to bypass Cloudflare -// if (!cloudflareBypassed) { -// // Prompt user to update WebView if it seems too outdated -// if (isWebViewOutdated) { -// context.toast(R.string.information_webview_outdated, Toast.LENGTH_LONG) -// } -// -// throw Exception(context.getString(R.string.information_cloudflare_bypass_failure)) -// } -// } -// -// companion object { -// private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare") -// private val COOKIE_NAMES = listOf("__cfduid", "cf_clearance") -// } -} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt index cea5814f..b57841ed 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt @@ -10,12 +10,16 @@ package eu.kanade.tachiyomi.network // import android.content.Context // import eu.kanade.tachiyomi.BuildConfig // import eu.kanade.tachiyomi.data.preference.PreferencesHelper -import android.content.Context // import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.OkHttpClient // import okhttp3.dnsoverhttps.DnsOverHttps // import okhttp3.logging.HttpLoggingInterceptor // import uy.kohesive.injekt.injectLazy +import android.content.Context +import eu.kanade.tachiyomi.network.interceptor.CloudflareInterceptor +import eu.kanade.tachiyomi.network.interceptor.UserAgentInterceptor +import okhttp3.OkHttpClient +import okhttp3.logging.HttpLoggingInterceptor +import suwayomi.tachidesk.server.serverConfig import java.util.concurrent.TimeUnit @Suppress("UNUSED_PARAMETER") @@ -25,55 +29,45 @@ class NetworkHelper(context: Context) { // private val cacheDir = File(context.cacheDir, "network_cache") - private val cacheSize = 5L * 1024 * 1024 // 5 MiB +// private val cacheSize = 5L * 1024 * 1024 // 5 MiB - val cookieManager = MemoryCookieJar() +// val cookieManager = MemoryCookieJar() + val cookieManager = PersistentCookieJar(context) - val client by lazy { - val builder = OkHttpClient.Builder() - .cookieJar(cookieManager) -// .cache(Cache(cacheDir, cacheSize)) - .connectTimeout(30, TimeUnit.SECONDS) - .readTimeout(5, TimeUnit.MINUTES) - .writeTimeout(5, TimeUnit.MINUTES) -// .dispatcher(Dispatcher(Executors.newFixedThreadPool(1))) + private val baseClientBuilder: OkHttpClient.Builder + get() { + val builder = OkHttpClient.Builder() + .cookieJar(cookieManager) + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .addInterceptor(UserAgentInterceptor()) -// .addInterceptor(UserAgentInterceptor()) + if (serverConfig.debugLogsEnabled) { + val httpLoggingInterceptor = HttpLoggingInterceptor().apply { + level = HttpLoggingInterceptor.Level.HEADERS + } + builder.addInterceptor(httpLoggingInterceptor) + } -// if (BuildConfig.DEBUG) { -// val httpLoggingInterceptor = HttpLoggingInterceptor().apply { -// level = HttpLoggingInterceptor.Level.HEADERS +// when (preferences.dohProvider()) { +// PREF_DOH_CLOUDFLARE -> builder.dohCloudflare() +// PREF_DOH_GOOGLE -> builder.dohGoogle() // } -// builder.addInterceptor(httpLoggingInterceptor) -// } -// if (preferences.enableDoh()) { -// builder.dns( -// DnsOverHttps.Builder().client(builder.build()) -// .url("https://cloudflare-dns.com/dns-query".toHttpUrl()) -// .bootstrapDnsHosts( -// listOf( -// InetAddress.getByName("162.159.36.1"), -// InetAddress.getByName("162.159.46.1"), -// InetAddress.getByName("1.1.1.1"), -// InetAddress.getByName("1.0.0.1"), -// InetAddress.getByName("162.159.132.53"), -// InetAddress.getByName("2606:4700:4700::1111"), -// InetAddress.getByName("2606:4700:4700::1001"), -// InetAddress.getByName("2606:4700:4700::0064"), -// InetAddress.getByName("2606:4700:4700::6400") -// ) -// ) -// .build() -// ) -// } + return builder + } - builder.build() - } +// val client by lazy { baseClientBuilder.cache(Cache(cacheDir, cacheSize)).build() } + val client by lazy { baseClientBuilder.build() } val cloudflareClient by lazy { client.newBuilder() .addInterceptor(CloudflareInterceptor()) .build() } + + // Tachidesk --> + val cookies: PersistentCookieStore + get() = cookieManager.store + // Tachidesk <-- } diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieJar.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieJar.kt new file mode 100644 index 00000000..120e8d6e --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieJar.kt @@ -0,0 +1,20 @@ +package eu.kanade.tachiyomi.network + +import android.content.Context +import okhttp3.Cookie +import okhttp3.CookieJar +import okhttp3.HttpUrl + +// from TachiWeb-Server +class PersistentCookieJar(context: Context) : CookieJar { + + val store = PersistentCookieStore(context) + + override fun saveFromResponse(url: HttpUrl, cookies: List) { + store.addAll(url, cookies) + } + + override fun loadForRequest(url: HttpUrl): List { + return store.get(url) + } +} \ No newline at end of file diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieStore.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieStore.kt new file mode 100644 index 00000000..82cde3a1 --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieStore.kt @@ -0,0 +1,80 @@ +package eu.kanade.tachiyomi.network + +import android.content.Context +import okhttp3.Cookie +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import java.net.URI +import java.util.concurrent.ConcurrentHashMap + +// from TachiWeb-Server +class PersistentCookieStore(context: Context) { + + private val cookieMap = ConcurrentHashMap>() + private val prefs = context.getSharedPreferences("cookie_store", Context.MODE_PRIVATE) + + init { + for ((key, value) in prefs.all) { + @Suppress("UNCHECKED_CAST") + val cookies = value as? Set + if (cookies != null) { + try { + val url = "http://$key".toHttpUrlOrNull() ?: continue + val nonExpiredCookies = cookies.mapNotNull { Cookie.parse(url, it) } + .filter { !it.hasExpired() } + cookieMap.put(key, nonExpiredCookies) + } catch (e: Exception) { + // Ignore + } + } + } + } + + @Synchronized + fun addAll(url: HttpUrl, cookies: List) { + val key = url.toUri().host + + // Append or replace the cookies for this domain. + val cookiesForDomain = cookieMap[key].orEmpty().toMutableList() + for (cookie in cookies) { + // Find a cookie with the same name. Replace it if found, otherwise add a new one. + val pos = cookiesForDomain.indexOfFirst { it.name == cookie.name } + if (pos == -1) { + cookiesForDomain.add(cookie) + } else { + cookiesForDomain[pos] = cookie + } + } + cookieMap.put(key, cookiesForDomain) + + // Get cookies to be stored in disk + val newValues = cookiesForDomain.asSequence() + .filter { it.persistent && !it.hasExpired() } + .map(Cookie::toString) + .toSet() + + prefs.edit().putStringSet(key, newValues).apply() + } + + @Synchronized + fun removeAll() { + prefs.edit().clear().apply() + cookieMap.clear() + } + + fun remove(uri: URI) { + prefs.edit().remove(uri.host).apply() + cookieMap.remove(uri.host) + } + + fun get(url: HttpUrl) = get(url.toUri().host) + + fun get(uri: URI) = get(uri.host) + + private fun get(url: String): List { + return cookieMap[url].orEmpty().filter { !it.hasExpired() } + } + + private fun Cookie.hasExpired() = System.currentTimeMillis() >= expiresAt + +} \ No newline at end of file diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt new file mode 100644 index 00000000..a6a0d5c0 --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt @@ -0,0 +1,95 @@ +package eu.kanade.tachiyomi.network.interceptor + +import com.gargoylesoftware.htmlunit.BrowserVersion +import com.gargoylesoftware.htmlunit.WebClient +import com.gargoylesoftware.htmlunit.html.HtmlPage +import eu.kanade.tachiyomi.network.NetworkHelper +import okhttp3.Cookie +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.Request +import okhttp3.Response +import uy.kohesive.injekt.injectLazy +import java.io.IOException + +// from TachiWeb-Server +class CloudflareInterceptor : Interceptor { + private val network: NetworkHelper by injectLazy() + + private val `serverCheck` = arrayOf("cloudflare-nginx", "cloudflare") + + @Synchronized + override fun intercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + + // Check if Cloudflare anti-bot is on + if (response.code == 503 && response.header("Server") in serverCheck) { + return try { + chain.proceed(resolveChallenge(response)) + } catch (e: Exception) { + // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that + // we don't crash the entire app + throw IOException(e) + } + } + + return response + } + + private fun resolveChallenge(response: Response): Request { + val browserVersion = BrowserVersion.BrowserVersionBuilder(BrowserVersion.BEST_SUPPORTED) + .setUserAgent(response.request.header("User-Agent") ?: BrowserVersion.BEST_SUPPORTED.userAgent) + .build() + val convertedCookies = WebClient(browserVersion).use { webClient -> + webClient.options.isThrowExceptionOnFailingStatusCode = false + webClient.options.isThrowExceptionOnScriptError = false + webClient.getPage(response.request.url.toString()) + webClient.waitForBackgroundJavaScript(10000) + // Challenge solved, process cookies + webClient.cookieManager.cookies.filter { + // Only include Cloudflare cookies + it.name.startsWith("__cf") || it.name.startsWith("cf_") + }.map { + // Convert cookies -> OkHttp format + Cookie.Builder() + .domain(it.domain.removePrefix(".")) + .expiresAt(it.expires.time) + .name(it.name) + .path(it.path) + .value(it.value).apply { + if (it.isHttpOnly) httpOnly() + if (it.isSecure) secure() + }.build() + } + } + + // Copy cookies to cookie store + convertedCookies.forEach { + network.cookies.addAll( + HttpUrl.Builder() + .scheme("http") + .host(it.domain) + .build(), + listOf(it) + ) + } + // Merge new and existing cookies for this request + // Find the cookies that we need to merge into this request + val convertedForThisRequest = convertedCookies.filter { + it.matches(response.request.url) + } + // Extract cookies from current request + val existingCookies = Cookie.parseAll( + response.request.url, + response.request.headers + ) + // Filter out existing values of cookies that we are about to merge in + val filteredExisting = existingCookies.filter { existing -> + convertedForThisRequest.none { converted -> converted.name == existing.name } + } + val newCookies = filteredExisting + convertedForThisRequest + return response.request.newBuilder() + .header("Cookie", newCookies.map { it.toString() }.joinToString("; ")) + .build() + } +} \ No newline at end of file diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/UserAgentInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/UserAgentInterceptor.kt new file mode 100644 index 00000000..5a3789ee --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/UserAgentInterceptor.kt @@ -0,0 +1,22 @@ +package eu.kanade.tachiyomi.network.interceptor + +import eu.kanade.tachiyomi.source.online.HttpSource +import okhttp3.Interceptor +import okhttp3.Response + +class UserAgentInterceptor : Interceptor { + override fun intercept(chain: Interceptor.Chain): Response { + val originalRequest = chain.request() + + return if (originalRequest.header("User-Agent").isNullOrEmpty()) { + val newRequest = originalRequest + .newBuilder() + .removeHeader("User-Agent") + .addHeader("User-Agent", HttpSource.DEFAULT_USER_AGENT) + .build() + chain.proceed(newRequest) + } else { + chain.proceed(originalRequest) + } + } +} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt index 079bafb9..5f801d12 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt @@ -372,6 +372,6 @@ abstract class HttpSource : CatalogueSource { override fun getFilterList() = FilterList() companion object { - const val DEFAULT_USERAGENT = "Mozilla/5.0 (Windows NT 6.3; WOW64)" + const val DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36 Edg/88.0.705.63" } }