add CloudflareInterceptor from TachiWeb-Server

server/build.gradle.kts

@@ -69,6 +69,9 @@ dependencies {
     // extracting zip files
     implementation("net.lingala.zip4j:zip4j:2.9.0")
 
+    // CloudflareInterceptor
+    implementation("net.sourceforge.htmlunit:htmlunit:2.52.0")
+
     // Source models and interfaces from Tachiyomi 1.x
     // using source class from tachiyomi commit 9493577de27c40ce8b2b6122cc447d025e34c477 to not depend on tachiyomi.sourceapi
 //    implementation("tachiyomi.sourceapi:source-api:1.1")

server/src/main/kotlin/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt

@@ -1,177 +0,0 @@
-package eu.kanade.tachiyomi.network
-
-/*
- * Copyright (C) Contributors to the Suwayomi project
- *
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
-
-// import android.annotation.SuppressLint
-// import android.content.Context
-// import android.os.Build
-// import android.os.Handler
-// import android.os.Looper
-// import android.webkit.WebSettings
-// import android.webkit.WebView
-// import android.widget.Toast
-// import eu.kanade.tachiyomi.R
-// import eu.kanade.tachiyomi.util.lang.launchUI
-// import eu.kanade.tachiyomi.util.system.WebViewClientCompat
-// import eu.kanade.tachiyomi.util.system.WebViewUtil
-// import eu.kanade.tachiyomi.util.system.isOutdated
-// import eu.kanade.tachiyomi.util.system.setDefaultSettings
-// import eu.kanade.tachiyomi.util.system.toast
-import okhttp3.Interceptor
-import okhttp3.Response
-// import uy.kohesive.injekt.injectLazy
-
-class CloudflareInterceptor() : Interceptor {
-
-//    private val handler = Handler(Looper.getMainLooper())
-
-//    private val networkHelper = NetworkHelper()
-
-    /**
-     * When this is called, it initializes the WebView if it wasn't already. We use this to avoid
-     * blocking the main thread too much. If used too often we could consider moving it to the
-     * Application class.
-     */
-//    private val initWebView by lazy {
-//        WebSettings.getDefaultUserAgent(context)
-//    }
-
-    @Synchronized
-    override fun intercept(chain: Interceptor.Chain): Response {
-        val originalRequest = chain.request()
-        return chain.proceed(originalRequest)
-
-//        if (!WebViewUtil.supportsWebView(context)) {
-//            launchUI {
-//                context.toast(R.string.information_webview_required, Toast.LENGTH_LONG)
-//            }
-//            return chain.proceed(originalRequest)
-//        }
-//
-//        initWebView
-//
-//        val response = chain.proceed(originalRequest)
-//
-//        // Check if Cloudflare anti-bot is on
-//        if (response.code != 503 || response.header("Server") !in SERVER_CHECK) {
-//            return response
-//        }
-//
-//        try {
-//            response.close()
-//            networkHelper.cookieManager.remove(originalRequest.url, COOKIE_NAMES, 0)
-//            val oldCookie = networkHelper.cookieManager.get(originalRequest.url)
-//                .firstOrNull { it.name == "cf_clearance" }
-//            resolveWithWebView(originalRequest, oldCookie)
-//
-//            return chain.proceed(originalRequest)
-//        } catch (e: Exception) {
-//            // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
-//            // we don't crash the entire app
-//            throw IOException(e)
-//        }
-    }
-//
-// //    @SuppressLint("SetJavaScriptEnabled")
-//    private fun resolveWithWebView(request: Request, oldCookie: Cookie?) {
-//        // We need to lock this thread until the WebView finds the challenge solution url, because
-//        // OkHttp doesn't support asynchronous interceptors.
-//        val latch = CountDownLatch(1)
-//
-//        var webView: WebView? = null
-//
-//        var challengeFound = false
-//        var cloudflareBypassed = false
-//        var isWebViewOutdated = false
-//
-//        val origRequestUrl = request.url.toString()
-//        val headers = request.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap()
-//        headers["X-Requested-With"] = WebViewUtil.REQUESTED_WITH
-//
-//        handler.post {
-//            val webview = WebView(context)
-//            webView = webview
-//            webview.setDefaultSettings()
-//
-//            // Avoid sending empty User-Agent, Chromium WebView will reset to default if empty
-//            webview.settings.userAgentString = request.header("User-Agent")
-//                ?: HttpSource.DEFAULT_USERAGENT
-//
-//            webview.webViewClient = object : WebViewClientCompat() {
-//                override fun onPageFinished(view: WebView, url: String) {
-//                    fun isCloudFlareBypassed(): Boolean {
-//                        return networkHelper.cookieManager.get(origRequestUrl.toHttpUrl())
-//                            .firstOrNull { it.name == "cf_clearance" }
-//                            .let { it != null && it != oldCookie }
-//                    }
-//
-//                    if (isCloudFlareBypassed()) {
-//                        cloudflareBypassed = true
-//                        latch.countDown()
-//                    }
-//
-//                    // HTTP error codes are only received since M
-//                    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M &&
-//                        url == origRequestUrl && !challengeFound
-//                    ) {
-//                        // The first request didn't return the challenge, abort.
-//                        latch.countDown()
-//                    }
-//                }
-//
-//                override fun onReceivedErrorCompat(
-//                    view: WebView,
-//                    errorCode: Int,
-//                    description: String?,
-//                    failingUrl: String,
-//                    isMainFrame: Boolean
-//                ) {
-//                    if (isMainFrame) {
-//                        if (errorCode == 503) {
-//                            // Found the Cloudflare challenge page.
-//                            challengeFound = true
-//                        } else {
-//                            // Unlock thread, the challenge wasn't found.
-//                            latch.countDown()
-//                        }
-//                    }
-//                }
-//            }
-//
-//            webView?.loadUrl(origRequestUrl, headers)
-//        }
-//
-//        // Wait a reasonable amount of time to retrieve the solution. The minimum should be
-//        // around 4 seconds but it can take more due to slow networks or server issues.
-//        latch.await(12, TimeUnit.SECONDS)
-//
-//        handler.post {
-//            if (!cloudflareBypassed) {
-//                isWebViewOutdated = webView?.isOutdated() == true
-//            }
-//
-//            webView?.stopLoading()
-//            webView?.destroy()
-//        }
-//
-//        // Throw exception if we failed to bypass Cloudflare
-//        if (!cloudflareBypassed) {
-//            // Prompt user to update WebView if it seems too outdated
-//            if (isWebViewOutdated) {
-//                context.toast(R.string.information_webview_outdated, Toast.LENGTH_LONG)
-//            }
-//
-//            throw Exception(context.getString(R.string.information_cloudflare_bypass_failure))
-//        }
-//    }
-//
-//    companion object {
-//        private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare")
-//        private val COOKIE_NAMES = listOf("__cfduid", "cf_clearance")
-//    }
-}

server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt

@@ -10,12 +10,16 @@ package eu.kanade.tachiyomi.network
 // import android.content.Context
 // import eu.kanade.tachiyomi.BuildConfig
 // import eu.kanade.tachiyomi.data.preference.PreferencesHelper
-import android.content.Context
 // import okhttp3.HttpUrl.Companion.toHttpUrl
-import okhttp3.OkHttpClient
 // import okhttp3.dnsoverhttps.DnsOverHttps
 // import okhttp3.logging.HttpLoggingInterceptor
 // import uy.kohesive.injekt.injectLazy
+import android.content.Context
+import eu.kanade.tachiyomi.network.interceptor.CloudflareInterceptor
+import eu.kanade.tachiyomi.network.interceptor.UserAgentInterceptor
+import okhttp3.OkHttpClient
+import okhttp3.logging.HttpLoggingInterceptor
+import suwayomi.tachidesk.server.serverConfig
 import java.util.concurrent.TimeUnit
 
 @Suppress("UNUSED_PARAMETER")
@@ -25,55 +29,45 @@ class NetworkHelper(context: Context) {
 
 //    private val cacheDir = File(context.cacheDir, "network_cache")
 
-    private val cacheSize = 5L * 1024 * 1024 // 5 MiB
+//    private val cacheSize = 5L * 1024 * 1024 // 5 MiB
 
-    val cookieManager = MemoryCookieJar()
+//    val cookieManager = MemoryCookieJar()
+    val cookieManager = PersistentCookieJar(context)
 
-    val client by lazy {
-        val builder = OkHttpClient.Builder()
-            .cookieJar(cookieManager)
-//            .cache(Cache(cacheDir, cacheSize))
-            .connectTimeout(30, TimeUnit.SECONDS)
-            .readTimeout(5, TimeUnit.MINUTES)
-            .writeTimeout(5, TimeUnit.MINUTES)
-//            .dispatcher(Dispatcher(Executors.newFixedThreadPool(1)))
+    private val baseClientBuilder: OkHttpClient.Builder
+        get() {
+            val builder = OkHttpClient.Builder()
+                .cookieJar(cookieManager)
+                .connectTimeout(30, TimeUnit.SECONDS)
+                .readTimeout(30, TimeUnit.SECONDS)
+                .addInterceptor(UserAgentInterceptor())
 
-//            .addInterceptor(UserAgentInterceptor())
+            if (serverConfig.debugLogsEnabled) {
+                val httpLoggingInterceptor = HttpLoggingInterceptor().apply {
+                    level = HttpLoggingInterceptor.Level.HEADERS
+                }
+                builder.addInterceptor(httpLoggingInterceptor)
+            }
 
-//        if (BuildConfig.DEBUG) {
-//            val httpLoggingInterceptor = HttpLoggingInterceptor().apply {
-//                level = HttpLoggingInterceptor.Level.HEADERS
+//            when (preferences.dohProvider()) {
+//                PREF_DOH_CLOUDFLARE -> builder.dohCloudflare()
+//                PREF_DOH_GOOGLE -> builder.dohGoogle()
 //            }
-//            builder.addInterceptor(httpLoggingInterceptor)
-//        }
 
-//        if (preferences.enableDoh()) {
-//            builder.dns(
-//                DnsOverHttps.Builder().client(builder.build())
-//                    .url("https://cloudflare-dns.com/dns-query".toHttpUrl())
-//                    .bootstrapDnsHosts(
-//                        listOf(
-//                            InetAddress.getByName("162.159.36.1"),
-//                            InetAddress.getByName("162.159.46.1"),
-//                            InetAddress.getByName("1.1.1.1"),
-//                            InetAddress.getByName("1.0.0.1"),
-//                            InetAddress.getByName("162.159.132.53"),
-//                            InetAddress.getByName("2606:4700:4700::1111"),
-//                            InetAddress.getByName("2606:4700:4700::1001"),
-//                            InetAddress.getByName("2606:4700:4700::0064"),
-//                            InetAddress.getByName("2606:4700:4700::6400")
-//                        )
-//                    )
-//                    .build()
-//            )
-//        }
+            return builder
+        }
 
-        builder.build()
-    }
+//    val client by lazy { baseClientBuilder.cache(Cache(cacheDir, cacheSize)).build() }
+    val client by lazy { baseClientBuilder.build() }
 
     val cloudflareClient by lazy {
         client.newBuilder()
             .addInterceptor(CloudflareInterceptor())
             .build()
     }
+
+    // Tachidesk -->
+    val cookies: PersistentCookieStore
+        get() = cookieManager.store
+    // Tachidesk <--
 }

server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieJar.kt

@@ -0,0 +1,20 @@
+package eu.kanade.tachiyomi.network
+
+import android.content.Context
+import okhttp3.Cookie
+import okhttp3.CookieJar
+import okhttp3.HttpUrl
+
+// from TachiWeb-Server
+class PersistentCookieJar(context: Context) : CookieJar {
+
+    val store = PersistentCookieStore(context)
+
+    override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
+        store.addAll(url, cookies)
+    }
+
+    override fun loadForRequest(url: HttpUrl): List<Cookie> {
+        return store.get(url)
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/network/PersistentCookieStore.kt

@@ -0,0 +1,80 @@
+package eu.kanade.tachiyomi.network
+
+import android.content.Context
+import okhttp3.Cookie
+import okhttp3.HttpUrl
+import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
+import java.net.URI
+import java.util.concurrent.ConcurrentHashMap
+
+// from TachiWeb-Server
+class PersistentCookieStore(context: Context) {
+
+    private val cookieMap = ConcurrentHashMap<String, List<Cookie>>()
+    private val prefs = context.getSharedPreferences("cookie_store", Context.MODE_PRIVATE)
+
+    init {
+        for ((key, value) in prefs.all) {
+            @Suppress("UNCHECKED_CAST")
+            val cookies = value as? Set<String>
+            if (cookies != null) {
+                try {
+                    val url = "http://$key".toHttpUrlOrNull() ?: continue
+                    val nonExpiredCookies = cookies.mapNotNull { Cookie.parse(url, it) }
+                        .filter { !it.hasExpired() }
+                    cookieMap.put(key, nonExpiredCookies)
+                } catch (e: Exception) {
+                    // Ignore
+                }
+            }
+        }
+    }
+
+    @Synchronized
+    fun addAll(url: HttpUrl, cookies: List<Cookie>) {
+        val key = url.toUri().host
+
+        // Append or replace the cookies for this domain.
+        val cookiesForDomain = cookieMap[key].orEmpty().toMutableList()
+        for (cookie in cookies) {
+            // Find a cookie with the same name. Replace it if found, otherwise add a new one.
+            val pos = cookiesForDomain.indexOfFirst { it.name == cookie.name }
+            if (pos == -1) {
+                cookiesForDomain.add(cookie)
+            } else {
+                cookiesForDomain[pos] = cookie
+            }
+        }
+        cookieMap.put(key, cookiesForDomain)
+
+        // Get cookies to be stored in disk
+        val newValues = cookiesForDomain.asSequence()
+            .filter { it.persistent && !it.hasExpired() }
+            .map(Cookie::toString)
+            .toSet()
+
+        prefs.edit().putStringSet(key, newValues).apply()
+    }
+
+    @Synchronized
+    fun removeAll() {
+        prefs.edit().clear().apply()
+        cookieMap.clear()
+    }
+
+    fun remove(uri: URI) {
+        prefs.edit().remove(uri.host).apply()
+        cookieMap.remove(uri.host)
+    }
+
+    fun get(url: HttpUrl) = get(url.toUri().host)
+
+    fun get(uri: URI) = get(uri.host)
+
+    private fun get(url: String): List<Cookie> {
+        return cookieMap[url].orEmpty().filter { !it.hasExpired() }
+    }
+
+    private fun Cookie.hasExpired() = System.currentTimeMillis() >= expiresAt
+
+}

server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt

@@ -0,0 +1,95 @@
+package eu.kanade.tachiyomi.network.interceptor
+
+import com.gargoylesoftware.htmlunit.BrowserVersion
+import com.gargoylesoftware.htmlunit.WebClient
+import com.gargoylesoftware.htmlunit.html.HtmlPage
+import eu.kanade.tachiyomi.network.NetworkHelper
+import okhttp3.Cookie
+import okhttp3.HttpUrl
+import okhttp3.Interceptor
+import okhttp3.Request
+import okhttp3.Response
+import uy.kohesive.injekt.injectLazy
+import java.io.IOException
+
+// from TachiWeb-Server
+class CloudflareInterceptor : Interceptor {
+    private val network: NetworkHelper by injectLazy()
+
+    private val `serverCheck` = arrayOf("cloudflare-nginx", "cloudflare")
+
+    @Synchronized
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val response = chain.proceed(chain.request())
+
+        // Check if Cloudflare anti-bot is on
+        if (response.code == 503 && response.header("Server") in serverCheck) {
+            return try {
+                chain.proceed(resolveChallenge(response))
+            } catch (e: Exception) {
+                // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
+                // we don't crash the entire app
+                throw IOException(e)
+            }
+        }
+
+        return response
+    }
+
+    private fun resolveChallenge(response: Response): Request {
+        val browserVersion = BrowserVersion.BrowserVersionBuilder(BrowserVersion.BEST_SUPPORTED)
+            .setUserAgent(response.request.header("User-Agent") ?: BrowserVersion.BEST_SUPPORTED.userAgent)
+            .build()
+        val convertedCookies = WebClient(browserVersion).use { webClient ->
+            webClient.options.isThrowExceptionOnFailingStatusCode = false
+            webClient.options.isThrowExceptionOnScriptError = false
+            webClient.getPage<HtmlPage>(response.request.url.toString())
+            webClient.waitForBackgroundJavaScript(10000)
+            // Challenge solved, process cookies
+            webClient.cookieManager.cookies.filter {
+                // Only include Cloudflare cookies
+                it.name.startsWith("__cf") || it.name.startsWith("cf_")
+            }.map {
+                // Convert cookies -> OkHttp format
+                Cookie.Builder()
+                    .domain(it.domain.removePrefix("."))
+                    .expiresAt(it.expires.time)
+                    .name(it.name)
+                    .path(it.path)
+                    .value(it.value).apply {
+                        if (it.isHttpOnly) httpOnly()
+                        if (it.isSecure) secure()
+                    }.build()
+            }
+        }
+
+        // Copy cookies to cookie store
+        convertedCookies.forEach {
+            network.cookies.addAll(
+                HttpUrl.Builder()
+                    .scheme("http")
+                    .host(it.domain)
+                    .build(),
+                listOf(it)
+            )
+        }
+        // Merge new and existing cookies for this request
+        // Find the cookies that we need to merge into this request
+        val convertedForThisRequest = convertedCookies.filter {
+            it.matches(response.request.url)
+        }
+        // Extract cookies from current request
+        val existingCookies = Cookie.parseAll(
+            response.request.url,
+            response.request.headers
+        )
+        // Filter out existing values of cookies that we are about to merge in
+        val filteredExisting = existingCookies.filter { existing ->
+            convertedForThisRequest.none { converted -> converted.name == existing.name }
+        }
+        val newCookies = filteredExisting + convertedForThisRequest
+        return response.request.newBuilder()
+            .header("Cookie", newCookies.map { it.toString() }.joinToString("; "))
+            .build()
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/UserAgentInterceptor.kt

@@ -0,0 +1,22 @@
+package eu.kanade.tachiyomi.network.interceptor
+
+import eu.kanade.tachiyomi.source.online.HttpSource
+import okhttp3.Interceptor
+import okhttp3.Response
+
+class UserAgentInterceptor : Interceptor {
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val originalRequest = chain.request()
+
+        return if (originalRequest.header("User-Agent").isNullOrEmpty()) {
+            val newRequest = originalRequest
+                .newBuilder()
+                .removeHeader("User-Agent")
+                .addHeader("User-Agent", HttpSource.DEFAULT_USER_AGENT)
+                .build()
+            chain.proceed(newRequest)
+        } else {
+            chain.proceed(originalRequest)
+        }
+    }
+}

server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSource.kt

@@ -372,6 +372,6 @@ abstract class HttpSource : CatalogueSource {
     override fun getFilterList() = FilterList()
 
     companion object {
-        const val DEFAULT_USERAGENT = "Mozilla/5.0 (Windows NT 6.3; WOW64)"
+        const val DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36 Edg/88.0.705.63"
     }
 }