From 4cf7512ee09729db8a3d8133fbbcf25ff77d2a1a Mon Sep 17 00:00:00 2001 From: Mitchell Syer Date: Sat, 7 Jan 2023 16:47:53 -0500 Subject: [PATCH] Improve Playwright handling (#479) * Improve playwright * Move DriverJar.java and Chromium.kt --- gradle/libs.versions.toml | 3 +- scripts/bundler.sh | 22 +++++++ .../tachidesk/server/util}/DriverJar.java | 38 +++++------- .../interceptor/CloudflareInterceptor.kt | 2 +- .../tachidesk/server/util/Chromium.kt | 58 +++++++++++++++++++ 5 files changed, 98 insertions(+), 25 deletions(-) rename server/src/main/java/{eu/kanade/tachiyomi/network/interceptor => suwayomi/tachidesk/server/util}/DriverJar.java (88%) create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/server/util/Chromium.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 42f209bd..ca2e6bfe 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -10,6 +10,7 @@ dex2jar = "v56" rhino = "1.7.14" settings = "1.0.0-RC" twelvemonkeys = "3.9.4" +playwright = "1.28.0" [libraries] # Kotlin @@ -98,7 +99,7 @@ zip4j = "net.lingala.zip4j:zip4j:2.11.2" junrar = "com.github.junrar:junrar:7.5.3" # CloudflareInterceptor -playwright = "com.microsoft.playwright:playwright:1.28.0" +playwright = { module = "com.microsoft.playwright:playwright", version.ref = "playwright" } # AES/CBC/PKCS7Padding Cypher provider bouncycastle = "org.bouncycastle:bcprov-jdk18on:1.72" diff --git a/scripts/bundler.sh b/scripts/bundler.sh index a72b28f5..952f65c6 100755 --- a/scripts/bundler.sh +++ b/scripts/bundler.sh @@ -26,6 +26,8 @@ main() { set -- "${POSITIONAL_ARGS[@]}" OS="$1" + PLAYWRIGHT_VERSION="$(cat gradle/libs.versions.toml | grep -oP "playwright = \"\K([0-9\.]*)(?=\")")" + PLAYWRIGHT_REVISION="$(curl --silent "https://raw.githubusercontent.com/microsoft/playwright/v$PLAYWRIGHT_VERSION/packages/playwright-core/browsers.json" 2>&1 | grep -ozP "\"name\": \"chromium\",\n *\"revision\": \"\K[0-9]*")" JAR="$(ls server/build/*.jar | tail -n1)" RELEASE_NAME="$(echo "${JAR%.*}" | xargs basename)-$OS" RELEASE_VERSION="$(tmp="${JAR%-*}"; echo "${tmp##*-}" | tr -d v)" @@ -57,6 +59,9 @@ main() { ELECTRON_URL="https://github.com/electron/electron/releases/download/$electron_version/$ELECTRON" download_jre_and_electron + PLAYWRIGHT_PLATFORM="linux" + setup_playwright + RELEASE="$RELEASE_NAME.tar.gz" make_linux_bundle move_release_to_output_dir @@ -70,6 +75,9 @@ main() { ELECTRON_URL="https://github.com/electron/electron/releases/download/$electron_version/$ELECTRON" download_jre_and_electron + PLAYWRIGHT_PLATFORM="mac" + setup_playwright + RELEASE="$RELEASE_NAME.zip" make_macos_bundle move_release_to_output_dir @@ -83,6 +91,9 @@ main() { ELECTRON_URL="https://github.com/electron/electron/releases/download/$electron_version/$ELECTRON" download_jre_and_electron + PLAYWRIGHT_PLATFORM="mac-arm64" + setup_playwright + RELEASE="$RELEASE_NAME.zip" make_macos_bundle move_release_to_output_dir @@ -96,6 +107,9 @@ main() { ELECTRON_URL="https://github.com/electron/electron/releases/download/$electron_version/$ELECTRON" download_jre_and_electron + PLAYWRIGHT_PLATFORM="win64" + setup_playwright + RELEASE="$RELEASE_NAME.zip" make_windows_bundle move_release_to_output_dir @@ -113,6 +127,9 @@ main() { ELECTRON_URL="https://github.com/electron/electron/releases/download/$electron_version/$ELECTRON" download_jre_and_electron + PLAYWRIGHT_PLATFORM="win64" + setup_playwright + RELEASE="$RELEASE_NAME.zip" make_windows_bundle move_release_to_output_dir @@ -268,6 +285,11 @@ make_windows_package() { "$RELEASE_NAME/jre.wxs" "$RELEASE_NAME/electron.wxs" -o "$RELEASE" } +setup_playwright() { + mkdir "$RELEASE_NAME/bin" + curl -L "https://playwright.azureedge.net/builds/chromium/$PLAYWRIGHT_REVISION/chromium-$PLAYWRIGHT_PLATFORM.zip" -o "$RELEASE_NAME/bin/chromium.zip" +} + # Error handler # set -u: Treat unset variables as an error when substituting. # set -o pipefail: Prevents errors in pipeline from being masked. diff --git a/server/src/main/java/eu/kanade/tachiyomi/network/interceptor/DriverJar.java b/server/src/main/java/suwayomi/tachidesk/server/util/DriverJar.java similarity index 88% rename from server/src/main/java/eu/kanade/tachiyomi/network/interceptor/DriverJar.java rename to server/src/main/java/suwayomi/tachidesk/server/util/DriverJar.java index e73ab75d..ea4a88fa 100644 --- a/server/src/main/java/eu/kanade/tachiyomi/network/interceptor/DriverJar.java +++ b/server/src/main/java/suwayomi/tachidesk/server/util/DriverJar.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package eu.kanade.tachiyomi.network.interceptor; +package suwayomi.tachidesk.server.util; import com.microsoft.playwright.impl.driver.Driver; @@ -26,22 +26,9 @@ import java.util.Collections; import java.util.Map; import java.util.concurrent.TimeUnit; -/* - exact copy of https://github.com/microsoft/playwright-java/blob/4d278c391e3c50738ddea6c3e324a4bbbf719d86/driver-bundle/src/main/java/com/microsoft/playwright/impl/driver/jar/DriverJar.java - with diff: - 108a109,116 - > - > private FileSystem initFileSystem(URI uri) throws IOException { - > try { - > return FileSystems.newFileSystem(uri, Collections.emptyMap()); - > } catch (FileSystemAlreadyExistsException e) { - > return null; - > } - > } - 116c124 - < try (FileSystem fileSystem = "jar".equals(uri.getScheme()) ? FileSystems.newFileSystem(uri, Collections.emptyMap()) : null) { - --- - > try (FileSystem fileSystem = "jar".equals(uri.getScheme()) ? initFileSystem(uri) : null) { +/** + * Copy of DriverJar + * with support for pre-installing chromium and only supports chromium playwright */ public class DriverJar extends Driver { private static final String PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = "PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD"; @@ -56,8 +43,8 @@ public class DriverJar extends Driver { String alternativeTmpdir = System.getProperty("playwright.driver.tmpdir"); String prefix = "playwright-java-"; driverTempDir = alternativeTmpdir == null - ? Files.createTempDirectory(prefix) - : Files.createTempDirectory(Paths.get(alternativeTmpdir), prefix); + ? Files.createTempDirectory(prefix) + : Files.createTempDirectory(Paths.get(alternativeTmpdir), prefix); driverTempDir.toFile().deleteOnExit(); String nodePath = System.getProperty("playwright.nodejs.path"); if (nodePath != null) { @@ -99,12 +86,14 @@ public class DriverJar extends Driver { logMessage("Skipping browsers download because `SELENIUM_REMOTE_URL` env variable is set"); return; } + Chromium.preinstall(platformDir()); Path driver = driverPath(); if (!Files.exists(driver)) { throw new RuntimeException("Failed to find driver: " + driver); } ProcessBuilder pb = createProcessBuilder(); pb.command().add("install"); + pb.command().add("chromium"); pb.redirectError(ProcessBuilder.Redirect.INHERIT); pb.redirectOutput(ProcessBuilder.Redirect.INHERIT); Process p = pb.start(); @@ -123,7 +112,6 @@ public class DriverJar extends Driver { return name.endsWith(".sh") || name.endsWith(".exe") || !name.contains("."); } - private FileSystem initFileSystem(URI uri) throws IOException { try { return FileSystems.newFileSystem(uri, Collections.emptyMap()); @@ -131,10 +119,14 @@ public class DriverJar extends Driver { return null; } } - void extractDriverToTempDir() throws URISyntaxException, IOException { + + public static URI getDriverResourceURI() throws URISyntaxException { ClassLoader classloader = Thread.currentThread().getContextClassLoader(); - URI originalUri = classloader.getResource( - "driver/" + platformDir()).toURI(); + return classloader.getResource("driver/" + platformDir()).toURI(); + } + + void extractDriverToTempDir() throws URISyntaxException, IOException { + URI originalUri = getDriverResourceURI(); URI uri = maybeExtractNestedJar(originalUri); // Create zip filesystem if loading from jar. diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt index 73e00de1..952e784f 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt @@ -72,7 +72,7 @@ object CFClearance { init { // Fix the default DriverJar issue by providing our own implementation // ref: https://github.com/microsoft/playwright-java/issues/1138 - System.setProperty("playwright.driver.impl", "eu.kanade.tachiyomi.network.interceptor.DriverJar") + System.setProperty("playwright.driver.impl", "suwayomi.tachidesk.server.util.DriverJar") } fun resolveWithWebView(originalRequest: Request): Request { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/util/Chromium.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/util/Chromium.kt new file mode 100644 index 00000000..adee2afc --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/util/Chromium.kt @@ -0,0 +1,58 @@ +package suwayomi.tachidesk.server.util + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.decodeFromStream +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import net.harawata.appdirs.AppDirsFactory +import java.nio.file.FileSystems +import java.nio.file.Files +import java.nio.file.StandardCopyOption +import kotlin.io.path.Path +import kotlin.io.path.absolutePathString +import kotlin.io.path.createDirectories +import kotlin.io.path.exists +import kotlin.io.path.notExists +import kotlin.streams.asSequence + +object Chromium { + @OptIn(ExperimentalSerializationApi::class) + @JvmStatic + fun preinstall(platformDir: String) { + val loader = Thread.currentThread().contextClassLoader + val resource = loader.getResource("driver/$platformDir/package/browsers.json") ?: return + val json = resource.openStream().use { + Json.decodeFromStream(it) + } + val revision = json["browsers"]?.jsonArray + ?.find { it.jsonObject["name"]?.jsonPrimitive?.contentOrNull == "chromium" } + ?.jsonObject + ?.get("revision") + ?.jsonPrimitive + ?.contentOrNull + ?: return + + val playwrightDir = AppDirsFactory.getInstance().getUserDataDir("ms-playwright", null, null) + val chromiumZip = Path(".").resolve("bin/chromium.zip") + val chromePath = Path(playwrightDir).resolve("chromium-$revision") + if (chromePath.exists() || chromiumZip.notExists()) return + chromePath.createDirectories() + + FileSystems.newFileSystem(chromiumZip, null).use { + val src = it.getPath("/") + Files.walk(src) + .asSequence() + .forEach { source -> + Files.copy( + source, + chromePath.resolve(source.absolutePathString().removePrefix("/")), + StandardCopyOption.REPLACE_EXISTING + ) + } + } + } +}