diff --git a/buildSrc/src/main/kotlin/RandomUaCheck.kt b/buildSrc/src/main/kotlin/RandomUaCheck.kt new file mode 100644 index 0000000000..266236ed7d --- /dev/null +++ b/buildSrc/src/main/kotlin/RandomUaCheck.kt @@ -0,0 +1,25 @@ +import com.diffplug.spotless.FormatterFunc +import com.diffplug.spotless.FormatterStep +import java.io.Serializable + +object RandomUaCheck { + fun create(): FormatterStep = FormatterStep.create( + "randomua-requires-getMangaUrl", + State(), + State::toFormatter, + ) + + private class State : Serializable { + fun toFormatter() = FormatterFunc { content -> + if ("package keiyoushi.lib.randomua" !in content && + "keiyoushi.lib.randomua" in content && + "override fun getMangaUrl(" !in content + ) { + throw AssertionError( + "usage of :lib:randomua requires override of getMangaUrl()", + ) + } + content + } + } +} diff --git a/buildSrc/src/main/kotlin/keiyoushi.lint.gradle.kts b/buildSrc/src/main/kotlin/keiyoushi.lint.gradle.kts index 26998cb328..b9aaf0a98c 100644 --- a/buildSrc/src/main/kotlin/keiyoushi.lint.gradle.kts +++ b/buildSrc/src/main/kotlin/keiyoushi.lint.gradle.kts @@ -12,6 +12,7 @@ spotless { )) trimTrailingWhitespace() endWithNewline() + addStep(RandomUaCheck.create()) } java { diff --git a/buildSrc/src/main/kotlin/lib-android.gradle.kts b/buildSrc/src/main/kotlin/lib-android.gradle.kts index 7173005beb..0cadda3ac4 100644 --- a/buildSrc/src/main/kotlin/lib-android.gradle.kts +++ b/buildSrc/src/main/kotlin/lib-android.gradle.kts @@ -30,6 +30,7 @@ android { kotlin { compilerOptions { freeCompilerArgs.add("-opt-in=kotlinx.serialization.ExperimentalSerializationApi") + freeCompilerArgs.add("-Xcontext-parameters") } } diff --git a/buildSrc/src/main/kotlin/lib-multisrc.gradle.kts b/buildSrc/src/main/kotlin/lib-multisrc.gradle.kts index 9043873fc0..f764ad624b 100644 --- a/buildSrc/src/main/kotlin/lib-multisrc.gradle.kts +++ b/buildSrc/src/main/kotlin/lib-multisrc.gradle.kts @@ -31,6 +31,7 @@ android { kotlin { compilerOptions { freeCompilerArgs.add("-opt-in=kotlinx.serialization.ExperimentalSerializationApi") + freeCompilerArgs.add("-Xcontext-parameters") } } diff --git a/common.gradle b/common.gradle index 0a06c10844..99c4108e6d 100644 --- a/common.gradle +++ b/common.gradle @@ -93,6 +93,7 @@ android { kotlin { compilerOptions { freeCompilerArgs.add("-opt-in=kotlinx.serialization.ExperimentalSerializationApi") + freeCompilerArgs.add("-Xcontext-parameters") } } diff --git a/core/build.gradle.kts b/core/build.gradle.kts index dc1ffd9563..0a425ae582 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -21,6 +21,7 @@ android { kotlin { compilerOptions { freeCompilerArgs.add("-opt-in=kotlinx.serialization.ExperimentalSerializationApi") + freeCompilerArgs.add("-Xcontext-parameters") } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ea6f619c69..6b8ae03eb2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -28,6 +28,7 @@ injekt-core = { module = "com.github.null2264.injekt:injekt-core", version = "41 rxjava = { module = "io.reactivex:rxjava", version = "1.3.8" } jsoup = { module = "org.jsoup:jsoup", version = "1.22.1" } okhttp = { module = "com.squareup.okhttp3:okhttp", version = "5.3.2" } +okhttp-brotli = { module = "com.squareup.okhttp3:okhttp-brotli", version = "5.3.2" } quickjs = { module = "app.cash.quickjs:quickjs-android", version = "0.9.2" } [bundles] diff --git a/lib/randomua/build.gradle.kts b/lib/randomua/build.gradle.kts index c26cbc8a82..89c0c1867a 100644 --- a/lib/randomua/build.gradle.kts +++ b/lib/randomua/build.gradle.kts @@ -1,3 +1,7 @@ plugins { id("lib-android") } + +dependencies { + compileOnly(libs.okhttp.brotli) +} diff --git a/lib/randomua/src/keiyoushi/lib/randomua/Helper.kt b/lib/randomua/src/keiyoushi/lib/randomua/Helper.kt new file mode 100644 index 0000000000..80494af684 --- /dev/null +++ b/lib/randomua/src/keiyoushi/lib/randomua/Helper.kt @@ -0,0 +1,85 @@ +package keiyoushi.lib.randomua + +import android.os.Looper +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.NetworkHelper +import eu.kanade.tachiyomi.network.await +import keiyoushi.utils.parseAs +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.Serializable +import okhttp3.CacheControl +import okhttp3.brotli.BrotliInterceptor +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get + +private var userAgent: String? = null +private val client = Injekt.get().client.newBuilder() + .addNetworkInterceptor { chain -> + chain.proceed(chain.request()).newBuilder() + .header("Cache-Control", "max-age=${24 * 60 * 60}") + .removeHeader("Pragma") + .removeHeader("Expires") + .build() + } + .apply { + val index = networkInterceptors().indexOfFirst { it is BrotliInterceptor } + if (index >= 0) interceptors().add(networkInterceptors().removeAt(index)) + } + .build() + +internal fun getRandomUserAgent( + userAgentType: UserAgentType, + filterInclude: List, + filterExclude: List, +): String? { + if (!userAgent.isNullOrEmpty()) return userAgent + + // avoid network on main thread when webview screen accesses headers + val uaRequest = if (Looper.myLooper() == Looper.getMainLooper()) { + GET(UA_DB_URL, cache = CacheControl.FORCE_CACHE) + } else { + GET(UA_DB_URL) + } + + val uaResponse = runBlocking(Dispatchers.IO) { client.newCall(uaRequest).await() } + + if (!uaResponse.isSuccessful) { + uaResponse.close() + return null + } + + val userAgentList = uaResponse.parseAs() + + return when (userAgentType) { + UserAgentType.DESKTOP -> userAgentList.desktop + UserAgentType.MOBILE -> userAgentList.mobile + else -> error("Expected UserAgentType.DESKTOP or UserAgentType.MOBILE but got UserAgentType.${userAgentType.name} instead") + } + .filter { + filterInclude.isEmpty() || filterInclude.any { filter -> + it.contains(filter, ignoreCase = true) + } + } + .filterNot { + filterExclude.any { filter -> + it.contains(filter, ignoreCase = true) + } + } + .randomOrNull() + .also { userAgent = it } +} + +private const val UA_DB_URL = "https://keiyoushi.github.io/user-agents/user-agents.json" + +enum class UserAgentType { + MOBILE, + DESKTOP, + OFF, +} + +@Serializable +private class UserAgentList( + val desktop: List, + val mobile: List, +) diff --git a/lib/randomua/src/keiyoushi/lib/randomua/RandomUserAgentInterceptor.kt b/lib/randomua/src/keiyoushi/lib/randomua/RandomUserAgentInterceptor.kt deleted file mode 100644 index 310087ecb3..0000000000 --- a/lib/randomua/src/keiyoushi/lib/randomua/RandomUserAgentInterceptor.kt +++ /dev/null @@ -1,121 +0,0 @@ -package keiyoushi.lib.randomua - -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.NetworkHelper -import kotlinx.serialization.Serializable -import kotlinx.serialization.decodeFromString -import kotlinx.serialization.json.Json -import okhttp3.Interceptor -import okhttp3.OkHttpClient -import okhttp3.Response -import uy.kohesive.injekt.injectLazy -import java.io.IOException - -private class RandomUserAgentInterceptor( - private val userAgentType: UserAgentType, - private val customUA: String?, - private val filterInclude: List, - private val filterExclude: List, -) : Interceptor { - - private var userAgent: String? = null - - private val json: Json by injectLazy() - - private val network: NetworkHelper by injectLazy() - - private val client = network.client - - override fun intercept(chain: Interceptor.Chain): Response { - try { - val originalRequest = chain.request() - - val newUserAgent = getUserAgent() - ?: return chain.proceed(originalRequest) - - val originalHeaders = originalRequest.headers - - val modifiedHeaders = originalHeaders.newBuilder() - .set("User-Agent", newUserAgent) - .build() - - return chain.proceed( - originalRequest.newBuilder() - .headers(modifiedHeaders) - .build(), - ) - } catch (e: Exception) { - throw IOException(e.message) - } - } - - private fun getUserAgent(): String? { - if (userAgentType == UserAgentType.OFF) { - return customUA?.ifBlank { null } - } - - if (!userAgent.isNullOrEmpty()) return userAgent - - val uaResponse = client.newCall(GET(UA_DB_URL)).execute() - - if (!uaResponse.isSuccessful) { - uaResponse.close() - return null - } - - val userAgentList = uaResponse.use { json.decodeFromString(it.body.string()) } - - return when (userAgentType) { - UserAgentType.DESKTOP -> userAgentList.desktop - UserAgentType.MOBILE -> userAgentList.mobile - else -> error("Expected UserAgentType.DESKTOP or UserAgentType.MOBILE but got UserAgentType.${userAgentType.name} instead") - } - .filter { - filterInclude.isEmpty() || filterInclude.any { filter -> - it.contains(filter, ignoreCase = true) - } - } - .filterNot { - filterExclude.any { filter -> - it.contains(filter, ignoreCase = true) - } - } - .randomOrNull() - .also { userAgent = it } - } - - companion object { - private const val UA_DB_URL = "https://keiyoushi.github.io/user-agents/user-agents.json" - } -} - -/** - * Helper function to add a latest random user agent interceptor. - * The interceptor will added at the first position in the chain, - * so the CloudflareInterceptor in the app will be able to make usage of it. - * - * @param userAgentType User Agent type one of (DESKTOP, MOBILE, OFF) - * @param customUA Optional custom user agent used when userAgentType is OFF - * @param filterInclude Filter to only include User Agents containing these strings - * @param filterExclude Filter to exclude User Agents containing these strings - */ -fun OkHttpClient.Builder.setRandomUserAgent( - userAgentType: UserAgentType, - customUA: String? = null, - filterInclude: List = emptyList(), - filterExclude: List = emptyList(), -) = apply { - interceptors().add(0, RandomUserAgentInterceptor(userAgentType, customUA, filterInclude, filterExclude)) -} - -enum class UserAgentType { - MOBILE, - DESKTOP, - OFF, -} - -@Serializable -private data class UserAgentList( - val desktop: List, - val mobile: List, -) diff --git a/lib/randomua/src/keiyoushi/lib/randomua/RandomUserAgentPreference.kt b/lib/randomua/src/keiyoushi/lib/randomua/RandomUserAgentPreference.kt deleted file mode 100644 index bd6b838f7e..0000000000 --- a/lib/randomua/src/keiyoushi/lib/randomua/RandomUserAgentPreference.kt +++ /dev/null @@ -1,66 +0,0 @@ -package keiyoushi.lib.randomua - -import android.content.SharedPreferences -import android.widget.Toast -import androidx.preference.EditTextPreference -import androidx.preference.ListPreference -import androidx.preference.PreferenceScreen -import okhttp3.Headers - -/** - * Helper function to return UserAgentType based on SharedPreference value - */ -fun SharedPreferences.getPrefUAType(): UserAgentType = when (getString(PREF_KEY_RANDOM_UA, "off")) { - "mobile" -> UserAgentType.MOBILE - "desktop" -> UserAgentType.DESKTOP - else -> UserAgentType.OFF -} - -/** - * Helper function to return custom UserAgent from SharedPreference - */ -fun SharedPreferences.getPrefCustomUA(): String? = getString(PREF_KEY_CUSTOM_UA, null) - -/** - * Helper function to add Random User-Agent settings to SharedPreference - * - * @param screen, PreferenceScreen from `setupPreferenceScreen` - */ -fun addRandomUAPreferenceToScreen( - screen: PreferenceScreen, -) { - val context = screen.context - - ListPreference(context).apply { - key = PREF_KEY_RANDOM_UA - title = TITLE_RANDOM_UA - entries = RANDOM_UA_ENTRIES - entryValues = RANDOM_UA_VALUES - summary = "%s" - setDefaultValue("off") - }.also(screen::addPreference) - - EditTextPreference(context).apply { - key = PREF_KEY_CUSTOM_UA - title = TITLE_CUSTOM_UA - summary = CUSTOM_UA_SUMMARY - setOnPreferenceChangeListener { _, newValue -> - try { - Headers.headersOf("User-Agent", newValue as String) - true - } catch (e: IllegalArgumentException) { - Toast.makeText(context, "Invalid user agent string: ${e.message}", Toast.LENGTH_LONG).show() - false - } - } - }.also(screen::addPreference) -} - -const val TITLE_RANDOM_UA = "Random user agent string (requires restart)" -const val PREF_KEY_RANDOM_UA = "pref_key_random_ua_" -val RANDOM_UA_ENTRIES = arrayOf("OFF", "Desktop", "Mobile") -val RANDOM_UA_VALUES = arrayOf("off", "desktop", "mobile") - -const val TITLE_CUSTOM_UA = "Custom user agent string (requires restart)" -const val PREF_KEY_CUSTOM_UA = "pref_key_custom_ua_" -const val CUSTOM_UA_SUMMARY = "Leave blank to use the default user agent string (ignored if random user agent string is enabled)" diff --git a/lib/randomua/src/keiyoushi/lib/randomua/UserAgentPreference.kt b/lib/randomua/src/keiyoushi/lib/randomua/UserAgentPreference.kt new file mode 100644 index 0000000000..ae5ccd24c2 --- /dev/null +++ b/lib/randomua/src/keiyoushi/lib/randomua/UserAgentPreference.kt @@ -0,0 +1,97 @@ +package keiyoushi.lib.randomua + +import android.content.SharedPreferences +import android.widget.Toast +import androidx.preference.EditTextPreference +import androidx.preference.ListPreference +import androidx.preference.PreferenceScreen +import eu.kanade.tachiyomi.source.online.HttpSource +import keiyoushi.utils.getPreferences +import okhttp3.Headers + +/** + * Helper function to return UserAgentType based on SharedPreference value + */ +private fun SharedPreferences.getPrefUAType(): UserAgentType = when (getString(PREF_KEY_RANDOM_UA, "off")) { + "mobile" -> UserAgentType.MOBILE + "desktop" -> UserAgentType.DESKTOP + else -> UserAgentType.OFF +} + +/** + * Helper function to return custom UserAgent from SharedPreference + */ +private fun SharedPreferences.getPrefCustomUA(): String? = getString(PREF_KEY_CUSTOM_UA, null) + ?.takeIf { it.isNotBlank() } + +/** + * Helper function to add user agent preference to the headers + * + * @param userAgentType only set if you want to not include or bypass the preference value + * @param filterInclude Filter to only include Random User Agents containing these strings + * @param filterExclude Filter to exclude Random User Agents containing these strings + */ +context(source: HttpSource) +fun Headers.Builder.setRandomUserAgent( + userAgentType: UserAgentType? = null, + filterInclude: List = emptyList(), + filterExclude: List = emptyList(), +) = apply { + val preferences = source.getPreferences() + val randomUserAgentType = userAgentType ?: preferences.getPrefUAType() + val customUserAgent = preferences.getPrefCustomUA() + + val userAgent = if (randomUserAgentType != UserAgentType.OFF) { + getRandomUserAgent(randomUserAgentType, filterInclude, filterExclude) + ?: return@apply + } else if (customUserAgent != null) { + customUserAgent + } else { + return@apply + } + + set("User-Agent", userAgent) +} + +/** + * Helper function to add Random User-Agent settings to SharedPreference + */ +context(source: HttpSource) +fun PreferenceScreen.addRandomUAPreference() { + val preferences = source.getPreferences() + val customUaPref = EditTextPreference(context).apply { + key = PREF_KEY_CUSTOM_UA + title = "Custom user agent string" + summary = "Leave blank to use the default user agent string" + setEnabled(preferences.getPrefUAType() == UserAgentType.OFF) + setOnPreferenceChangeListener { _, newValue -> + try { + Headers.headersOf("User-Agent", newValue as String) + Toast.makeText(context, "Restart the app to apply changes", Toast.LENGTH_SHORT).show() + true + } catch (e: IllegalArgumentException) { + Toast.makeText(context, "Invalid user agent string: ${e.message}", Toast.LENGTH_LONG).show() + false + } + } + } + ListPreference(context).apply { + key = PREF_KEY_RANDOM_UA + title = "Random user agent string" + entries = RANDOM_UA_ENTRIES + entryValues = RANDOM_UA_VALUES + summary = "%s" + setDefaultValue("off") + setOnPreferenceChangeListener { _, newValue -> + customUaPref.setEnabled(newValue == "off") + Toast.makeText(context, "Restart the app to apply changes", Toast.LENGTH_SHORT).show() + true + } + }.also(::addPreference) + customUaPref.also(::addPreference) +} + +private const val PREF_KEY_RANDOM_UA = "pref_key_random_ua_" +private val RANDOM_UA_ENTRIES = arrayOf("OFF", "Desktop", "Mobile") +private val RANDOM_UA_VALUES = arrayOf("off", "desktop", "mobile") +private const val PREF_KEY_CUSTOM_UA = "pref_key_custom_ua_" diff --git a/src/all/nhentai/build.gradle b/src/all/nhentai/build.gradle index 8a2fcbfe2a..ff02caa19f 100644 --- a/src/all/nhentai/build.gradle +++ b/src/all/nhentai/build.gradle @@ -1,7 +1,7 @@ ext { extName = 'NHentai' extClass = '.NHFactory' - extVersionCode = 54 + extVersionCode = 57 isNsfw = true } diff --git a/src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHDto.kt b/src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHDto.kt index 9c1927d43c..96d4376ec6 100644 --- a/src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHDto.kt +++ b/src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHDto.kt @@ -1,16 +1,39 @@ package eu.kanade.tachiyomi.extension.all.nhentai +import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable @Serializable -class Hentai( +class NHConfig( + @SerialName("image_servers") val imageServers: List, + @SerialName("thumb_servers") val thumbServers: List, +) + +@Serializable +class PaginatedResponse( + val result: List = listOf(), + @SerialName("per_page") val perPage: Int = 0, + @SerialName("num_pages") val numPages: Int? = null, + val total: Int? = null, +) + +@Serializable +class GallerySearchItem( var id: Int, - val images: Images, - val media_id: String, + @SerialName("english_title") val englishTitle: String? = null, + @SerialName("japanese_title") val japaneseTitle: String? = null, + val thumbnail: String, +) + +@Serializable +class Gallery( + var id: Int, + val pages: List, + val thumbnail: Image, val tags: List, val title: Title, - val upload_date: Long, - val num_favorites: Long, + @SerialName("upload_date") val uploadDate: Long, + @SerialName("num_favorites") val numFavorites: Long, ) @Serializable @@ -21,21 +44,7 @@ class Title( ) @Serializable -class Images( - val pages: List, -) - -@Serializable -class Image( - private val t: String, -) { - val extension get() = when (t) { - "w" -> "webp" - "p" -> "png" - "g" -> "gif" - else -> "jpg" - } -} +class Image(val path: String) @Serializable class Tag( diff --git a/src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHUtils.kt b/src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHUtils.kt index fdd6cd5656..b8c0dcaaad 100644 --- a/src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHUtils.kt +++ b/src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHUtils.kt @@ -1,19 +1,17 @@ package eu.kanade.tachiyomi.extension.all.nhentai -import org.jsoup.nodes.Element - object NHUtils { - fun getArtists(data: Hentai): String { + fun getArtists(data: Gallery): String { val artists = data.tags.filter { it.type == "artist" } return artists.joinToString(", ") { it.name } } - fun getGroups(data: Hentai): String? { + fun getGroups(data: Gallery): String? { val groups = data.tags.filter { it.type == "group" } - return groups.joinToString(", ") { it.name }.takeIf { it.isBlank() } + return groups.joinToString(", ") { it.name }.takeIf { it.isNotBlank() } } - fun getTagDescription(data: Hentai): String { + fun getTagDescription(data: Gallery): String { val tags = data.tags.groupBy { it.type } return buildString { tags["category"]?.joinToString { it.name }?.let { @@ -28,10 +26,8 @@ object NHUtils { } } - fun getTags(data: Hentai): String { + fun getTags(data: Gallery): String { val artists = data.tags.filter { it.type == "tag" } return artists.joinToString(", ") { it.name } } - - private fun Element.cleanTag(): String = text().replace(Regex("\\(.*\\)"), "").trim() } diff --git a/src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHentai.kt b/src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHentai.kt index 1671375240..ddfac1389e 100644 --- a/src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHentai.kt +++ b/src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHentai.kt @@ -1,6 +1,8 @@ package eu.kanade.tachiyomi.extension.all.nhentai import android.content.SharedPreferences +import android.webkit.CookieManager +import androidx.preference.EditTextPreference import androidx.preference.ListPreference import androidx.preference.PreferenceScreen import eu.kanade.tachiyomi.extension.all.nhentai.NHUtils.getArtists @@ -8,7 +10,6 @@ import eu.kanade.tachiyomi.extension.all.nhentai.NHUtils.getGroups import eu.kanade.tachiyomi.extension.all.nhentai.NHUtils.getTagDescription import eu.kanade.tachiyomi.extension.all.nhentai.NHUtils.getTags import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.network.interceptor.rateLimit import eu.kanade.tachiyomi.source.ConfigurableSource import eu.kanade.tachiyomi.source.model.Filter @@ -18,31 +19,27 @@ import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.model.UpdateStrategy -import eu.kanade.tachiyomi.source.online.ParsedHttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.lib.randomua.addRandomUAPreferenceToScreen -import keiyoushi.lib.randomua.getPrefCustomUA -import keiyoushi.lib.randomua.getPrefUAType -import keiyoushi.lib.randomua.setRandomUserAgent +import eu.kanade.tachiyomi.source.online.HttpSource +import keiyoushi.lib.randomua.addRandomUAPreference import keiyoushi.utils.getPreferencesLazy -import kotlinx.serialization.decodeFromString +import keiyoushi.utils.parseAs import kotlinx.serialization.json.Json import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import rx.Observable import uy.kohesive.injekt.injectLazy +import java.io.IOException open class NHentai( override val lang: String, private val nhLang: String, -) : ParsedHttpSource(), +) : HttpSource(), ConfigurableSource { final override val baseUrl = "https://nhentai.net" + val apiUrl = "$baseUrl/api/v2" override val id by lazy { if (lang == "all") 7309872737163460316 else super.id } @@ -52,27 +49,81 @@ open class NHentai( private val json: Json by injectLazy() + private val webViewCookieManager: CookieManager by lazy { CookieManager.getInstance() } + private val preferences: SharedPreferences by getPreferencesLazy() override val client: OkHttpClient by lazy { network.cloudflareClient.newBuilder() - .setRandomUserAgent( - userAgentType = preferences.getPrefUAType(), - customUA = preferences.getPrefCustomUA(), - filterInclude = listOf("chrome"), - ) .rateLimit(4) + .addNetworkInterceptor(::authorizationInterceptor) .build() } + // Authentication + + val apiKey + get() = preferences.getString(API_KEY, "") + val cookieToken + get() = webViewCookieManager.getCookie(baseUrl) + .split("; ") + .firstOrNull { it.startsWith("access_token=") } + ?.replace("access_token=", "") ?: "" + var accessToken: String = "" + + fun authorizationInterceptor(chain: Interceptor.Chain): Response { + var request = chain.request() + if (!apiKey.isNullOrBlank()) { + request = request.newBuilder().addHeader("Authorization", "Key $apiKey").build() + val response = chain.proceed(request) + if (response.code == 401) { + response.close() + throw IOException("Invalid API key") + } + return response + } else if (request.url.toString().contains("/favorites")) { + val newToken = cookieToken + if (accessToken.isBlank() || accessToken != newToken) accessToken = newToken + request = request.newBuilder().addHeader("Authorization", "User $accessToken").build() + val response = chain.proceed(request) + if (response.code == 401) { + response.close() + accessToken = "" + throw IOException("Log in via WebView or add API key in the settings to view favorites") + } + return response + } + + val response = chain.proceed(request) + return response + } + + // Cdns + + val nhConfig: NHConfig by lazy { + try { + client.newCall(GET("$apiUrl/config", headers)).execute().body.string().parseAs() + } catch (_: IOException) { + NHConfig( + (1..4).map { n -> "https://i$n.nhentai.net" }.toList(), + (1..4).map { n -> "https://t$n.nhentai.net" }.toList(), + ) + } + } + val imageServer + get() = nhConfig.imageServers.random() + + val thumbServer + get() = nhConfig.thumbServers.random() + + // Preferences + private var displayFullTitle: Boolean = when (preferences.getString(TITLE_PREF, "full")) { "full" -> true else -> false } private val shortenTitleRegex = Regex("""(\[[^]]*]|[({][^)}]*[)}])""") - private val dataRegex = Regex("""JSON\.parse\(\s*"(.*)"\s*\)""") - private val hentaiSelector = "script:containsData(JSON.parse):not(:containsData(media_server)):not(:containsData(avatar_url))" private fun String.shortenTitle() = this.replace(shortenTitleRegex, "").trim() override fun setupPreferenceScreen(screen: PreferenceScreen) { @@ -93,48 +144,78 @@ open class NHentai( } }.also(screen::addPreference) - addRandomUAPreferenceToScreen(screen) + ListPreference(screen.context).apply { + key = SORT_PREF + title = SORT_PREF + entries = SORT_OPTIONS.map { it.first }.toTypedArray() + entryValues = SORT_OPTIONS.map { it.second }.toTypedArray() + summary = "%s" + setDefaultValue("popular") + }.also(screen::addPreference) + + EditTextPreference(screen.context).apply { + key = API_KEY + title = "API key" + summary = "Profile > Settings > API Keys" + setDefaultValue("") + }.let(screen::addPreference) + + screen.addRandomUAPreference() } - override fun latestUpdatesRequest(page: Int) = GET(if (nhLang.isBlank()) "$baseUrl/?page=$page" else "$baseUrl/language/$nhLang/?page=$page", headers) + // Latest - override fun latestUpdatesSelector() = "#content .container:not(.index-popular) .gallery" + override fun latestUpdatesRequest(page: Int) = GET( + if (nhLang.isBlank()) { + "$apiUrl/galleries?page=$page" + } else { + "$apiUrl/search?query=language%3A$nhLang&page=$page" + }, + headers, + ) - override fun latestUpdatesFromElement(element: Element) = SManga.create().apply { - setUrlWithoutDomain(element.select("a").attr("href")) - title = element.select("a > div").text().replace("\"", "").let { - if (displayFullTitle) it.trim() else it.shortenTitle() - } - thumbnail_url = element.selectFirst(".cover img")!!.let { img -> - if (img.hasAttr("data-src")) img.attr("abs:data-src") else img.attr("abs:src") - } + override fun latestUpdatesParse(response: Response): MangasPage { + val res = response.parseAs() + val mangas = res.result.map { parseSearchData(it) } + val page = response.request.url.queryParameter("page")?.toIntOrNull() ?: 1 + val hasNextPage = + (res.numPages != null && res.numPages > page) || (res.numPages == null && res.total != null && res.total < page * res.perPage) + return MangasPage(mangas, hasNextPage) } - override fun latestUpdatesNextPageSelector() = "#content > section.pagination > a.next" + // Popular - override fun popularMangaRequest(page: Int) = GET(if (nhLang.isBlank()) "$baseUrl/search/?q=\"\"&sort=popular&page=$page" else "$baseUrl/language/$nhLang/popular?page=$page", headers) - - override fun popularMangaFromElement(element: Element) = latestUpdatesFromElement(element) + override fun popularMangaRequest(page: Int) = GET( + if (nhLang.isBlank()) { + "$apiUrl/search/?query=\"\"&sort=popular&page=$page" + } else { + "$apiUrl/search?sort=popular&query=language%3A$nhLang&page=$page" + }, + headers, + ) - override fun popularMangaSelector() = latestUpdatesSelector() + override fun popularMangaParse(response: Response): MangasPage { + val res = response.parseAs() + val mangas = res.result.map { parseSearchData(it) } + val page = response.request.url.queryParameter("page")?.toIntOrNull() ?: 1 + val hasNextPage = + (res.numPages != null && res.numPages > page) || (res.numPages == null && res.total != null && res.total < page * res.perPage) + return MangasPage(mangas, hasNextPage) + } - override fun popularMangaNextPageSelector() = latestUpdatesNextPageSelector() + // Search - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable = when { + override suspend fun getSearchManga(page: Int, query: String, filters: FilterList): MangasPage = when { query.startsWith(PREFIX_ID_SEARCH) -> { val id = query.removePrefix(PREFIX_ID_SEARCH) - client.newCall(searchMangaByIdRequest(id)) - .asObservableSuccess() - .map { response -> searchMangaByIdParse(response, id) } + searchMangaByIdParse(client.newCall(searchMangaByIdRequest(id)).execute(), id) } query.toIntOrNull() != null -> { - client.newCall(searchMangaByIdRequest(query)) - .asObservableSuccess() - .map { response -> searchMangaByIdParse(response, query) } + searchMangaByIdParse(client.newCall(searchMangaByIdRequest(query)).execute(), query) } - else -> super.fetchSearchManga(page, query, filters) + else -> super.getSearchManga(page, query, filters) } override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { @@ -142,48 +223,41 @@ open class NHentai( val nhLangSearch = if (nhLang.isBlank()) "" else "language:$nhLang " val advQuery = combineQuery(filterList) val favoriteFilter = filterList.findInstance() - val offsetPage = - filterList.findInstance()?.state?.toIntOrNull()?.plus(page) ?: page if (favoriteFilter?.state == true) { - val url = "$baseUrl/favorites/".toHttpUrl().newBuilder() + val url = "$apiUrl/favorites".toHttpUrl().newBuilder() .addQueryParameter("q", "$query $advQuery") - .addQueryParameter("page", offsetPage.toString()) - + .addQueryParameter("page", page.toString()) return GET(url.build(), headers) } else { - val url = "$baseUrl/search/".toHttpUrl().newBuilder() + val url = "$apiUrl/search".toHttpUrl().newBuilder() // Blank query (Multi + sort by popular month/week/day) shows a 404 page // Searching for `""` is a hacky way to return everything without any filtering - .addQueryParameter("q", "$query $nhLangSearch$advQuery".ifBlank { "\"\"" }) - .addQueryParameter("page", offsetPage.toString()) + .addQueryParameter("query", "$query $nhLangSearch$advQuery".ifBlank { "\"\"" }) + .addQueryParameter("page", page.toString()) filterList.findInstance()?.let { f -> url.addQueryParameter("sort", f.toUriPart()) } - return GET(url.build(), headers) } } private fun combineQuery(filters: FilterList): String = buildString { filters.filterIsInstance().forEach { filter -> - filter.state.split(",") - .map(String::trim) - .filterNot(String::isBlank) - .forEach { tag -> - val y = !(filter.name == "Pages" || filter.name == "Uploaded") - if (tag.startsWith("-")) append("-") - append(filter.name, ':') - if (y) append('"') - append(tag.removePrefix("-")) - if (y) append('"') - append(" ") - } + filter.state.split(",").map(String::trim).filterNot(String::isBlank).forEach { tag -> + val y = !(filter.name == "Pages" || filter.name == "Uploaded") + if (tag.startsWith("-")) append("-") + append(filter.name, ':') + if (y) append('"') + append(tag.removePrefix("-")) + if (y) append('"') + append(" ") + } } } - private fun searchMangaByIdRequest(id: String) = GET("$baseUrl/g/$id", headers) + private fun searchMangaByIdRequest(id: String) = GET("$apiUrl/galleries/$id", headers) private fun searchMangaByIdParse(response: Response, id: String): MangasPage { val details = mangaDetailsParse(response) @@ -192,93 +266,84 @@ open class NHentai( } override fun searchMangaParse(response: Response): MangasPage { - if (response.request.url.toString().contains("/login/")) { - val document = response.asJsoup() - if (document.select(".fa-sign-in").isNotEmpty()) { - throw Exception("Log in via WebView to view favorites") - } - } + val res = response.parseAs() + val mangas = res.result.map { parseSearchData(it) } + val page = response.request.url.queryParameter("page")?.toIntOrNull() ?: 1 + val hasNextPage = + (res.numPages != null && res.numPages > page) || (res.numPages == null && res.total != null && res.total < page * res.perPage) + return MangasPage(mangas, hasNextPage) + } - return super.searchMangaParse(response) + fun parseSearchData(data: GallerySearchItem): SManga = SManga.create().apply { + url = "/g/${data.id}/" + title = if (displayFullTitle) { + data.englishTitle ?: data.japaneseTitle!! + } else { + (data.englishTitle ?: data.japaneseTitle)!!.shortenTitle() + } + thumbnail_url = "$thumbServer/${data.thumbnail}" + status = SManga.COMPLETED + update_strategy = UpdateStrategy.ONLY_FETCH_ONCE } - override fun searchMangaFromElement(element: Element) = latestUpdatesFromElement(element) - - override fun searchMangaSelector() = latestUpdatesSelector() - - override fun searchMangaNextPageSelector() = latestUpdatesNextPageSelector() - - override fun mangaDetailsParse(document: Document): SManga { - val data = document.getHentaiData() - val cdnUrl = document.getCdnUrls(thumbnail = true).random() - return SManga.create().apply { - title = if (displayFullTitle) data.title.english ?: data.title.japanese ?: data.title.pretty!! else data.title.pretty ?: (data.title.english ?: data.title.japanese)!!.shortenTitle() - thumbnail_url = "https://$cdnUrl/galleries/${data.media_id}/1t.${data.images.pages[0].extension}" - status = SManga.COMPLETED - artist = getArtists(data) - author = getGroups(data) ?: getArtists(data) - // Some people want these additional details in description - description = "Full English and Japanese titles:\n" - .plus("${data.title.english ?: data.title.japanese ?: data.title.pretty ?: ""}\n") - .plus(data.title.japanese ?: "") - .plus("\n\n") - .plus("Pages: ${data.images.pages.size}\n") - .plus("Favorited by: ${data.num_favorites}\n") - .plus(getTagDescription(data)) - genre = getTags(data) - update_strategy = UpdateStrategy.ONLY_FETCH_ONCE + // Manga + + override fun getMangaUrl(manga: SManga) = "$baseUrl${manga.url}" + + override fun mangaDetailsRequest(manga: SManga): Request = searchMangaByIdRequest(manga.url.removeSurrounding("/g/", "/")) + + override fun mangaDetailsParse(response: Response): SManga = parseMangaData(response.parseAs()) + + fun parseMangaData(data: Gallery): SManga = SManga.create().apply { + url = "/g/${data.id}/" + title = if (displayFullTitle) { + data.title.english ?: data.title.japanese ?: data.title.pretty!! + } else { + data.title.pretty ?: (data.title.english ?: data.title.japanese)!!.shortenTitle() } + thumbnail_url = "$thumbServer/${data.thumbnail.path}" + status = SManga.COMPLETED + artist = getArtists(data) + author = getGroups(data) ?: getArtists(data) + // Some people want these additional details in description + description = + "Full English and Japanese titles:\n".plus("${data.title.english ?: data.title.japanese ?: data.title.pretty ?: ""}\n") + .plus(data.title.japanese ?: "").plus("\n\n").plus("Pages: ${data.pages.size}\n") + .plus("Favorited by: ${data.numFavorites}\n").plus(getTagDescription(data)) + genre = getTags(data) + update_strategy = UpdateStrategy.ONLY_FETCH_ONCE } - override fun chapterListRequest(manga: SManga): Request = GET("$baseUrl${manga.url}", headers) + // Chapter List + + override fun chapterListRequest(manga: SManga): Request = GET("$apiUrl/galleries/${manga.url.removeSurrounding("/g/", "/")}", headers) override fun chapterListParse(response: Response): List { - val data = response.asJsoup().getHentaiData() + val data = response.parseAs() return listOf( SChapter.create().apply { + url = "/g/${data.id}/" name = "Chapter" scanlator = getGroups(data) - date_upload = data.upload_date * 1000 - setUrlWithoutDomain(response.request.url.encodedPath) + date_upload = data.uploadDate * 1000 }, ) } - override fun chapterFromElement(element: Element) = throw UnsupportedOperationException() - - override fun chapterListSelector() = throw UnsupportedOperationException() + // Pages - override fun pageListParse(document: Document): List { - val data = document.getHentaiData() - val cdnUrls = document.getCdnUrls(thumbnail = false) + override fun pageListRequest(chapter: SChapter): Request = GET("$apiUrl/galleries/${chapter.url.removeSurrounding("/g/", "/")}", headers) - return data.images.pages.mapIndexed { i, image -> - Page( - index = i, - imageUrl = "https://${cdnUrls.random()}/galleries/${data.media_id}/${i + 1}.${image.extension}", - ) + override fun pageListParse(response: Response): List { + val data = response.parseAs(json) + return data.pages.mapIndexed { i, page -> + Page(i, imageUrl = "$imageServer/${page.path}") } } - private fun Document.getHentaiData(): Hentai { - val script = selectFirst(hentaiSelector)!!.data() - return dataRegex.find(script)!!.groupValues[1].parseAs() - } - - private fun Document.getCdnUrls(thumbnail: Boolean): List { - val regex = Regex( - if (thumbnail) { - """thumb_cdn_urls:\s*(\[.*])""" - } else { - """image_cdn_urls:\s*(\[.*])""" - }, - ) - val html = body().html() - val cdnJson = regex.find(html)!!.groupValues[1] - - return cdnJson.parseAs>() - } + override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() + // Filters override fun getFilterList(): FilterList = FilterList( Filter.Header("Separate tags with commas (,)"), Filter.Header("Prepend with dash (-) to exclude"), @@ -296,7 +361,7 @@ open class NHentai( PagesFilter(), Filter.Separator(), - SortFilter(), + SortFilter(SORT_OPTIONS.indexOfFirst { it.second == preferences.getString(SORT_PREF, "popular") }), OffsetPageFilter(), Filter.Header("Sort is ignored if favorites only"), FavoriteFilter(), @@ -314,38 +379,35 @@ open class NHentai( class OffsetPageFilter : Filter.Text("Offset results by # pages") - override fun imageUrlParse(document: Document) = throw UnsupportedOperationException() - private class FavoriteFilter : Filter.CheckBox("Show favorites only", false) - private class SortFilter : - UriPartFilter( - "Sort By", - arrayOf( - Pair("Popular: All Time", "popular"), - Pair("Popular: Month", "popular-month"), - Pair("Popular: Week", "popular-week"), - Pair("Popular: Today", "popular-today"), - Pair("Recent", "date"), - ), - ) + private class SortFilter(default: Int) : UriPartFilter("Sort By", SORT_OPTIONS, default) - private inline fun String.parseAs(): T { - val data = Regex("""\\u([0-9A-Fa-f]{4})""").replace(this) { - it.groupValues[1].toInt(16).toChar().toString() - } - return json.decodeFromString( - data, - ) - } - private open class UriPartFilter(displayName: String, val vals: Array>) : Filter.Select(displayName, vals.map { it.first }.toTypedArray()) { + private open class UriPartFilter(displayName: String, val vals: Array>, state: Int) : Filter.Select(displayName, vals.map { it.first }.toTypedArray(), state) { fun toUriPart() = vals[state].second } + // Utils + + private inline fun String.parseAs(): T { + val data = Regex("""\\u([0-9A-Fa-f]{4})""").replace(this) { it.groupValues[1].toInt(16).toChar().toString() } + return json.decodeFromString(data) + } private inline fun Iterable<*>.findInstance() = find { it is T } as? T companion object { + const val API_KEY = "api_key" const val PREFIX_ID_SEARCH = "id:" private const val TITLE_PREF = "Display manga title as:" + + private val SORT_OPTIONS = arrayOf( + Pair("Popular: All Time", "popular"), + Pair("Popular: Month", "popular-month"), + Pair("Popular: Week", "popular-week"), + Pair("Popular: Today", "popular-today"), + Pair("Recent", "date"), + ) + + private const val SORT_PREF = "Default sort preference when searching" } }