en/Donghuastream: Fix video loading and improve extractors

src/en/donghuastream/build.gradle

@@ -3,12 +3,14 @@ ext {
     extClass = '.DonghuaStream'
     themePkg = 'animestream'
     baseUrl = 'https://donghuastream.org'
-    overrideVersionCode = 15
+    overrideVersionCode = 16
 }
 
 apply from: "$rootDir/common.gradle"
 
 dependencies {
     implementation(project(':lib:dailymotionextractor'))
+    implementation(project(':lib:okruextractor'))
     implementation(project(':lib:playlistutils'))
+    implementation(project(':lib:unpacker'))
 }

src/en/donghuastream/src/eu/kanade/tachiyomi/animeextension/en/donghuastream/DonghuaStream.kt

@@ -1,9 +1,25 @@
 package eu.kanade.tachiyomi.animeextension.en.donghuastream
 
+import android.content.SharedPreferences
+import androidx.preference.PreferenceScreen
 import aniyomi.lib.dailymotionextractor.DailymotionExtractor
+import aniyomi.lib.okruextractor.OkruExtractor
+import eu.kanade.tachiyomi.animeextension.en.donghuastream.extractors.RumbleExtractor
 import eu.kanade.tachiyomi.animeextension.en.donghuastream.extractors.StreamPlayExtractor
+import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
+import eu.kanade.tachiyomi.animesource.model.SEpisode
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.multisrc.animestream.AnimeStream
+import eu.kanade.tachiyomi.network.GET
+import keiyoushi.utils.LazyMutable
+import keiyoushi.utils.UrlUtils
+import keiyoushi.utils.addSetPreference
+import keiyoushi.utils.addSwitchPreference
+import kotlinx.coroutines.runBlocking
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.Request
+import okhttp3.Response
+import kotlin.getValue
 
 class DonghuaStream :
     AnimeStream(
@@ -14,29 +30,105 @@ class DonghuaStream :
     override val fetchFilters: Boolean
         get() = false
 
+    // ============================ Manual Changes ==========================
+
+    override fun popularAnimeNextPageSelector() = "div.mrgn a.r"
+
+    override fun latestUpdatesNextPageSelector() = popularAnimeNextPageSelector()
+
+    override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request {
+        val url = baseUrl.toHttpUrl().newBuilder().apply {
+            addPathSegment("pagg")
+            addPathSegment(page.toString())
+            addPathSegment("")
+            addQueryParameter("s", query)
+        }.build()
+        return GET(url)
+    }
+
+    private var SharedPreferences.ignorePreview
+        by LazyMutable { preferences.getBoolean(IGNORE_PREVIEW_KEY, IGNORE_PREVIEW_DEFAULT) }
+
+    private var SharedPreferences.getHosters
+        by LazyMutable { preferences.getStringSet(PREF_HOSTER_KEY, PREF_HOSTER_DEFAULT)!! }
+
+    private companion object {
+        private const val PREF_HOSTER_KEY = "dm_hoster_selection"
+        private val INTERNAL_HOSTER_NAMES = listOf("Dailymotion", "Streamplay", "Rumble", "Ok.ru")
+        private val PREF_HOSTER_ENTRY_VALUES = INTERNAL_HOSTER_NAMES.map { it.lowercase() }.toList()
+        private val PREF_HOSTER_DEFAULT = INTERNAL_HOSTER_NAMES.map { it.lowercase() }.toSet()
+
+        private const val IGNORE_PREVIEW_KEY = "dm_ignore_preview"
+        private const val IGNORE_PREVIEW_DEFAULT = true
+    }
+
+    override fun setupPreferenceScreen(screen: PreferenceScreen) {
+        super.setupPreferenceScreen(screen)
+
+        screen.addSetPreference(
+            key = PREF_HOSTER_KEY,
+            title = "Enable/Disable Hosts",
+            summary = "",
+            entries = INTERNAL_HOSTER_NAMES,
+            entryValues = PREF_HOSTER_ENTRY_VALUES,
+            default = PREF_HOSTER_DEFAULT,
+        ) {
+            preferences.getHosters = it
+        }
+
+        screen.addSwitchPreference(
+            key = IGNORE_PREVIEW_KEY,
+            title = "Skip Preview episodes",
+            summary = "",
+            default = IGNORE_PREVIEW_DEFAULT,
+        ) {
+            preferences.ignorePreview = it
+        }
+    }
+
+    override val prefQualityValues = arrayOf("2160p", "1440p", "1080p", "720p", "480p", "360p")
+    override val prefQualityEntries = prefQualityValues
+
+    override fun episodeListParse(response: Response): List<SEpisode> = super.episodeListParse(response)
+        .filter { !it.name.contains("Preview", ignoreCase = true) || !preferences.ignorePreview }
+
     // ============================ Video Links =============================
 
     private val dailymotionExtractor by lazy { DailymotionExtractor(client, headers) }
     private val streamPlayExtractor by lazy { StreamPlayExtractor(client, headers) }
+    private val okruExtractor by lazy { OkruExtractor(client) }
+
+    private val rumbleExtractor by lazy { RumbleExtractor(client, headers) }
 
     override fun getVideoList(url: String, name: String): List<Video> {
         val prefix = "$name - "
-        return when {
-            url.contains("dailymotion") -> dailymotionExtractor.videosFromUrl(url, prefix = prefix)
-            url.contains("streamplay") -> streamPlayExtractor.videosFromUrl(url, prefix = prefix)
-            else -> emptyList()
+        return runBlocking {
+            when {
+                preferences.getHosters.contains("dailymotion") && url.contains("dailymotion") ->
+                    dailymotionExtractor.videosFromUrl(url, prefix = prefix)
+                preferences.getHosters.contains("streamplay") && url.contains("streamplay") ->
+                    streamPlayExtractor.videosFromUrl(url, prefix = prefix)
+                preferences.getHosters.contains("ok.ru") && url.contains("ok.ru") ->
+                    UrlUtils.fixUrl(url)?.let { okruExtractor.videosFromUrl(url = it, prefix = prefix) }
+                        ?: emptyList()
+                preferences.getHosters.contains("rumble") && url.contains("rumble") ->
+                    rumbleExtractor.videosFromUrl(url, prefix = prefix)
+                else -> emptyList()
+            }
         }
     }
 
     // ============================= Utilities ==============================
 
+    private val qualityRegex by lazy { Regex("""(\d+)p""") }
+
     override fun List<Video>.sort(): List<Video> {
         val quality = preferences.getString(videoSortPrefKey, videoSortPrefDefault)!!
         return sortedWith(
-            compareBy(
+            compareBy<Video>(
                 { it.quality.contains(quality) },
-                { Regex("""(\d+)p""").find(it.quality)?.groupValues?.get(1)?.toIntOrNull() ?: 0 },
-            ),
+                { qualityRegex.find(it.quality)?.groupValues?.get(1)?.toIntOrNull() ?: 0 },
+            ).thenByDescending { it.quality },
         ).reversed()
     }
 }

src/en/donghuastream/src/eu/kanade/tachiyomi/animeextension/en/donghuastream/extractors/RumbleExtractor.kt

@@ -0,0 +1,21 @@
+package eu.kanade.tachiyomi.animeextension.en.donghuastream.extractors
+
+import aniyomi.lib.playlistutils.PlaylistUtils
+import eu.kanade.tachiyomi.animesource.model.Video
+import okhttp3.Headers
+import okhttp3.OkHttpClient
+
+class RumbleExtractor(private val client: OkHttpClient, private val headers: Headers) {
+
+    private val playlistUtils by lazy { PlaylistUtils(client, headers) }
+
+    fun videosFromUrl(url: String, prefix: String = ""): List<Video> {
+        val id = extractRumbleId(url) ?: return emptyList()
+        val sourceUrl = "https://rumble.com/hls-vod/$id/playlist.m3u8"
+        return playlistUtils.extractFromHls(sourceUrl, referer = url, subtitleList = emptyList(), videoNameGen = { q -> "$prefix $q" })
+    }
+
+    private val regex by lazy { Regex("""rumble\.com/embed/v([a-zA-Z0-9]+)""") }
+
+    private fun extractRumbleId(url: String): String? = regex.find(url)?.groupValues?.get(1)
+}

src/en/donghuastream/src/eu/kanade/tachiyomi/animeextension/en/donghuastream/extractors/StreamPlayExtractor.kt

@@ -1,55 +1,109 @@
 package eu.kanade.tachiyomi.animeextension.en.donghuastream.extractors
 
+import aniyomi.lib.jsunpacker.JsUnpacker
 import aniyomi.lib.playlistutils.PlaylistUtils
 import eu.kanade.tachiyomi.animesource.model.Track
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.util.asJsoup
+import eu.kanade.tachiyomi.network.POST
+import eu.kanade.tachiyomi.network.awaitSuccess
+import keiyoushi.utils.UrlUtils
+import keiyoushi.utils.parallelCatchingFlatMap
 import keiyoushi.utils.parseAs
+import keiyoushi.utils.useAsJsoup
 import kotlinx.serialization.Serializable
-import kotlinx.serialization.json.Json
 import okhttp3.Headers
-import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
+import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.OkHttpClient
-import uy.kohesive.injekt.injectLazy
+import okhttp3.RequestBody.Companion.toRequestBody
 
 class StreamPlayExtractor(private val client: OkHttpClient, private val headers: Headers) {
 
-    private val json: Json by injectLazy()
-
     private val playlistUtils by lazy { PlaylistUtils(client, headers) }
 
-    fun videosFromUrl(url: String, prefix: String = ""): List<Video> {
+    suspend fun videosFromUrl(url: String, prefix: String = ""): List<Video> {
         val document = client.newCall(
             GET(url, headers),
-        ).execute().asJsoup()
+        ).awaitSuccess().useAsJsoup()
+
+        return document.select("#servers a").parallelCatchingFlatMap { element ->
+            extractAndDecodeFromDocument(element.attr("href"), "$prefix ${element.text()}")
+        }
+    }
 
-        val apiUrl = document.selectFirst("script:containsData(/api/)")
-            ?.data()
-            ?.substringAfter("url:")
-            ?.substringAfter("\"")
-            ?.substringBefore("\"")
-            ?: return emptyList()
+    private val kakenRegex by lazy { Regex("window\\.kaken ?= ?\"([^\"]+)\";") }
+
+    /**
+     * Server 3 has issue with playlist compatibility, it only plays the first segment
+     */
+    private suspend fun extractAndDecodeFromDocument(url: String, prefix: String): List<Video> {
+        val document = client.newCall(
+            GET(url, headers),
+        ).awaitSuccess().useAsJsoup()
+
+        // Find script containing the packed code
+        val packedScript = document.selectFirst("script:containsData(function(p,a,c,k,e,d))")
+
+        val kaken = if (packedScript != null) {
+            val scriptContent = packedScript.data()
+            JsUnpacker.unpackAndCombine(scriptContent)?.let {
+                kakenRegex.find(it)?.groupValues?.get(1)
+            }
+        } else {
+            // For mobile UA, it's non-packed
+            document.selectFirst("script:containsData(window.kaken)")
+                ?.data()?.let {
+                    // Extract kaken
+                    kakenRegex.find(it)?.groupValues?.get(1)
+                }
+        } ?: return emptyList()
+
+        val httpUrl = url.toHttpUrlOrNull() ?: return emptyList()
 
         val apiHeaders = headers.newBuilder().apply {
             add("Accept", "application/json, text/javascript, */*; q=0.01")
-            add("Host", apiUrl.toHttpUrl().host)
+            add("Origin", "${httpUrl.scheme}://${httpUrl.host}")
             add("Referer", url)
             add("X-Requested-With", "XMLHttpRequest")
         }.build()
 
         val apiResponse = client.newCall(
-            GET("$apiUrl&_=${System.currentTimeMillis() / 1000}", headers = apiHeaders),
-        ).execute().parseAs<APIResponse>()
+            POST(
+                "https://play.streamplay.co.in/api/",
+                headers = apiHeaders,
+                body = kaken.toRequestBody("text/plain".toMediaType()),
+            ),
+        ).awaitSuccess().parseAs<APIResponse>()
 
         val subtitleList = apiResponse.tracks?.let { t ->
             t.map { Track(it.file, it.label) }
         } ?: emptyList()
 
-        return apiResponse.sources.flatMap { source ->
-            val sourceUrl = source.file.replace("^//".toRegex(), "https://")
-            playlistUtils.extractFromHls(sourceUrl, referer = url, subtitleList = subtitleList, videoNameGen = { q -> "$prefix$q (StreamPlay)" })
+        val videos = apiResponse.sources.parallelCatchingFlatMap { source ->
+            val sourceUrl = UrlUtils.fixUrl(source.videoUrl) ?: return@parallelCatchingFlatMap emptyList()
+            if (source.type == "hls" && sourceUrl.endsWith("master.m3u8")) {
+                playlistUtils.extractFromHls(
+                    playlistUrl = sourceUrl,
+                    referer = url,
+                    subtitleList = subtitleList,
+                    videoNameGen = { q -> "$prefix $q (StreamPlay)" },
+                )
+            } else {
+                listOf(
+                    Video(
+                        sourceUrl,
+                        "$prefix (StreamPlay) Original",
+                        sourceUrl,
+                        headers = headers.newBuilder()
+                            .set("Referer", url)
+                            .build(),
+                        subtitleTracks = subtitleList,
+                    ),
+                )
+            }
         }
+        return videos
     }
 
     @Serializable
@@ -60,7 +114,12 @@ class StreamPlayExtractor(private val client: OkHttpClient, private val headers:
         @Serializable
         data class SourceObject(
             val file: String,
-        )
+            val label: String,
+            val type: String,
+        ) {
+            val videoUrl: String
+                get() = file.replace("master.txt", "master.m3u8")
+        }
 
         @Serializable
         data class TrackObject(