[YouTube] Remove fetch of YouTube service workers and improve WEB client version and API key extraction

Client version and API key are not available anymore in service workers, so it is unneeded to fetch them.

Common code in WEB client version extraction has been deduplicated, usage of the Java 8 Stream API has been made and initial data fallback has been used as a last resort.
This means that the client version extraction from regexes will be used before this fallback, as it doesn't contain the full client version.
This can be used as a way to fingerprint the extractor, even if it seems to be not the case.

Author: AudricV

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -73,6 +73,7 @@
 import java.util.Optional;
 import java.util.Random;
 import java.util.regex.Pattern;
+import java.util.stream.Stream;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -612,87 +613,85 @@ public static boolean areHardcodedClientVersionAndKeyValid()
         return hardcodedClientVersionAndKeyValid.get();
     }
 
-
-    private static void extractClientVersionAndKeyFromSwJs()
-            throws IOException, ExtractionException {
-        if (keyAndVersionExtracted) {
-            return;
-        }
-        final String url = "https://www.youtube.com/sw.js";
-        final Map<String, List<String>> headers = new HashMap<>();
-        headers.put("Origin", singletonList("https://www.youtube.com"));
-        headers.put("Referer", singletonList("https://www.youtube.com"));
-        final String response = getDownloader().get(url, headers).responseBody();
-        try {
-            clientVersion = getStringResultFromRegexArray(response,
-                    INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
-            key = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
-        } catch (final Parser.RegexException e) {
-            throw new ParsingException("Could not extract YouTube WEB InnerTube client version "
-                    + "and API key from sw.js", e);
-        }
-        keyAndVersionExtracted = true;
-    }
-
     private static void extractClientVersionAndKeyFromHtmlSearchResultsPage()
             throws IOException, ExtractionException {
         // Don't extract the client version and the InnerTube key if it has been already extracted
         if (keyAndVersionExtracted) {
             return;
         }
+
         // Don't provide a search term in order to have a smaller response
         final String url = "https://www.youtube.com/results?search_query=&ucbcb=1";
         final String html = getDownloader().get(url, getCookieHeader()).responseBody();
         final JsonObject initialData = getInitialData(html);
         final JsonArray serviceTrackingParams = initialData.getObject("responseContext")
                 .getArray("serviceTrackingParams");
-        String shortClientVersion = null;
 
         // Try to get version from initial data first
-        for (final Object service : serviceTrackingParams) {
-            final JsonObject s = (JsonObject) service;
-            if (s.getString("service").equals("CSI")) {
-                final JsonArray params = s.getArray("params");
-                for (final Object param : params) {
-                    final JsonObject p = (JsonObject) param;
-                    final String paramKey = p.getString("key");
-                    if (paramKey != null && paramKey.equals("cver")) {
-                        clientVersion = p.getString("value");
-                    }
-                }
-            } else if (s.getString("service").equals("ECATCHER")) {
-                // Fallback to get a shortened client version which does not contain the last two
-                // digits
-                final JsonArray params = s.getArray("params");
-                for (final Object param : params) {
-                    final JsonObject p = (JsonObject) param;
-                    final String paramKey = p.getString("key");
-                    if (paramKey != null && paramKey.equals("client.version")) {
-                        shortClientVersion = p.getString("value");
-                    }
-                }
+        final Stream<JsonObject> serviceTrackingParamsStream = serviceTrackingParams.stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast);
+
+        clientVersion = getClientVersionFromServiceTrackingParam(
+                serviceTrackingParamsStream, "CSI", "cver");
+
+        if (clientVersion == null) {
+            try {
+                clientVersion = getStringResultFromRegexArray(html,
+                        INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
+            } catch (final Parser.RegexException ignored) {
             }
         }
 
+        // Fallback to get a shortened client version which does not contain the last two
+        // digits
+        if (isNullOrEmpty(clientVersion)) {
+            clientVersion = getClientVersionFromServiceTrackingParam(
+                    serviceTrackingParamsStream, "ECATCHER", "client.version");
+        }
+
         try {
-            clientVersion = getStringResultFromRegexArray(html,
-                    INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
+            key = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1);
         } catch (final Parser.RegexException ignored) {
         }
 
-        if (!isNullOrEmpty(clientVersion) && !isNullOrEmpty(shortClientVersion)) {
-            clientVersion = shortClientVersion;
+        if (isNullOrEmpty(key)) {
+            throw new ParsingException(
+                    // CHECKSTYLE:OFF
+                    "Could not extract YouTube WEB InnerTube API key from HTML search results page");
+                    // CHECKSTYLE:ON
         }
 
-        try {
-            key = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1);
-        } catch (final Parser.RegexException e) {
-            throw new ParsingException("Could not extract YouTube WEB InnerTube client version "
-                    + "and API key from HTML search results page", e);
+        if (clientVersion == null) {
+            throw new ParsingException(
+                    // CHECKSTYLE:OFF
+                    "Could not extract YouTube WEB InnerTube client version from HTML search results page");
+                    // CHECKSTYLE:ON
         }
+
         keyAndVersionExtracted = true;
     }
 
+    @Nullable
+    private static String getClientVersionFromServiceTrackingParam(
+            @Nonnull final Stream<JsonObject> serviceTrackingParamsStream,
+            @Nonnull final String serviceName,
+            @Nonnull final String clientVersionKey) {
+        return serviceTrackingParamsStream.filter(serviceTrackingParam ->
+                        serviceTrackingParam.getString("service", "")
+                                .equals(serviceName))
+                .flatMap(serviceTrackingParam -> serviceTrackingParam.getArray("params")
+                        .stream())
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .filter(param -> param.getString("key", "")
+                        .equals(clientVersionKey))
+                .map(param -> param.getString("value"))
+                .filter(paramValue -> !isNullOrEmpty(paramValue))
+                .findFirst()
+                .orElse(null);
+    }
+
     /**
      * Get the client version used by YouTube website on InnerTube requests.
      */
@@ -701,20 +700,15 @@ public static String getClientVersion() throws IOException, ExtractionException
             return clientVersion;
         }
 
-        // Always extract latest client version, by trying first to extract it from the JavaScript
-        // service worker, then from HTML search results page as a fallback, to prevent
+        // Always extract the latest client version from HTML search results page to prevent
         // fingerprinting based on the client version used
-        try {
-            extractClientVersionAndKeyFromSwJs();
-        } catch (final Exception e) {
-            extractClientVersionAndKeyFromHtmlSearchResultsPage();
-        }
+        extractClientVersionAndKeyFromHtmlSearchResultsPage();
 
         if (keyAndVersionExtracted) {
             return clientVersion;
         }
 
-        // Fallback to the hardcoded one if it's valid
+        // Fallback to the hardcoded one if it is valid
         if (areHardcodedClientVersionAndKeyValid()) {
             clientVersion = HARDCODED_CLIENT_VERSION;
             return clientVersion;
@@ -731,14 +725,9 @@ public static String getKey() throws IOException, ExtractionException {
             return key;
         }
 
-        // Always extract the key used by the webiste, by trying first to extract it from the
-        // JavaScript service worker, then from HTML search results page as a fallback, to prevent
-        // fingerprinting based on the key and/or invalid key issues
-        try {
-            extractClientVersionAndKeyFromSwJs();
-        } catch (final Exception e) {
-            extractClientVersionAndKeyFromHtmlSearchResultsPage();
-        }
+        // Always extract the latest client version from HTML search results page to prevent
+        // fingerprinting based on the client version used
+        extractClientVersionAndKeyFromHtmlSearchResultsPage();
 
         if (keyAndVersionExtracted) {
             return key;
@@ -751,7 +740,8 @@ public static String getKey() throws IOException, ExtractionException {
         }
 
         // The ANDROID API key is also valid with the WEB client so return it if we couldn't
-        // extract the WEB API key.
+        // extract the WEB API key. This can be used as a way to fingerprint the extractor in this
+        // case
         return ANDROID_YOUTUBE_KEY;
     }
 
@@ -843,29 +833,13 @@ public static String[] getYoutubeMusicKey()
             return youtubeMusicKey;
         }
 
-        String musicClientVersion;
-        String musicKey;
-        String musicClientName;
-
-        try {
-            final String url = "https://music.youtube.com/sw.js";
-            final Map<String, List<String>> headers = new HashMap<>();
-            headers.put("Origin", singletonList("https://music.youtube.com"));
-            headers.put("Referer", singletonList("https://music.youtube.com"));
-            final String response = getDownloader().get(url, headers).responseBody();
-                musicClientVersion = getStringResultFromRegexArray(response,
-                        INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
-                musicKey = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
-                musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, response);
-        } catch (final Exception e) {
-            final String url = "https://music.youtube.com/?ucbcb=1";
-            final String html = getDownloader().get(url, getCookieHeader()).responseBody();
+        final String url = "https://music.youtube.com/?ucbcb=1";
+        final String html = getDownloader().get(url, getCookieHeader()).responseBody();
 
-            musicKey = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1);
-            musicClientVersion = getStringResultFromRegexArray(html,
-                    INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES);
-            musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, html);
-        }
+        final String musicKey = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1);
+        final String musicClientVersion = getStringResultFromRegexArray(html,
+                INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES);
+        final String musicClientName = Parser.matchGroup1(INNERTUBE_CLIENT_NAME_REGEX, html);
 
         youtubeMusicKey = new String[] {musicKey, musicClientName, musicClientVersion};
         return youtubeMusicKey;