Merge pull request #891 from Theta-Dev/fix-throttling-decrypter

[YouTube] Fix extraction of more complex nsig functions

Author: AudricV

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java

@@ -12,24 +12,24 @@
 import java.util.regex.Pattern;
 
 /**
+ * YouTube's streaming URLs of HTML5 clients are protected with a cipher, which modifies their
+ * {@code n} query parameter.
+ *
  * <p>
- * YouTube's media is protected with a cipher,
- * which modifies the "n" query parameter of it's video playback urls.
- * This class handles extracting that "n" query parameter,
- * applying the cipher on it and returning the resulting url which is not throttled.
+ * This class handles extracting that {@code n} query parameter, applying the cipher on it and
+ * returning the resulting URL which is not throttled.
  * </p>
  *
- * <pre>
- * https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=VVF2xyZLVRZZxHXZ&amp;other=other
- * </pre>
+ * <p>
+ * For instance,
+ * {@code https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=VVF2xyZLVRZZxHXZ&other=other}
  * becomes
- * <pre>
- * https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=iHywZkMipkszqA&amp;other=other
- * </pre>
- * <br>
+ * {@code https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=iHywZkMipkszqA&other=other}.
+ * </p>
+ *
  * <p>
- * Decoding the "n" parameter is time intensive. For this reason, the results are cached.
- * The cache can be cleared using {@link #clearCache()}
+ * Decoding the {@code n} parameter is time intensive. For this reason, the results are cached.
+ * The cache can be cleared using {@link #clearCache()}.
  * </p>
  *
  */
@@ -73,13 +73,35 @@ public YoutubeThrottlingDecrypter() throws ParsingException {
     }
 
     /**
+     * Try to decrypt a YouTube streaming URL protected with a throttling parameter.
+     *
      * <p>
-     * The videoId is only used to fetch the decryption function.
-     * It can be a constant value of any existing video.
-     * A constant value is discouraged, because it could allow tracking.
+     * If the streaming URL provided doesn't contain a throttling parameter, it is returned as it
+     * is; otherwise, the encrypted value is decrypted and this value is replaced by the decrypted
+     * one.
+     * </p>
+     *
+     * <p>
+     * If the JavaScript code has been not extracted, it is extracted with the given video ID using
+     * {@link YoutubeJavaScriptExtractor#extractJavaScriptCode(String)}.
+     * </p>
+     *
+     * @param streamingUrl The streaming URL to decrypt, if needed.
+     * @param videoId      A video ID, used to fetch the JavaScript code to get the decryption
+     *                     function. It can be a constant value of any existing video, but a
+     *                     constant value is discouraged, because it could allow tracking.
+     * @return A streaming URL with the decrypted parameter or the streaming URL itself if no
+     * throttling parameter has been found
+     * @throws ParsingException If the streaming URL contains a throttling parameter and its
+     *                          decryption failed
      */
-    public static String apply(final String url, final String videoId) throws ParsingException {
-        if (containsNParam(url)) {
+    public static String apply(@Nonnull final String streamingUrl,
+                               @Nonnull final String videoId) throws ParsingException {
+        if (!containsNParam(streamingUrl)) {
+            return streamingUrl;
+        }
+
+        try {
             if (FUNCTION == null) {
                 final String playerJsCode
                         = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId);
@@ -88,11 +110,11 @@ public static String apply(final String url, final String videoId) throws Parsin
                 FUNCTION = parseDecodeFunction(playerJsCode, FUNCTION_NAME);
             }
 
-            final String oldNParam = parseNParam(url);
+            final String oldNParam = parseNParam(streamingUrl);
             final String newNParam = decryptNParam(FUNCTION, FUNCTION_NAME, oldNParam);
-            return replaceNParam(url, oldNParam, newNParam);
-        } else {
-            return url;
+            return replaceNParam(streamingUrl, oldNParam, newNParam);
+        } catch (final Exception e) {
+            throw new ParsingException("Could not parse, decrypt or replace n parameter", e);
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -602,15 +602,21 @@ public List<VideoStream> getVideoOnlyStreams() throws ExtractionException {
     }
 
     /**
-     * Try to decrypt url and fallback to given url, because decryption is not
-     * always needed.
+     * Try to decrypt a streaming URL and fallback to the given URL, because decryption may fail if
+     * YouTube do breaking changes.
+     *
+     * <p>
      * This way a breaking change from YouTube does not result in a broken extractor.
+     * </p>
+     *
+     * @param streamingUrl the streaming URL to decrypt with {@link YoutubeThrottlingDecrypter}
+     * @param videoId      the video ID to use when extracting JavaScript player code, if needed
      */
-    private String tryDecryptUrl(final String url, final String videoId) {
+    private String tryDecryptUrl(final String streamingUrl, final String videoId) {
         try {
-            return YoutubeThrottlingDecrypter.apply(url, videoId);
+            return YoutubeThrottlingDecrypter.apply(streamingUrl, videoId);
         } catch (final ParsingException e) {
-            return url;
+            return streamingUrl;
         }
     }
 

extractor/src/main/java/org/schabi/newpipe/extractor/utils/StringUtils.java

@@ -23,14 +23,13 @@ public static String matchToClosingParenthesis(@Nonnull final String string,
         }
 
         startIndex += start.length();
-        int endIndex = startIndex;
-        while (string.charAt(endIndex) != '{') {
-            ++endIndex;
-        }
+        int endIndex = findNextParenthesis(string, startIndex, true);
         ++endIndex;
 
         int openParenthesis = 1;
         while (openParenthesis > 0) {
+            endIndex = findNextParenthesis(string, endIndex, false);
+
             switch (string.charAt(endIndex)) {
                 case '{':
                     ++openParenthesis;
@@ -46,4 +45,47 @@ public static String matchToClosingParenthesis(@Nonnull final String string,
 
         return string.substring(startIndex, endIndex);
     }
+
+    private static int findNextParenthesis(@Nonnull final String string,
+                                           final int offset,
+                                           final boolean onlyOpen) {
+        boolean lastEscaped = false;
+        char quote = ' ';
+
+        for (int i = offset; i < string.length(); i++) {
+            boolean thisEscaped = false;
+            final char c = string.charAt(i);
+
+            switch (c) {
+                case '{':
+                    if (quote == ' ') {
+                        return i;
+                    }
+                    break;
+                case '}':
+                    if (!onlyOpen && quote == ' ') {
+                        return i;
+                    }
+                    break;
+                case '\\':
+                    if (!lastEscaped) {
+                        thisEscaped = true;
+                    }
+                    break;
+                case '\'':
+                case '"':
+                    if (!lastEscaped) {
+                        if (quote == ' ') {
+                            quote = c;
+                        } else if (quote == c) {
+                            quote = ' ';
+                        }
+                    }
+            }
+
+            lastEscaped = thisEscaped;
+        }
+
+        return -1;
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/utils/StringUtilsTest.java

@@ -58,4 +58,14 @@ public void lessClosing__success() {
 
         assertEquals(expected, substring);
     }
+
+    @Test
+    void find_closing_with_quotes() {
+        final String expected = "{return \",}\\\"/\"}";
+        final String string = "function(d){return \",}\\\"/\"}";
+
+        final String substring = matchToClosingParenthesis(string, "function(d)");
+
+        assertEquals(expected, substring);
+    }
 }