Throw upon trying to fetch unencrypted URL

Author: fynngodau

extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java

@@ -3,6 +3,7 @@
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.localization.Localization;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -25,10 +26,10 @@ public abstract class Downloader {
      * localization. It should only be used when the resource that will be fetched won't be affected
      * by the localization.
      *
-     * @param url the URL that is pointing to the wanted resource
+     * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
      * @return the result of the GET request
      */
-    public Response get(final String url) throws IOException, ReCaptchaException {
+    public final Response get(final String url) throws IOException, ReCaptchaException {
         return get(url, null, NewPipe.getPreferredLocalization());
     }
 
@@ -37,24 +38,24 @@ public Response get(final String url) throws IOException, ReCaptchaException {
      * <br>
      * It will set the {@code Accept-Language} header to the language of the localization parameter.
      *
-     * @param url          the URL that is pointing to the wanted resource
+     * @param url          the URL that is pointing to the wanted resource (must start with HTTPS)
      * @param localization the source of the value of the {@code Accept-Language} header
      * @return the result of the GET request
      */
-    public Response get(final String url, final Localization localization)
+    public final Response get(final String url, final Localization localization)
             throws IOException, ReCaptchaException {
         return get(url, null, localization);
     }
 
     /**
      * Do a GET request with the specified headers.
      *
-     * @param url     the URL that is pointing to the wanted resource
+     * @param url     the URL that is pointing to the wanted resource (must start with HTTPS)
      * @param headers a list of headers that will be used in the request.
      *                Any default headers <b>should</b> be overridden by these.
      * @return the result of the GET request
      */
-    public Response get(final String url, @Nullable final Map<String, List<String>> headers)
+    public final Response get(final String url, @Nullable final Map<String, List<String>> headers)
             throws IOException, ReCaptchaException {
         return get(url, headers, NewPipe.getPreferredLocalization());
     }
@@ -64,44 +65,42 @@ public Response get(final String url, @Nullable final Map<String, List<String>>
      * <br>
      * It will set the {@code Accept-Language} header to the language of the localization parameter.
      *
-     * @param url          the URL that is pointing to the wanted resource
+     * @param url          the URL that is pointing to the wanted resource (must start with HTTPS)
      * @param headers      a list of headers that will be used in the request.
      *                     Any default headers <b>should</b> be overridden by these.
      * @param localization the source of the value of the {@code Accept-Language} header
      * @return the result of the GET request
      */
-    public Response get(final String url,
+    public final Response get(final String url,
                         @Nullable final Map<String, List<String>> headers,
                         final Localization localization)
             throws IOException, ReCaptchaException {
-        return execute(Request.newBuilder()
+        return executeIfHttps(Request.newBuilder()
                 .get(url)
-                .headers(headers)
-                .localization(localization)
                 .build());
     }
 
     /**
      * Do a HEAD request.
      *
-     * @param url the URL that is pointing to the wanted resource
+     * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
      * @return the result of the HEAD request
      */
-    public Response head(final String url) throws IOException, ReCaptchaException {
+    public final Response head(final String url) throws IOException, ReCaptchaException {
         return head(url, null);
     }
 
     /**
      * Do a HEAD request with the specified headers.
      *
-     * @param url     the URL that is pointing to the wanted resource
+     * @param url     the URL that is pointing to the wanted resource (must start with HTTPS)
      * @param headers a list of headers that will be used in the request.
      *                Any default headers <b>should</b> be overridden by these.
      * @return the result of the HEAD request
      */
-    public Response head(final String url, @Nullable final Map<String, List<String>> headers)
+    public final Response head(final String url, @Nullable final Map<String, List<String>> headers)
             throws IOException, ReCaptchaException {
-        return execute(Request.newBuilder()
+        return executeIfHttps(Request.newBuilder()
                 .head(url)
                 .headers(headers)
                 .build());
@@ -110,13 +109,13 @@ public Response head(final String url, @Nullable final Map<String, List<String>>
     /**
      * Do a POST request with the specified headers, sending the data array.
      *
-     * @param url        the URL that is pointing to the wanted resource
+     * @param url        the URL that is pointing to the wanted resource (must start with HTTPS)
      * @param headers    a list of headers that will be used in the request.
      *                   Any default headers <b>should</b> be overridden by these.
      * @param dataToSend byte array that will be sent when doing the request.
      * @return the result of the POST request
      */
-    public Response post(final String url,
+    public final Response post(final String url,
                          @Nullable final Map<String, List<String>> headers,
                          @Nullable final byte[] dataToSend)
             throws IOException, ReCaptchaException {
@@ -128,19 +127,19 @@ public Response post(final String url,
      * <br>
      * It will set the {@code Accept-Language} header to the language of the localization parameter.
      *
-     * @param url          the URL that is pointing to the wanted resource
+     * @param url          the URL that is pointing to the wanted resource (must start with HTTPS)
      * @param headers      a list of headers that will be used in the request.
      *                     Any default headers <b>should</b> be overridden by these.
      * @param dataToSend   byte array that will be sent when doing the request.
      * @param localization the source of the value of the {@code Accept-Language} header
      * @return the result of the POST request
      */
-    public Response post(final String url,
+    public final Response post(final String url,
                          @Nullable final Map<String, List<String>> headers,
                          @Nullable final byte[] dataToSend,
                          final Localization localization)
             throws IOException, ReCaptchaException {
-        return execute(Request.newBuilder()
+        return executeIfHttps(Request.newBuilder()
                 .post(url, dataToSend)
                 .headers(headers)
                 .localization(localization)
@@ -151,7 +150,7 @@ public Response post(final String url,
      * Convenient method to send a POST request using the specified value of the
      * {@code Content-Type} header with a given {@link Localization}.
      *
-     * @param url          the URL that is pointing to the wanted resource
+     * @param url          the URL that is pointing to the wanted resource (must start with HTTPS)
      * @param headers      a list of headers that will be used in the request.
      *                     Any default headers <b>should</b> be overridden by these.
      * @param dataToSend   byte array that will be sent when doing the request.
@@ -161,7 +160,7 @@ public Response post(final String url,
      * @return the result of the POST request
      * @see #post(String, Map, byte[], Localization)
      */
-    public Response postWithContentType(final String url,
+    public final Response postWithContentType(final String url,
                                         @Nullable final Map<String, List<String>> headers,
                                         @Nullable final byte[] dataToSend,
                                         final Localization localization,
@@ -179,7 +178,7 @@ public Response postWithContentType(final String url,
      * Convenient method to send a POST request using the specified value of the
      * {@code Content-Type} header.
      *
-     * @param url         the URL that is pointing to the wanted resource
+     * @param url         the URL that is pointing to the wanted resource (must start with HTTPS)
      * @param headers     a list of headers that will be used in the request.
      *                    Any default headers <b>should</b> be overridden by these.
      * @param dataToSend  byte array that will be sent when doing the request.
@@ -188,7 +187,7 @@ public Response postWithContentType(final String url,
      * @return the result of the POST request
      * @see #post(String, Map, byte[], Localization)
      */
-    public Response postWithContentType(final String url,
+    public final Response postWithContentType(final String url,
                                         @Nullable final Map<String, List<String>> headers,
                                         @Nullable final byte[] dataToSend,
                                         final String contentType)
@@ -201,15 +200,15 @@ public Response postWithContentType(final String url,
      * Convenient method to send a POST request the JSON mime type as the value of the
      * {@code Content-Type} header with a given {@link Localization}.
      *
-     * @param url          the URL that is pointing to the wanted resource
+     * @param url          the URL that is pointing to the wanted resource (must start with HTTPS)
      * @param headers      a list of headers that will be used in the request.
      *                     Any default headers <b>should</b> be overridden by these.
      * @param dataToSend   byte array that will be sent when doing the request.
      * @param localization the source of the value of the {@code Accept-Language} header
      * @return the result of the POST request
      * @see #post(String, Map, byte[], Localization)
      */
-    public Response postWithContentTypeJson(final String url,
+    public final Response postWithContentTypeJson(final String url,
                                             @Nullable final Map<String, List<String>> headers,
                                             @Nullable final byte[] dataToSend,
                                             final Localization localization)
@@ -221,26 +220,39 @@ public Response postWithContentTypeJson(final String url,
      * Convenient method to send a POST request the JSON mime type as the value of the
      * {@code Content-Type} header.
      *
-     * @param url         the URL that is pointing to the wanted resource
+     * @param url         the URL that is pointing to the wanted resource (must start with HTTPS)
      * @param headers     a list of headers that will be used in the request.
      *                    Any default headers <b>should</b> be overridden by these.
      * @param dataToSend  byte array that will be sent when doing the request.
      * @return the result of the POST request
      * @see #post(String, Map, byte[], Localization)
      */
-    public Response postWithContentTypeJson(final String url,
+    public final Response postWithContentTypeJson(final String url,
                                             @Nullable final Map<String, List<String>> headers,
                                             @Nullable final byte[] dataToSend)
             throws IOException, ReCaptchaException {
         return postWithContentTypeJson(url, headers, dataToSend,
                 NewPipe.getPreferredLocalization());
     }
 
+    public final Response executeIfHttps(final @Nonnull Request request)
+            throws IOException, ReCaptchaException {
+
+        if (!request.url().equals(Utils.replaceHttpWithHttps(request.url()))) {
+            throw new IOException(
+                    "All queries must be made using HTTPS. Extractors must guarantee "
+                            + "that HTTPS links are provided."
+            );
+        } else {
+            return execute(request);
+        }
+    }
+
     /**
      * Do a request using the specified {@link Request} object.
      *
      * @return the result of the request
      */
-    public abstract Response execute(@Nonnull Request request)
+    protected abstract Response execute(@Nonnull Request request)
             throws IOException, ReCaptchaException;
 }

extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java

@@ -47,7 +47,7 @@ public static DownloaderTestImpl getInstance() {
     }
 
     @Override
-    public Response execute(@Nonnull final Request request)
+    protected Response execute(@Nonnull final Request request)
             throws IOException, ReCaptchaException {
         final String httpMethod = request.httpMethod();
         final String url = request.url();

extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java

@@ -46,7 +46,7 @@ public MockDownloader(@Nonnull final String path) throws IOException {
     }
 
     @Override
-    public Response execute(@Nonnull final Request request) {
+    protected Response execute(@Nonnull final Request request) {
         final Response result = mocks.get(request);
         if (result == null) {
             throw new NullPointerException("No mock response for request with url '" + request

extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java

@@ -70,7 +70,7 @@ public RecordingDownloader(final String stringPath) throws IOException {
     public Response execute(@Nonnull final Request request) throws IOException,
             ReCaptchaException {
         final Downloader downloader = DownloaderTestImpl.getInstance();
-        Response response = downloader.execute(request);
+        Response response = downloader.executeIfHttps(request);
         String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1");
         response = new Response(
                 response.responseCode(),