Add Content-Type header to all POST requests without an empty body

extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java

@@ -7,6 +7,8 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -39,7 +41,7 @@ public Response get(final String url) throws IOException, ReCaptchaException {
      * @param localization the source of the value of the {@code Accept-Language} header
      * @return the result of the GET request
      */
-    public Response get(final String url, @Nullable final Localization localization)
+    public Response get(final String url, final Localization localization)
             throws IOException, ReCaptchaException {
         return get(url, null, localization);
     }
@@ -70,7 +72,7 @@ public Response get(final String url, @Nullable final Map<String, List<String>>
      */
     public Response get(final String url,
                         @Nullable final Map<String, List<String>> headers,
-                        @Nullable final Localization localization)
+                        final Localization localization)
             throws IOException, ReCaptchaException {
         return execute(Request.newBuilder()
                 .get(url)
@@ -112,7 +114,7 @@ public Response head(final String url, @Nullable final Map<String, List<String>>
      * @param headers    a list of headers that will be used in the request.
      *                   Any default headers <b>should</b> be overridden by these.
      * @param dataToSend byte array that will be sent when doing the request.
-     * @return the result of the GET request
+     * @return the result of the POST request
      */
     public Response post(final String url,
                          @Nullable final Map<String, List<String>> headers,
@@ -131,12 +133,12 @@ public Response post(final String url,
      *                     Any default headers <b>should</b> be overridden by these.
      * @param dataToSend   byte array that will be sent when doing the request.
      * @param localization the source of the value of the {@code Accept-Language} header
-     * @return the result of the GET request
+     * @return the result of the POST request
      */
     public Response post(final String url,
                          @Nullable final Map<String, List<String>> headers,
                          @Nullable final byte[] dataToSend,
-                         @Nullable final Localization localization)
+                         final Localization localization)
             throws IOException, ReCaptchaException {
         return execute(Request.newBuilder()
                 .post(url, dataToSend)
@@ -145,6 +147,95 @@ public Response post(final String url,
                 .build());
     }
 
+    /**
+     * Convenient method to send a POST request using the specified value of the
+     * {@code Content-Type} header with a given {@link Localization}.
+     *
+     * @param url          the URL that is pointing to the wanted resource
+     * @param headers      a list of headers that will be used in the request.
+     *                     Any default headers <b>should</b> be overridden by these.
+     * @param dataToSend   byte array that will be sent when doing the request.
+     * @param localization the source of the value of the {@code Accept-Language} header
+     * @param contentType  the mime type of the body sent, which will be set as the value of the
+     *                     {@code Content-Type} header
+     * @return the result of the POST request
+     * @see #post(String, Map, byte[], Localization)
+     */
+    public Response postWithContentType(final String url,
+                                        @Nullable final Map<String, List<String>> headers,
+                                        @Nullable final byte[] dataToSend,
+                                        final Localization localization,
+                                        final String contentType)
+            throws IOException, ReCaptchaException {
+        final Map<String, List<String>> actualHeaders = new HashMap<>();
+        if (headers != null) {
+            actualHeaders.putAll(headers);
+        }
+        actualHeaders.put("Content-Type", Collections.singletonList(contentType));
+        return post(url, actualHeaders, dataToSend, localization);
+    }
+
+    /**
+     * Convenient method to send a POST request using the specified value of the
+     * {@code Content-Type} header.
+     *
+     * @param url         the URL that is pointing to the wanted resource
+     * @param headers     a list of headers that will be used in the request.
+     *                    Any default headers <b>should</b> be overridden by these.
+     * @param dataToSend  byte array that will be sent when doing the request.
+     * @param contentType the mime type of the body sent, which will be set as the value of the
+     *                    {@code Content-Type} header
+     * @return the result of the POST request
+     * @see #post(String, Map, byte[], Localization)
+     */
+    public Response postWithContentType(final String url,
+                                        @Nullable final Map<String, List<String>> headers,
+                                        @Nullable final byte[] dataToSend,
+                                        final String contentType)
+            throws IOException, ReCaptchaException {
+        return postWithContentType(url, headers, dataToSend, NewPipe.getPreferredLocalization(),
+                contentType);
+    }
+
+    /**
+     * Convenient method to send a POST request the JSON mime type as the value of the
+     * {@code Content-Type} header with a given {@link Localization}.
+     *
+     * @param url          the URL that is pointing to the wanted resource
+     * @param headers      a list of headers that will be used in the request.
+     *                     Any default headers <b>should</b> be overridden by these.
+     * @param dataToSend   byte array that will be sent when doing the request.
+     * @param localization the source of the value of the {@code Accept-Language} header
+     * @return the result of the POST request
+     * @see #post(String, Map, byte[], Localization)
+     */
+    public Response postWithContentTypeJson(final String url,
+                                            @Nullable final Map<String, List<String>> headers,
+                                            @Nullable final byte[] dataToSend,
+                                            final Localization localization)
+            throws IOException, ReCaptchaException {
+        return postWithContentType(url, headers, dataToSend, localization, "application/json");
+    }
+
+    /**
+     * Convenient method to send a POST request the JSON mime type as the value of the
+     * {@code Content-Type} header.
+     *
+     * @param url         the URL that is pointing to the wanted resource
+     * @param headers     a list of headers that will be used in the request.
+     *                    Any default headers <b>should</b> be overridden by these.
+     * @param dataToSend  byte array that will be sent when doing the request.
+     * @return the result of the POST request
+     * @see #post(String, Map, byte[], Localization)
+     */
+    public Response postWithContentTypeJson(final String url,
+                                            @Nullable final Map<String, List<String>> headers,
+                                            @Nullable final byte[] dataToSend)
+            throws IOException, ReCaptchaException {
+        return postWithContentTypeJson(url, headers, dataToSend,
+                NewPipe.getPreferredLocalization());
+    }
+
     /**
      * Do a request using the specified {@link Request} object.
      *

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java

@@ -6,22 +6,23 @@
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 import com.grack.nanojson.JsonWriter;
-
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
+import org.schabi.newpipe.extractor.utils.Utils;
 
+import javax.annotation.Nullable;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.time.DateTimeException;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import java.util.Collections;
 import java.util.Locale;
 
-import javax.annotation.Nullable;
-
 public final class BandcampExtractorHelper {
 
     public static final String BASE_URL = "https://bandcamp.com";
@@ -43,8 +44,8 @@ public static String getStreamUrlFromIds(final long bandId,
                             + "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.charAt(0))
                     .responseBody();
 
-            return JsonParser.object().from(jsonString)
-                    .getString("bandcamp_url").replace("http://", "https://");
+            return Utils.replaceHttpWithHttps(JsonParser.object().from(jsonString)
+                    .getString("bandcamp_url"));
 
         } catch (final JsonParserException | ReCaptchaException | IOException e) {
             throw new ParsingException("Ids could not be translated to URL", e);
@@ -60,19 +61,15 @@ public static String getStreamUrlFromIds(final long bandId,
      */
     public static JsonObject getArtistDetails(final String id) throws ParsingException {
         try {
-            return
-                    JsonParser.object().from(
-                            NewPipe.getDownloader().post(
-                                    BASE_API_URL + "/mobile/22/band_details",
-                                    null,
-                                    JsonWriter.string()
-                                            .object()
-                                            .value("band_id", id)
-                                            .end()
-                                            .done()
-                                            .getBytes()
-                            ).responseBody()
-                    );
+            return JsonParser.object().from(NewPipe.getDownloader().postWithContentTypeJson(
+                    BASE_API_URL + "/mobile/22/band_details",
+                    Collections.emptyMap(),
+                    JsonWriter.string()
+                            .object()
+                            .value("band_id", id)
+                            .end()
+                            .done()
+                            .getBytes(StandardCharsets.UTF_8)).responseBody());
         } catch (final IOException | ReCaptchaException | JsonParserException e) {
             throw new ParsingException("Could not download band details", e);
         }
@@ -123,7 +120,7 @@ public static boolean isSupportedDomain(final String url) throws ParsingExceptio
     /**
      * Whether the URL points to a radio kiosk.
      * @param url the URL to check
-     * @return true if the URL matches <code>https://bandcamp.com/?show=SHOW_ID</code>
+     * @return true if the URL matches {@code https://bandcamp.com/?show=SHOW_ID}
      */
     public static boolean isRadioUrl(final String url) {
         return url.toLowerCase().matches("https?://bandcamp\\.com/\\?show=\\d+");

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java

@@ -18,6 +18,8 @@
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
 
 import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL;
 
@@ -40,11 +42,11 @@ public BandcampFeaturedExtractor(final StreamingService streamingService,
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
         try {
-            json = JsonParser.object().from(
-                    getDownloader().post(
-                            FEATURED_API_URL, null, "{\"platform\":\"\",\"version\":0}".getBytes()
-                    ).responseBody()
-            );
+            json = JsonParser.object().from(getDownloader().postWithContentTypeJson(
+                            FEATURED_API_URL,
+                            Collections.emptyMap(),
+                            "{\"platform\":\"\",\"version\":0}".getBytes(StandardCharsets.UTF_8))
+                    .responseBody());
         } catch (final JsonParserException e) {
             throw new ParsingException("Could not parse Bandcamp featured API response", e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -556,13 +556,13 @@ public static boolean areHardcodedClientVersionAndKeyValid()
 
         final Map<String, List<String>> headers = new HashMap<>();
         headers.put("X-YouTube-Client-Name", singletonList("1"));
-        headers.put("X-YouTube-Client-Version",
-                singletonList(HARDCODED_CLIENT_VERSION));
+        headers.put("X-YouTube-Client-Version", singletonList(HARDCODED_CLIENT_VERSION));
 
         // This endpoint is fetched by the YouTube website to get the items of its main menu and is
         // pretty lightweight (around 30kB)
-        final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
-                        + HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, headers, body);
+        final Response response = getDownloader().postWithContentTypeJson(
+                YOUTUBEI_V1_URL + "guide?key=" + HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER,
+                headers, body);
         final String responseBody = response.responseBody();
         final int responseCode = response.responseCode();
 
@@ -800,15 +800,12 @@ public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException,
         // @formatter:on
 
         final Map<String, List<String>> headers = new HashMap<>();
-        headers.put("X-YouTube-Client-Name", singletonList(
-                HARDCODED_YOUTUBE_MUSIC_KEY[1]));
-        headers.put("X-YouTube-Client-Version", singletonList(
-                HARDCODED_YOUTUBE_MUSIC_KEY[2]));
+        headers.put("X-YouTube-Client-Name", singletonList(HARDCODED_YOUTUBE_MUSIC_KEY[1]));
+        headers.put("X-YouTube-Client-Version", singletonList(HARDCODED_YOUTUBE_MUSIC_KEY[2]));
         headers.put("Origin", singletonList("https://music.youtube.com"));
-        headers.put("Referer", singletonList("music.youtube.com"));
-        headers.put("Content-Type", singletonList("application/json"));
+        headers.put("Referer", singletonList("https://music.youtube.com"));
 
-        final Response response = getDownloader().post(url, headers, json);
+        final Response response = getDownloader().postWithContentTypeJson(url, headers, json);
         // Ensure to have a valid response
         return response.responseBody().length() > 500 && response.responseCode() == 200;
     }
@@ -1148,13 +1145,11 @@ public static JsonObject getJsonPostResponse(final String endpoint,
                                                  final Localization localization)
             throws IOException, ExtractionException {
         final Map<String, List<String>> headers = new HashMap<>();
-        addClientInfoHeaders(headers);
-        headers.put("Content-Type", singletonList("application/json"));
-
-        final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
-                + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
+        addYouTubeHeaders(headers);
 
-        return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
+        return JsonUtils.toJsonObject(getValidJsonResponseBody(
+                getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint + "?key="
+                        + getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
     }
 
     public static JsonObject getJsonAndroidPostResponse(
@@ -1183,17 +1178,17 @@ private static JsonObject getMobilePostResponse(
             @Nonnull final String innerTubeApiKey,
             @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
         final Map<String, List<String>> headers = new HashMap<>();
-        headers.put("Content-Type", singletonList("application/json"));
         headers.put("User-Agent", singletonList(userAgent));
         headers.put("X-Goog-Api-Format-Version", singletonList("2"));
 
         final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey
                 + DISABLE_PRETTY_PRINT_PARAMETER;
 
-        final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
-                        ? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
-                headers, body, localization);
-        return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
+        return JsonUtils.toJsonObject(getValidJsonResponseBody(
+                getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest)
+                                ? baseEndpointUrl
+                                : baseEndpointUrl + endPartOfUrlRequest,
+                        headers, body, localization)));
     }
 
     @Nonnull
@@ -1396,6 +1391,16 @@ public static String getIosUserAgent(@Nullable final Localization localization)
                 + ")";
     }
 
+    @Nonnull
+    public static Map<String, List<String>> getYoutubeMusicHeaders() {
+        final Map<String, List<String>> headers = new HashMap<>();
+        headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKey[1]));
+        headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKey[2]));
+        headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
+        headers.put("Referer", Collections.singletonList("https://music.youtube.com"));
+        return headers;
+    }
+
     /**
      * Add required headers and cookies to an existing headers Map.
      * @see #addClientInfoHeaders(Map)

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java

@@ -35,7 +35,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
@@ -707,7 +706,8 @@ private static Response getStreamingWebUrlWithoutRedirects(
             throws CreationException {
         try {
             final Map<String, List<String>> headers = new HashMap<>();
-            addClientInfoHeaders(headers);
+            headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
+            headers.put("Referer", Collections.singletonList("https://www.youtube.com"));
 
             String responseMimeType = "";