Skip to content

Commit d5437e0

Browse files
AudricVAudricV
authored
Merge pull request #863 from AudricV/add-content-type-and-content-length-headers-to-post-requests
Add Content-Type header to all POST requests without an empty body
2 parents 88e07e5 + 34d79bd commit d5437e0

File tree

307 files changed

+9302
-9222
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

307 files changed

+9302
-9222
lines changed

‎extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java‎

Lines changed: 96 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
import javax.annotation.Nonnull;
88
import javax.annotation.Nullable;
99
import java.io.IOException;
10+
import java.util.Collections;
11+
import java.util.HashMap;
1012
import java.util.List;
1113
import java.util.Map;
1214

@@ -39,7 +41,7 @@ public Response get(final String url) throws IOException, ReCaptchaException {
3941
* @param localization the source of the value of the {@code Accept-Language} header
4042
* @return the result of the GET request
4143
*/
42-
public Response get(final String url, @Nullable final Localization localization)
44+
public Response get(final String url, final Localization localization)
4345
throws IOException, ReCaptchaException {
4446
return get(url, null, localization);
4547
}
@@ -70,7 +72,7 @@ public Response get(final String url, @Nullable final Map<String, List<String>>
7072
*/
7173
public Response get(final String url,
7274
@Nullable final Map<String, List<String>> headers,
73-
@Nullable final Localization localization)
75+
final Localization localization)
7476
throws IOException, ReCaptchaException {
7577
return execute(Request.newBuilder()
7678
.get(url)
@@ -112,7 +114,7 @@ public Response head(final String url, @Nullable final Map<String, List<String>>
112114
* @param headers a list of headers that will be used in the request.
113115
* Any default headers <b>should</b> be overridden by these.
114116
* @param dataToSend byte array that will be sent when doing the request.
115-
* @return the result of the GET request
117+
* @return the result of the POST request
116118
*/
117119
public Response post(final String url,
118120
@Nullable final Map<String, List<String>> headers,
@@ -131,12 +133,12 @@ public Response post(final String url,
131133
* Any default headers <b>should</b> be overridden by these.
132134
* @param dataToSend byte array that will be sent when doing the request.
133135
* @param localization the source of the value of the {@code Accept-Language} header
134-
* @return the result of the GET request
136+
* @return the result of the POST request
135137
*/
136138
public Response post(final String url,
137139
@Nullable final Map<String, List<String>> headers,
138140
@Nullable final byte[] dataToSend,
139-
@Nullable final Localization localization)
141+
final Localization localization)
140142
throws IOException, ReCaptchaException {
141143
return execute(Request.newBuilder()
142144
.post(url, dataToSend)
@@ -145,6 +147,95 @@ public Response post(final String url,
145147
.build());
146148
}
147149

150+
/**
151+
* Convenient method to send a POST request using the specified value of the
152+
* {@code Content-Type} header with a given {@link Localization}.
153+
*
154+
* @param url the URL that is pointing to the wanted resource
155+
* @param headers a list of headers that will be used in the request.
156+
* Any default headers <b>should</b> be overridden by these.
157+
* @param dataToSend byte array that will be sent when doing the request.
158+
* @param localization the source of the value of the {@code Accept-Language} header
159+
* @param contentType the mime type of the body sent, which will be set as the value of the
160+
* {@code Content-Type} header
161+
* @return the result of the POST request
162+
* @see #post(String, Map, byte[], Localization)
163+
*/
164+
public Response postWithContentType(final String url,
165+
@Nullable final Map<String, List<String>> headers,
166+
@Nullable final byte[] dataToSend,
167+
final Localization localization,
168+
final String contentType)
169+
throws IOException, ReCaptchaException {
170+
final Map<String, List<String>> actualHeaders = new HashMap<>();
171+
if (headers != null) {
172+
actualHeaders.putAll(headers);
173+
}
174+
actualHeaders.put("Content-Type", Collections.singletonList(contentType));
175+
return post(url, actualHeaders, dataToSend, localization);
176+
}
177+
178+
/**
179+
* Convenient method to send a POST request using the specified value of the
180+
* {@code Content-Type} header.
181+
*
182+
* @param url the URL that is pointing to the wanted resource
183+
* @param headers a list of headers that will be used in the request.
184+
* Any default headers <b>should</b> be overridden by these.
185+
* @param dataToSend byte array that will be sent when doing the request.
186+
* @param contentType the mime type of the body sent, which will be set as the value of the
187+
* {@code Content-Type} header
188+
* @return the result of the POST request
189+
* @see #post(String, Map, byte[], Localization)
190+
*/
191+
public Response postWithContentType(final String url,
192+
@Nullable final Map<String, List<String>> headers,
193+
@Nullable final byte[] dataToSend,
194+
final String contentType)
195+
throws IOException, ReCaptchaException {
196+
return postWithContentType(url, headers, dataToSend, NewPipe.getPreferredLocalization(),
197+
contentType);
198+
}
199+
200+
/**
201+
* Convenient method to send a POST request the JSON mime type as the value of the
202+
* {@code Content-Type} header with a given {@link Localization}.
203+
*
204+
* @param url the URL that is pointing to the wanted resource
205+
* @param headers a list of headers that will be used in the request.
206+
* Any default headers <b>should</b> be overridden by these.
207+
* @param dataToSend byte array that will be sent when doing the request.
208+
* @param localization the source of the value of the {@code Accept-Language} header
209+
* @return the result of the POST request
210+
* @see #post(String, Map, byte[], Localization)
211+
*/
212+
public Response postWithContentTypeJson(final String url,
213+
@Nullable final Map<String, List<String>> headers,
214+
@Nullable final byte[] dataToSend,
215+
final Localization localization)
216+
throws IOException, ReCaptchaException {
217+
return postWithContentType(url, headers, dataToSend, localization, "application/json");
218+
}
219+
220+
/**
221+
* Convenient method to send a POST request the JSON mime type as the value of the
222+
* {@code Content-Type} header.
223+
*
224+
* @param url the URL that is pointing to the wanted resource
225+
* @param headers a list of headers that will be used in the request.
226+
* Any default headers <b>should</b> be overridden by these.
227+
* @param dataToSend byte array that will be sent when doing the request.
228+
* @return the result of the POST request
229+
* @see #post(String, Map, byte[], Localization)
230+
*/
231+
public Response postWithContentTypeJson(final String url,
232+
@Nullable final Map<String, List<String>> headers,
233+
@Nullable final byte[] dataToSend)
234+
throws IOException, ReCaptchaException {
235+
return postWithContentTypeJson(url, headers, dataToSend,
236+
NewPipe.getPreferredLocalization());
237+
}
238+
148239
/**
149240
* Do a request using the specified {@link Request} object.
150241
*

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java‎

Lines changed: 16 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,23 @@
66
import com.grack.nanojson.JsonParser;
77
import com.grack.nanojson.JsonParserException;
88
import com.grack.nanojson.JsonWriter;
9-
109
import org.jsoup.Jsoup;
1110
import org.jsoup.nodes.Element;
1211
import org.schabi.newpipe.extractor.NewPipe;
1312
import org.schabi.newpipe.extractor.exceptions.ParsingException;
1413
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
1514
import org.schabi.newpipe.extractor.localization.DateWrapper;
15+
import org.schabi.newpipe.extractor.utils.Utils;
1616

17+
import javax.annotation.Nullable;
1718
import java.io.IOException;
19+
import java.nio.charset.StandardCharsets;
1820
import java.time.DateTimeException;
1921
import java.time.ZonedDateTime;
2022
import java.time.format.DateTimeFormatter;
23+
import java.util.Collections;
2124
import java.util.Locale;
2225

23-
import javax.annotation.Nullable;
24-
2526
public final class BandcampExtractorHelper {
2627

2728
public static final String BASE_URL = "https://bandcamp.com";
@@ -43,8 +44,8 @@ public static String getStreamUrlFromIds(final long bandId,
4344
+ "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.charAt(0))
4445
.responseBody();
4546

46-
return JsonParser.object().from(jsonString)
47-
.getString("bandcamp_url").replace("http://", "https://");
47+
return Utils.replaceHttpWithHttps(JsonParser.object().from(jsonString)
48+
.getString("bandcamp_url"));
4849

4950
} catch (final JsonParserException | ReCaptchaException | IOException e) {
5051
throw new ParsingException("Ids could not be translated to URL", e);
@@ -60,19 +61,15 @@ public static String getStreamUrlFromIds(final long bandId,
6061
*/
6162
public static JsonObject getArtistDetails(final String id) throws ParsingException {
6263
try {
63-
return
64-
JsonParser.object().from(
65-
NewPipe.getDownloader().post(
66-
BASE_API_URL + "/mobile/22/band_details",
67-
null,
68-
JsonWriter.string()
69-
.object()
70-
.value("band_id", id)
71-
.end()
72-
.done()
73-
.getBytes()
74-
).responseBody()
75-
);
64+
return JsonParser.object().from(NewPipe.getDownloader().postWithContentTypeJson(
65+
BASE_API_URL + "/mobile/22/band_details",
66+
Collections.emptyMap(),
67+
JsonWriter.string()
68+
.object()
69+
.value("band_id", id)
70+
.end()
71+
.done()
72+
.getBytes(StandardCharsets.UTF_8)).responseBody());
7673
} catch (final IOException | ReCaptchaException | JsonParserException e) {
7774
throw new ParsingException("Could not download band details", e);
7875
}
@@ -123,7 +120,7 @@ public static boolean isSupportedDomain(final String url) throws ParsingExceptio
123120
/**
124121
* Whether the URL points to a radio kiosk.
125122
* @param url the URL to check
126-
* @return true if the URL matches <code>https://bandcamp.com/?show=SHOW_ID</code>
123+
* @return true if the URL matches {@code https://bandcamp.com/?show=SHOW_ID}
127124
*/
128125
public static boolean isRadioUrl(final String url) {
129126
return url.toLowerCase().matches("https?://bandcamp\\.com/\\?show=\\d+");

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
import javax.annotation.Nonnull;
2020
import java.io.IOException;
21+
import java.nio.charset.StandardCharsets;
22+
import java.util.Collections;
2123

2224
import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL;
2325

@@ -40,11 +42,11 @@ public BandcampFeaturedExtractor(final StreamingService streamingService,
4042
public void onFetchPage(@Nonnull final Downloader downloader)
4143
throws IOException, ExtractionException {
4244
try {
43-
json = JsonParser.object().from(
44-
getDownloader().post(
45-
FEATURED_API_URL, null, "{\"platform\":\"\",\"version\":0}".getBytes()
46-
).responseBody()
47-
);
45+
json = JsonParser.object().from(getDownloader().postWithContentTypeJson(
46+
FEATURED_API_URL,
47+
Collections.emptyMap(),
48+
"{\"platform\":\"\",\"version\":0}".getBytes(StandardCharsets.UTF_8))
49+
.responseBody());
4850
} catch (final JsonParserException e) {
4951
throw new ParsingException("Could not parse Bandcamp featured API response", e);
5052
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 27 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -556,13 +556,13 @@ public static boolean areHardcodedClientVersionAndKeyValid()
556556

557557
final Map<String, List<String>> headers = new HashMap<>();
558558
headers.put("X-YouTube-Client-Name", singletonList("1"));
559-
headers.put("X-YouTube-Client-Version",
560-
singletonList(HARDCODED_CLIENT_VERSION));
559+
headers.put("X-YouTube-Client-Version", singletonList(HARDCODED_CLIENT_VERSION));
561560

562561
// This endpoint is fetched by the YouTube website to get the items of its main menu and is
563562
// pretty lightweight (around 30kB)
564-
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
565-
+ HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, headers, body);
563+
final Response response = getDownloader().postWithContentTypeJson(
564+
YOUTUBEI_V1_URL + "guide?key=" + HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER,
565+
headers, body);
566566
final String responseBody = response.responseBody();
567567
final int responseCode = response.responseCode();
568568

@@ -800,15 +800,12 @@ public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException,
800800
// @formatter:on
801801

802802
final Map<String, List<String>> headers = new HashMap<>();
803-
headers.put("X-YouTube-Client-Name", singletonList(
804-
HARDCODED_YOUTUBE_MUSIC_KEY[1]));
805-
headers.put("X-YouTube-Client-Version", singletonList(
806-
HARDCODED_YOUTUBE_MUSIC_KEY[2]));
803+
headers.put("X-YouTube-Client-Name", singletonList(HARDCODED_YOUTUBE_MUSIC_KEY[1]));
804+
headers.put("X-YouTube-Client-Version", singletonList(HARDCODED_YOUTUBE_MUSIC_KEY[2]));
807805
headers.put("Origin", singletonList("https://music.youtube.com"));
808-
headers.put("Referer", singletonList("music.youtube.com"));
809-
headers.put("Content-Type", singletonList("application/json"));
806+
headers.put("Referer", singletonList("https://music.youtube.com"));
810807

811-
final Response response = getDownloader().post(url, headers, json);
808+
final Response response = getDownloader().postWithContentTypeJson(url, headers, json);
812809
// Ensure to have a valid response
813810
return response.responseBody().length() > 500 && response.responseCode() == 200;
814811
}
@@ -1180,13 +1177,11 @@ public static JsonObject getJsonPostResponse(final String endpoint,
11801177
final Localization localization)
11811178
throws IOException, ExtractionException {
11821179
final Map<String, List<String>> headers = new HashMap<>();
1183-
addClientInfoHeaders(headers);
1184-
headers.put("Content-Type", singletonList("application/json"));
1185-
1186-
final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
1187-
+ getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
1180+
addYouTubeHeaders(headers);
11881181

1189-
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
1182+
return JsonUtils.toJsonObject(getValidJsonResponseBody(
1183+
getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint + "?key="
1184+
+ getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
11901185
}
11911186

11921187
public static JsonObject getJsonAndroidPostResponse(
@@ -1215,17 +1210,17 @@ private static JsonObject getMobilePostResponse(
12151210
@Nonnull final String innerTubeApiKey,
12161211
@Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
12171212
final Map<String, List<String>> headers = new HashMap<>();
1218-
headers.put("Content-Type", singletonList("application/json"));
12191213
headers.put("User-Agent", singletonList(userAgent));
12201214
headers.put("X-Goog-Api-Format-Version", singletonList("2"));
12211215

12221216
final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey
12231217
+ DISABLE_PRETTY_PRINT_PARAMETER;
12241218

1225-
final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
1226-
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
1227-
headers, body, localization);
1228-
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
1219+
return JsonUtils.toJsonObject(getValidJsonResponseBody(
1220+
getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest)
1221+
? baseEndpointUrl
1222+
: baseEndpointUrl + endPartOfUrlRequest,
1223+
headers, body, localization)));
12291224
}
12301225

12311226
@Nonnull
@@ -1428,6 +1423,16 @@ public static String getIosUserAgent(@Nullable final Localization localization)
14281423
+ ")";
14291424
}
14301425

1426+
@Nonnull
1427+
public static Map<String, List<String>> getYoutubeMusicHeaders() {
1428+
final Map<String, List<String>> headers = new HashMap<>();
1429+
headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKey[1]));
1430+
headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKey[2]));
1431+
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
1432+
headers.put("Referer", Collections.singletonList("https://music.youtube.com"));
1433+
return headers;
1434+
}
1435+
14311436
/**
14321437
* Add required headers and cookies to an existing headers Map.
14331438
* @see #addClientInfoHeaders(Map)

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import java.util.Map;
3636
import java.util.Objects;
3737

38-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
3938
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
4039
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
4140
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
@@ -707,7 +706,8 @@ private static Response getStreamingWebUrlWithoutRedirects(
707706
throws CreationException {
708707
try {
709708
final Map<String, List<String>> headers = new HashMap<>();
710-
addClientInfoHeaders(headers);
709+
headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
710+
headers.put("Referer", Collections.singletonList("https://www.youtube.com"));
711711

712712
String responseMimeType = "";
713713

0 commit comments

Comments
 (0)