Skip to content

Commit 59b620c

Browse files
TobiGrTobiGr
authored
Merge pull request #1397 from FineFindus/fix/multiple-uploaders-url
[YouTube] Extract uploader URL for videos with multiple uploaders
2 parents 0affe14 + d051342 commit 59b620c

File tree

14 files changed

+3131
-13
lines changed

14 files changed

+3131
-13
lines changed

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -759,6 +759,22 @@ public static String getUrlFromNavigationEndpoint(
759759
.getString("playlistId");
760760
}
761761

762+
if (navigationEndpoint.has("showDialogCommand")) {
763+
try {
764+
final JsonArray listItems = JsonUtils.getArray(navigationEndpoint,
765+
"showDialogCommand.panelLoadingStrategy.inlineContent.dialogViewModel"
766+
+ ".customContent.listViewModel.listItems");
767+
768+
// the first item seems to always be the channel that actually uploaded the video,
769+
// i.e. it appears in their video feed
770+
final JsonObject command = JsonUtils.getObject(listItems.getObject(0),
771+
"listItemViewModel.rendererContext.commandContext.onTap.innertubeCommand");
772+
return getUrlFromNavigationEndpoint(command);
773+
} catch (final ParsingException p) {
774+
}
775+
}
776+
777+
762778
if (navigationEndpoint.has("commandMetadata")) {
763779
final JsonObject metadata = navigationEndpoint.getObject("commandMetadata")
764780
.getObject("webCommandMetadata");
@@ -1572,4 +1588,24 @@ public static JsonBuilder<JsonObject> prepareJsonBuilder(
15721588

15731589
return builder;
15741590
}
1591+
1592+
/**
1593+
* Gets the first collaborator, which is the channel that owns the video,
1594+
* i.e. the video is displayed on their channel page.
1595+
*
1596+
* @param navigationEndpoint JSON object for the navigationEndpoint
1597+
* @return The first collaborator in the JSON object or {@code null}
1598+
*/
1599+
@Nullable
1600+
public static JsonObject getFirstCollaborator(final JsonObject navigationEndpoint)
1601+
throws ParsingException {
1602+
try {
1603+
// CHECKSTYLE:OFF
1604+
final JsonArray listItems = JsonUtils.getArray(navigationEndpoint, "showDialogCommand.panelLoadingStrategy.inlineContent.dialogViewModel.customContent.listViewModel.listItems");
1605+
// CHECKSTYLE:ON
1606+
return listItems.getObject(0).getObject("listItemViewModel");
1607+
} catch (final ParsingException e) {
1608+
return null;
1609+
}
1610+
}
15751611
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 49 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -541,23 +541,47 @@ public String getUploaderName() throws ParsingException {
541541

542542
@Override
543543
public boolean isUploaderVerified() throws ParsingException {
544-
return YoutubeParsingHelper.isVerified(
545-
getVideoSecondaryInfoRenderer()
544+
final JsonObject videoOwnerRenderer = getVideoSecondaryInfoRenderer()
546545
.getObject("owner")
547-
.getObject("videoOwnerRenderer")
548-
.getArray("badges"));
546+
.getObject("videoOwnerRenderer");
547+
548+
if (videoOwnerRenderer.has("badges")) {
549+
return YoutubeParsingHelper.isVerified(videoOwnerRenderer
550+
.getArray("badges"));
551+
}
552+
553+
554+
final JsonObject channel = YoutubeParsingHelper.getFirstCollaborator(
555+
videoOwnerRenderer.getObject("navigationEndpoint"));
556+
if (channel == null) {
557+
return false;
558+
}
559+
560+
return YoutubeParsingHelper.hasArtistOrVerifiedIconBadgeAttachment(
561+
channel.getObject("title").getArray("attachmentRuns"));
549562
}
550563

551564
@Nonnull
552565
@Override
553566
public List<Image> getUploaderAvatars() throws ParsingException {
554567
assertPageFetched();
555-
556-
final List<Image> imageList = getImagesFromThumbnailsArray(
557-
getVideoSecondaryInfoRenderer().getObject("owner")
558-
.getObject("videoOwnerRenderer")
559-
.getObject("thumbnail")
560-
.getArray("thumbnails"));
568+
final JsonObject owner = getVideoSecondaryInfoRenderer().getObject("owner")
569+
.getObject("videoOwnerRenderer");
570+
571+
final List<Image> imageList;
572+
if (owner.has("avatarStack")) {
573+
imageList = getImagesFromThumbnailsArray(
574+
owner.getObject("avatarStack").getObject("avatarStackViewModel")
575+
.getArray("avatars")
576+
// only consider the first collaborator, which is the video owner
577+
.getObject(0)
578+
.getObject("avatarViewModel")
579+
.getObject("image")
580+
.getArray("sources"));
581+
} else {
582+
imageList = getImagesFromThumbnailsArray(
583+
owner.getObject("thumbnail").getArray("thumbnails"));
584+
}
561585

562586
if (imageList.isEmpty() && ageLimit == NO_AGE_LIMIT) {
563587
throw new ParsingException("Could not get uploader avatars");
@@ -570,12 +594,24 @@ public List<Image> getUploaderAvatars() throws ParsingException {
570594
public long getUploaderSubscriberCount() throws ParsingException {
571595
final JsonObject videoOwnerRenderer = JsonUtils.getObject(videoSecondaryInfoRenderer,
572596
"owner.videoOwnerRenderer");
573-
if (!videoOwnerRenderer.has("subscriberCountText")) {
597+
598+
String subscriberCountText = null;
599+
if (videoOwnerRenderer.has("subscriberCountText")) {
600+
subscriberCountText = getTextFromObject(videoOwnerRenderer
601+
.getObject("subscriberCountText"));
602+
} else {
603+
final String content = YoutubeParsingHelper.getFirstCollaborator(
604+
videoOwnerRenderer.getObject("navigationEndpoint")
605+
).getObject("subtitle").getString("content");
606+
subscriberCountText = content.split("•")[1];
607+
}
608+
609+
if (isNullOrEmpty(subscriberCountText)) {
574610
return UNKNOWN_SUBSCRIBER_COUNT;
575611
}
612+
576613
try {
577-
return Utils.mixedNumberWordToLong(getTextFromObject(videoOwnerRenderer
578-
.getObject("subscriberCountText")));
614+
return Utils.mixedNumberWordToLong(subscriberCountText);
579615
} catch (final NumberFormatException e) {
580616
throw new ParsingException("Could not get uploader subscriber count", e);
581617
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,56 @@ public void testSearchSuggestion() throws Exception {
335335
}
336336
}
337337

338+
public static class MultipleUploader extends DefaultSearchExtractorTest implements InitYoutubeTest {
339+
private static final String QUERY = "Nxk6aRHi664";
340+
341+
@Override
342+
protected SearchExtractor createExtractor() throws Exception {
343+
return YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
344+
}
345+
346+
@Override public StreamingService expectedService() { return YouTube; }
347+
@Override public String expectedName() { return QUERY; }
348+
@Override public String expectedId() { return QUERY; }
349+
@Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
350+
@Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
351+
@Override public String expectedSearchString() { return QUERY; }
352+
@Nullable @Override public String expectedSearchSuggestion() { return null; }
353+
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
354+
355+
@Test
356+
void testUploaderName() throws IOException, ExtractionException {
357+
final List<InfoItem> items = extractor().getInitialPage().getItems();
358+
assertEquals("Le Vortex - ARTE and Thomas Gauthier",
359+
((StreamInfoItem) items.get(0)).getUploaderName());
360+
}
361+
362+
@Test
363+
void testUploaderUrl() throws IOException, ExtractionException {
364+
final List<InfoItem> items = extractor().getInitialPage().getItems();
365+
assertEquals("https://www.youtube.com/channel/UCZxLew-WXWm5dhRZBgEFl-Q",
366+
((StreamInfoItem) items.get(0)).getUploaderUrl());
367+
}
368+
@Test
369+
void testUploaderAvatars() throws IOException, ExtractionException {
370+
final List<InfoItem> items = extractor().getInitialPage().getItems();
371+
assertNotNull(((StreamInfoItem) items.get(0)).getUploaderAvatars());
372+
}
373+
374+
@Disabled("Irrelevant - sometimes suggestions show up, sometimes not")
375+
@Override
376+
public void testSearchSuggestion() throws Exception {
377+
super.testSearchSuggestion();
378+
}
379+
380+
@Disabled("Irrelevant - sometimes suggestions show up, sometimes not")
381+
@Override
382+
public void testMoreRelatedItems() throws Exception {
383+
super.testMoreRelatedItems();
384+
}
385+
}
386+
387+
338388
public static class ShortFormContent extends DefaultSearchExtractorTest implements InitYoutubeTest {
339389
private static final String QUERY = "#shorts";
340390

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorCollaboratorsTest.java‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package org.schabi.newpipe.extractor.services.youtube.stream;
2+
3+
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
4+
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertNotEmpty;
5+
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestImageCollection;
6+
7+
import org.junit.jupiter.api.Disabled;
8+
import org.schabi.newpipe.extractor.Image;
9+
import org.schabi.newpipe.extractor.StreamingService;
10+
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
11+
import org.schabi.newpipe.extractor.services.youtube.InitYoutubeTest;
12+
import org.schabi.newpipe.extractor.stream.StreamExtractor;
13+
import org.schabi.newpipe.extractor.stream.StreamType;
14+
15+
import java.util.Collections;
16+
import java.util.List;
17+
18+
import javax.annotation.Nullable;
19+
20+
public class YoutubeStreamExtractorCollaboratorsTest extends DefaultStreamExtractorTest
21+
implements InitYoutubeTest {
22+
private static final String ID = "3sbYbckT1VY";
23+
private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID;
24+
25+
@Override
26+
protected StreamExtractor createExtractor() throws Exception {
27+
return YouTube.getStreamExtractor(URL);
28+
}
29+
30+
@Override public StreamingService expectedService() { return YouTube; }
31+
@Override public String expectedName() { return "Engineers vs Pumpkin Carving 2.0"; }
32+
@Override public String expectedId() { return ID; }
33+
@Override public String expectedUrlContains() { return YoutubeStreamExtractorDefaultTest.BASE_URL + ID; }
34+
@Override public String expectedOriginalUrlContains() { return URL; }
35+
36+
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
37+
@Override public String expectedUploaderName() { return "CrunchLabs"; }
38+
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UC513PdAP2-jWkJunTh5kXRw"; }
39+
@Override public long expectedUploaderSubscriberCountAtLeast() { return 227_0000; }
40+
@Override public boolean expectedUploaderVerified() { return true; }
41+
@Override public boolean expectedDescriptionIsEmpty() { return false; }
42+
@Override public List<String> expectedDescriptionContains() { return Collections.emptyList(); }
43+
@Override public long expectedLength() { return 696; }
44+
@Override public long expectedViewCountAtLeast() { return 1_400_000; }
45+
@Nullable @Override public String expectedUploadDate() { return "2025-10-25 15:33:05.000"; }
46+
@Nullable @Override public String expectedTextualUploadDate() { return "2025-10-25T08:33:05-07:00"; }
47+
@Override public long expectedLikeCountAtLeast() { return 20_000; }
48+
@Override public long expectedDislikeCountAtLeast() { return -1; }
49+
@Override public boolean expectedHasSubtitles() { return true; }
50+
@Override public boolean expectedHasFrames() { return true; }
51+
52+
@Override public String expectedCategory() { return "Science & Technology"; }
53+
54+
@Override public String expectedLicence() { return "YouTube licence"; }
55+
@Override
56+
public List<String> expectedTags() {
57+
return Collections.emptyList();
58+
}
59+
60+
@Override
61+
public void testUploaderAvatars() throws Exception {
62+
List<Image> avatars = extractor().getUploaderAvatars();
63+
assertNotEmpty(avatars);
64+
defaultTestImageCollection(avatars);
65+
}
66+
}

‎extractor/src/test/resources/mocks/v1/org/schabi/newpipe/extractor/services/youtube/search/youtubesearchextractor/multipleuploader/generated_mock_0.json‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
{
2+
"request": {
3+
"httpMethod": "GET",
4+
"url": "https://www.youtube.com/sw.js",
5+
"headers": {
6+
"Referer": [
7+
"https://www.youtube.com"
8+
],
9+
"Origin": [
10+
"https://www.youtube.com"
11+
],
12+
"Accept-Language": [
13+
"en-GB, en;q\u003d0.9"
14+
]
15+
},
16+
"localization": {
17+
"languageCode": "en",
18+
"countryCode": "GB"
19+
}
20+
},
21+
"response": {
22+
"responseCode": 200,
23+
"responseMessage": "",
24+
"responseHeaders": {
25+
"access-control-allow-credentials": [
26+
"true"
27+
],
28+
"access-control-allow-origin": [
29+
"https://www.youtube.com"
30+
],
31+
"alt-svc": [
32+
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
33+
],
34+
"cache-control": [
35+
"private, max-age\u003d0"
36+
],
37+
"content-security-policy": [
38+
"require-trusted-types-for \u0027script\u0027"
39+
],
40+
"content-security-policy-report-only": [
41+
"script-src \u0027unsafe-eval\u0027 \u0027self\u0027 \u0027unsafe-inline\u0027 https://www.google.com https://apis.google.com https://ssl.gstatic.com https://www.gstatic.com https://www.googletagmanager.com https://www.google-analytics.com https://*.youtube.com https://*.google.com https://*.gstatic.com https://youtube.com https://www.youtube.com https://google.com https://*.doubleclick.net https://*.googleapis.com https://www.googleadservices.com https://tpc.googlesyndication.com https://www.youtubekids.com https://www.youtube-nocookie.com https://www.youtubeeducation.com https://www-onepick-opensocial.googleusercontent.com;report-uri /cspreport/allowlist"
42+
],
43+
"content-type": [
44+
"text/javascript; charset\u003dutf-8"
45+
],
46+
"cross-origin-opener-policy": [
47+
"same-origin; report-to\u003d\"youtube_main\""
48+
],
49+
"date": [
50+
"Fri, 31 Oct 2025 15:26:13 GMT"
51+
],
52+
"document-policy": [
53+
"include-js-call-stacks-in-crash-reports"
54+
],
55+
"expires": [
56+
"Fri, 31 Oct 2025 15:26:13 GMT"
57+
],
58+
"origin-trial": [
59+
"AmhMBR6zCLzDDxpW+HfpP67BqwIknWnyMOXOQGfzYswFmJe+fgaI6XZgAzcxOrzNtP7hEDsOo1jdjFnVr2IdxQ4AAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTc1ODA2NzE5OSwiaXNTdWJkb21haW4iOnRydWV9",
60+
"AiDEBptUfVeO93q48VdVMe/ubupazdAl8AaHP+NBzdnW8quUcHdzJUyGSfrmtpKJu7EOvwRp9ug2rEo3XU+WMAMAAAB2eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJEZXZpY2VCb3VuZFNlc3Npb25DcmVkZW50aWFsczIiLCJleHBpcnkiOjE3NzQzMTA0MDAsImlzU3ViZG9tYWluIjp0cnVlfQ\u003d\u003d"
61+
],
62+
"p3p": [
63+
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
64+
],
65+
"permissions-policy": [
66+
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-form-factors\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
67+
],
68+
"report-to": [
69+
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
70+
],
71+
"reporting-endpoints": [
72+
"default\u003d\"/web-reports?context\u003deJwNzmtsjGkYxnFv_3Z0Z6bzzvvcjxQlbLU2sabaJiWOH0irS4Zu4jw2bTEtQquzM0MtiU1JEMdYp9o6bHZV1CllJXRtgw_OuqF0lwqpD9IQrETVsft8-H24k_u6crmbvyjuWmKl5ZdZrm-XWidXVVhzvo5YuU0_WBvzolbHoah1rCFqPR8Qt0KpcWv33bi1-NFya59_ZsLyPjMTGma7KSlx82aPm_9a3VS-dBP6xsOdiR6eT_ZwptZDQ72Hh089xEJeXhV5yYl5Ke_w0l6QxPk9SQyZ6uPwWR8Fl31sM1588pGWZbNqhM2wMTbHJ9jEwzaeuE31epusXTaJB22Sr9msb7M5n-YnsciP27i910-PE35a3_oZ9d7P9-JQlOPwyzSHh4UOwVKHlSsd1hiT1jqM2-wwdqdD4xGHAXUOKccdpNmhb7tDW6eD70tF_lDFipGK-3mKromKYEjRK66wVivUTsWCWsUiY4VRZVxsVIy5pUhpUoy-rfjqriK9RVH_RJHcYbKdig2fFS0uodkjDO0pBPsYKUJdf0FShfx0Yb5RmyEEcoR_RwkV44XioBCbInQae2cL40PCwLDJlQrucmG78ahCWBsxd5VQsk6YtUPI2CX8US2sqRF8B4Sag8KWX4V-v4vZLVyoExKOCleNP08Jj88KReeEk5eE6BVh_1Xzc134-4bw3U3T3SxYLULeP0KlkfzAbDbuGaWtQrfHJtMmbHoqpD8Tfn4tHO0QkjqFd0buO-Evo91oei98_GA2fhQiXcL07hq8mp98mqijuag1h3trXAM1wUGaQ4M104do6odrto3VVOVqTk_WVM_QvC3U_JjSzaW8iTuObL_i8m9t3PSblRqoLI9FY3PDGcvCcwMlkfKyaCBcNj8wL7IwunBe8eLC7MzsnKzM7BEZWZmFSzP_B5923l0\""
73+
],
74+
"server": [
75+
"ESF"
76+
],
77+
"set-cookie": [
78+
"YSC\u003dEPX_t6VvDa8; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
79+
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSat, 04-Feb-2023 15:26:13 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
80+
],
81+
"strict-transport-security": [
82+
"max-age\u003d31536000"
83+
],
84+
"x-content-type-options": [
85+
"nosniff"
86+
],
87+
"x-frame-options": [
88+
"SAMEORIGIN"
89+
],
90+
"x-xss-protection": [
91+
"0"
92+
]
93+
},
94+
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
95+
"latestUrl": "https://www.youtube.com/sw.js"
96+
}
97+
}

0 commit comments

Comments
 (0)