Skip to content

Commit 95f05c4

Browse files
evermind-zzevermind-zz
committed
searchfilters: convert youtube to new framework
Available content filters: Youtube - all - videos - channels - playlists Youtube Music - Songs - Videos - Albums - Playlists - Artists Available sort filters: - 'Sort by' - Upload Date - Duration - Features
1 parent fd52082 commit 95f05c4

11 files changed

+798
-116
lines changed

‎extractor/build.gradle‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
plugins {
22
id 'checkstyle'
3+
id 'com.squareup.wire' version '4.4.1'
34
}
45

56
test {
@@ -18,6 +19,15 @@ checkstyle {
1819
toolVersion checkstyleVersion
1920
}
2021

22+
checkstyleMain
23+
// exclude the wire generated youtube protobuf files
24+
.exclude ('org/schabi/newpipe/extractor/services/youtube/search/filter/protobuf/')
25+
26+
wire {
27+
java {
28+
}
29+
}
30+
2131
checkstyleTest {
2232
enabled false // do not checkstyle test files
2333
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
package org.schabi.newpipe.extractor.services.youtube;
22

3+
import org.schabi.newpipe.extractor.search.filter.FilterItem;
4+
import org.schabi.newpipe.extractor.services.youtube.search.filter.YoutubeFilters;
5+
36
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO;
47
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.COMMENTS;
58
import static org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE;
@@ -42,6 +45,7 @@
4245
import org.schabi.newpipe.extractor.stream.StreamExtractor;
4346
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
4447
import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor;
48+
import org.schabi.newpipe.extractor.utils.Utils;
4549

4650
import java.util.List;
4751

@@ -120,9 +124,10 @@ public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler)
120124

121125
@Override
122126
public SearchExtractor getSearchExtractor(final SearchQueryHandler query) {
123-
final List<String> contentFilters = query.getContentFilters();
127+
final FilterItem filterItem =
128+
Utils.getFirstContentFilterItem(query);
124129

125-
if (!contentFilters.isEmpty() && contentFilters.get(0).startsWith("music_")) {
130+
if (filterItem instanceof YoutubeFilters.MusicYoutubeContentFilterItem) {
126131
return new YoutubeMusicSearchExtractor(this, query);
127132
} else {
128133
return new YoutubeSearchExtractor(this, query);

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java‎

Lines changed: 18 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
1414
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
1515

16+
import org.schabi.newpipe.extractor.search.SearchExtractor;
17+
import org.schabi.newpipe.extractor.services.youtube.search.filter.YoutubeFilters;
1618
import com.grack.nanojson.JsonArray;
1719
import com.grack.nanojson.JsonObject;
1820
import com.grack.nanojson.JsonParser;
@@ -31,7 +33,6 @@
3133
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
3234
import org.schabi.newpipe.extractor.localization.DateWrapper;
3335
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
34-
import org.schabi.newpipe.extractor.search.SearchExtractor;
3536
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
3637
import org.schabi.newpipe.extractor.utils.JsonUtils;
3738
import org.schabi.newpipe.extractor.utils.Parser;
@@ -55,6 +56,15 @@ public YoutubeMusicSearchExtractor(final StreamingService service,
5556
super(service, linkHandler);
5657
}
5758

59+
private String getSearchType() {
60+
final YoutubeFilters.MusicYoutubeContentFilterItem contentFilterItem =
61+
Utils.getFirstContentFilterItem(getLinkHandler());
62+
if (contentFilterItem != null && contentFilterItem.getName() != null) {
63+
return contentFilterItem.getName();
64+
}
65+
return "";
66+
}
67+
5868
@Override
5969
public void onFetchPage(@Nonnull final Downloader downloader)
6070
throws IOException, ExtractionException {
@@ -63,28 +73,12 @@ public void onFetchPage(@Nonnull final Downloader downloader)
6373
final String url = "https://music.youtube.com/youtubei/v1/search?alt=json&key="
6474
+ youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
6575

66-
final String params;
67-
68-
switch (getLinkHandler().getContentFilters().get(0)) {
69-
case MUSIC_SONGS:
70-
params = "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D";
71-
break;
72-
case MUSIC_VIDEOS:
73-
params = "Eg-KAQwIABABGAAgACgAMABqChAEEAUQAxAKEAk%3D";
74-
break;
75-
case MUSIC_ALBUMS:
76-
params = "Eg-KAQwIABAAGAEgACgAMABqChAEEAUQAxAKEAk%3D";
77-
break;
78-
case MUSIC_PLAYLISTS:
79-
params = "Eg-KAQwIABAAGAAgACgBMABqChAEEAUQAxAKEAk%3D";
80-
break;
81-
case MUSIC_ARTISTS:
82-
params = "Eg-KAQwIABAAGAAgASgAMABqChAEEAUQAxAKEAk%3D";
83-
break;
84-
default:
85-
params = null;
86-
break;
87-
}
76+
77+
final YoutubeFilters.MusicYoutubeContentFilterItem contentFilterItem =
78+
Utils.getFirstContentFilterItem(getLinkHandler());
79+
// Get the search parameter for the request. If getParams() be null
80+
// (which should never happen - only in test cases), JsonWriter.string() can handle it
81+
final String params = (contentFilterItem != null) ? contentFilterItem.getParams() : null;
8882

8983
// @formatter:off
9084
final byte[] json = JsonWriter.string()
@@ -297,7 +291,7 @@ private void collectMusicStreamsFrom(final MultiInfoItemsCollector collector,
297291
.getObject("musicResponsiveListItemFlexColumnRenderer");
298292
final JsonArray descriptionElements = flexColumnRenderer.getObject("text")
299293
.getArray("runs");
300-
final String searchType = getLinkHandler().getContentFilters().get(0);
294+
final String searchType = getSearchType();
301295
if (searchType.equals(MUSIC_SONGS) || searchType.equals(MUSIC_VIDEOS)) {
302296
collector.commit(new YoutubeStreamInfoItemExtractor(info, timeAgoParser) {
303297
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,11 @@
77
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
88
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
99
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
10-
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter;
1110
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
1211
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
1312

13+
import org.schabi.newpipe.extractor.search.SearchExtractor;
14+
import org.schabi.newpipe.extractor.services.youtube.search.filter.YoutubeFilters;
1415
import com.grack.nanojson.JsonArray;
1516
import com.grack.nanojson.JsonBuilder;
1617
import com.grack.nanojson.JsonObject;
@@ -29,9 +30,9 @@
2930
import org.schabi.newpipe.extractor.localization.Localization;
3031
import org.schabi.newpipe.extractor.localization.TimeAgoParser;
3132
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
32-
import org.schabi.newpipe.extractor.search.SearchExtractor;
3333
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
3434
import org.schabi.newpipe.extractor.utils.JsonUtils;
35+
import org.schabi.newpipe.extractor.utils.Utils;
3536

3637
import java.io.IOException;
3738
import java.util.HashMap;
@@ -73,15 +74,11 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
7374
final String query = super.getSearchString();
7475
final Localization localization = getExtractorLocalization();
7576

76-
// Get the search parameter of the request
77-
final List<String> contentFilters = super.getLinkHandler().getContentFilters();
78-
final String params;
79-
if (!isNullOrEmpty(contentFilters)) {
80-
final String searchType = contentFilters.get(0);
81-
params = getSearchParameter(searchType);
82-
} else {
83-
params = "";
84-
}
77+
final YoutubeFilters.YoutubeContentFilterItem contentFilterItem =
78+
Utils.getFirstContentFilterItem(getLinkHandler());
79+
// Get the search parameter for the request. If getParams() be null
80+
// (which should never happen - only in test cases), JsonWriter.string() can handle it
81+
final String params = (contentFilterItem != null) ? contentFilterItem.getParams() : null;
8582

8683
final JsonBuilder<JsonObject> jsonBody = prepareDesktopJsonBuilder(localization,
8784
getExtractorContentCountry())

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeChannelLinkHandlerFactory.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
22

3+
import org.schabi.newpipe.extractor.search.filter.FilterItem;
4+
35
import java.util.regex.Pattern;
46
import org.schabi.newpipe.extractor.exceptions.ParsingException;
57
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
@@ -48,12 +50,13 @@ public static YoutubeChannelLinkHandlerFactory getInstance() {
4850
* Returns URL to channel from an ID
4951
*
5052
* @param id Channel ID including e.g. 'channel/'
53+
* @param searchFilter
5154
* @return URL to channel
5255
*/
5356
@Override
5457
public String getUrl(final String id,
55-
final List<String> contentFilters,
56-
final String searchFilter) {
58+
final List<FilterItem> contentFilters,
59+
final List<FilterItem> searchFilter) {
5760
return "https://www.youtube.com/" + id;
5861
}
5962

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeCommentsLinkHandlerFactory.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
22

3+
import org.schabi.newpipe.extractor.search.filter.FilterItem;
4+
35
import org.schabi.newpipe.extractor.exceptions.FoundAdException;
46
import org.schabi.newpipe.extractor.exceptions.ParsingException;
57
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory;
@@ -43,8 +45,8 @@ public boolean onAcceptUrl(final String url) throws FoundAdException {
4345

4446
@Override
4547
public String getUrl(final String id,
46-
final List<String> contentFilter,
47-
final String sortFilter) throws ParsingException {
48+
final List<FilterItem> contentFilter,
49+
final List<FilterItem> sortFilter) throws ParsingException {
4850
return getUrl(id);
4951
}
5052
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubePlaylistLinkHandlerFactory.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
22

3+
import org.schabi.newpipe.extractor.search.filter.FilterItem;
4+
35
import java.net.MalformedURLException;
46
import java.net.URL;
57
import java.util.List;
@@ -24,8 +26,8 @@ public static YoutubePlaylistLinkHandlerFactory getInstance() {
2426
}
2527

2628
@Override
27-
public String getUrl(final String id, final List<String> contentFilters,
28-
final String sortFilter) {
29+
public String getUrl(final String id, final List<FilterItem> contentFilters,
30+
final List<FilterItem> sortFilter) {
2931
return "https://www.youtube.com/playlist?list=" + id;
3032
}
3133

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java‎

Lines changed: 20 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
package org.schabi.newpipe.extractor.services.youtube.linkHandler;
22

3+
import org.schabi.newpipe.extractor.search.filter.FilterContainer;
4+
import org.schabi.newpipe.extractor.search.filter.FilterItem;
5+
import org.schabi.newpipe.extractor.services.youtube.search.filter.YoutubeFilters;
6+
37
import org.schabi.newpipe.extractor.exceptions.ParsingException;
48
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
59

610
import javax.annotation.Nonnull;
7-
import java.io.UnsupportedEncodingException;
8-
import java.net.URLEncoder;
9-
import java.util.List;
1011

11-
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
12-
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
12+
import java.util.List;
1313

1414
public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
1515

@@ -24,8 +24,7 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa
2424
public static final String MUSIC_PLAYLISTS = "music_playlists";
2525
public static final String MUSIC_ARTISTS = "music_artists";
2626

27-
private static final String SEARCH_URL = "https://www.youtube.com/results?search_query=";
28-
private static final String MUSIC_SEARCH_URL = "https://music.youtube.com/search?q=";
27+
private final YoutubeFilters searchFilters = new YoutubeFilters();
2928

3029
@Nonnull
3130
public static YoutubeSearchQueryHandlerFactory getInstance() {
@@ -34,75 +33,25 @@ public static YoutubeSearchQueryHandlerFactory getInstance() {
3433

3534
@Override
3635
public String getUrl(final String searchString,
37-
@Nonnull final List<String> contentFilters,
38-
final String sortFilter) throws ParsingException {
39-
try {
40-
if (!contentFilters.isEmpty()) {
41-
final String contentFilter = contentFilters.get(0);
42-
switch (contentFilter) {
43-
case ALL:
44-
default:
45-
break;
46-
case VIDEOS:
47-
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8)
48-
+ "&sp=EgIQAQ%253D%253D";
49-
case CHANNELS:
50-
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8)
51-
+ "&sp=EgIQAg%253D%253D";
52-
case PLAYLISTS:
53-
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8)
54-
+ "&sp=EgIQAw%253D%253D";
55-
case MUSIC_SONGS:
56-
case MUSIC_VIDEOS:
57-
case MUSIC_ALBUMS:
58-
case MUSIC_PLAYLISTS:
59-
case MUSIC_ARTISTS:
60-
return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
61-
}
62-
}
63-
64-
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
65-
} catch (final UnsupportedEncodingException e) {
66-
throw new ParsingException("Could not encode query", e);
67-
}
36+
@Nonnull final List<FilterItem> selectedContentFilter,
37+
final List<FilterItem> selectedSortFilter) throws ParsingException {
38+
searchFilters.setSelectedContentFilter(selectedContentFilter);
39+
searchFilters.setSelectedSortFilter(selectedSortFilter);
40+
return searchFilters.evaluateSelectedFilters(searchString);
6841
}
6942

7043
@Override
71-
public String[] getAvailableContentFilter() {
72-
return new String[]{
73-
ALL,
74-
VIDEOS,
75-
CHANNELS,
76-
PLAYLISTS,
77-
MUSIC_SONGS,
78-
MUSIC_VIDEOS,
79-
MUSIC_ALBUMS,
80-
MUSIC_PLAYLISTS
81-
// MUSIC_ARTISTS
82-
};
44+
public FilterContainer getAvailableContentFilter() {
45+
return searchFilters.getContentFilters();
8346
}
8447

85-
@Nonnull
86-
public static String getSearchParameter(final String contentFilter) {
87-
if (isNullOrEmpty(contentFilter)) {
88-
return "";
89-
}
48+
@Override
49+
public FilterContainer getContentFilterSortFilterVariant(final int contentFilterId) {
50+
return searchFilters.getContentFilterSortFilterVariant(contentFilterId);
51+
}
9052

91-
switch (contentFilter) {
92-
case VIDEOS:
93-
return "EgIQAQ%3D%3D";
94-
case CHANNELS:
95-
return "EgIQAg%3D%3D";
96-
case PLAYLISTS:
97-
return "EgIQAw%3D%3D";
98-
case ALL:
99-
case MUSIC_SONGS:
100-
case MUSIC_VIDEOS:
101-
case MUSIC_ALBUMS:
102-
case MUSIC_PLAYLISTS:
103-
case MUSIC_ARTISTS:
104-
default:
105-
return "";
106-
}
53+
@Override
54+
public FilterItem getFilterItem(final int filterId) {
55+
return searchFilters.getFilterItem(filterId);
10756
}
10857
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeTrendingLinkHandlerFactory.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
2121
*/
2222

23+
import org.schabi.newpipe.extractor.search.filter.FilterItem;
24+
2325
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL;
2426
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL;
2527

@@ -33,8 +35,8 @@
3335
public class YoutubeTrendingLinkHandlerFactory extends ListLinkHandlerFactory {
3436

3537
public String getUrl(final String id,
36-
final List<String> contentFilters,
37-
final String sortFilter) {
38+
final List<FilterItem> contentFilters,
39+
final List<FilterItem> sortFilter) {
3840
return "https://www.youtube.com/feed/trending";
3941
}
4042

0 commit comments

Comments
 (0)