Skip to content

Commit 5a18730

Browse files
litetexlitetex
authored
Merge pull request #813 from litetex/fix-test-2022-03
Fixed tests (+ Youtube shorts in channels)
2 parents bb49f7d + af82edf commit 5a18730

35 files changed

+856
-558
lines changed

‎extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,16 +41,12 @@ private NewPipe() {
4141
}
4242

4343
public static void init(final Downloader d) {
44-
downloader = d;
45-
preferredLocalization = Localization.DEFAULT;
46-
preferredContentCountry = ContentCountry.DEFAULT;
44+
init(d, Localization.DEFAULT);
4745
}
4846

4947
public static void init(final Downloader d, final Localization l) {
50-
downloader = d;
51-
preferredLocalization = l;
52-
preferredContentCountry = l.getCountryCode().isEmpty()
53-
? ContentCountry.DEFAULT : new ContentCountry(l.getCountryCode());
48+
init(d, l, l.getCountryCode().isEmpty()
49+
? ContentCountry.DEFAULT : new ContentCountry(l.getCountryCode()));
5450
}
5551

5652
public static void init(final Downloader d, final Localization l, final ContentCountry c) {

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKiosk.java‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.grack.nanojson.JsonObject;
55
import com.grack.nanojson.JsonParser;
66
import com.grack.nanojson.JsonParserException;
7+
78
import org.schabi.newpipe.extractor.Page;
89
import org.schabi.newpipe.extractor.StreamingService;
910
import org.schabi.newpipe.extractor.downloader.Downloader;
@@ -14,10 +15,11 @@
1415
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
1516
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
1617

17-
import javax.annotation.Nonnull;
1818
import java.io.IOException;
1919
import java.util.Comparator;
2020

21+
import javax.annotation.Nonnull;
22+
2123
public class MediaCCCRecentKiosk extends KioskExtractor<StreamInfoItem> {
2224

2325
private JsonObject doc;
@@ -51,11 +53,17 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
5153
streamInfoItem -> streamInfoItem.getUploadDate().offsetDateTime());
5254
comparator = comparator.reversed();
5355

54-
final StreamInfoItemsCollector collector
55-
= new StreamInfoItemsCollector(getServiceId(), comparator);
56-
for (int i = 0; i < events.size(); i++) {
57-
collector.commit(new MediaCCCRecentKioskExtractor(events.getObject(i)));
58-
}
56+
final StreamInfoItemsCollector collector =
57+
new StreamInfoItemsCollector(getServiceId(), comparator);
58+
59+
events.stream()
60+
.filter(JsonObject.class::isInstance)
61+
.map(JsonObject.class::cast)
62+
.map(MediaCCCRecentKioskExtractor::new)
63+
// #813 / voc/voctoweb#609 -> returns faulty data -> filter it out
64+
.filter(extractor -> extractor.getDuration() > 0)
65+
.forEach(collector::commit);
66+
5967
return new InfoItemsPage<>(collector, null);
6068
}
6169

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKioskExtractor.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
11
package org.schabi.newpipe.extractor.services.media_ccc.extractors;
22

33
import com.grack.nanojson.JsonObject;
4+
45
import org.schabi.newpipe.extractor.exceptions.ParsingException;
56
import org.schabi.newpipe.extractor.localization.DateWrapper;
67
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
78
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
89
import org.schabi.newpipe.extractor.stream.StreamType;
910

10-
import javax.annotation.Nullable;
1111
import java.time.ZonedDateTime;
1212
import java.time.format.DateTimeFormatter;
1313

14+
import javax.annotation.Nullable;
15+
1416
public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor {
1517

1618
private final JsonObject event;
@@ -45,7 +47,7 @@ public boolean isAd() {
4547
}
4648

4749
@Override
48-
public long getDuration() throws ParsingException {
50+
public long getDuration() {
4951
// duration and length have the same value, see
5052
// https://github.com/voc/voctoweb/blob/master/app/views/public/shared/_event.json.jbuilder
5153
return event.getInt("duration");

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -219,10 +219,34 @@ public static int parseDurationString(@Nonnull final String input)
219219
throw new ParsingException("Error duration string with unknown format: " + input);
220220
}
221221

222-
return ((Integer.parseInt(Utils.removeNonDigitCharacters(days)) * 24
223-
+ Integer.parseInt(Utils.removeNonDigitCharacters(hours))) * 60
224-
+ Integer.parseInt(Utils.removeNonDigitCharacters(minutes))) * 60
225-
+ Integer.parseInt(Utils.removeNonDigitCharacters(seconds));
222+
return ((convertDurationToInt(days) * 24
223+
+ convertDurationToInt(hours)) * 60
224+
+ convertDurationToInt(minutes)) * 60
225+
+ convertDurationToInt(seconds);
226+
}
227+
228+
/**
229+
* Tries to convert a duration string to an integer without throwing an exception.
230+
* <br/>
231+
* Helper method for {@link #parseDurationString(String)}.
232+
* <br/>
233+
* Note: This method is also used as a workaround for NewPipe#8034 (YT shorts no longer
234+
* display any duration in channels).
235+
*
236+
* @param input The string to process
237+
* @return The converted integer or 0 if the conversion failed.
238+
*/
239+
private static int convertDurationToInt(final String input) {
240+
if (input == null || input.isEmpty()) {
241+
return 0;
242+
}
243+
244+
final String clearedInput = Utils.removeNonDigitCharacters(input);
245+
try {
246+
return Integer.parseInt(clearedInput);
247+
} catch (final NumberFormatException ex) {
248+
return 0;
249+
}
226250
}
227251

228252
@Nonnull

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,11 @@ public long getDuration() throws ParsingException {
138138
}
139139
}
140140

141+
// NewPipe#8034 - YT returns not a correct duration for "YT shorts" videos
142+
if ("SHORTS".equalsIgnoreCase(duration)) {
143+
return 0;
144+
}
145+
141146
return YoutubeParsingHelper.parseDurationString(duration);
142147
}
143148

‎extractor/src/test/java/org/schabi/newpipe/downloader/MockOnly.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,9 @@
1717
@Retention(RetentionPolicy.RUNTIME)
1818
@ExtendWith(MockOnlyCondition.class)
1919
public @interface MockOnly {
20+
21+
/**
22+
* The reason why the test is mockonly.
23+
*/
24+
String value();
2025
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCRecentListExtractorTest.java‎

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,21 @@
11
package org.schabi.newpipe.extractor.services.media_ccc;
22

3+
import static org.junit.jupiter.api.Assertions.assertAll;
4+
import static org.junit.jupiter.api.Assertions.assertFalse;
5+
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertGreater;
6+
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
7+
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
8+
39
import org.junit.jupiter.api.BeforeAll;
410
import org.junit.jupiter.api.Test;
11+
import org.junit.jupiter.api.function.Executable;
512
import org.schabi.newpipe.downloader.DownloaderTestImpl;
613
import org.schabi.newpipe.extractor.NewPipe;
714
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
815
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
916

1017
import java.util.List;
11-
12-
import static org.junit.jupiter.api.Assertions.*;
13-
import static org.schabi.newpipe.extractor.ServiceList.MediaCCC;
14-
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
18+
import java.util.stream.Stream;
1519

1620
public class MediaCCCRecentListExtractorTest {
1721
private static KioskExtractor extractor;
@@ -24,16 +28,23 @@ public static void setUpClass() throws Exception {
2428
}
2529

2630
@Test
27-
public void testStreamList() throws Exception {
31+
void testStreamList() throws Exception {
2832
final List<StreamInfoItem> items = extractor.getInitialPage().getItems();
29-
assertEquals(100, items.size());
30-
for (final StreamInfoItem item: items) {
31-
assertFalse(isNullOrEmpty(item.getName()));
32-
assertTrue(item.getDuration() > 0);
33-
// Disabled for now, because sometimes videos are uploaded, but their release date is in the future
34-
// assertTrue(item.getUploadDate().offsetDateTime().isBefore(OffsetDateTime.now()));
35-
}
36-
}
33+
assertFalse(items.isEmpty(), "No items returned");
3734

35+
assertAll(items.stream().flatMap(this::getAllConditionsForItem));
36+
}
3837

38+
private Stream<Executable> getAllConditionsForItem(final StreamInfoItem item) {
39+
return Stream.of(
40+
() -> assertFalse(
41+
isNullOrEmpty(item.getName()),
42+
"Name=[" + item.getName() + "] of " + item + " is empty or null"
43+
),
44+
() -> assertGreater(0,
45+
item.getDuration(),
46+
"Duration[=" + item.getDuration() + "] of " + item + " is <= 0"
47+
)
48+
);
49+
}
3950
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -28,42 +28,47 @@ public static class Default {
2828
public static void setUp() throws Exception {
2929
NewPipe.init(DownloaderTestImpl.getInstance());
3030
extractor = (PeertubeCommentsExtractor) PeerTube
31-
.getCommentsExtractor("https://framatube.org/videos/watch/9c9de5e8-0a1e-484a-b099-e80766180a6d");
31+
.getCommentsExtractor("https://framatube.org/w/kkGMgK9ZtnKfYAgnEtQxbv");
3232
}
3333

3434
@Test
35-
public void testGetComments() throws IOException, ExtractionException {
35+
void testGetComments() throws IOException, ExtractionException {
36+
final String comment = "I love this";
37+
3638
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
37-
boolean result = findInComments(comments, "Cool.");
39+
boolean result = findInComments(comments, comment);
3840

3941
while (comments.hasNextPage() && !result) {
4042
comments = extractor.getPage(comments.getNextPage());
41-
result = findInComments(comments, "Cool.");
43+
result = findInComments(comments, comment);
4244
}
4345

4446
assertTrue(result);
4547
}
4648

4749
@Test
48-
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
49-
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
50+
void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
51+
final String comment = "great video";
52+
53+
final CommentsInfo commentsInfo =
54+
CommentsInfo.getInfo("https://framatube.org/w/kkGMgK9ZtnKfYAgnEtQxbv");
5055
assertEquals("Comments", commentsInfo.getName());
5156

52-
boolean result = findInComments(commentsInfo.getRelatedItems(), "Cool");
57+
boolean result = findInComments(commentsInfo.getRelatedItems(), comment);
5358

5459
Page nextPage = commentsInfo.getNextPage();
5560
InfoItemsPage<CommentsInfoItem> moreItems = new InfoItemsPage<>(null, nextPage, null);
5661
while (moreItems.hasNextPage() && !result) {
5762
moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
58-
result = findInComments(moreItems.getItems(), "Cool");
63+
result = findInComments(moreItems.getItems(), comment);
5964
nextPage = moreItems.getNextPage();
6065
}
6166

6267
assertTrue(result);
6368
}
6469

6570
@Test
66-
public void testGetCommentsAllData() throws IOException, ExtractionException {
71+
void testGetCommentsAllData() throws IOException, ExtractionException {
6772
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
6873
for (CommentsInfoItem c : comments.getItems()) {
6974
assertFalse(Utils.isBlank(c.getUploaderUrl()));
@@ -105,13 +110,13 @@ public static void setUp() throws Exception {
105110
}
106111

107112
@Test
108-
public void testGetComments() throws IOException, ExtractionException {
113+
void testGetComments() throws IOException, ExtractionException {
109114
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
110115
assertTrue(comments.getErrors().isEmpty());
111116
}
112117

113118
@Test
114-
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
119+
void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
115120
final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
116121
assertTrue(commentsInfo.getErrors().isEmpty());
117122
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubePlaylistLinkHandlerFactoryTest.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import org.schabi.newpipe.extractor.exceptions.ParsingException;
88
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubePlaylistLinkHandlerFactory;
99

10+
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
1011
import static org.junit.jupiter.api.Assertions.assertEquals;
1112
import static org.junit.jupiter.api.Assertions.assertTrue;
1213

@@ -24,7 +25,7 @@ public static void setUp() {
2425
}
2526

2627
@Test
27-
public void acceptUrlTest() throws ParsingException {
28+
void acceptUrlTest() throws ParsingException {
2829
assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909"));
2930
assertTrue(linkHandler.acceptUrl("https://framatube.org/w/p/d8ca79f9-e4c7-4269-8183-d78ed269c909"));
3031
assertTrue(linkHandler.acceptUrl("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909/videos"));
@@ -35,7 +36,7 @@ public void acceptUrlTest() throws ParsingException {
3536
}
3637

3738
@Test
38-
public void getIdFromUrl() throws ParsingException {
39+
void getIdFromUrl() throws ParsingException {
3940
assertEquals("d8ca79f9-e4c7-4269-8183-d78ed269c909", linkHandler.getId("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909"));
4041
assertEquals("d8ca79f9-e4c7-4269-8183-d78ed269c909", linkHandler.getId("https://framatube.org/w/p/d8ca79f9-e4c7-4269-8183-d78ed269c909"));
4142
assertEquals("dacdc4ef-5160-4846-9b70-a655880da667", linkHandler.getId("https://framatube.org/videos/watch/playlist/dacdc4ef-5160-4846-9b70-a655880da667"));
@@ -47,9 +48,9 @@ public void getIdFromUrl() throws ParsingException {
4748
}
4849

4950
@Test
50-
public void getUrl() throws ParsingException {
51-
System.out.println(linkHandler.fromUrl("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909").getUrl());;
52-
System.out.println(linkHandler.fromUrl("https://framatube.org/w/p/d8ca79f9-e4c7-4269-8183-d78ed269c909").getUrl());;
53-
System.out.println(linkHandler.fromUrl("https://framatube.org/w/p/sLFbqXsw7sPR3AfvqQSBZB").getUrl());;
51+
void getUrl() {
52+
assertDoesNotThrow(() -> linkHandler.fromUrl("https://framatube.org/videos/watch/playlist/d8ca79f9-e4c7-4269-8183-d78ed269c909").getUrl());
53+
assertDoesNotThrow(() -> linkHandler.fromUrl("https://framatube.org/w/p/d8ca79f9-e4c7-4269-8183-d78ed269c909").getUrl());
54+
assertDoesNotThrow(() -> linkHandler.fromUrl("https://framatube.org/w/p/sLFbqXsw7sPR3AfvqQSBZB").getUrl());
5455
}
5556
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/peertube/PeertubeStreamExtractorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public void testGetLanguageInformation() throws ParsingException {
8686
@Override public long expectedViewCountAtLeast() { return 38600; }
8787
@Nullable @Override public String expectedUploadDate() { return "2018-10-01 10:52:46.396"; }
8888
@Nullable @Override public String expectedTextualUploadDate() { return "2018-10-01T10:52:46.396Z"; }
89-
@Override public long expectedLikeCountAtLeast() { return 120; }
89+
@Override public long expectedLikeCountAtLeast() { return 50; }
9090
@Override public long expectedDislikeCountAtLeast() { return 0; }
9191
@Override public String expectedHost() { return "framatube.org"; }
9292
@Override public String expectedCategory() { return "Science & Technology"; }

0 commit comments

Comments
 (0)