diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java index b0ac2e14f7..9e04238bf4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java @@ -78,7 +78,7 @@ public void reset() { * Add an error * @param error the error */ - protected void addError(final Exception error) { + public void addError(final Exception error) { errors.add(error); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java index e1b19e7fb9..091c5e7675 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java @@ -1,18 +1,26 @@ package org.schabi.newpipe.extractor; +import javax.annotation.Nullable; import java.io.Serializable; import java.util.List; import java.util.Map; -import javax.annotation.Nullable; - import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +/** + * The {@link Page} class is used for storing information on future requests + * for retrieving content. + *
+ * A page has an {@link #id}, an {@link #url}, as well as information on possible {@link #cookies}. + * In case the data behind the URL has already been retrieved, + * it can be accessed by using {@link #getBody()} or {@link #getContent()}. + */ public class Page implements Serializable { private final String url; private final String id; private final List ids; private final Map cookies; + private Serializable content; @Nullable private final byte[] body; @@ -78,4 +86,28 @@ public static boolean isValid(final Page page) { public byte[] getBody() { return body; } + + public boolean hasContent() { + return content != null; + } + + /** + * Get the page's content if it has been set, returns {@code null} otherwise. + * @return the page's content + */ + @Nullable + public Serializable getContent() { + return content; + } + + /** + * Set the page's content. + * The page's content can either be retrieved manually by requesting the resource + * behind the page's URL (see {@link #url} and {@link #getUrl()}) + * or storing it in a {@link Page}s instance in case the content has already been downloaded. + * @param content the page's content + */ + public void setContent(@Nullable final Serializable content) { + this.content = content; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 57deb64a21..8a1abc68c6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -323,4 +323,11 @@ public static String getAvatarUrl(final JsonObject object) { public static String getUploaderName(final JsonObject object) { return object.getObject("user").getString("username", ""); } + + public static boolean isReplyTo(@Nonnull final JsonObject originalComment, + @Nonnull final JsonObject otherComment) { + return originalComment.getInt("timestamp") == otherComment.getInt("timestamp"); + + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java index b02a3ea802..148ec92004 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; @@ -16,14 +18,29 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; - +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; +import org.schabi.newpipe.extractor.utils.cache.SoundCloudCommentsCache; +import org.schabi.newpipe.extractor.utils.cache.SoundCloudCommentsCache.CachedCommentInfo; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import javax.annotation.Nonnull; - -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import javax.annotation.Nullable; public class SoundcloudCommentsExtractor extends CommentsExtractor { + public static final String COLLECTION = "collection"; + public static final String NEXT_HREF = "next_href"; + + /** + * The last comment which was a top level comment. + * Next pages might start with replies to the last top level comment + * and therefore the {@link SoundcloudCommentsInfoItemExtractor#replyCount} + * of the last top level comment cannot be determined certainly. + */ + private static final SoundCloudCommentsCache LAST_TOP_LEVEL_COMMENTS = + new SoundCloudCommentsCache(10); + public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) { super(service, uiHandler); @@ -46,44 +63,205 @@ public InfoItemsPage getInitialPage() throws ExtractionExcepti final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( getServiceId()); - collectStreamsFrom(collector, json.getArray("collection")); + collectCommentsFrom(collector, json, null); - return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); + return new InfoItemsPage<>(collector, new Page(json.getString(NEXT_HREF))); } @Override - public InfoItemsPage getPage(final Page page) throws ExtractionException, - IOException { + public InfoItemsPage getPage(final Page page) + throws ExtractionException, IOException { + if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } - - final Downloader downloader = NewPipe.getDownloader(); - final Response response = downloader.get(page.getUrl()); - final JsonObject json; - try { - json = JsonParser.object().from(response.responseBody()); - } catch (final JsonParserException e) { - throw new ParsingException("Could not parse json", e); - } - final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( getServiceId()); - collectStreamsFrom(collector, json.getArray("collection")); + // Replies typically do not have a next page, but that's not always the case. + final boolean hasNextPage; + if (page.hasContent()) { + // This page contains the whole previously fetched comments. + // We need to get the comments which are replies to the comment with the page's id. + json = (JsonObject) page.getContent(); + try { + final int commentId = Integer.parseInt(page.getId()); + hasNextPage = collectRepliesFrom(collector, json, commentId, page.getUrl()); + } catch (final NumberFormatException e) { + throw new ParsingException("Got invalid comment id", e); + } + } else { + + final Downloader downloader = NewPipe.getDownloader(); + final Response response = downloader.get(page.getUrl()); - return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); + try { + json = JsonParser.object().from(response.responseBody()); + hasNextPage = json.has(NEXT_HREF); + } catch (final JsonParserException e) { + throw new ParsingException("Could not parse json", e); + } + + final CachedCommentInfo topLevelCommentElement = LAST_TOP_LEVEL_COMMENTS.get(getUrl()); + if (topLevelCommentElement == null) { + if (LAST_TOP_LEVEL_COMMENTS.isEmpty()) { + collector.addError(new RuntimeException( + "Could not get last top level comment. It has been removed from cache." + + " Increase the cache size to not loose any comments")); + } + collectCommentsFrom(collector, json, null); + } else { + collectCommentsFrom(collector, json, topLevelCommentElement); + } + } + + if (hasNextPage) { + return new InfoItemsPage<>(collector, new Page(json.getString(NEXT_HREF))); + } else { + return new InfoItemsPage<>(collector, null); + } } @Override - public void onFetchPage(@Nonnull final Downloader downloader) { } + public void onFetchPage(@Nonnull final Downloader downloader) { + } - private void collectStreamsFrom(final CommentsInfoItemsCollector collector, - final JsonArray entries) throws ParsingException { + /** + * Collect top level comments from a SoundCloud API response. + * + * @param collector the collector which collects the the top level comments + * @param json the JsonObject of the API response + * @param lastTopLevelComment the last top level comment from the previous page or {@code null} + * if this method is run for the initial page. + * @throws ParsingException + */ + private void collectCommentsFrom(@Nonnull final CommentsInfoItemsCollector collector, + @Nonnull final JsonObject json, + @Nullable final CachedCommentInfo lastTopLevelComment) + throws ParsingException { + final List extractors = new ArrayList<>(); final String url = getUrl(); - for (final Object comment : entries) { - collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url)); + + JsonObject currentTopLevelComment = null; + int currentTopLevelCommentIndex = 0; + boolean isLastCommentReply = true; + boolean isFirstCommentReply = false; + boolean addedLastTopLevelComment = lastTopLevelComment == null; + // Check whether the first comment in the list is a reply to the last top level comment + // from the previous page if there was a previous page. + if (lastTopLevelComment != null) { + final JsonObject firstComment = json.getArray(COLLECTION).getObject(0); + if (SoundcloudParsingHelper.isReplyTo(lastTopLevelComment.comment, firstComment)) { + currentTopLevelComment = lastTopLevelComment.comment; + isFirstCommentReply = true; + merge(json, lastTopLevelComment.json, lastTopLevelComment.index); + } else { + extractors.add(new SoundcloudCommentsInfoItemExtractor( + lastTopLevelComment.json, + lastTopLevelComment.index, + lastTopLevelComment.comment, url, null)); + addedLastTopLevelComment = true; + } + } + + final JsonArray entries = json.getArray(COLLECTION); + for (int i = 0; i < entries.size(); i++) { + final JsonObject entry = entries.getObject(i); + // Extract all top level comments + // The first comment is a top level co + // if it is not a reply to the last top level comment + // + if ((i == 0 && !isFirstCommentReply) + || ( + i != 0 && !SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry) + && !SoundcloudParsingHelper.isReplyTo(currentTopLevelComment, entry))) { + currentTopLevelComment = entry; + currentTopLevelCommentIndex = i; + if (!addedLastTopLevelComment) { + // There is a new top level comment. This also means that we can now determine + // the reply count and get all replies for the top level comment. + extractors.add(new SoundcloudCommentsInfoItemExtractor( + json, 0, lastTopLevelComment.comment, url, null)); + addedLastTopLevelComment = true; + } + if (i == entries.size() - 1) { + isLastCommentReply = false; + LAST_TOP_LEVEL_COMMENTS.put(getUrl(), currentTopLevelComment, json, i); + + // Do not collect the last comment if it is a top level comment + // because it might have replies. + // That is information we cannot get from the comment itself + // (thanks SoundCloud...) but needs to be obtained from the next comment. + // The comment will therefore be collected + // when collecting the items from the next page. + break; + } + extractors.add(new SoundcloudCommentsInfoItemExtractor( + json, i, entry, url, null)); + } } + if (isLastCommentReply) { + // Do not collect the last top level comment if it has replies and the retrieved + // comment list ends with a reply. We do not know whether the next page starts + // with more replies to the last top level comment. + LAST_TOP_LEVEL_COMMENTS.put( + getUrl(), + extractors.remove(extractors.size() - 1).item, + json, currentTopLevelCommentIndex); + } + extractors.stream().forEach(collector::commit); + + } + + /** + * Collect replies to a top level comment from a SoundCloud API response. + * + * @param collector the collector which collects the the replies + * @param json the SoundCloud API response + * @param id the comment's id for which the replies are collected + * @param url the corresponding page's URL + * @return {code true} if there might be more replies to the comment; + * {@code false} if there are definitely no more replies + */ + private boolean collectRepliesFrom(@Nonnull final CommentsInfoItemsCollector collector, + @Nonnull final JsonObject json, + final int id, + @Nonnull final String url) { + JsonObject originalComment = null; + final JsonArray entries = json.getArray(COLLECTION); + boolean moreReplies = false; + for (int i = 0; i < entries.size(); i++) { + final JsonObject comment = entries.getObject(i); + if (comment.getInt("id") == id) { + originalComment = comment; + continue; + } + if (originalComment != null + && SoundcloudParsingHelper.isReplyTo(originalComment, comment)) { + collector.commit(new SoundcloudCommentsInfoItemExtractor( + json, i, entries.getObject(i), url, originalComment)); + // There might be more replies to the originalComment + // if the original comment is at the end of the list. + if (i == entries.size() - 1 && json.has(NEXT_HREF)) { + moreReplies = true; + } + } + } + return moreReplies; } + + private void merge(@Nonnull final JsonObject target, @Nonnull final JsonObject subject, + final int index) { + final JsonArray targetArray = target.getArray(COLLECTION); + final JsonArray subjectArray = subject.getArray(COLLECTION); + final JsonArray newArray = new JsonArray( + targetArray.size() + subjectArray.size() - index - 1); + for (int i = index; i < subjectArray.size(); i++) { + newArray.add(subjectArray.getObject(i)); + } + newArray.addAll(targetArray); + target.put(COLLECTION, newArray); + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index ec3f353e62..78afff6fd5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -1,62 +1,144 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCommentsExtractor.COLLECTION; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.Description; -import javax.annotation.Nullable; import java.util.Objects; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { - private final JsonObject json; + public static final int PREVIOUS_PAGE_INDEX = -1; + public static final String BODY = "body"; + public static final String USER_PERMALINK = "permalink"; + public static final String USER_FULL_NAME = "full_name"; + public static final String USER_USERNAME = "username"; + + @Nonnull private final JsonObject json; + private final int index; + @Nonnull public final JsonObject item; private final String url; - - public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final String url) { + @Nonnull private final JsonObject user; + /** + * A comment to which this comment is a reply. + * Is {@code null} if this comment is itself a top level comment. + */ + @Nullable private final JsonObject topLevelComment; + + /** + * The reply count is not given by the SoundCloud API, but needs to be obtained + * by counting the comments which come directly after this item and have the same timestamp. + */ + private int replyCount = CommentsInfoItem.UNKNOWN_REPLY_COUNT; + private Page repliesPage = null; + + public SoundcloudCommentsInfoItemExtractor(@Nonnull final JsonObject json, final int index, + @Nonnull final JsonObject item, final String url, + @Nullable final JsonObject topLevelComment) { this.json = json; + this.index = index; + this.item = item; this.url = url; + this.topLevelComment = topLevelComment; + this.user = item.getObject("user"); + } + + public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int index, + final JsonObject item, final String url) { + this(json, index, item, url, null); + } + + public void addInfoFromNextPage(@Nonnull final JsonArray newItems, final int itemCount) { + final JsonArray currentItems = this.json.getArray(COLLECTION); + for (int i = 0; i < itemCount; i++) { + currentItems.add(newItems.getObject(i)); + } } @Override public String getCommentId() { - return Objects.toString(json.getLong("id"), null); + return Objects.toString(item.getLong("id"), null); } - @Override public Description getCommentText() { - return new Description(json.getString("body"), Description.PLAIN_TEXT); + String commentContent = item.getString(BODY); + if (topLevelComment == null) { + return new Description(commentContent, Description.PLAIN_TEXT); + } + // This comment is a reply to another comment. + // Therefore, the comment starts with the mention of the original comment's author. + // The account is automatically linked by the SoundCloud web UI. + // We need to do this manually. + if (commentContent.startsWith("@")) { + final String authorName = commentContent.split(" ", 2)[0].replace("@", ""); + final JsonArray comments = json.getArray(COLLECTION); + JsonObject author = null; + for (int i = index - 1; i >= 0 && author == null; i--) { + final JsonObject commentsAuthor = comments.getObject(i).getObject("user"); + // use startsWith because sometimes the mention of the user + // is followed by a punctuation character. + if (authorName.startsWith(commentsAuthor.getString(USER_PERMALINK))) { + author = commentsAuthor; + } + } + if (author == null) { + author = topLevelComment.getObject("user"); + } + final String name = isNullOrEmpty(author.getString(USER_FULL_NAME)) + ? author.getString(USER_USERNAME) : author.getString(USER_FULL_NAME); + final String link = "" + + "@" + name + ""; + commentContent = commentContent + .replace("@" + author.getString(USER_PERMALINK), link) + .replace("@" + author.getInt("user_id"), link); + } + + return new Description(commentContent, Description.HTML); } @Override public String getUploaderName() { - return json.getObject("user").getString("username"); + if (isNullOrEmpty(user.getString(USER_FULL_NAME))) { + return user.getString(USER_USERNAME); + } + return user.getString(USER_FULL_NAME); } @Override public String getUploaderAvatarUrl() { - return json.getObject("user").getString("avatar_url"); + return user.getString("avatar_url"); } @Override public boolean isUploaderVerified() throws ParsingException { - return json.getObject("user").getBoolean("verified"); + return user.getBoolean("verified"); } @Override public int getStreamPosition() throws ParsingException { - return json.getInt("timestamp") / 1000; // convert milliseconds to seconds + return item.getInt("timestamp") / 1000; // convert milliseconds to seconds } @Override public String getUploaderUrl() { - return json.getObject("user").getString("permalink_url"); + return user.getString("permalink_url"); } @Override public String getTextualUploadDate() { - return json.getString("created_at"); + return item.getString("created_at"); } @Nullable @@ -67,7 +149,7 @@ public DateWrapper getUploadDate() throws ParsingException { @Override public String getName() throws ParsingException { - return json.getObject("user").getString("permalink"); + return user.getString(USER_PERMALINK); } @Override @@ -77,6 +159,46 @@ public String getUrl() { @Override public String getThumbnailUrl() { - return json.getObject("user").getString("avatar_url"); + return user.getString("avatar_url"); + } + + @Override + public Page getReplies() { + if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) { + replyCount = 0; + // SoundCloud has only comments and top level replies, but not nested replies. + // Therefore, replies cannot have further replies. + if (topLevelComment == null) { + // Loop through all comments which come after the original comment + // to find its replies. + final JsonArray allItems = json.getArray(COLLECTION); + for (int i = index + 1; i < allItems.size(); i++) { + if (SoundcloudParsingHelper.isReplyTo(item, allItems.getObject(i))) { + replyCount++; + } else { + // Only the comments directly after the original comment + // having the same timestamp are replies to the original comment. + // The first comment not having the same timestamp + // is the next top-level comment. + break; + } + } + } + if (replyCount == 0) { + return null; + } + repliesPage = new Page(getUrl(), getCommentId()); + repliesPage.setContent(json); + } + + return repliesPage; + } + + @Override + public int getReplyCount() { + if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) { + getReplies(); + } + return replyCount; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java index 23c6a29392..775ee10486 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java @@ -3,6 +3,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; import java.io.IOException; import java.util.List; @@ -14,6 +15,9 @@ public final class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerF private static final SoundcloudCommentsLinkHandlerFactory INSTANCE = new SoundcloudCommentsLinkHandlerFactory(); + private static final String OFFSET_PATTERN = "https://api-v2.soundcloud.com/tracks/" + + "([0-9a-z]+)/comments?([0-9a-z/&])?offset=([0-9])+"; + private SoundcloudCommentsLinkHandlerFactory() { } @@ -27,7 +31,7 @@ public String getUrl(final String id, final String sortFilter) throws ParsingException { try { return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id=" - + clientId() + "&threaded=0" + "&filter_replies=1"; + + clientId() + "&threaded=1" + "&filter_replies=1"; // Anything but 1 = sort by new // + "&limit=NUMBER_OF_ITEMS_PER_REQUEST". We let the API control (default = 10) // + "&offset=OFFSET". We let the API control (default = 0, then we use nextPageUrl) @@ -36,12 +40,29 @@ public String getUrl(final String id, } } + public String getUrl(final String id, + final List contentFilter, + final String sortFilter, + final int offset) throws ParsingException { + return getUrl(id, contentFilter, sortFilter) + "&offset=" + offset; + } + @Override public String getId(final String url) throws ParsingException { // Delegation to avoid duplicate code, as we need the same id return SoundcloudStreamLinkHandlerFactory.getInstance().getId(url); } + public int getReplyOffset(final String url) throws ParsingException { + try { + return Integer.parseInt(Parser.matchGroup(OFFSET_PATTERN, url, 3)); + } catch (Parser.RegexException | NumberFormatException e) { + throw new ParsingException("Could not get offset from URL: " + url, e); + } + } + + + @Override public boolean onAcceptUrl(final String url) { try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java index 9af4be09b9..14ee29b0c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java @@ -33,7 +33,7 @@ public String getUrl(final String id) throws ParsingException { @Override public String getId(final String url) throws ParsingException { if (Parser.isMatch(API_URL_PATTERN, url)) { - return Parser.matchGroup1(API_URL_PATTERN, url); + return Parser.matchGroup(API_URL_PATTERN, url, 2); } Utils.checkUrl(URL_PATTERN, url); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java index 46bd324204..fcf07d0937 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.services.youtube.DeliveryType; import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; +import org.schabi.newpipe.extractor.utils.cache.ManifestCreatorCache; import org.w3c.dom.Attr; import org.w3c.dom.DOMException; import org.w3c.dom.Document; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java index 46e84df1db..9226d8d2fd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java @@ -15,7 +15,7 @@ import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.services.youtube.DeliveryType; import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; +import org.schabi.newpipe.extractor.utils.cache.ManifestCreatorCache; import org.schabi.newpipe.extractor.utils.Utils; import org.w3c.dom.DOMException; import org.w3c.dom.Document; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java index 3a5a7dd23d..5c23138f08 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java @@ -15,7 +15,7 @@ import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.services.youtube.DeliveryType; import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; +import org.schabi.newpipe.extractor.utils.cache.ManifestCreatorCache; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java index 0f69895bba..1c1e04c37d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java @@ -2,7 +2,7 @@ import org.schabi.newpipe.extractor.services.youtube.DeliveryType; import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; +import org.schabi.newpipe.extractor.utils.cache.ManifestCreatorCache; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/Cache.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/Cache.java new file mode 100644 index 0000000000..6e8180e93f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/Cache.java @@ -0,0 +1,9 @@ +package org.schabi.newpipe.extractor.utils.cache; + +public interface Cache { + void put(K key, V value); + V get(K key); + int size(); + boolean isEmpty(); + void clear(); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCache.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/ManifestCreatorCache.java similarity index 98% rename from extractor/src/main/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCache.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/ManifestCreatorCache.java index ac12f83f95..149369c53e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCache.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/ManifestCreatorCache.java @@ -1,4 +1,6 @@ -package org.schabi.newpipe.extractor.utils; +package org.schabi.newpipe.extractor.utils.cache; + +import org.schabi.newpipe.extractor.utils.Pair; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCache.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCache.java new file mode 100644 index 0000000000..5c367ce49f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCache.java @@ -0,0 +1,74 @@ +package org.schabi.newpipe.extractor.utils.cache; + +import com.grack.nanojson.JsonObject; + +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * LRU cache which can contain a few items. + */ +public class SoundCloudCommentsCache { + + private final int maxSize; + private final Map store; + public SoundCloudCommentsCache(final int size) { + if (size < 1) { + throw new IllegalArgumentException("Size must be at least 1"); + } + store = new HashMap<>(size); + maxSize = size; + } + + public void put(@Nonnull final String key, @Nonnull final JsonObject comment, + @Nonnull final JsonObject json, final int index) { + if (store.size() == maxSize) { + store.remove( + store.entrySet().stream() + .reduce((a, b) -> a.getValue().lastHit < b.getValue().lastHit ? a : b) + .get().getKey()); + } + store.put(key, new CachedCommentInfo(comment, json, index)); + } + + @Nullable + public CachedCommentInfo get(final String key) { + final CachedCommentInfo result = store.get(key); + if (result == null) { + return null; + } + result.lastHit = System.nanoTime(); + return result; + } + + public int size() { + return store.size(); + } + + public boolean isEmpty() { + return store.isEmpty(); + } + + public void clear() { + store.clear(); + } + + public final class CachedCommentInfo { + @Nonnull public final JsonObject comment; + @Nonnull public final JsonObject json; + public final int index; + private long lastHit = System.nanoTime(); + + private CachedCommentInfo(@Nonnull final JsonObject comment, + @Nonnull final JsonObject json, + final int index) { + this.comment = comment; + this.json = json; + this.index = index; + } + } + +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCacheTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/ManifestCreatorCacheTest.java similarity index 96% rename from extractor/src/test/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCacheTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/ManifestCreatorCacheTest.java index 83c5c1dfb1..a28d745ebd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCacheTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/ManifestCreatorCacheTest.java @@ -1,6 +1,7 @@ -package org.schabi.newpipe.extractor.utils; +package org.schabi.newpipe.extractor.utils.cache; import org.junit.jupiter.api.Test; +import org.schabi.newpipe.extractor.utils.cache.ManifestCreatorCache; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCacheTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCacheTest.java new file mode 100644 index 0000000000..bd985905a5 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCacheTest.java @@ -0,0 +1,83 @@ +package org.schabi.newpipe.extractor.utils.cache; + +import com.grack.nanojson.JsonObject; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class SoundCloudCommentsCacheTest { + @Test + void testInstantiation() { + assertThrows(RuntimeException.class, () -> new SoundCloudCommentsCache(-15)); + assertThrows(RuntimeException.class, () -> new SoundCloudCommentsCache(0)); + assertDoesNotThrow(() -> new SoundCloudCommentsCache(1)); + assertDoesNotThrow(() -> new SoundCloudCommentsCache(10)); + } + + @Test + void testSize() { + SoundCloudCommentsCache cache = new SoundCloudCommentsCache(10); + assertEquals(0, cache.size()); + assertTrue(cache.isEmpty()); + cache.put("a", new JsonObject(), new JsonObject(), 1); + assertEquals(1, cache.size()); + cache.put("b", new JsonObject(), new JsonObject(), 1); + assertEquals(2, cache.size()); + cache.put("c", new JsonObject(), new JsonObject(), 1); + assertEquals(3, cache.size()); + cache.put("a", new JsonObject(), new JsonObject(), 1); + assertEquals(3, cache.size()); + cache.put("b", new JsonObject(), new JsonObject(), 1); + assertEquals(3, cache.size()); + cache.clear(); + assertEquals(0, cache.size()); + } + + @Test + void testLRUStrategy() { + final SoundCloudCommentsCache cache = new SoundCloudCommentsCache(4); + cache.put("1", new JsonObject(), new JsonObject(), 1); + cache.put("2", new JsonObject(), new JsonObject(), 2); + cache.put("3", new JsonObject(), new JsonObject(), 3); + cache.put("4", new JsonObject(), new JsonObject(), 4); + cache.put("5", new JsonObject(), new JsonObject(), 5); + assertNull(cache.get("1")); + final SoundCloudCommentsCache.CachedCommentInfo cci = cache.get("2"); + assertNotNull(cci); + cache.put("6", new JsonObject(), new JsonObject(), 6); + assertNotNull(cache.get("2")); + assertNull(cache.get("3")); + cache.put("7", new JsonObject(), new JsonObject(), 7); + cache.put("8", new JsonObject(), new JsonObject(), 8); + cache.put("9", new JsonObject(), new JsonObject(), 9); + assertNull(cache.get("1")); + assertNull(cache.get("3")); + assertNull(cache.get("4")); + assertNull(cache.get("5")); + assertNotNull(cache.get("2")); + } + + @Test + void testStorage() { + final SoundCloudCommentsCache cache = new SoundCloudCommentsCache(10); + cache.put("1", new JsonObject(), new JsonObject(), 1); + cache.put("1", new JsonObject(), new JsonObject(), 2); + assertEquals(2, cache.get("1").index); + cache.put("1", new JsonObject(), new JsonObject(), 3); + assertEquals(3, cache.get("1").index); + } + + @Test + void testClear() { + final SoundCloudCommentsCache cache = new SoundCloudCommentsCache(10); + cache.put("1", new JsonObject(), new JsonObject(), 1); + cache.put("2", new JsonObject(), new JsonObject(), 2); + cache.put("3", new JsonObject(), new JsonObject(), 3); + cache.put("4", new JsonObject(), new JsonObject(), 4); + cache.put("5", new JsonObject(), new JsonObject(), 5); + cache.clear(); + assertTrue(cache.isEmpty()); + assertEquals(0, cache.size()); + } + +}