From 0d6b01108e9f2da86bec77096ed757681776171c Mon Sep 17 00:00:00 2001 From: Stypox Date: Fri, 3 Oct 2025 14:43:03 +0200 Subject: [PATCH 1/3] Update nanojson to include upstream and FireMasterK's optimizations --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 9d6f384b2c..c981b87943 100644 --- a/build.gradle +++ b/build.gradle @@ -27,7 +27,7 @@ allprojects { } ext { - nanojsonVersion = "e9d656ddb49a412a5a0a5d5ef20ca7ef09549996" + nanojsonVersion = "c7a6c1c08d16b6d5ecded34758e6415e07be2166" jsr305Version = "3.0.2" junitVersion = "5.14.0" checkstyleVersion = "10.26.1" From 13c61d3e692855085674664b1c42b44b83843eb8 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Sat, 23 Aug 2025 10:56:52 +0530 Subject: [PATCH 2/3] Fixes for LazyString parsing. --- .../java/org/schabi/newpipe/extractor/utils/JsonUtils.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index 850eba778e..279a1a9b5a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -5,6 +5,7 @@ import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.LazyString; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.schabi.newpipe.extractor.exceptions.ParsingException; @@ -52,7 +53,7 @@ private static T getInstanceOf(@Nonnull final JsonObject object, @Nonnull public static String getString(@Nonnull final JsonObject object, @Nonnull final String path) throws ParsingException { - return getInstanceOf(object, path, String.class); + return getInstanceOf(object, path, LazyString.class).toString(); } @Nonnull @@ -157,8 +158,8 @@ public static JsonObject getJsonData(final String html, final String variable) public static List getStringListFromJsonArray(@Nonnull final JsonArray array) { return array.stream() - .filter(String.class::isInstance) - .map(String.class::cast) + .filter(LazyString.class::isInstance) + .map(Object::toString) .collect(Collectors.toList()); } } From 1f6cb35de98238c2c95078a4d2b8da74c02e5d49 Mon Sep 17 00:00:00 2001 From: Stypox Date: Fri, 3 Oct 2025 15:51:57 +0200 Subject: [PATCH 3/3] Avoid using raw get() on JsonObject get() may return objects with types that are used internally in the nanojson library, such as LazyString or Number. Use specialized methods instead. This commit removes all usages (except in SoundcloudParsingHelper::resolveIdWithWidgetApi()). By doing so, timeago-generator now compiles again, and YoutubeChannelExtractor::getTags() and JsonUtils tests pass again. The compilation and tests failed because nanojson now internally uses LazyString instead of plain String. --- .../peertube/PeertubeParsingHelper.java | 2 +- .../extractors/PeertubeStreamExtractor.java | 2 +- .../soundcloud/SoundcloudParsingHelper.java | 3 +-- .../extractors/YoutubeChannelExtractor.java | 11 +++----- .../newpipe/extractor/utils/JsonUtils.java | 14 ++++++++--- .../extractor/utils/JsonUtilsTest.java | 13 +++++----- .../GeneratePatternClasses.java | 25 ++++++++----------- 7 files changed, 34 insertions(+), 36 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java index 4e8bd2d350..1aaea0de52 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java @@ -108,7 +108,7 @@ public static void collectItemsFrom(final InfoItemsCollector collector, final boolean sepia) throws ParsingException { final JsonArray contents; try { - contents = (JsonArray) JsonUtils.getValue(json, "data"); + contents = JsonUtils.getArray(json, "data"); } catch (final Exception e) { throw new ParsingException("Unable to extract list info", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 5406368854..08946d3fbd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -409,7 +409,7 @@ private void collectStreamsFrom(final StreamInfoItemsCollector collector, final JsonObject jsonObject) throws ParsingException { final JsonArray contents; try { - contents = (JsonArray) JsonUtils.getValue(jsonObject, "data"); + contents = JsonUtils.getArray(jsonObject, "data"); } catch (final Exception e) { throw new ParsingException("Could not extract related videos", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index ae8fd77d6d..6af0ebd61c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -29,7 +29,6 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.ImageSuffix; -import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; import org.schabi.newpipe.extractor.utils.Utils; @@ -227,7 +226,7 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc final String response = NewPipe.getDownloader().get(widgetUrl, SoundCloud.getLocalization()).responseBody(); final JsonObject o = JsonParser.object().from(response); - return String.valueOf(JsonUtils.getValue(o, "id")); + return o.get("id").toString(); } catch (final JsonParserException e) { throw new ParsingException("Could not parse JSON response", e); } catch (final ExtractionException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 147345526a..b82bad7362 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -43,6 +43,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelTabExtractor.VideosTabExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -51,7 +52,6 @@ import java.util.List; import java.util.Optional; import java.util.function.Consumer; -import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -490,12 +490,7 @@ public List getTags() throws ParsingException { return List.of(); } - return jsonResponse.getObject("microformat") - .getObject("microformatDataRenderer") - .getArray("tags") - .stream() - .filter(String.class::isInstance) - .map(String.class::cast) - .collect(Collectors.toUnmodifiableList()); + return JsonUtils.getStringListFromJsonArray(jsonResponse.getObject("microformat") + .getObject("microformatDataRenderer").getArray("tags")); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java index 279a1a9b5a..19a9800430 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/JsonUtils.java @@ -22,9 +22,13 @@ public final class JsonUtils { private JsonUtils() { } + /** + * Note that this accesses JsonObject's internal types directly, including LazyString and + * Number. Don't rely on the types of the returned object! + */ @Nonnull - public static Object getValue(@Nonnull final JsonObject object, - @Nonnull final String path) throws ParsingException { + static Object getValue(@Nonnull final JsonObject object, + @Nonnull final String path) throws ParsingException { final List keys = Arrays.asList(path.split("\\.")); final JsonObject parentObject = getObject(object, keys.subList(0, keys.size() - 1)); @@ -81,8 +85,12 @@ public static JsonArray getArray(@Nonnull final JsonObject object, @Nonnull fina return getInstanceOf(object, path, JsonArray.class); } + /** + * Note that this accesses JsonObject's internal types directly, including LazyString and + * Number. Don't rely on the types of the returned objects! + */ @Nonnull - public static List getValues(@Nonnull final JsonArray array, @Nonnull final String path) + static List getValues(@Nonnull final JsonArray array, @Nonnull final String path) throws ParsingException { final List result = new ArrayList<>(); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java index 11394bcf3f..3bed0dc064 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/JsonUtilsTest.java @@ -8,8 +8,7 @@ import org.junit.jupiter.api.Test; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import java.util.List; - +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -18,13 +17,13 @@ public class JsonUtilsTest { @Test public void testGetValueFlat() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}"); - assertTrue("John".equals(JsonUtils.getValue(obj, "name"))); + assertEquals("John", JsonUtils.getValue(obj, "name").toString()); } @Test public void testGetValueNested() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"name\":\"John\",\"age\":30,\"cars\":{\"car1\":\"Ford\",\"car2\":\"BMW\",\"car3\":\"Fiat\"}}"); - assertTrue("BMW".equals(JsonUtils.getValue(obj, "cars.car2"))); + assertEquals("BMW", JsonUtils.getValue(obj, "cars.car2").toString()); } @Test @@ -38,9 +37,9 @@ public void testGetArray() throws JsonParserException, ParsingException { public void testGetValues() throws JsonParserException, ParsingException { JsonObject obj = JsonParser.object().from("{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}"); JsonArray arr = (JsonArray) JsonUtils.getValue(obj, "topping"); - List types = JsonUtils.getValues(arr, "type"); - assertTrue(types.contains("Chocolate with Sprinkles")); - + assertTrue( + JsonUtils.getValues(arr, "type").stream() + .anyMatch(o -> "Chocolate with Sprinkles".equals(o.toString()))); } } diff --git a/timeago-generator/src/main/java/org/schabi/newpipe/timeago_generator/GeneratePatternClasses.java b/timeago-generator/src/main/java/org/schabi/newpipe/timeago_generator/GeneratePatternClasses.java index a2ef2ffa4f..335316b454 100644 --- a/timeago-generator/src/main/java/org/schabi/newpipe/timeago_generator/GeneratePatternClasses.java +++ b/timeago-generator/src/main/java/org/schabi/newpipe/timeago_generator/GeneratePatternClasses.java @@ -9,7 +9,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.TreeMap; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -21,24 +20,22 @@ public static void main(String[] args) throws FileNotFoundException, JsonParserE final InputStream resourceAsStream = new FileInputStream("timeago-parser/raw/unique_patterns.json"); - final JsonObject from = JsonParser.object().from(resourceAsStream); - final TreeMap map = new TreeMap<>(from); - + final JsonObject map = JsonParser.object().from(resourceAsStream); final StringBuilder patternMapEntries = new StringBuilder(); for (Map.Entry entry : map.entrySet()) { final String languageCode = entry.getKey().replace('-', '_'); - final Map unitsList = (Map) entry.getValue(); + final JsonObject unitsList = (JsonObject) entry.getValue(); - final String wordSeparator = (String) unitsList.get("word_separator"); + final String wordSeparator = unitsList.getString("word_separator"); - final JsonArray seconds = (JsonArray) unitsList.get("seconds"); - final JsonArray minutes = (JsonArray) unitsList.get("minutes"); - final JsonArray hours = (JsonArray) unitsList.get("hours"); - final JsonArray days = (JsonArray) unitsList.get("days"); - final JsonArray weeks = (JsonArray) unitsList.get("weeks"); - final JsonArray months = (JsonArray) unitsList.get("months"); - final JsonArray years = (JsonArray) unitsList.get("years"); + final JsonArray seconds = unitsList.getArray("seconds"); + final JsonArray minutes = unitsList.getArray("minutes"); + final JsonArray hours = unitsList.getArray("hours"); + final JsonArray days = unitsList.getArray("days"); + final JsonArray weeks = unitsList.getArray("weeks"); + final JsonArray months = unitsList.getArray("months"); + final JsonArray years = unitsList.getArray("years"); final StringBuilder specialCasesString = new StringBuilder(); specialCasesConstruct(ChronoUnit.SECONDS, seconds, specialCasesString); @@ -126,7 +123,7 @@ private static void specialCasesConstruct(ChronoUnit unit, JsonArray array, Stri final JsonObject caseObject = (JsonObject) o; for (Map.Entry caseEntry : caseObject.entrySet()) { final int caseAmount = Integer.parseInt(caseEntry.getKey()); - final String caseText = (String) caseEntry.getValue(); + final String caseText = caseEntry.getValue().toString(); iterator.remove(); stringBuilder.append(" ")