Update YouTube stream mocks and disable YoutubeStreamExtractorRelatedMixTest

Mixes seems to be not given by YouTube anymore if you use a PENDING consent cookie value.
As mocks needs to updated, the test is always failing because of this change.

Author: AudricV

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorRelatedMixTest.java

@@ -9,25 +9,22 @@
 import static org.schabi.newpipe.extractor.services.youtube.stream.YoutubeStreamExtractorDefaultTest.YOUTUBE_LICENCE;
 
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.schabi.newpipe.downloader.DownloaderFactory;
-import org.schabi.newpipe.downloader.MockOnly;
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfo.PlaylistType;
 import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeTestsUtils;
-import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
-import java.util.Random;
 import java.util.stream.Collectors;
 
 import javax.annotation.Nullable;
@@ -87,7 +84,7 @@ public static void setUp() throws Exception {
     // @formatter:on
 
     @Test
-    @MockOnly("related items keep changing, and so do the mixes contained within them")
+    @Disabled("Mixes are not available in related items anymore")
     @Override
     public void testRelatedItems() throws Exception {
         super.testRelatedItems();

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/stream/ageRestricted/generated_mock_0.json

@@ -26,16 +26,16 @@
         "text/javascript; charset\u003dutf-8"
       ],
       "cross-origin-opener-policy-report-only": [
-        "same-origin; report-to\u003d\"ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\""
+        "same-origin; report-to\u003d\"youtube_main\""
       ],
       "cross-origin-resource-policy": [
         "cross-origin"
       ],
       "date": [
-        "Tue, 15 Mar 2022 18:05:31 GMT"
+        "Sat, 02 Apr 2022 17:14:54 GMT"
       ],
       "expires": [
-        "Tue, 15 Mar 2022 18:05:31 GMT"
+        "Sat, 02 Apr 2022 17:14:54 GMT"
       ],
       "p3p": [
         "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
@@ -44,15 +44,15 @@
         "ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
       ],
       "report-to": [
-        "{\"group\":\"ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/encsid_ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\"}]}"
+        "{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
       ],
       "server": [
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dZaxye481UVs; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003dvj5u87YI-gQ; Domain\u003d.youtube.com; Expires\u003dSun, 11-Sep-2022 18:05:31 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "CONSENT\u003dPENDING+306; expires\u003dThu, 14-Mar-2024 18:05:31 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "YSC\u003dr-r-p9CtzRs; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003d1NOqEiunPX8; Domain\u003d.youtube.com; Expires\u003dThu, 29-Sep-2022 17:14:54 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "CONSENT\u003dPENDING+972; expires\u003dMon, 01-Apr-2024 17:14:54 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"
@@ -67,7 +67,7 @@
         "0"
       ]
     },
-    "responseBody": "var scriptUrl \u003d \u0027https:\\/\\/www.youtube.com\\/s\\/player\\/bd67d609\\/www-widgetapi.vflset\\/www-widgetapi.js\u0027;try{var ttPolicy\u003dwindow.trustedTypes.createPolicy(\"youtube-widget-api\",{createScriptURL:function(x){return x}});scriptUrl\u003dttPolicy.createScriptURL(scriptUrl)}catch(e){}if(!window[\"YT\"])var YT\u003d{loading:0,loaded:0};if(!window[\"YTConfig\"])var YTConfig\u003d{\"host\":\"https://www.youtube.com\"};\nif(!YT.loading){YT.loading\u003d1;(function(){var l\u003d[];YT.ready\u003dfunction(f){if(YT.loaded)f();else l.push(f)};window.onYTReady\u003dfunction(){YT.loaded\u003d1;for(var i\u003d0;i\u003cl.length;i++)try{l[i]()}catch(e$0){}};YT.setConfig\u003dfunction(c){for(var k in c)if(c.hasOwnProperty(k))YTConfig[k]\u003dc[k]};var a\u003ddocument.createElement(\"script\");a.type\u003d\"text/javascript\";a.id\u003d\"www-widgetapi-script\";a.src\u003dscriptUrl;a.async\u003dtrue;var c\u003ddocument.currentScript;if(c){var n\u003dc.nonce||c.getAttribute(\"nonce\");if(n)a.setAttribute(\"nonce\",n)}var b\u003d\ndocument.getElementsByTagName(\"script\")[0];b.parentNode.insertBefore(a,b)})()};\n",
+    "responseBody": "var scriptUrl \u003d \u0027https:\\/\\/www.youtube.com\\/s\\/player\\/449ea0a5\\/www-widgetapi.vflset\\/www-widgetapi.js\u0027;try{var ttPolicy\u003dwindow.trustedTypes.createPolicy(\"youtube-widget-api\",{createScriptURL:function(x){return x}});scriptUrl\u003dttPolicy.createScriptURL(scriptUrl)}catch(e){}if(!window[\"YT\"])var YT\u003d{loading:0,loaded:0};if(!window[\"YTConfig\"])var YTConfig\u003d{\"host\":\"https://www.youtube.com\"};\nif(!YT.loading){YT.loading\u003d1;(function(){var l\u003d[];YT.ready\u003dfunction(f){if(YT.loaded)f();else l.push(f)};window.onYTReady\u003dfunction(){YT.loaded\u003d1;for(var i\u003d0;i\u003cl.length;i++)try{l[i]()}catch(e$0){}};YT.setConfig\u003dfunction(c){for(var k in c)if(c.hasOwnProperty(k))YTConfig[k]\u003dc[k]};var a\u003ddocument.createElement(\"script\");a.type\u003d\"text/javascript\";a.id\u003d\"www-widgetapi-script\";a.src\u003dscriptUrl;a.async\u003dtrue;var c\u003ddocument.currentScript;if(c){var n\u003dc.nonce||c.getAttribute(\"nonce\");if(n)a.setAttribute(\"nonce\",n)}var b\u003d\ndocument.getElementsByTagName(\"script\")[0];b.parentNode.insertBefore(a,b)})()};\n",
     "latestUrl": "https://www.youtube.com/iframe_api"
   }
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/stream/ageRestricted/generated_mock_1.json

@@ -3,14 +3,14 @@
     "httpMethod": "GET",
     "url": "https://www.youtube.com/sw.js",
     "headers": {
-      "Accept-Language": [
-        "en-GB, en;q\u003d0.9"
-      ],
       "Origin": [
         "https://www.youtube.com"
       ],
       "Referer": [
         "https://www.youtube.com"
+      ],
+      "Accept-Language": [
+        "en-GB, en;q\u003d0.9"
       ]
     },
     "localization": {
@@ -38,13 +38,13 @@
         "text/javascript; charset\u003dutf-8"
       ],
       "cross-origin-opener-policy-report-only": [
-        "same-origin; report-to\u003d\"ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\""
+        "same-origin; report-to\u003d\"youtube_main\""
       ],
       "date": [
-        "Tue, 15 Mar 2022 18:05:31 GMT"
+        "Sat, 02 Apr 2022 17:14:55 GMT"
       ],
       "expires": [
-        "Tue, 15 Mar 2022 18:05:31 GMT"
+        "Sat, 02 Apr 2022 17:14:55 GMT"
       ],
       "p3p": [
         "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
@@ -53,15 +53,15 @@
         "ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
       ],
       "report-to": [
-        "{\"group\":\"ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/encsid_ATmXEA_XZXH6CdbrmjUzyTbVgxu22C8KYH7NsxKbRt94\"}]}"
+        "{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
       ],
       "server": [
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003d5JkDJJ7c0Ac; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 19-Jun-2019 18:05:31 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "CONSENT\u003dPENDING+629; expires\u003dThu, 14-Mar-2024 18:05:31 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "YSC\u003dY-iSkSnbr98; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dSun, 07-Jul-2019 17:14:55 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "CONSENT\u003dPENDING+804; expires\u003dMon, 01-Apr-2024 17:14:55 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"
@@ -76,7 +76,7 @@
         "0"
       ]
     },
-    "responseBody": "/** 13448315194419459673 */self.document \u003d self; self.window \u003d self;var ytcfg\u003d{d:function(){return window.yt\u0026\u0026yt.config_||ytcfg.data_||(ytcfg.data_\u003d{})},get:function(k,o){return k in ytcfg.d()?ytcfg.d()[k]:o},set:function(){var a\u003darguments;if(a.length\u003e1)ytcfg.d()[a[0]]\u003da[1];else for(var k in a[0])ytcfg.d()[k]\u003da[0][k]}};\nytcfg.set({\"EXPERIMENT_FLAGS\":{\"allow_music_base_url\":true,\"allow_skip_networkless\":true,\"clear_user_partitioned_ls\":true,\"deprecate_two_way_binding_child\":true,\"deprecate_two_way_binding_parent\":true,\"desktop_image_cta_no_background\":true,\"desktop_text_ads_gray_visurl\":true,\"disable_child_node_auto_formatted_strings\":true,\"disable_simple_mixed_direction_formatted_strings\":true,\"disable_thumbnail_preloading\":true,\"enable_client_sli_logging\":true,\"enable_gel_log_commands\":true,\"enable_gray_visurl\":true,\"enable_mixed_direction_formatted_strings\":true,\"enable_server_stitched_dai\":true,\"enable_sli_flush\":true,\"enable_tectonic_ad_ux_for_halftime\":true,\"enable_topsoil_wta_for_halftime_live_infra\":true,\"enable_web_eom_visitor_data\":true,\"export_networkless_options\":true,\"forward_domain_admin_state_on_embeds\":true,\"html5_enable_single_video_vod_ivar_on_pacf\":true,\"html5_enable_video_overlay_on_inplayer_slot_for_tv\":true,\"html5_pacf_enable_dai\":true,\"html5_recognize_predict_start_cue_point\":true,\"html5_server_stitched_dai_group\":true,\"json_condensed_response\":true,\"kevlar_dropdown_fix\":true,\"kevlar_gel_error_routing\":true,\"log_heartbeat_with_lifecycles\":true,\"log_web_endpoint_to_layer\":true,\"networkless_gel\":true,\"networkless_logging\":true,\"nwl_send_fast_on_unload\":true,\"offline_error_handling\":true,\"pageid_as_header_web\":true,\"pes_migrate_association_data\":true,\"polymer_bad_build_labels\":true,\"polymer_verifiy_app_state\":true,\"qoe_send_and_write\":true,\"record_app_crashed_web\":true,\"render_unicode_emojis_as_images_counterfactual\":true,\"search_image_menu_alignment\":true,\"skip_ls_gel_retry\":true,\"sponsorships_upsell_in_picker_check_eligibility\":true,\"suppress_error_204_logging\":true,\"sw_nav_request_network_first\":true,\"use_bg_facade\":true,\"use_document_lifecycles\":true,\"use_screen_manager_util\":true,\"use_undefined_csn_any_layer\":true,\"vss_final_ping_send_and_write\":true,\"vss_playback_use_send_and_write\":true,\"web_api_url\":true,\"web_click_command_fallback\":true,\"web_dedupe_ve_grafting\":true,\"web_deprecate_service_ajax_map_dependency\":true,\"web_enable_ad_signals_in_it_context\":true,\"web_fetch_promise_cleanup_killswitch\":true,\"web_forward_command_on_pbj\":true,\"web_log_memory_total_kbytes\":true,\"web_ordered_response_processors\":true,\"web_prefetch_preload_video\":true,\"web_yt_config_context\":true,\"ytidb_fetch_datasync_ids_for_data_cleanup\":true,\"addto_ajax_log_warning_fraction\":0.1,\"log_window_onerror_fraction\":0.1,\"ytidb_transaction_ended_event_rate_limit\":0.02,\"botguard_async_snapshot_timeout_ms\":3000,\"check_navigator_accuracy_timeout_ms\":0,\"initial_gel_batch_timeout\":2000,\"network_polling_interval\":30000,\"polymer_log_prop_change_observer_percent\":0,\"web_foreground_heartbeat_interval_ms\":28000,\"web_logging_max_batch\":150,\"ytidb_remake_db_retries\":1,\"ytidb_reopen_db_retries\":0,\"cb_v2_uxe\":\"23983171\",\"web_client_version_override\":\"\",\"kevlar_command_handler_command_banlist\":[],\"web_op_continuation_type_banlist\":[],\"web_op_endpoint_banlist\":[],\"web_op_signal_type_banlist\":[]},\"INNERTUBE_API_KEY\":\"AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\",\"INNERTUBE_API_VERSION\":\"v1\",\"INNERTUBE_CLIENT_NAME\":\"WEB\",\"INNERTUBE_CLIENT_VERSION\":\"2.20220315.01.00\",\"INNERTUBE_CONTEXT\":{\"client\":{\"clientName\":\"WEB\",\"clientVersion\":\"2.20220315.01.00\"}},\"INNERTUBE_CONTEXT_CLIENT_NAME\":1,\"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"2.20220315.01.00\",\"LATEST_ECATCHER_SERVICE_TRACKING_PARAMS\":{\"client.name\":\"WEB\"}});window.ytcfg.obfuscatedData_ \u003d [];if (self.trustedTypes \u0026\u0026 self.trustedTypes.createPolicy) {const swPolicy \u003d trustedTypes.createPolicy(\u0027youtubeServiceWorkerPolicy\u0027, {createScriptURL: function(ignored) { return \u0027https:\\/\\/www.youtube.com\\/s\\/desktop\\/d3f3a825\\/jsbin\\/serviceworker-notifications.vflset\\/serviceworker-notifications.js\u0027; }}); importScripts(swPolicy.createScriptURL(\u0027\u0027));} else {importScripts(\u0027https:\\/\\/www.youtube.com\\/s\\/desktop\\/d3f3a825\\/jsbin\\/serviceworker-notifications.vflset\\/serviceworker-notifications.js\u0027);}",
+    "responseBody": "/** 16014900561114119181 */self.document \u003d self; self.window \u003d self;var ytcfg\u003d{d:function(){return window.yt\u0026\u0026yt.config_||ytcfg.data_||(ytcfg.data_\u003d{})},get:function(k,o){return k in ytcfg.d()?ytcfg.d()[k]:o},set:function(){var a\u003darguments;if(a.length\u003e1)ytcfg.d()[a[0]]\u003da[1];else for(var k in a[0])ytcfg.d()[k]\u003da[0][k]}};\nytcfg.set({\"EXPERIMENT_FLAGS\":{\"allow_music_base_url\":true,\"allow_skip_networkless\":true,\"clear_user_partitioned_ls\":true,\"deprecate_two_way_binding_child\":true,\"deprecate_two_way_binding_parent\":true,\"desktop_image_cta_no_background\":true,\"desktop_search_image_hover_state\":true,\"desktop_search_image_no_cta\":true,\"desktop_text_ads_gray_visurl\":true,\"disable_child_node_auto_formatted_strings\":true,\"disable_simple_mixed_direction_formatted_strings\":true,\"disable_thumbnail_preloading\":true,\"enable_client_sli_logging\":true,\"enable_gel_log_commands\":true,\"enable_gray_visurl\":true,\"enable_mixed_direction_formatted_strings\":true,\"enable_server_stitched_dai\":true,\"enable_sli_flush\":true,\"enable_tectonic_ad_ux_for_halftime\":true,\"enable_topsoil_wta_for_halftime_live_infra\":true,\"enable_web_eom_visitor_data\":true,\"export_networkless_options\":true,\"forward_domain_admin_state_on_embeds\":true,\"html5_enable_single_video_vod_ivar_on_pacf\":true,\"html5_enable_video_overlay_on_inplayer_slot_for_tv\":true,\"html5_pacf_enable_dai\":true,\"html5_recognize_predict_start_cue_point\":true,\"html5_server_stitched_dai_group\":true,\"json_condensed_response\":true,\"kevlar_dropdown_fix\":true,\"kevlar_gel_error_routing\":true,\"log_heartbeat_with_lifecycles\":true,\"log_web_endpoint_to_layer\":true,\"networkless_gel\":true,\"networkless_logging\":true,\"nwl_send_fast_on_unload\":true,\"offline_error_handling\":true,\"pageid_as_header_web\":true,\"pes_migrate_association_data\":true,\"polymer_bad_build_labels\":true,\"polymer_verifiy_app_state\":true,\"qoe_send_and_write\":true,\"record_app_crashed_web\":true,\"search_image_menu_alignment\":true,\"skip_ls_gel_retry\":true,\"suppress_error_204_logging\":true,\"sw_nav_request_network_first\":true,\"use_bg_facade\":true,\"use_document_lifecycles\":true,\"use_screen_manager_util\":true,\"use_undefined_csn_any_layer\":true,\"vss_final_ping_send_and_write\":true,\"vss_playback_use_send_and_write\":true,\"web_api_url\":true,\"web_click_command_fallback\":true,\"web_dedupe_ve_grafting\":true,\"web_deprecate_service_ajax_map_dependency\":true,\"web_enable_ad_signals_in_it_context\":true,\"web_forward_command_on_pbj\":true,\"web_inject_fetch_manager\":true,\"web_log_memory_total_kbytes\":true,\"web_ordered_response_processors\":true,\"web_prefetch_preload_video\":true,\"web_yt_config_context\":true,\"ytidb_fetch_datasync_ids_for_data_cleanup\":true,\"addto_ajax_log_warning_fraction\":0.1,\"log_window_onerror_fraction\":0.1,\"ytidb_transaction_ended_event_rate_limit\":0.02,\"botguard_async_snapshot_timeout_ms\":3000,\"check_navigator_accuracy_timeout_ms\":0,\"initial_gel_batch_timeout\":2000,\"network_polling_interval\":30000,\"polymer_log_prop_change_observer_percent\":0,\"web_foreground_heartbeat_interval_ms\":28000,\"web_logging_max_batch\":150,\"ytidb_remake_db_retries\":1,\"ytidb_reopen_db_retries\":0,\"cb_v2_uxe\":\"23983171\",\"web_client_version_override\":\"\",\"kevlar_command_handler_command_banlist\":[],\"web_op_continuation_type_banlist\":[],\"web_op_endpoint_banlist\":[],\"web_op_signal_type_banlist\":[]},\"INNERTUBE_API_KEY\":\"AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8\",\"INNERTUBE_API_VERSION\":\"v1\",\"INNERTUBE_CLIENT_NAME\":\"WEB\",\"INNERTUBE_CLIENT_VERSION\":\"2.20220331.06.00\",\"INNERTUBE_CONTEXT\":{\"client\":{\"clientName\":\"WEB\",\"clientVersion\":\"2.20220331.06.00\"}},\"INNERTUBE_CONTEXT_CLIENT_NAME\":1,\"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"2.20220331.06.00\",\"LATEST_ECATCHER_SERVICE_TRACKING_PARAMS\":{\"client.name\":\"WEB\"}});window.ytcfg.obfuscatedData_ \u003d [];if (self.trustedTypes \u0026\u0026 self.trustedTypes.createPolicy) {const swPolicy \u003d trustedTypes.createPolicy(\u0027youtubeServiceWorkerPolicy\u0027, {createScriptURL: function(ignored) { return \u0027https:\\/\\/www.youtube.com\\/s\\/desktop\\/01530da7\\/jsbin\\/serviceworker-notifications.vflset\\/serviceworker-notifications.js\u0027; }}); importScripts(swPolicy.createScriptURL(\u0027\u0027));} else {importScripts(\u0027https:\\/\\/www.youtube.com\\/s\\/desktop\\/01530da7\\/jsbin\\/serviceworker-notifications.vflset\\/serviceworker-notifications.js\u0027);}",
     "latestUrl": "https://www.youtube.com/sw.js"
   }
 }