refactor: innertube api refactor

Author: trldvix

README.md

@@ -9,14 +9,14 @@
 
 ### This library uses undocumented YouTube API, so it's possible that it will stop working at any time. Use at your own risk.
 
-> **Note:** If you want to use this library on Android platform, refer to
+> **Note:** If you want to use this library on an Android platform, refer to
 > [Android compatibility](#-android-compatibility).
 
 ## 📖 Introduction
 
 Java library which allows you to retrieve subtitles/transcripts for a YouTube video.
 It supports manual and automatically generated subtitles, bulk transcript retrieval for all videos in the playlist or
-on the channel and does not use headless browser for scraping.
+on the channel and does not use a headless browser for scraping.
 Inspired by [Python library](https://github.com/jdepoix/youtube-transcript-api).
 
 ## ☑️ Features
@@ -60,6 +60,15 @@ implementation 'io.github.thoroldvix:youtube-transcript-api:0.3.6'
 implementation("io.github.thoroldvix:youtube-transcript-api:0.3.6")
 ```
 
+## ❗ IMPORTANT ❗
+
+YouTube has started blocking most IPs that belong to cloud providers (like AWS, Google Cloud Platform, Azure, etc.),
+which means you most likely will get access errors when deploying to any cloud solution. It is also possible that
+YouTube will block you even if you run it locally, it will happen if you make too many requests, mainly when
+using [bulk transcript retrieval](#bulk-transcript-retrieval).
+To avoid this, you will need to use rotating proxies like [Webshare](https://www.webshare.io/?referral_code=g0ylrg6pzy7f) (referral link) or similar solutions.
+You can read on how to make a library use your proxy [here](#youtubeclient-customization-and-proxy).
+
 ## 🔰 Getting Started
 
 To start using YouTube Transcript API, you need to create an instance of `YoutubeTranscriptApi` by
@@ -81,15 +90,15 @@ for [finding specific transcripts](#find-transcripts) by language or by type (ma
 ```java
 TranscriptList transcriptList = youtubeTranscriptApi.listTranscripts("videoId");
 
-// Iterate over transcript list
-for(Transcript transcript : transcriptList) {
-        System.out.println(transcript);
+// Iterate over a transcript list
+for(Transcript transcript : transcriptList){
+    System.out.println(transcript);
 }
 
 // Find transcript in specific language
 Transcript transcript = transcriptList.findTranscript("en");
 
-// Find manually created transcript
+// Find a manually created transcript
 Transcript manualyCreatedTranscript = transcriptList.findManualTranscript("en");
 
 // Find automatically generated transcript
@@ -138,18 +147,19 @@ TranscriptContent transcriptContent = youtubeTranscriptApi.listTranscripts("vide
 Given that English is the most common language, you can omit the language code, and it will default to English:
 
 ```java
-// Retrieve transcript content in english
+// Retrieve transcript content in English
 TranscriptContent transcriptContent = youtubeTranscriptApi.listTranscripts("videoId")
-        //no language code defaults to english
-        .findTranscript()
-        .fetch();
+                //no language code defaults to English
+                .findTranscript()
+                .fetch();
 // Or
 TranscriptContent transcriptContent = youtubeTranscriptApi.getTranscript("videoId");
 ```
 
 For bulk transcript retrieval see [Bulk Transcript Retrieval](#bulk-transcript-retrieval).
 
 ## 🤖 Android compatibility
+
 This library uses Java 11 HttpClient for making YouTube requests by default, it was done so it depends on minimal amount
 of 3rd party libraries. Since Android SDK doesn't include Java 11 HttpClient, you will have to implement
 your own `YoutubeClient` for it to work.
@@ -160,7 +170,8 @@ You can check how to do it in [YoutubeClient Customization and Proxy](#youtubecl
 
 ### Use fallback language
 
-In case if desired language is not available, instead of getting an exception you can pass some other languages that
+In case if the desired language is not available, instead of getting an exception, you can pass some other languages
+that
 will be used as a fallback.
 
 For example:
@@ -260,15 +271,14 @@ By default, `YoutubeTranscriptApi` uses Java 11 HttpClient for making requests t
 different client or use a proxy,
 you can create your own YouTube client by implementing the `YoutubeClient` interface.
 
-Here is example implementation using OkHttp:
+Here is an example implementation using OkHttp:
 
 ```java
 public class OkHttpYoutubeClient implements YoutubeClient {
-
     private final OkHttpClient client;
 
     public OkHttpYoutubeClient() {
-      this.client = new OkHttpClient();
+        this.client = new OkHttpClient();
     }
 
     @Override
@@ -278,67 +288,61 @@ public class OkHttpYoutubeClient implements YoutubeClient {
                 .url(url)
                 .build();
 
-        return sendGetRequest(request);
+        return executeRequest(request);
     }
 
     @Override
-    public String get(YtApiV3Endpoint endpoint, Map<String, String> params) throws TranscriptRetrievalException {
+    public String post(String url, String json) throws TranscriptRetrievalException {
+        RequestBody requestBody = RequestBody.create(json, MediaType.parse("application/json; charset=utf-8"));
+
         Request request = new Request.Builder()
-                .url(endpoint.url(params))
+                .url(url)
+                .post(requestBody)
                 .build();
 
-        return sendGetRequest(request);
+        return executeRequest(request);
     }
 
-    private String sendGetRequest(Request request) throws TranscriptRetrievalException {
+    private String executeRequest(Request request) throws TranscriptRetrievalException {
         try (Response response = client.newCall(request).execute()) {
             if (response.isSuccessful()) {
-                ResponseBody body = response.body();
-                if (body == null) {
+                ResponseBody responseBody = response.body();
+                if (responseBody == null) {
                     throw new TranscriptRetrievalException("Response body is null");
                 }
-                return body.string();
+                return responseBody.string();
             }
         } catch (IOException e) {
-            throw new TranscriptRetrievalException("Failed to retrieve data from YouTube", e);
+            throw new TranscriptRetrievalException("HTTP request failed", e);
         }
-        throw new TranscriptRetrievalException("Failed to retrieve data from YouTube");
+
+        throw new TranscriptRetrievalException("HTTP request failed with non-successful response");
     }
 }
 ```
-After implementing your custom `YouTubeClient` you will need to pass it to `TranscriptApiFactory` `createWithClient` method.
+
+After implementing your custom `YouTubeClient` you will need to pass it to `TranscriptApiFactory` `createWithClient`
+method.
 
 ```java
 YoutubeClient okHttpClient = new OkHttpYoutubeClient();
 YoutubeTranscriptApi youtubeTranscriptApi = TranscriptApiFactory.createWithClient(okHttpClient);
 ```
 
 ### Cookies
-
-Some videos may be age-restricted, requiring authentication to access the transcript.
-To achieve this, obtain access to the desired video in a browser and download the cookies in Netscape format, storing
-them as a TXT file.
-You can use extensions
-like [Get cookies.txt LOCALLY](https://chromewebstore.google.com/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc)
-for Chrome or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) for Firefox to do this.
-`YoutubeTranscriptApi` contains `listTranscriptsWithCookies` and `getTranscriptWithCookies` which accept a path to the
-cookies.txt file.
-
-```java
-// Retrieve transcript list
-TranscriptList transcriptList = youtubeTranscriptApi.listTranscriptsWithCookies("videoId", "path/to/cookies.txt");
-
-// Get transcript content
-TranscriptContent transcriptContent = youtubeTranscriptApi.getTranscriptWithCookies("videoId", "path/to/cookies.txt", "en");
-```
+Some videos are age-restricted, so this library won't be able to access those videos without some sort of authentication.
+Unfortunately, some recent changes to the YouTube API have broken the current implementation of cookie-based
+authentication, so this feature is currently not available.
 
 ### Bulk Transcript Retrieval
 
-There are a few methods for bulk transcript retrieval in `YoutubeTranscriptApi` 
+#### ❗You will most likely get [IP blocked](#-important-) by YouTube if you use this❗
+
+There are a few methods for bulk transcript retrieval in `YoutubeTranscriptApi`
 
-Playlists and channels information is retrieved from
+Playlists and channels information are retrieved from
 the [YouTube V3 API](https://developers.google.com/youtube/v3/docs/),
-so you will need to provide API key for all methods.
+so you will need to provide an API key for all methods.
 
 All methods take a `TranscriptRequest` object as a parameter,
 which contains the following fields:
@@ -348,8 +352,6 @@ which contains the following fields:
   fail fast by throwing an error if one of the transcripts could not be retrieved,
   otherwise it will ignore failed transcripts.
 
-- `cookies` (optional) - Path to [cookies.txt](#cookies) file.
-
 All methods return a map which contains the video ID as a key and the corresponding result as a value.
 
 ```java
@@ -426,10 +428,28 @@ undocumented API URL embedded within its HTML. This JSON looks like this:
 }
 ```
 
-This library works by making a single GET request to the YouTube page of the specified video, extracting the JSON data
-from the HTML, and parsing it to obtain a list of all available transcripts. To fetch the transcript content, it then
-sends a GET request to the API URL extracted from the JSON. The YouTube API returns the transcript content in XML
-format, like this:
+Before you could directly extract this JSON from video page HTML and call extracted API URL, but YouTube fixed this by
+not allowing
+requests to the URL that is embedded in this JSON,
+but there is a workaround. Each video page also contains an INNERTUBE_API_KEY field, which can be used to access
+internal YouTube API. Because of this you can make POST request to this URL
+`https://www.youtube.com/youtubei/v1/player?key=INNERTUBE_API_KEY` with a body like this:
+
+```json
+{
+  "context": {
+    "client": {
+      "clientName": "ANDROID",
+      "clientVersion": "20.10.38"
+    }
+  },
+  "videoId": "dQw4w9WgXcQ"
+}
+```
+
+To retrieve JSON that is similar to the JSON contained in the video page HTML. Extracted API URL is then
+called to retrieve the content of the transcript,
+it has an XML format and looks like this
 
 ```xml
 <?xml version="1.0" encoding="utf-8" ?>

gradle/libs.versions.toml

@@ -6,6 +6,7 @@ jackson = "2.17.2"
 apache-commons-text = "1.12.0"
 maven-publish = "0.29.0"
 gradle-release = "3.0.2"
+jspecify = "1.0.0"
 
 [libraries]
 junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" }
@@ -14,6 +15,7 @@ assertj-core = { module = "org.assertj:assertj-core", version.ref = "assertj" }
 mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version.ref = "mockito" }
 jackson-dataformat-xml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-xml", version.ref = "jackson" }
 apache-commons-text = { module = "org.apache.commons:commons-text", version.ref = "apache-commons-text" }
+jspecify = { module = "org.jspecify:jspecify", version.ref = "jspecify"}
 
 [plugins]
 maven-publish = { id = "com.vanniktech.maven.publish", version.ref = "maven-publish" }

lib/build.gradle.kts

@@ -6,8 +6,8 @@ object Metadata {
     const val GROUP_ID = "io.github.thoroldvix"
     const val LICENSE = "MIT"
     const val LICENSE_URL = "https://opensource.org/licenses/MIT"
-    const val GITHUB_REPO = "thoroldvix/youtube-transcript-api"
-    const val DEVELOPER_ID = "thoroldvix"
+    const val GITHUB_REPO = "trldvix/youtube-transcript-api"
+    const val DEVELOPER_ID = "trldvix"
     const val DEVELOPER_NAME = "Alexey Bobkov"
     const val DEVELOPER_EMAIL = "[email protected]"
 }
@@ -34,6 +34,7 @@ tasks.getByName<Test>("test") {
 dependencies {
     implementation(libs.jackson.dataformat.xml)
     implementation(libs.apache.commons.text)
+    implementation(libs.jspecify)
 
     testRuntimeOnly(libs.junit.jupiter.platform.launcher)
     testImplementation(libs.junit.jupiter)

lib/src/main/java/io/github/thoroldvix/api/BulkTranscriptRequest.java

@@ -0,0 +1,41 @@
+package io.github.thoroldvix.api;
+
+/**
+ * Request object for retrieving transcripts.
+ * <p>
+ * Contains an API key required for the YouTube V3 API,
+ * and optionally a file path to the text file containing the authentication cookies. If cookies are not provided, the API will not be able to access age restricted videos.
+ * Also contains a flag to stop on error or continue on error. Defaults to false if not provided.
+ * </p>
+ */
+public class BulkTranscriptRequest {
+    private final String apiKey;
+    private final boolean stopOnError;
+
+    public BulkTranscriptRequest(String apiKey, boolean stopOnError) {
+        if (apiKey.isBlank()) {
+            throw new IllegalArgumentException("API key cannot be empty");
+        }
+        this.apiKey = apiKey;
+        this.stopOnError = stopOnError;
+    }
+
+    public BulkTranscriptRequest(String apiKey) {
+        this(apiKey, true);
+    }
+
+    /**
+     * @return API key for the YouTube V3 API (see <a href="https://developers.google.com/youtube/v3/getting-started">Getting started</a>)
+     */
+    public String getApiKey() {
+        return apiKey;
+    }
+
+    /**
+     * @return Whether to stop if transcript retrieval fails for a video. If false, all transcripts that could not be retrieved will be skipped,
+     * otherwise an exception will be thrown on the first error.
+     */
+    public boolean isStopOnError() {
+        return stopOnError;
+    }
+}