Merge pull request #644 from adam-miller/adam/parameterize-yt-dlp-process-command

feat: Parameterize yt-dlp process command arguments

Author: ato

contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeDL.java

@@ -34,6 +34,7 @@
 import java.net.URI;
 import java.nio.channels.Channels;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
@@ -67,7 +68,7 @@
  * Extracts links to media by running yt-dlp in a subprocess. Runs only on html.
  *
  * <p>
- * Also implements {@link WARCRecordBuilder} to write youtube-dl json to the
+ * Also implements {@link WARCRecordBuilder} to write yt-dlp json to the
  * warc.
  * 
  * <p>
@@ -81,7 +82,7 @@
  * fetch chain, and to the end of the warc writer chain.
  * 
  * <p>
- * Keeps a log of containing pages and media captured as a result of youtube-dl
+ * Keeps a log of containing pages and media captured as a result of yt-dlp
  * extraction. The format of the log is as follows:
  *
  * <pre>
@@ -140,7 +141,7 @@ protected RandomAccessFile getLocalTempFile() {
             localTemp = openNewTempFile();
             tempfile.set(localTemp);
         }
-	logger.info("Getting youtube-dl temp file ");
+	logger.info("Getting yt-dlp temp file ");
         return localTemp;
     }
     protected boolean isOpen(RandomAccessFile f) {
@@ -149,12 +150,12 @@ protected boolean isOpen(RandomAccessFile f) {
             return true;
         }
         catch (IOException e) {
-	    logger.info("youtube-dl temp file is not open");
+	    logger.info("yt-dlp temp file is not open");
             return false ;
         }
     }
     protected RandomAccessFile openNewTempFile() {
-	logger.info("Opening New youtube-dl temp file ");
+	logger.info("Opening New yt-dlp temp file ");
 	File t;
         try {
             t = File.createTempFile("ydl", ".json");
@@ -194,6 +195,18 @@ public void setLogMetadataRecord(boolean logMetadataRecord) {
         kp.put("logMetadataRecord",logMetadataRecord);
     }
 
+    {
+        setProcessArguments(Arrays.asList("yt-dlp", "--ignore-config",
+                "--simulate", "--dump-single-json", "-S vcodec:h264,res:720,acodec:aac",
+                "--no-cache-dir", "--no-playlist", "--playlist-end=" + MAX_VIDEOS_PER_PAGE ));
+    }
+    public List<String> getProcessArguments() {
+        return (List<String>) kp.get("processArguments");
+    }
+    public void setProcessArguments(List<String> processArguments) {
+        kp.put("processArguments", processArguments);
+    }
+
     @Override
     public void start() {
         if (!isRunning) {
@@ -229,7 +242,7 @@ protected String readToEnd(Reader r) throws IOException {
      * - If {@code uri} is annotated "youtube-dl" and is an actual video
      *   download, log a line to ExtractorYoutubeDL.log
      *
-     * - If {@link #shouldExtract(CrawlURI)}, do youtube-dl extraction.
+     * - If {@link #shouldExtract(CrawlURI)}, do yt-dlp extraction.
      */
     @Override
     protected void extract(CrawlURI uri) {
@@ -400,7 +413,7 @@ public void write(int b) throws IOException {}
     }
 
     /**
-     * Streams through youtube-dl json output. Sticks video urls in
+     * Streams through yt-dlp json output. Sticks video urls in
      * <code>results.videoUrls</code>, web page urls in
      * <code>results.pageUrls</code>, and saves the json in anonymous temp file
      * <code>results.jsonFile</code>.
@@ -468,17 +481,16 @@ protected YoutubeDLResults runYoutubeDL(CrawlURI uri) {
          * the best audio with best acodec no better than aac and
          * with the smallest dimension no larger than 720.
          */
-        ProcessBuilder pb = new ProcessBuilder("yt-dlp", "--ignore-config",
-                "--simulate", "--dump-single-json", "-S vcodec:h264,res:720,acodec:aac",
-                "--no-cache-dir", "--no-playlist",
-                "--playlist-end=" + MAX_VIDEOS_PER_PAGE, uri.toString());
+        ArrayList<String> processArguments = new ArrayList<String>(getProcessArguments());
+        processArguments.add(uri.toString());
+        ProcessBuilder pb = new ProcessBuilder(processArguments);
         logger.info("running: " + String.join(" ", pb.command()));
 
         Process proc = null;
         try {
             proc = pb.start();
         } catch (IOException e) {
-            logger.log(Level.WARNING, "youtube-dl failed " + pb.command(), e);
+            logger.log(Level.WARNING, "yt-dlp failed " + pb.command(), e);
             return null;
         }
 
@@ -506,7 +518,7 @@ public String call() throws IOException {
                 try {
                     // this happens when there was no json output, which means no videos
                     // were found, totally normal
-                    logger.log(Level.FINE, "problem parsing json from youtube-dl " + pb.command() + " " + future.get());
+                    logger.log(Level.FINE, "problem parsing json from yt-dlp " + pb.command() + " " + future.get());
                 } catch (InterruptedException e1) {
                     throw new IOException(e1);
                 } catch (ExecutionException e1) {
@@ -515,15 +527,15 @@ public String call() throws IOException {
             }
         } catch (IOException e) {
             logger.log(Level.WARNING,
-                    "problem reading output from youtube-dl " + pb.command(),
+                    "problem reading output from yt-dlp " + pb.command(),
                     e);
             return null;
         } finally {
             try {
                 // the process should already have completed
                 proc.waitFor(1, TimeUnit.SECONDS);
             } catch (InterruptedException e) {
-                logger.warning("youtube-dl still running? killing it");
+                logger.warning("yt-dlp still running? killing it");
                 proc.destroyForcibly();
             }
             threadPool.shutdown();
@@ -540,13 +552,13 @@ protected boolean shouldProcess(CrawlURI uri) {
             return true;
         }
 
-        // Otherwise, check if we want to run youtube-dl on the url.
+        // Otherwise, check if we want to run yt-dlp on the url.
         return shouldExtract(uri);
     }
 
     /**
-     * Returns {@code true} if we should run youtube-dl on this url. We run
-     * youtube-dl on html 200s that are not too huge.
+     * Returns {@code true} if we should run yt-dlp on this url. We run
+     * yt-dlp on html 200s that are not too huge.
      */
     protected boolean shouldExtract(CrawlURI uri) {
         if (uri.getFetchStatus() != 200) {