Browser: Disable downloads in Firefox and Chrome

This hopefully will stop us filling up ~/Downloads with random junk.

Author: ato

commons/src/main/java/org/archive/net/webdriver/LocalWebDriverBiDi.java

@@ -58,18 +58,15 @@ public class LocalWebDriverBiDi implements WebDriverBiDi, Closeable {
     private static final String[] BROWSERS = new String[]{
             "firefox", "/Applications/Firefox.app/Contents/MacOS/firefox", "chromedriver"};
 
-    public LocalWebDriverBiDi(String executable, List<String> options, int proxyPort, Path profileDir)
+    public LocalWebDriverBiDi(String executable, List<String> options, Session.CapabilitiesRequest capabilities, Path profileDir)
             throws ExecutionException, InterruptedException, IOException {
         this.process = executable == null ? launchAnyBrowser(options, profileDir) : launchBrowser(executable, options, profileDir);
         Runtime.getRuntime().addShutdownHook(new Thread(this.process::destroyForcibly));
         new Thread(this::handleStderr).start();
         webSocket = HttpClient.newHttpClient().newWebSocketBuilder()
                 .buildAsync(URI.create(webSocketUrl.get() + "/session"), new Listener())
                 .get();
-        var alwaysMatch = Map.of("acceptInsecureCerts", true,
-                "proxy", new Session.ProxyConfiguration("manual",
-                        "127.0.0.1:" + proxyPort, "127.0.0.1:" + proxyPort));
-        sessionId = session().new_(new Session.CapabilitiesRequest(alwaysMatch, null)).sessionId();
+        sessionId = session().new_(capabilities).sessionId();
     }
 
     private static Process launchAnyBrowser(List<String> options, Path profileDir) throws IOException {

engine/src/main/java/org/archive/crawler/processor/Browser.java

@@ -187,10 +187,34 @@ public void start() {
             Path profileDir = crawlController.getScratchDir().getFile().toPath().resolve("profile");
             Files.createDirectories(profileDir);
 
-            // Firefox: send localhost requests via the proxy too (for tests and local crawling)
-            Files.writeString(profileDir.resolve("user.js"), "user_pref('network.proxy.allow_hijacking_localhost', true);");
-
-            this.webdriver = new LocalWebDriverBiDi(executable, options, proxy.getPort(), profileDir);
+            // Firefox doesn't seem to allow setting prefs via capabilities with bidi
+            // so drop them in user.js instead
+            Files.writeString(profileDir.resolve("user.js"), """
+            // send localhost requests via the proxy too (for tests and local crawling)
+            user_pref('network.proxy.allow_hijacking_localhost', true);
+            
+            // disable downloads by setting to something that can't be created as a directory
+            user_pref('browser.download.dir', '/dev/null');
+            user_pref('browser.download.folderList', 2);
+            """);
+
+            int proxyPort = proxy.getPort();
+            var alwaysMatch = Map.of("acceptInsecureCerts", true,
+                    "proxy", new Session.ProxyConfiguration("manual",
+                            "127.0.0.1:" + proxyPort, "127.0.0.1:" + proxyPort));
+
+            List<Map<String,Object>> firstMatch = List.of(
+                    Map.of("browserName", "chrome",
+                            "goog:chromeOptions", Map.of(
+                                    "args", List.of("headless=new", "user-data-dir=" + profileDir),
+                                    "prefs", Map.of("download_restrictions", 3))),
+                    // Fallback for other browsers
+                    Map.of()
+            );
+            var capabilities = new Session.CapabilitiesRequest(alwaysMatch, firstMatch);
+
+
+            this.webdriver = new LocalWebDriverBiDi(executable, options, capabilities, profileDir);
         } catch (Exception e) {
             logger.log(ERROR, "Error starting browser", e);
             throw new RuntimeException(e);

engine/src/test/java/org/archive/crawler/processor/BrowserTest.java

@@ -6,10 +6,9 @@
 import org.archive.modules.fetcher.DefaultServerCache;
 import org.archive.modules.fetcher.FetchHTTP2;
 import org.archive.net.UURIFactory;
+import org.archive.url.URIException;
 import org.archive.util.Recorder;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
 import org.junit.jupiter.api.io.TempDir;
 
@@ -18,73 +17,77 @@
 import java.net.InetSocketAddress;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import static java.lang.System.Logger.Level.DEBUG;
 import static org.junit.jupiter.api.Assertions.*;
 
+@EnabledIfSystemProperty(named = "runBrowserTests", matches = "true")
 class BrowserTest {
     private static final System.Logger logger = System.getLogger(BrowserTest.class.getName());
 
     private static HttpServer httpServer;
-
+    private static FetchHTTP2 fetcher;
+    private static Browser browser;
+    private static String baseUrl;
+    private static ArrayList<CrawlURI> subrequests;
+    private static CrawlController crawlController;
+    private Set<Recorder> recorders = new HashSet<>();
     @TempDir
     Path tempDir;
 
     @Test
-    @EnabledIfSystemProperty(named = "runBrowserTests", matches = "true")
+    @Disabled
     public void test() throws IOException, InterruptedException {
-        String url = "http://" + httpServer.getAddress().getAddress().getHostAddress() + ":" +
-                httpServer.getAddress().getPort() + "/";
-        var fetcher = new FetchHTTP2(new DefaultServerCache(), null);
-        fetcher.setUserAgentProvider(new CrawlMetadata());
-        fetcher.start();
-        try {
-            var crawlController = new CrawlController();
-            FetchChain fetchChain = new FetchChain();
-            fetchChain.setProcessors(List.of());
-            crawlController.setFetchChain(fetchChain);
-
-            var subrequests = new ArrayList<CrawlURI>();
-
-            DispositionChain dispositionChain = new DispositionChain();
-            dispositionChain.setProcessors(List.of(new Processor() {
-                @Override
-                protected boolean shouldProcess(CrawlURI uri) {
-                    return true;
-                }
+        CrawlURI crawlURI = newCrawlURI(baseUrl);
+        fetcher.process(crawlURI);
+        assertEquals(200, crawlURI.getFetchStatus());
+        browser.innerProcess(crawlURI);
 
-                @Override
-                protected void innerProcess(CrawlURI uri) throws InterruptedException {
-                    subrequests.add(uri);
-                }
-            }));
-            crawlController.setDispositionChain(dispositionChain);
-            crawlController.getScratchDir().setPath(tempDir.toString());
-            var browserProcessor = new Browser(fetcher, crawlController, event -> {}, null);
-            try {
-                browserProcessor.start();
-
-                CrawlURI crawlURI = new CrawlURI(UURIFactory.getInstance(url));
-                crawlURI.setRecorder(new Recorder(tempDir.toFile(), "fetcher"));
-                fetcher.process(crawlURI);
-                assertEquals(200, crawlURI.getFetchStatus());
-                browserProcessor.innerProcess(crawlURI);
-
-                var outLinks = new ArrayList<>(crawlURI.getOutLinks());
-                assertEquals("/link", outLinks.get(0).getUURI().getPath());
-                assertTrue(crawlURI.getAnnotations().contains("browser"));
-
-                logger.log(DEBUG, "Subrequests: {0}", subrequests);
-            } finally {
-                browserProcessor.stop();
-            }
-        } finally {
-            fetcher.stop();
+        var outLinks = new ArrayList<>(crawlURI.getOutLinks());
+        assertEquals("/link", outLinks.get(0).getUURI().getPath());
+        assertTrue(crawlURI.getAnnotations().contains("browser"));
+
+        logger.log(DEBUG, "Subrequests: {0}", subrequests);
+    }
+
+    @Test
+    public void testDownload() throws IOException, InterruptedException {
+        CrawlURI crawlURI = newCrawlURI(baseUrl + "download.bin");
+        fetcher.process(crawlURI);
+        assertEquals(200, crawlURI.getFetchStatus());
+        browser.innerProcess(crawlURI);
+    }
+
+    private CrawlURI newCrawlURI(String uri) throws URIException {
+        CrawlURI curi = new CrawlURI(UURIFactory.getInstance(uri));
+        Recorder recorder = new Recorder(tempDir.toFile(), "fetcher");
+        recorders.add(recorder);
+        curi.setRecorder(recorder);
+        return curi;
+    }
+
+    @BeforeEach
+    void setUp() {
+        crawlController.getScratchDir().setPath(tempDir.toString());
+    }
+
+    @AfterEach
+    void tearDown() {
+        subrequests.clear();
+        for (Recorder recorder : recorders) {
+            recorder.cleanup();
         }
     }
 
     @BeforeAll
+    static void setUpAll() throws Exception {
+        startHttpServer();
+        startProcessors();
+    }
+
     static void startHttpServer() throws IOException {
         httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), -1);
         httpServer.createContext("/", exchange -> {
@@ -101,6 +104,10 @@ static void startHttpServer() throws IOException {
                     body = "body { color: red; background: url(bg.jpg); }";
                     contentType = "text/css";
                 }
+                case "/download.bin" -> {
+                    body = "sample-download-file";
+                    contentType = "application/octet-stream";
+                }
                 default -> status = 404;
             }
             exchange.getResponseHeaders().add("Content-Type", contentType);
@@ -109,11 +116,43 @@ static void startHttpServer() throws IOException {
             exchange.close();
         });
         httpServer.start();
+        baseUrl = "http://" + httpServer.getAddress().getAddress().getHostAddress() + ":" +
+                     httpServer.getAddress().getPort() + "/";
+    }
+
+    static void startProcessors() {
+        fetcher = new FetchHTTP2(new DefaultServerCache(), null);
+        fetcher.setUserAgentProvider(new CrawlMetadata());
+        fetcher.start();
+        crawlController = new CrawlController();
+        FetchChain fetchChain = new FetchChain();
+        fetchChain.setProcessors(List.of());
+        crawlController.setFetchChain(fetchChain);
+
+        subrequests = new ArrayList<CrawlURI>();
+
+        DispositionChain dispositionChain = new DispositionChain();
+        dispositionChain.setProcessors(List.of(new Processor() {
+            @Override
+            protected boolean shouldProcess(CrawlURI uri) {
+                return true;
+            }
+
+            @Override
+            protected void innerProcess(CrawlURI uri) throws InterruptedException {
+                subrequests.add(uri);
+            }
+        }));
+        crawlController.setDispositionChain(dispositionChain);
+        browser = new Browser(fetcher, crawlController, event -> {}, null);
+            browser.start();
     }
 
     @AfterAll
-    static void stopHttpServer() {
+    static void tearDownAll() {
         if (httpServer != null) httpServer.stop(0);
+        if (browser != null) browser.stop();
+        if (fetcher != null) fetcher.stop();
     }
 
 }