Browser: Handle navigation abort from downloads starting

Author: ato

commons/src/main/java/org/archive/net/webdriver/LocalWebDriverBiDi.java

@@ -173,7 +173,12 @@ public <T extends BiDiModule> T module(Class<T> moduleClass) {
                                     parameterizedType.getActualTypeArguments()[0] instanceof Class<?> typeClass) {
                                 yield future.thenApply(result -> BiDiJson.fromJson(result, typeClass));
                             } else {
-                                yield BiDiJson.fromJson(future.get(30, TimeUnit.SECONDS), method.getReturnType());
+                                try {
+                                    yield BiDiJson.fromJson(future.get(30, TimeUnit.SECONDS), method.getReturnType());
+                                } catch (ExecutionException e) {
+                                    if (e.getCause() instanceof RuntimeException re) throw re;
+                                    throw e;
+                                }
                             }
                         }
                     };

engine/src/main/java/org/archive/crawler/processor/Browser.java

@@ -19,6 +19,7 @@
 
 package org.archive.crawler.processor;
 
+import org.apache.commons.lang.StringUtils;
 import org.archive.crawler.event.CrawlURIDispositionEvent;
 import org.archive.crawler.framework.CrawlController;
 import org.archive.crawler.framework.Frontier;
@@ -115,6 +116,9 @@ protected boolean shouldProcess(CrawlURI curi) {
         if (!scheme.equals("https") && !scheme.equals("http")) return false;
         String mime = curi.getContentType().toLowerCase();
         if (!mime.startsWith("text/html")) return false;
+        if (StringUtils.startsWithIgnoreCase(curi.getHttpResponseHeader("Content-Disposition"), "attachment")) {
+            return false;
+        }
         return true;
     }
 
@@ -143,7 +147,16 @@ private void visit(CrawlURI curi) {
             webdriver.network().addIntercept(List.of(Network.InterceptPhase.beforeRequestSent), List.of(tab),
                     List.of(new Network.UrlPatternPattern("pattern", "http"),
                             new Network.UrlPatternPattern("pattern", "https")));
-            var navigation = webdriver.browsingContext().navigate(tab, curi.getURI(), BrowsingContext.ReadinessState.complete);
+            BrowsingContext.NavigateResult navigation;
+
+            try {
+                navigation = webdriver.browsingContext().navigate(tab, curi.getURI(), BrowsingContext.ReadinessState.complete);
+            } catch (WebDriverException e) {
+                if (e.getMessage().equals("net::ERR_ABORTED")) return; // Chrome: probably download started
+                throw e;
+            }
+            if (navigation.url().equals("about:blank")) return; // Firefox: probably download started
+
             logger.log(System.Logger.Level.DEBUG, "Navigated to {0}", navigation);
 
             // Wait for network activity to stop

engine/src/test/java/org/archive/crawler/processor/BrowserTest.java

@@ -58,7 +58,11 @@ public void testDownload() throws IOException, InterruptedException {
         CrawlURI crawlURI = newCrawlURI(baseUrl + "download.bin");
         fetcher.process(crawlURI);
         assertEquals(200, crawlURI.getFetchStatus());
+        assertFalse(browser.shouldProcess(crawlURI), "content-disposition header should skip processing");
+
+        // force processing anyway to test the behavior for other download reasons (e.g. non-HTML)
         browser.innerProcess(crawlURI);
+        assertFalse(crawlURI.getAnnotations().contains("browser"), "navigation should have aborted");
     }
 
     private CrawlURI newCrawlURI(String uri) throws URIException {
@@ -106,7 +110,7 @@ static void startHttpServer() throws IOException {
                 }
                 case "/download.bin" -> {
                     body = "sample-download-file";
-                    contentType = "application/octet-stream";
+                    exchange.getResponseHeaders().add("Content-Disposition", "attachment; filename=heritrix-test.bin");
                 }
                 default -> status = 404;
             }