Browser: Add processor report

Author: ato

engine/src/main/java/org/archive/crawler/processor/Browser.java

@@ -43,6 +43,8 @@
 import org.archive.util.IdleBarrier;
 import org.archive.util.Recorder;
 import org.eclipse.jetty.client.Result;
+import org.json.JSONException;
+import org.json.JSONObject;
 import org.springframework.context.ApplicationEventPublisher;
 
 import java.io.IOException;
@@ -57,6 +59,7 @@
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Stream;
 
 import static java.lang.System.Logger.Level.ERROR;
@@ -80,6 +83,7 @@ public class Browser extends Processor {
     protected final Map<String, BrowserPage> pages = new ConcurrentHashMap<>();
     protected final Map<BrowsingContext.Context, String> pageIdsByContext = new ConcurrentHashMap<>();
     protected final ProcessorChain extractorChain = new ProcessorChain();
+    protected final AtomicLong subresourcesRecorded = new AtomicLong();
     protected List<Behavior> behaviors;
     protected String executable;
     protected List<String> options = List.of("--headless");
@@ -137,6 +141,29 @@ public void innerProcess(CrawlURI curi) {
         }
     }
 
+    @Override
+    protected JSONObject toCheckpointJson() throws JSONException {
+        return super.toCheckpointJson().put("subresourcesRecorded", subresourcesRecorded.get());
+    }
+
+    @Override
+    protected void fromCheckpointJson(JSONObject json) throws JSONException {
+        super.fromCheckpointJson(json);
+        subresourcesRecorded.set(json.getLong("subresourcesRecorded"));
+    }
+
+    @Override
+    public String report() {
+        StringBuilder builder = new StringBuilder();
+        builder.append(super.report());
+        builder.append("  Pages visited: ").append(getURICount()).append("\n");
+        builder.append("  Subresources recorded: ").append(subresourcesRecorded.get()).append("\n");
+        for (var behavior : behaviors) {
+            builder.append(behavior.report());
+        }
+        return builder.toString();
+    }
+
     private void visit(CrawlURI curi) {
         String pageId = UUID.randomUUID().toString();
         var tab = webdriver.browsingContext().create(BrowsingContext.CreateType.tab).context();
@@ -398,6 +425,7 @@ public void onComplete(Result result) {
                 if (recordingFailed) {
                     curi.setFetchStatus(FetchStatusCodes.S_RUNTIME_EXCEPTION);
                 } else {
+                    subresourcesRecorded.incrementAndGet();
                     curi.getOverlayNames(); // for sideeffect of creating the overlayNames list
 
                     Frontier frontier = crawlController.getFrontier();

modules/src/main/java/org/archive/modules/behaviors/Behavior.java

@@ -21,4 +21,8 @@
 
 public interface Behavior {
     void run(Page page);
+
+    default String report() {
+        return "  Behavior: " + getClass().getName() + "\n";
+    }
 }

modules/src/main/java/org/archive/modules/behaviors/ExtractLinks.java

@@ -28,8 +28,10 @@
 import org.archive.net.UURIFactory;
 
 import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
 
 public class ExtractLinks implements Behavior {
+    private final AtomicLong numberOfLinksExtracted = new AtomicLong(0);
     private final UriErrorLoggerModule loggerModule;
 
     public ExtractLinks(UriErrorLoggerModule loggerModule) {
@@ -56,9 +58,15 @@ public void run(Page page) {
                 UURI dest = UURIFactory.getInstance(page.curi().getUURI(), url);
                 CrawlURI link = page.curi().createCrawlURI(dest, LinkContext.NAVLINK_MISC, Hop.NAVLINK);
                 page.curi().getOutLinks().add(link);
+                numberOfLinksExtracted.incrementAndGet();
             } catch (URIException e) {
                 loggerModule.logUriError(e, page.curi().getUURI(), url);
             }
         }
     }
+
+    @Override
+    public String report() {
+        return Behavior.super.report() + "    Links extracted: " + numberOfLinksExtracted.get() + "\n";
+    }
 }