Merge pull request #30 from SWAT-engineering/improved-overflow-support/update-readme

Improved overflow support: Update README

Author: DavyLandman

README.md

@@ -7,13 +7,14 @@ a java file watcher that works across platforms and supports recursion, single f
 Features:
 
 - monitor a single file (or directory) for changes
-- monitor a directory for changes to it's direct descendants
-- monitor a directory for changes for all it's descendants (aka recursive directory watch)
+- monitor a directory for changes to its direct descendants
+- monitor a directory for changes for all its descendants (aka recursive directory watch)
 - edge cases dealt with:
-  - in case of overflow we will still generate events for new descendants
   - recursive watches will also continue in new directories
   - multiple watches for the same directory are merged to avoid overloading the kernel
   - events are processed in a configurable worker pool
+  - when an overflow happens, automatically approximate the events that were
+    missed using a configurable approximation policy
 
 Planned features:
 
@@ -39,6 +40,7 @@ Start using java-watch:
 var directory = Path.of("tmp", "test-dir");
 var watcherSetup = Watcher.watch(directory, WatchScope.PATH_AND_CHILDREN)
     .withExecutor(Executors.newCachedThreadPool()) // optionally configure a custom thread pool
+    .onOverflow(Approximation.DIRTY) // optionally configure a handler for overflows
     .on(watchEvent -> {
         System.err.println(watchEvent);
     });

src/main/java/engineering/swat/watch/OnOverflow.java renamed to src/main/java/engineering/swat/watch/Approximation.java

@@ -38,7 +38,7 @@
  * overflow issue), but it doesn't have to (e.g., it may carry out additional
  * overflow bookkeeping).
  */
-public enum OnOverflow {
+public enum Approximation {
 
     /**
      * Synthetic events are issued for <b>no regular files/directories</b> in
@@ -66,8 +66,8 @@ public enum OnOverflow {
      *
      * <p>
      * This approach is relatively cheap in terms of memory usage (cf.
-     * {@link #DIRTY}), but it results in a large over/underapproximation of the
-     * actual events (cf. DIRTY).
+     * {@link #DIFF}), but it results in a large over/underapproximation of the
+     * actual events (cf. DIFF).
      */
     ALL,
 
@@ -76,7 +76,8 @@ public enum OnOverflow {
      * <p>
      * Synthetic events of kinds {@link WatchEvent.Kind#CREATED},
      * {@link WatchEvent.Kind#MODIFIED}, and {@link WatchEvent.Kind#DELETED} are
-     * issued for dirty regular files/directories in the scope of the watch, as
+     * issued for regular files/directories in the scope of the watch, when
+     * their current versions are different from their previous versions, as
      * determined using <i>last-modified-times</i>. Specifically, when an
      * overflow event happens:
      *
@@ -103,5 +104,5 @@ public enum OnOverflow {
      * but it is relatively expensive in terms of memory usage (cf. ALL), as the
      * watch needs to keep track of last-modified-times.
      */
-    DIRTY
+    DIFF
 }

src/main/java/engineering/swat/watch/Watcher.java

@@ -57,7 +57,7 @@ public class Watcher {
     private final Logger logger = LogManager.getLogger();
     private final Path path;
     private final WatchScope scope;
-    private volatile OnOverflow approximateOnOverflow = OnOverflow.ALL;
+    private volatile Approximation approximateOnOverflow = Approximation.ALL;
     private volatile Executor executor = CompletableFuture::runAsync;
 
     private static final BiConsumer<EventHandlingWatch, WatchEvent> EMPTY_HANDLER = (w, e) -> {};
@@ -174,12 +174,12 @@ public Watcher withExecutor(Executor callbackHandler) {
      * {@link WatchEvent.Kind#CREATED}, {@link WatchEvent.Kind#MODIFIED}, and/or
      * {@link WatchEvent.Kind#DELETED}) should be issued when an overflow event
      * happens. If not defined before this watcher is started, the
-     * {@link engineering.swat.watch.OnOverflow#ALL} approach will be used.
+     * {@link Approximation#ALL} approach will be used.
      * @param whichFiles Constant to indicate for which regular
      * files/directories to approximate
      * @return This watcher for optional method chaining
      */
-    public Watcher approximate(OnOverflow whichFiles) {
+    public Watcher onOverflow(Approximation whichFiles) {
         this.approximateOnOverflow = whichFiles;
         return this;
     }
@@ -233,7 +233,7 @@ private BiConsumer<EventHandlingWatch, WatchEvent> applyApproximateOnOverflow()
                 return eventHandler;
             case ALL:
                 return eventHandler.andThen(new MemorylessRescanner(executor));
-            case DIRTY:
+            case DIFF:
                 return eventHandler.andThen(new IndexingRescanner(executor, path, scope));
             default:
                 throw new UnsupportedOperationException("No event handler has been defined yet for this overflow policy");

src/test/java/engineering/swat/watch/RecursiveWatchTests.java

@@ -149,8 +149,8 @@ void deleteOfFileInDirectoryShouldBeVisible() throws IOException {
     }
 
     @ParameterizedTest
-    @EnumSource // Repeat test for each `OnOverflow` value
-    void overflowsAreRecoveredFrom(OnOverflow whichFiles) throws IOException, InterruptedException {
+    @EnumSource // Repeat test for each `Approximation` value
+    void overflowsAreRecoveredFrom(Approximation whichFiles) throws IOException, InterruptedException {
         var parent = testDir.getTestDirectory();
         var descendants = new Path[] {
             Path.of("foo"),
@@ -180,7 +180,7 @@ void overflowsAreRecoveredFrom(OnOverflow whichFiles) throws IOException, Interr
         var dropEvents = new AtomicBoolean(false); // Toggles overflow simulation
         var watchConfig = Watcher.watch(parent, WatchScope.PATH_AND_ALL_DESCENDANTS)
             .withExecutor(ForkJoinPool.commonPool())
-            .approximate(whichFiles)
+            .onOverflow(whichFiles)
             .filter(e -> !dropEvents.get())
             .on(events::add);
 
@@ -207,7 +207,7 @@ void overflowsAreRecoveredFrom(OnOverflow whichFiles) throws IOException, Interr
             var overflow = new WatchEvent(WatchEvent.Kind.OVERFLOW, parent);
             watch.handleEvent(overflow);
 
-            if (whichFiles != OnOverflow.NONE) { // Auto-handler is configured
+            if (whichFiles != Approximation.NONE) { // Auto-handler is configured
                 for (var descendant : descendants) {
                     awaitCreation.accept(descendant);
                     awaitCreation.accept(descendant.resolve(file1));

src/test/java/engineering/swat/watch/SingleDirectoryTests.java

@@ -134,7 +134,7 @@ void memorylessRescanOnOverflow() throws IOException, InterruptedException {
         var nModified = new AtomicInteger();
         var nOverflow = new AtomicInteger();
         var watchConfig = Watcher.watch(directory, WatchScope.PATH_AND_CHILDREN)
-            .approximate(OnOverflow.ALL)
+            .onOverflow(Approximation.ALL)
             .on(e -> {
                 switch (e.getKind()) {
                     case CREATED:
@@ -179,7 +179,7 @@ void indexingRescanOnOverflow() throws IOException, InterruptedException {
         var nDeleted = new AtomicInteger();
 
         var watchConfig = Watcher.watch(directory, WatchScope.PATH_AND_CHILDREN)
-            .approximate(OnOverflow.DIRTY)
+            .onOverflow(Approximation.DIFF)
             .on(e -> {
                 var kind = e.getKind();
                 if (kind != OVERFLOW) {

src/test/java/engineering/swat/watch/SingleFileTests.java

@@ -135,7 +135,7 @@ void singleFileThatMonitorsOnlyADirectory() throws IOException, InterruptedExcep
     @Test
     void noRescanOnOverflow() throws IOException, InterruptedException {
         var bookkeeper = new Bookkeeper();
-        try (var watch = startWatchAndTriggerOverflow(OnOverflow.NONE, bookkeeper)) {
+        try (var watch = startWatchAndTriggerOverflow(Approximation.NONE, bookkeeper)) {
             Thread.sleep(TestHelper.SHORT_WAIT.toMillis());
 
             await("Overflow shouldn't trigger created, modified, or deleted events")
@@ -148,7 +148,7 @@ void noRescanOnOverflow() throws IOException, InterruptedException {
     @Test
     void memorylessRescanOnOverflow() throws IOException, InterruptedException {
         var bookkeeper = new Bookkeeper();
-        try (var watch = startWatchAndTriggerOverflow(OnOverflow.ALL, bookkeeper)) {
+        try (var watch = startWatchAndTriggerOverflow(Approximation.ALL, bookkeeper)) {
             Thread.sleep(TestHelper.SHORT_WAIT.toMillis());
 
             var isFile = Predicate.isEqual(watch.getPath());
@@ -167,13 +167,13 @@ void memorylessRescanOnOverflow() throws IOException, InterruptedException {
         }
     }
 
-    private ActiveWatch startWatchAndTriggerOverflow(OnOverflow whichFiles, Bookkeeper bookkeeper) throws IOException {
+    private ActiveWatch startWatchAndTriggerOverflow(Approximation whichFiles, Bookkeeper bookkeeper) throws IOException {
         var parent = testDir.getTestDirectory();
         var file = parent.resolve("a.txt");
 
         var watch = Watcher
             .watch(file, WatchScope.PATH_ONLY)
-            .approximate(whichFiles)
+            .onOverflow(whichFiles)
             .on(bookkeeper)
             .start();
 

src/test/java/engineering/swat/watch/TortureTests.java

@@ -145,8 +145,8 @@ Set<Path> stop() throws InterruptedException {
     private static final int THREADS = 4;
 
     @ParameterizedTest
-    @EnumSource(names = { "ALL", "DIRTY" })
-    void pressureOnFSShouldNotMissNewFilesAnything(OnOverflow whichFiles) throws InterruptedException, IOException {
+    @EnumSource(names = { "ALL", "DIFF" })
+    void pressureOnFSShouldNotMissNewFilesAnything(Approximation whichFiles) throws InterruptedException, IOException {
         final var root = testDir.getTestDirectory();
         var pool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 4);
 
@@ -155,7 +155,7 @@ void pressureOnFSShouldNotMissNewFilesAnything(OnOverflow whichFiles) throws Int
         var seenCreates = ConcurrentHashMap.<Path>newKeySet();
         var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.PATH_AND_ALL_DESCENDANTS)
             .withExecutor(pool)
-            .approximate(whichFiles)
+            .onOverflow(whichFiles)
             .on(ev -> {
                 var fullPath = ev.calculateFullPath();
                 switch (ev.getKind()) {
@@ -267,14 +267,14 @@ void manyRegistrationsForSamePath() throws InterruptedException, IOException {
         }
     }
 
-    static Stream<OnOverflow> manyRegisterAndUnregisterSameTimeSource() {
-        OnOverflow[] values = { OnOverflow.ALL, OnOverflow.DIRTY };
+    static Stream<Approximation> manyRegisterAndUnregisterSameTimeSource() {
+        Approximation[] values = { Approximation.ALL, Approximation.DIFF };
         return TestHelper.streamOf(values, 5);
     }
 
     @ParameterizedTest
     @MethodSource("manyRegisterAndUnregisterSameTimeSource")
-    void manyRegisterAndUnregisterSameTime(OnOverflow whichFiles) throws InterruptedException, IOException {
+    void manyRegisterAndUnregisterSameTime(Approximation whichFiles) throws InterruptedException, IOException {
         var startRegistering = new Semaphore(0);
         var startedWatching = new Semaphore(0);
         var stopAll = new Semaphore(0);
@@ -296,7 +296,7 @@ void manyRegisterAndUnregisterSameTime(OnOverflow whichFiles) throws Interrupted
                         for (int k = 0; k < 1000; k++) {
                             var watcher = Watcher
                                 .watch(testDir.getTestDirectory(), WatchScope.PATH_AND_CHILDREN)
-                                .approximate(whichFiles)
+                                .onOverflow(whichFiles)
                                 .on(e -> {
                                     if (e.calculateFullPath().equals(target)) {
                                         seen.add(id);
@@ -342,10 +342,10 @@ void manyRegisterAndUnregisterSameTime(OnOverflow whichFiles) throws Interrupted
     }
 
     @ParameterizedTest
-    @EnumSource(names = { "ALL", "DIRTY" })
+    @EnumSource(names = { "ALL", "DIFF" })
     //Deletes can race the filesystem, so you might miss a few files in a dir, if that dir is already deleted
     @EnabledIfEnvironmentVariable(named="TORTURE_DELETE", matches="true")
-    void pressureOnFSShouldNotMissDeletes(OnOverflow whichFiles) throws InterruptedException, IOException {
+    void pressureOnFSShouldNotMissDeletes(Approximation whichFiles) throws InterruptedException, IOException {
         final var root = testDir.getTestDirectory();
         var pool = Executors.newCachedThreadPool();
 
@@ -361,7 +361,7 @@ void pressureOnFSShouldNotMissDeletes(OnOverflow whichFiles) throws InterruptedE
             final var happened = new Semaphore(0);
             var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.PATH_AND_ALL_DESCENDANTS)
                 .withExecutor(pool)
-                .approximate(whichFiles)
+                .onOverflow(whichFiles)
                 .on(ev -> {
                     events.getAndIncrement();
                     happened.release();

src/test/java/engineering/swat/watch/impl/overflows/IndexingRescannerTests.java

@@ -38,7 +38,7 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-import engineering.swat.watch.OnOverflow;
+import engineering.swat.watch.Approximation;
 import engineering.swat.watch.TestDirectory;
 import engineering.swat.watch.TestHelper;
 import engineering.swat.watch.WatchEvent;
@@ -75,7 +75,7 @@ void onlyEventsForFilesInScopeAreIssued() throws IOException, InterruptedExcepti
         // children (not all descendants) of `path`
         var eventsOnlyForChildren = new AtomicBoolean(true);
         var watchConfig = Watcher.watch(path, WatchScope.PATH_AND_CHILDREN)
-            .approximate(OnOverflow.NONE) // Disable the auto-handler here; we'll have an explicit one below
+            .onOverflow(Approximation.NONE) // Disable the auto-handler here; we'll have an explicit one below
             .on(e -> {
                 if (e.getRelativePath().getNameCount() > 1) {
                     eventsOnlyForChildren.set(false);