Merge pull request #24 from SWAT-engineering/improved-overflow-support/first-overflow-policy

Improved overflow support: Overflow auto-handling for directory watches (non-recursive)

Author: sungshik

src/main/java/engineering/swat/watch/ActiveWatch.java

@@ -40,4 +40,9 @@ public interface ActiveWatch extends Closeable {
      * Gets the path watched by this watch.
      */
     Path getPath();
+
+    /**
+     * Gets the scope of this watch.
+     */
+    WatchScope getScope();
 }

src/main/java/engineering/swat/watch/OnOverflow.java

@@ -0,0 +1,107 @@
+/*
+ * BSD 2-Clause License
+ *
+ * Copyright (c) 2023, Swat.engineering
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package engineering.swat.watch;
+
+/**
+ * Constants to indicate for which regular files/directories in the scope of the
+ * watch an <i>approximation</i> of synthetic events (of kinds
+ * {@link WatchEvent.Kind#CREATED}, {@link WatchEvent.Kind#MODIFIED}, and/or
+ * {@link WatchEvent.Kind#DELETED}) should be issued when an overflow event
+ * happens. These synthetic events, as well as the overflow event itself, are
+ * subsequently passed to the user-defined event handler of the watch.
+ * Typically, the user-defined event handler can ignore the original overflow
+ * event (i.e., handling the synthetic events is sufficient to address the
+ * overflow issue), but it doesn't have to (e.g., it may carry out additional
+ * overflow bookkeeping).
+ */
+public enum OnOverflow {
+
+    /**
+     * Synthetic events are issued for <b>no regular files/directories</b> in
+     * the scope of the watch. Thus, the user-defined event handler is fully
+     * responsible to handle overflow events.
+     */
+    NONE,
+
+    /**
+     * <p>
+     * Synthetic events of kinds {@link WatchEvent.Kind#CREATED} and
+     * {@link WatchEvent.Kind#MODIFIED}, but not
+     * {@link WatchEvent.Kind#DELETED}, are issued for all regular
+     * files/directories in the scope of the watch. Specifically, when an
+     * overflow event happens:
+     *
+     * <ul>
+     * <li>CREATED events are issued for all regular files/directories
+     * (overapproximation).
+     * <li>MODIFIED events are issued for all non-empty, regular files
+     * (overapproximation) but for no directories (underapproximation).
+     * <li>DELETED events are issued for no regular files/directories
+     * (underapproximation).
+     * </ul>
+     *
+     * <p>
+     * This approach is relatively cheap in terms of memory usage (cf.
+     * {@link #DIRTY}), but it results in a large over/underapproximation of the
+     * actual events (cf. DIRTY).
+     */
+    ALL,
+
+
+    /**
+     * <p>
+     * Synthetic events of kinds {@link WatchEvent.Kind#CREATED},
+     * {@link WatchEvent.Kind#MODIFIED}, and {@link WatchEvent.Kind#DELETED} are
+     * issued for dirty regular files/directories in the scope of the watch, as
+     * determined using <i>last-modified-times</i>. Specifically, when an
+     * overflow event happens:
+     *
+     * <ul>
+     * <li>CREATED events are issued for all regular files/directories when the
+     * previous last-modified-time is unknown, but the current
+     * last-modified-time is known (i.e., the file started existing).
+     * <li>MODIFIED events are issued for all regular files/directories when the
+     * previous last-modified-time is before the current last-modified-time.
+     * <li>DELETED events are issued for all regular files/directories when the
+     * previous last-modified-time is known, but the current
+     * last-modified-time is unknown (i.e., the file stopped existing).
+     * </ul>
+     *
+     * <p>
+     * To keep track of last-modified-times, an internal <i>index</i> is
+     * populated with last-modified-times of all regular files/directories in
+     * the scope of the watch when the watch is started. Each time when any
+     * event happens, the index is updated accordingly, so when an overflow
+     * event happens, last-modified-times can be compared as described above.
+     *
+     * <p>
+     * This approach results in a small overapproximation (cf. {@link #ALL}),
+     * but it is relatively expensive in terms of memory usage (cf. ALL), as the
+     * watch needs to keep track of last-modified-times.
+     */
+    DIRTY
+}

src/main/java/engineering/swat/watch/Watcher.java

@@ -42,6 +42,7 @@
 import engineering.swat.watch.impl.jdk.JDKDirectoryWatch;
 import engineering.swat.watch.impl.jdk.JDKFileWatch;
 import engineering.swat.watch.impl.jdk.JDKRecursiveDirectoryWatch;
+import engineering.swat.watch.impl.overflows.MemorylessRescanner;
 
 /**
  * <p>Watch a path for changes.</p>
@@ -52,17 +53,17 @@
  */
 public class Watcher {
     private final Logger logger = LogManager.getLogger();
-    private final WatchScope scope;
     private final Path path;
+    private final WatchScope scope;
+    private volatile OnOverflow approximateOnOverflow = OnOverflow.ALL;
     private volatile Executor executor = CompletableFuture::runAsync;
 
     private static final BiConsumer<EventHandlingWatch, WatchEvent> EMPTY_HANDLER = (w, e) -> {};
     private volatile BiConsumer<EventHandlingWatch, WatchEvent> eventHandler = EMPTY_HANDLER;
 
-
-    private Watcher(WatchScope scope, Path path) {
-        this.scope = scope;
+    private Watcher(Path path, WatchScope scope) {
         this.path = path;
+        this.scope = scope;
     }
 
     /**
@@ -89,9 +90,8 @@ public static Watcher watch(Path path, WatchScope scope) {
                 break;
             default:
                 throw new IllegalArgumentException("Unsupported scope: " + scope);
-
         }
-        return new Watcher(scope, path);
+        return new Watcher(path, scope);
     }
 
     /**
@@ -148,6 +148,22 @@ public Watcher withExecutor(Executor callbackHandler) {
         return this;
     }
 
+    /**
+     * Optionally configure which regular files/directories in the scope of the
+     * watch an <i>approximation</i> of synthetic events (of kinds
+     * {@link WatchEvent.Kind#CREATED}, {@link WatchEvent.Kind#MODIFIED}, and/or
+     * {@link WatchEvent.Kind#DELETED}) should be issued when an overflow event
+     * happens. If not defined before this watcher is started, the
+     * {@link engineering.swat.watch.OnOverflow#ALL} approach will be used.
+     * @param whichFiles Constant to indicate for which regular
+     * files/directories to approximate
+     * @return This watcher for optional method chaining
+     */
+    public Watcher approximate(OnOverflow whichFiles) {
+        this.approximateOnOverflow = whichFiles;
+        return this;
+    }
+
     /**
      * Start watch the path for events.
      * @return a subscription for the watch, when closed, new events will stop being registered to the worker pool.
@@ -159,9 +175,11 @@ public ActiveWatch start() throws IOException {
             throw new IllegalStateException("There is no onEvent handler defined");
         }
 
+        var h = applyApproximateOnOverflow();
+
         switch (scope) {
             case PATH_AND_CHILDREN: {
-                var result = new JDKDirectoryWatch(path, executor, eventHandler, false);
+                var result = new JDKDirectoryWatch(path, executor, h);
                 result.open();
                 return result;
             }
@@ -188,4 +206,15 @@ public ActiveWatch start() throws IOException {
                 throw new IllegalStateException("Not supported yet");
         }
     }
+
+    private BiConsumer<EventHandlingWatch, WatchEvent> applyApproximateOnOverflow() {
+        switch (approximateOnOverflow) {
+            case NONE:
+                return eventHandler;
+            case ALL:
+                return eventHandler.andThen(new MemorylessRescanner(executor));
+            default:
+                throw new UnsupportedOperationException("No event handler has been defined yet for this overflow policy");
+        }
+    }
 }

src/main/java/engineering/swat/watch/impl/jdk/JDKDirectoryWatch.java

@@ -38,6 +38,7 @@
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 import engineering.swat.watch.WatchEvent;
+import engineering.swat.watch.WatchScope;
 import engineering.swat.watch.impl.EventHandlingWatch;
 import engineering.swat.watch.impl.util.BundledSubscription;
 import engineering.swat.watch.impl.util.SubscriptionKey;
@@ -74,6 +75,11 @@ private void handleJDKEvents(List<java.nio.file.WatchEvent<?>> events) {
 
     // -- JDKBaseWatch --
 
+    @Override
+    public WatchScope getScope() {
+        return nativeRecursive ? WatchScope.PATH_AND_ALL_DESCENDANTS : WatchScope.PATH_AND_CHILDREN;
+    }
+
     @Override
     public synchronized void close() throws IOException {
         if (bundledJDKWatcher != null) {

src/main/java/engineering/swat/watch/impl/jdk/JDKFileWatch.java

@@ -36,6 +36,7 @@
 import org.checkerframework.checker.nullness.qual.Nullable;
 
 import engineering.swat.watch.WatchEvent;
+import engineering.swat.watch.WatchScope;
 import engineering.swat.watch.impl.EventHandlingWatch;
 
 /**
@@ -73,6 +74,11 @@ private static Path requireNonNull(@Nullable Path p, String message) {
 
     // -- JDKBaseWatch --
 
+    @Override
+    public WatchScope getScope() {
+        return WatchScope.PATH_ONLY;
+    }
+
     @Override
     public void handleEvent(WatchEvent event) {
         internal.handleEvent(event);

src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatch.java

@@ -48,6 +48,7 @@
 import org.apache.logging.log4j.Logger;
 
 import engineering.swat.watch.WatchEvent;
+import engineering.swat.watch.WatchScope;
 import engineering.swat.watch.impl.EventHandlingWatch;
 
 public class JDKRecursiveDirectoryWatch extends JDKBaseWatch {
@@ -298,6 +299,11 @@ private void detectedMissingEntries(Path dir, ArrayList<WatchEvent> events, Hash
 
     // -- JDKBaseWatch --
 
+    @Override
+    public WatchScope getScope() {
+        return WatchScope.PATH_AND_ALL_DESCENDANTS;
+    }
+
     @Override
     public void handleEvent(WatchEvent event) {
         processEvents(event);