From 17f0cf7c297050276201b80f5b23434f13936751 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 20 Nov 2023 16:59:57 +0100 Subject: [PATCH 01/89] Started initial implementation --- .editorconfig | 17 ++ .gitignore | 2 + .vscode/settings.json | 3 + pom.xml | 170 ++++++++++++++++++ .../swat/watch/WatchSubscription.java | 14 ++ .../java/engineering/swat/watch/Watcher.java | 74 ++++++++ .../swat/watch/impl/JDKDirectoryWatcher.java | 44 +++++ .../swat/watch/impl/JDKPoller.java | 65 +++++++ .../engineering/swat/watch/SmokeTests.java | 56 ++++++ 9 files changed, 445 insertions(+) create mode 100644 .editorconfig create mode 100644 .vscode/settings.json create mode 100644 pom.xml create mode 100644 src/main/java/engineering/swat/watch/WatchSubscription.java create mode 100644 src/main/java/engineering/swat/watch/Watcher.java create mode 100644 src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java create mode 100644 src/main/java/engineering/swat/watch/impl/JDKPoller.java create mode 100644 src/test/java/engineering/swat/watch/SmokeTests.java diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..1f494b0c --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +# Editor configuration, see http://editorconfig.org +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +insert_final_newline = true +trim_trailing_whitespace = true +max_line_length = 80 + +[*.sh] +end_of_line = lf + +[*.java] +indent_size = 4 +max_line_length = 120 diff --git a/.gitignore b/.gitignore index 524f0963..8f85f7b7 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* replay_pid* + +/target diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..050505ce --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "java.configuration.updateBuildConfiguration": "interactive" +} diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..b23a0895 --- /dev/null +++ b/pom.xml @@ -0,0 +1,170 @@ + + + 4.0.0 + + engineering.swat + java-watch + 0.0.1-SNAPSHOT + jar + + + UTF-8 + 3.40.0 + 5.10.1 + 11 + 11 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.10.1 + + 11 + -parameters + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + attach-sources + + jar + + + + + + org.apache.maven.plugins + maven-release-plugin + + v@{project.version} + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.4.0 + + -Xdoclint:none + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + + org.jacoco + jacoco-maven-plugin + 0.8.8 + + + + prepare-agent + + + + report + test + + report + + + + + + + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + org.junit.jupiter + junit-jupiter-params + ${junit.version} + test + + + org.checkerframework + checker-qual + ${checkerframework.version} + + + + + + checker-framework + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + + properties + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.10.1 + + true + 11 + + 1000 + 1000 + + + + org.checkerframework + checker + ${checkerframework.version} + + + + + + org.checkerframework.checker.nullness.NullnessChecker + + + + + + + + + + org.checkerframework + checker + ${checkerframework.version} + provided + + + + + + + + diff --git a/src/main/java/engineering/swat/watch/WatchSubscription.java b/src/main/java/engineering/swat/watch/WatchSubscription.java new file mode 100644 index 00000000..ad9bc3f5 --- /dev/null +++ b/src/main/java/engineering/swat/watch/WatchSubscription.java @@ -0,0 +1,14 @@ +package engineering.swat.watch; + +import java.io.Closeable; +import java.io.IOException; + +public class WatchSubscription implements Closeable { + + @Override + public void close() throws IOException { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'close'"); + } + +} diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java new file mode 100644 index 00000000..1abe8888 --- /dev/null +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -0,0 +1,74 @@ +package engineering.swat.watch; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.function.Consumer; + +public class Watcher { + private final WatcherKind kind; + private final Path path; + private Executor executor = CompletableFuture::runAsync; + + private static final Consumer NO_OP = p -> {}; + + private Consumer createHandler = NO_OP; + private Consumer changeHandler = NO_OP; + private Consumer removeHandler = NO_OP; + + + private Watcher(WatcherKind kind, Path path) { + this.kind = kind; + this.path = path; + } + + private enum WatcherKind { + FILE, + DIRECTORY, + RECURSIVE_DIRECTORY + } + + public static Watcher singleFile(Path path) throws IOException { + if (!path.isAbsolute()) { + throw new IOException("We can only watch absolute paths"); + } + return new Watcher(WatcherKind.FILE, path); + } + + public static Watcher singleDirectory(Path path) throws IOException { + if (!path.isAbsolute()) { + throw new IOException("We can only watch absolute paths"); + } + return new Watcher(WatcherKind.DIRECTORY, path); + } + + public static Watcher recursiveDirectory(Path path) throws IOException { + if (!path.isAbsolute()) { + throw new IOException("We can only watch absolute paths"); + } + return new Watcher(WatcherKind.RECURSIVE_DIRECTORY, path); + } + + public Watcher onCreate(Consumer createHandler) { + this.createHandler = createHandler; + return this; + } + + public Watcher onChange(Consumer changeHandler) { + this.changeHandler = changeHandler; + return this; + } + + public Watcher onRemove(Consumer removeHandler) { + this.removeHandler = removeHandler; + return this; + } + + public WatchSubscription start() { + return null; + } + + + +} diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java new file mode 100644 index 00000000..6eee3a3a --- /dev/null +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -0,0 +1,44 @@ +package engineering.swat.watch.impl; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.function.Consumer; + +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +public class JDKDirectoryWatcher implements Closeable { + private final Path directory; + private final Executor exec; + private final Consumer> eventHandler; + private @MonotonicNonNull Closeable activeWatch; + + public JDKDirectoryWatcher(Path directory, Executor exec, Consumer> eventHandler) { + this.directory = directory; + this.exec = exec; + this.eventHandler = eventHandler; + } + + public void start() throws IOException { + try { + activeWatch = JDKPoller.INSTANCE.register(directory, this::handleChanges); + } catch (IOException e) { + throw new IOException("Could not register directory watcher for: " + directory, e); + } + } + + private void handleChanges(List> events) { + } + + @Override + public void close() throws IOException { + if (activeWatch != null) { + activeWatch.close(); + } + } + + +} diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java new file mode 100644 index 00000000..c37dfa40 --- /dev/null +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -0,0 +1,65 @@ +package engineering.swat.watch.impl; + +import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +enum JDKPoller { + INSTANCE; + + private final WatchService service; + private final Map>>> watchers = new ConcurrentHashMap<>(); + private final Thread pollThread; + + private JDKPoller() { + try { + service = FileSystems.getDefault().newWatchService(); + } catch (IOException e) { + throw new RuntimeException("Could not start watcher", e); + } + pollThread = new Thread(this::poller); + pollThread.setDaemon(true); + pollThread.setName("file-watcher poll events thread"); + pollThread.start(); + } + + private void poller() { + while (true) { + try { + WatchKey hit; + if ((hit = service.poll(1, TimeUnit.MILLISECONDS)) != null) { + var watchHandler = watchers.get(hit); + if (watchHandler != null) { + watchHandler.accept(hit.pollEvents()); + } + } + } catch (InterruptedException e) { + return; + } + } + } + + public Closeable register(Path path, Consumer>> changes) throws IOException { + var key = path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY); + watchers.put(key, changes); + return new Closeable() { + @Override + public void close() throws IOException { + key.cancel(); + watchers.remove(key); + } + }; + } +} diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java new file mode 100644 index 00000000..0849af8d --- /dev/null +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -0,0 +1,56 @@ +package engineering.swat.watch; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class SmokeTests { + static Path testDirectory; + static List testFiles = new ArrayList<>(); + + @BeforeAll + static void setupTestDirectory() throws IOException { + testDirectory = Files.createTempDirectory("smoke-test"); + for (var f : Arrays.asList("a.txt", "b.txt", "c.txt")) { + testFiles.add(Files.createFile(testDirectory.resolve(f))); + } + } + + @AfterAll + static void cleanupDirectory() throws IOException { + if (testDirectory != null) { + Files.walk(testDirectory) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } + + @Test + void watchSingleFile() throws IOException, InterruptedException { + var changed = new AtomicBoolean(false); + var target = testFiles.get(0); + var watchConfig = Watcher.singleDirectory(target) + .onChange(p -> {if (p.equals(target)) { changed.set(true); }}) + ; + + try (var activeWatch = watchConfig.start() ) { + Files.writeString(target, "Hello world"); + Thread.sleep(1000); + assertTrue(changed.get(), "The file change should be detected"); + } + } + +} From bb6c56f8fd9ec61287cb85704448f873b245ab5c Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 25 Dec 2023 12:25:02 +0100 Subject: [PATCH 02/89] First working test of directory watcher --- pom.xml | 11 ++++ .../engineering/swat/watch/WatchEvent.java | 37 +++++++++++++ .../java/engineering/swat/watch/Watcher.java | 55 ++++++++++++++++--- .../swat/watch/impl/JDKDirectoryWatcher.java | 49 ++++++++++++++--- .../swat/watch/impl/JDKPoller.java | 20 ++++++- .../engineering/swat/watch/SmokeTests.java | 5 +- src/test/resources/log4j2-test.xml | 16 ++++++ 7 files changed, 173 insertions(+), 20 deletions(-) create mode 100644 src/main/java/engineering/swat/watch/WatchEvent.java create mode 100644 src/test/resources/log4j2-test.xml diff --git a/pom.xml b/pom.xml index b23a0895..5e320f22 100644 --- a/pom.xml +++ b/pom.xml @@ -12,6 +12,7 @@ UTF-8 3.40.0 5.10.1 + 2.22.0 11 11 @@ -100,6 +101,16 @@ checker-qual ${checkerframework.version} + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + diff --git a/src/main/java/engineering/swat/watch/WatchEvent.java b/src/main/java/engineering/swat/watch/WatchEvent.java new file mode 100644 index 00000000..1fd3ae0a --- /dev/null +++ b/src/main/java/engineering/swat/watch/WatchEvent.java @@ -0,0 +1,37 @@ +package engineering.swat.watch; + +import java.nio.file.Path; + +public class WatchEvent { + + public enum Kind { + CREATED, MODIFIED, DELETED + } + + private final Kind kind; + private final Path rootPath; + private final Path relativePath; + + public WatchEvent(Kind kind, Path rootPath, Path relativePath) { + this.kind = kind; + this.rootPath = rootPath; + this.relativePath = relativePath; + } + + public Kind getKind() { + return this.kind; + } + + public Path getRelativePath() { + return relativePath; + } + + public Path getRootPath() { + return rootPath; + } + + public Path calculateFullPath() { + return rootPath.resolve(relativePath); + } + +} diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 1abe8888..e5bb55bd 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -1,12 +1,19 @@ package engineering.swat.watch; +import java.io.Closeable; import java.io.IOException; import java.nio.file.Path; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.function.Consumer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import engineering.swat.watch.impl.JDKDirectoryWatcher; + public class Watcher { + private final Logger logger = LogManager.getLogger(); private final WatcherKind kind; private final Path path; private Executor executor = CompletableFuture::runAsync; @@ -14,13 +21,14 @@ public class Watcher { private static final Consumer NO_OP = p -> {}; private Consumer createHandler = NO_OP; - private Consumer changeHandler = NO_OP; - private Consumer removeHandler = NO_OP; + private Consumer modifiedHandler = NO_OP; + private Consumer deletedHandler = NO_OP; private Watcher(WatcherKind kind, Path path) { this.kind = kind; this.path = path; + logger.info("Constructor logger for: {} at {} level", path, kind); } private enum WatcherKind { @@ -55,20 +63,51 @@ public Watcher onCreate(Consumer createHandler) { return this; } - public Watcher onChange(Consumer changeHandler) { - this.changeHandler = changeHandler; + public Watcher onModified(Consumer changeHandler) { + this.modifiedHandler = changeHandler; return this; } - public Watcher onRemove(Consumer removeHandler) { - this.removeHandler = removeHandler; + public Watcher onDeleted(Consumer removeHandler) { + this.deletedHandler = removeHandler; return this; } - public WatchSubscription start() { - return null; + public Watcher withExecutor(Executor callbackHandler) { + this.executor = callbackHandler; + return this; } + public Closeable start() throws IOException { + switch (kind) { + case DIRECTORY: + var result = new JDKDirectoryWatcher(path, executor, this::handleEvent); + result.start(); + return result; + case FILE: + case RECURSIVE_DIRECTORY: + default: + throw new IllegalArgumentException("Not supported yet"); + } + } + private void handleEvent(WatchEvent ev) { + switch (ev.getKind()) { + case CREATED: + callIfDefined(createHandler, ev); + break; + case DELETED: + callIfDefined(deletedHandler, ev); + break; + case MODIFIED: + callIfDefined(modifiedHandler, ev); + break; + } + } + private void callIfDefined(Consumer target, WatchEvent ev) { + if (target != NO_OP) { + executor.execute(() -> target.accept(ev.calculateFullPath())); + } + } } diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java index 6eee3a3a..3e510834 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -3,20 +3,25 @@ import java.io.Closeable; import java.io.IOException; import java.nio.file.Path; -import java.nio.file.WatchEvent; +import java.nio.file.StandardWatchEventKinds; import java.util.List; import java.util.concurrent.Executor; import java.util.function.Consumer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import engineering.swat.watch.WatchEvent; + public class JDKDirectoryWatcher implements Closeable { + private final Logger logger = LogManager.getLogger(); private final Path directory; private final Executor exec; - private final Consumer> eventHandler; - private @MonotonicNonNull Closeable activeWatch; + private final Consumer eventHandler; + private volatile @MonotonicNonNull Closeable activeWatch; - public JDKDirectoryWatcher(Path directory, Executor exec, Consumer> eventHandler) { + public JDKDirectoryWatcher(Path directory, Executor exec, Consumer eventHandler) { this.directory = directory; this.exec = exec; this.eventHandler = eventHandler; @@ -24,21 +29,51 @@ public JDKDirectoryWatcher(Path directory, Executor exec, Consumer public void start() throws IOException { try { + if (activeWatch != null) { + // TODO make sure there is no cross thread race possible here. + throw new IOException("Cannot start a watcher twice"); + } activeWatch = JDKPoller.INSTANCE.register(directory, this::handleChanges); + logger.debug("Started watch for: {}", directory); } catch (IOException e) { throw new IOException("Could not register directory watcher for: " + directory, e); } } - private void handleChanges(List> events) { + private void handleChanges(List> events) { + for (var ev: events) { + logger.trace("Handling event: {} for {}", ev, directory); + exec.execute(() -> eventHandler.accept(translate(ev))); + } + } + + private WatchEvent translate(java.nio.file.WatchEvent ev) { + WatchEvent.Kind kind; + if (ev.kind() == StandardWatchEventKinds.ENTRY_CREATE) { + kind = WatchEvent.Kind.CREATED; + } + else if (ev.kind() == StandardWatchEventKinds.ENTRY_MODIFY) { + kind = WatchEvent.Kind.MODIFIED; + } + else if (ev.kind() == StandardWatchEventKinds.ENTRY_DELETE) { + kind = WatchEvent.Kind.DELETED; + } + else { + throw new IllegalArgumentException("Unexpected watch event: " + ev); + } + var path = (Path)ev.context(); + if (path == null) { + throw new IllegalArgumentException("Missing path in event: " + ev); + } + logger.trace("Translated: {} to {} at {}", ev, kind, path); + return new WatchEvent(kind, directory, path); } @Override public void close() throws IOException { if (activeWatch != null) { + logger.debug("Closing watch for: {}", this.directory); activeWatch.close(); } } - - } diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index c37dfa40..3b599f79 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -16,9 +16,13 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + enum JDKPoller { INSTANCE; + private final Logger logger = LogManager.getLogger(); private final WatchService service; private final Map>>> watchers = new ConcurrentHashMap<>(); private final Thread pollThread; @@ -40,9 +44,17 @@ private void poller() { try { WatchKey hit; if ((hit = service.poll(1, TimeUnit.MILLISECONDS)) != null) { - var watchHandler = watchers.get(hit); - if (watchHandler != null) { - watchHandler.accept(hit.pollEvents()); + logger.trace("Got hit: {}", hit); + try { + var watchHandler = watchers.get(hit); + if (watchHandler != null) { + var events = hit.pollEvents(); + logger.trace("Found watcher for hit: {}, sending: {} (size: {})", watchHandler, events, events.size()); + watchHandler.accept(events); + } + } + finally{ + hit.reset(); } } } catch (InterruptedException e) { @@ -52,11 +64,13 @@ private void poller() { } public Closeable register(Path path, Consumer>> changes) throws IOException { + logger.debug("Register watch for: {}", path); var key = path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY); watchers.put(key, changes); return new Closeable() { @Override public void close() throws IOException { + logger.debug("Closing watch for: {}", path); key.cancel(); watchers.remove(key); } diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index 0849af8d..c30575b9 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.IOException; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -42,8 +43,8 @@ static void cleanupDirectory() throws IOException { void watchSingleFile() throws IOException, InterruptedException { var changed = new AtomicBoolean(false); var target = testFiles.get(0); - var watchConfig = Watcher.singleDirectory(target) - .onChange(p -> {if (p.equals(target)) { changed.set(true); }}) + var watchConfig = Watcher.singleDirectory(target.getParent()) + .onModified(p -> {if (p.equals(target)) { changed.set(true); }}) ; try (var activeWatch = watchConfig.start() ) { diff --git a/src/test/resources/log4j2-test.xml b/src/test/resources/log4j2-test.xml new file mode 100644 index 00000000..d341ee55 --- /dev/null +++ b/src/test/resources/log4j2-test.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + From 860ac759ca75c27ac7188d37b211870ffeb6798b Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 25 Dec 2023 12:45:23 +0100 Subject: [PATCH 03/89] Improved tests by removing sleeps --- pom.xml | 6 +++ .../engineering/swat/watch/SmokeTests.java | 43 ++++++++++++++++--- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 5e320f22..54a91c26 100644 --- a/pom.xml +++ b/pom.xml @@ -96,6 +96,12 @@ ${junit.version} test + + org.awaitility + awaitility + 4.2.0 + test + org.checkerframework checker-qual diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index c30575b9..94a55b8b 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -11,12 +11,19 @@ import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; + +import static org.awaitility.Awaitility.*; +import static java.time.Duration.*; + + public class SmokeTests { static Path testDirectory; static List testFiles = new ArrayList<>(); @@ -24,8 +31,17 @@ public class SmokeTests { @BeforeAll static void setupTestDirectory() throws IOException { testDirectory = Files.createTempDirectory("smoke-test"); + add3Files(testDirectory); + for (var d: Arrays.asList("d1", "d2", "d3")) { + Files.createDirectories(testDirectory.resolve(d)); + add3Files(testDirectory.resolve(d)); + } + Awaitility.setDefaultTimeout(1, TimeUnit.SECONDS); + } + + private static void add3Files(Path root) throws IOException { for (var f : Arrays.asList("a.txt", "b.txt", "c.txt")) { - testFiles.add(Files.createFile(testDirectory.resolve(f))); + testFiles.add(Files.createFile(root.resolve(f))); } } @@ -40,18 +56,35 @@ static void cleanupDirectory() throws IOException { } @Test - void watchSingleFile() throws IOException, InterruptedException { + void watchDirectory() throws IOException, InterruptedException { var changed = new AtomicBoolean(false); var target = testFiles.get(0); - var watchConfig = Watcher.singleDirectory(target.getParent()) + var watchConfig = Watcher.singleDirectory(testDirectory) .onModified(p -> {if (p.equals(target)) { changed.set(true); }}) ; try (var activeWatch = watchConfig.start() ) { Files.writeString(target, "Hello world"); - Thread.sleep(1000); - assertTrue(changed.get(), "The file change should be detected"); + await().alias("Target file change").until(changed::get); } } + @Test + void watchRecursiveDirectory() throws IOException, InterruptedException { + var changed = new AtomicBoolean(false); + var target = testFiles.stream() + .filter(p -> !p.getParent().equals(testDirectory)) + .findFirst() + .orElseThrow(); + var watchConfig = Watcher.recursiveDirectory(testDirectory) + .onModified(p -> {if (p.equals(target)) { changed.set(true); }}) + ; + + try (var activeWatch = watchConfig.start() ) { + Files.writeString(target, "Hello world"); + await().alias("Nested file change").until(changed::get); + } + } + + } From d3a47b452607da2b8e1d1b0a4dc46e510e3f0685 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 26 Dec 2023 13:21:11 +0100 Subject: [PATCH 04/89] Implemented initial recursive support --- .../engineering/swat/watch/WatchEvent.java | 6 +- .../java/engineering/swat/watch/Watcher.java | 15 +- .../swat/watch/impl/JDKDirectoryWatcher.java | 38 ++-- .../swat/watch/impl/JDKPoller.java | 6 +- .../impl/JDKRecursiveDirectoryWatcher.java | 168 ++++++++++++++++++ .../engineering/swat/watch/SmokeTests.java | 2 +- 6 files changed, 216 insertions(+), 19 deletions(-) create mode 100644 src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java diff --git a/src/main/java/engineering/swat/watch/WatchEvent.java b/src/main/java/engineering/swat/watch/WatchEvent.java index 1fd3ae0a..e753462b 100644 --- a/src/main/java/engineering/swat/watch/WatchEvent.java +++ b/src/main/java/engineering/swat/watch/WatchEvent.java @@ -2,6 +2,8 @@ import java.nio.file.Path; +import org.checkerframework.checker.nullness.qual.Nullable; + public class WatchEvent { public enum Kind { @@ -12,10 +14,10 @@ public enum Kind { private final Path rootPath; private final Path relativePath; - public WatchEvent(Kind kind, Path rootPath, Path relativePath) { + public WatchEvent(Kind kind, Path rootPath, @Nullable Path relativePath) { this.kind = kind; this.rootPath = rootPath; - this.relativePath = relativePath; + this.relativePath = relativePath == null ? Path.of("") : relativePath; } public Kind getKind() { diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index e5bb55bd..df373746 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -10,7 +10,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import engineering.swat.watch.WatchEvent.Kind; import engineering.swat.watch.impl.JDKDirectoryWatcher; +import engineering.swat.watch.impl.JDKRecursiveDirectoryWatcher; public class Watcher { private final Logger logger = LogManager.getLogger(); @@ -80,12 +82,17 @@ public Watcher withExecutor(Executor callbackHandler) { public Closeable start() throws IOException { switch (kind) { - case DIRECTORY: + case DIRECTORY: { var result = new JDKDirectoryWatcher(path, executor, this::handleEvent); - result.start(); + result.start(Kind.CREATED, Kind.MODIFIED, Kind.DELETED); return result; + } + case RECURSIVE_DIRECTORY: { + var result = new JDKRecursiveDirectoryWatcher(path, executor, this::handleEvent); + result.start(Kind.CREATED, Kind.MODIFIED, Kind.DELETED); + return result; + } case FILE: - case RECURSIVE_DIRECTORY: default: throw new IllegalArgumentException("Not supported yet"); } @@ -107,7 +114,7 @@ private void handleEvent(WatchEvent ev) { private void callIfDefined(Consumer target, WatchEvent ev) { if (target != NO_OP) { - executor.execute(() -> target.accept(ev.calculateFullPath())); + target.accept(ev.calculateFullPath()); } } } diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java index 3e510834..3331f008 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -1,9 +1,15 @@ package engineering.swat.watch.impl; +import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; + import java.io.Closeable; import java.io.IOException; import java.nio.file.Path; import java.nio.file.StandardWatchEventKinds; +import java.nio.file.WatchEvent.Kind; +import java.util.Arrays; import java.util.List; import java.util.concurrent.Executor; import java.util.function.Consumer; @@ -11,6 +17,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.Nullable; import engineering.swat.watch.WatchEvent; @@ -27,26 +34,40 @@ public JDKDirectoryWatcher(Path directory, Executor exec, Consumer e this.eventHandler = eventHandler; } - public void start() throws IOException { + public void start(WatchEvent.Kind... eventKinds) throws IOException { try { if (activeWatch != null) { // TODO make sure there is no cross thread race possible here. throw new IOException("Cannot start a watcher twice"); } - activeWatch = JDKPoller.INSTANCE.register(directory, this::handleChanges); + var kinds = Arrays.stream(eventKinds).map(JDKDirectoryWatcher::convertKind) + .toArray(Kind[]::new); + + activeWatch = JDKPoller.INSTANCE.register(directory, this::handleChanges, kinds); logger.debug("Started watch for: {}", directory); } catch (IOException e) { throw new IOException("Could not register directory watcher for: " + directory, e); } } - private void handleChanges(List> events) { - for (var ev: events) { - logger.trace("Handling event: {} for {}", ev, directory); - exec.execute(() -> eventHandler.accept(translate(ev))); + private static java.nio.file.WatchEvent.Kind convertKind(engineering.swat.watch.WatchEvent.Kind k) { + switch (k) { + case CREATED: + return ENTRY_CREATE; + case DELETED: + return ENTRY_DELETE; + case MODIFIED: + return ENTRY_MODIFY; + default: throw new IllegalArgumentException("Missing case for: " + k); } } + private void handleChanges(List> events) { + exec.execute(() -> + events.forEach(ev -> eventHandler.accept(translate(ev))) + ); + } + private WatchEvent translate(java.nio.file.WatchEvent ev) { WatchEvent.Kind kind; if (ev.kind() == StandardWatchEventKinds.ENTRY_CREATE) { @@ -61,10 +82,7 @@ else if (ev.kind() == StandardWatchEventKinds.ENTRY_DELETE) { else { throw new IllegalArgumentException("Unexpected watch event: " + ev); } - var path = (Path)ev.context(); - if (path == null) { - throw new IllegalArgumentException("Missing path in event: " + ev); - } + var path = (@Nullable Path)ev.context(); logger.trace("Translated: {} to {} at {}", ev, kind, path); return new WatchEvent(kind, directory, path); } diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index 3b599f79..6b26255d 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -10,6 +10,7 @@ import java.nio.file.WatchEvent; import java.nio.file.WatchKey; import java.nio.file.WatchService; +import java.nio.file.WatchEvent.Kind; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -63,9 +64,10 @@ private void poller() { } } - public Closeable register(Path path, Consumer>> changes) throws IOException { + public Closeable register(Path path, Consumer>> changes, Kind[] kinds) throws IOException { logger.debug("Register watch for: {}", path); - var key = path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY); + var key = path.register(service, kinds); + logger.trace("Got watch key: {}", key); watchers.put(key, changes); return new Closeable() { @Override diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java new file mode 100644 index 00000000..2a8c4cf0 --- /dev/null +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -0,0 +1,168 @@ +package engineering.swat.watch.impl; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import engineering.swat.watch.WatchEvent; +import engineering.swat.watch.WatchEvent.Kind; + +public class JDKRecursiveDirectoryWatcher implements Closeable { + private final Logger logger = LogManager.getLogger(); + private final Path directory; + private final Executor exec; + private final Consumer eventHandler; + private final ConcurrentMap activeWatches = new ConcurrentHashMap<>(); + + public JDKRecursiveDirectoryWatcher(Path directory, Executor exec, Consumer eventHandler) { + this.directory = directory; + this.exec = exec; + this.eventHandler = eventHandler; + } + + public void start(WatchEvent.Kind... eventKinds) throws IOException { + try { + var hadCreate = contains(eventKinds, Kind.CREATED); + var hadModify = contains(eventKinds, Kind.MODIFIED); + var hadDelete = contains(eventKinds, Kind.DELETED); + + var modifiedKinds = new Kind[hadModify ? 3 : 2]; + modifiedKinds[0] = Kind.CREATED; + modifiedKinds[1] = Kind.DELETED; + if (hadModify) { + modifiedKinds[2] = Kind.MODIFIED; + } + logger.debug("Starting recursive watch for: {} watching {}", directory, modifiedKinds); + startRecursive(directory, wrappedHandler(hadCreate, hadModify, hadDelete, modifiedKinds), modifiedKinds); + } catch (IOException e) { + throw new IOException("Could not register directory watcher for: " + directory, e); + } + } + + private Consumer wrappedHandler(boolean hadCreate, boolean hadModify, boolean hadDelete, Kind[] kinds) { + return ev -> { + logger.trace("Unwrapping event: {}", ev); + switch (ev.getKind()) { + case CREATED: + addNewDirectoryWatch(hadCreate, hadModify, hadDelete, kinds, ev); + if (hadCreate) { + eventHandler.accept(ev); + } + break; + case DELETED: + handleDeleteDirectory(ev); + if (hadDelete) { + eventHandler.accept(ev); + } + break; + case MODIFIED: + if (hadModify) { + eventHandler.accept(ev); + } + break; + } + }; + } + + private void handleDeleteDirectory(WatchEvent ev) { + var removedPath = ev.calculateFullPath(); + var existingWatch = activeWatches.get(removedPath); + try { + if (existingWatch != null) { + logger.debug("Clearing watch on removed directory: {}", removedPath); + existingWatch.close(); + } + } catch (IOException ex) { + logger.error("Error clearing: {} {}", removedPath, ex); + } + } + + private void addNewDirectoryWatch(boolean hadCreate, boolean hadModify, boolean hadDelete, Kind[] kinds, WatchEvent ev) { + var newPath = ev.calculateFullPath(); + if (Files.isDirectory(newPath)) { + try { + logger.debug("New directory found, adding a watch for it: {}", newPath); + startRecursive(newPath, wrappedHandler(hadCreate, hadModify, hadDelete, kinds), kinds); + } catch (IOException ex) { + logger.error("Error adding watch for: {} {}", newPath, ex); + } + } + } + + private void startRecursive(Path root, Consumer handler, Kind[] modifiedKinds) throws IOException { + Files.walkFileTree(root, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + logger.error("We could not visit {} to schedule recursive file watches: {}", file, exc); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + var watcher = new JDKDirectoryWatcher(dir, exec, handler); + activeWatches.put(dir, watcher); + try { + watcher.start(modifiedKinds); + return FileVisitResult.CONTINUE; + } catch (IOException ex) { + activeWatches.remove(dir); + throw ex; + } + } + + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (exc != null) { + logger.error("Error during directory iteration: {} = {}", dir, exc); + } + return FileVisitResult.CONTINUE; + } + }); + } + + + + + private boolean contains(Kind[] eventKinds, Kind needle) { + for (var k : eventKinds) { + if (k == needle) { + return true; + } + } + return false; + } + + @Override + public void close() throws IOException { + IOException firstFail = null; + for (var e : activeWatches.entrySet()) { + try { + e.getValue().close(); + } catch (IOException ex) { + logger.error("Could not close watch", ex); + if (firstFail == null) { + firstFail = ex; + } + } + } + if (firstFail != null) { + throw firstFail; + } + } +} diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index 94a55b8b..766b7ad8 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -36,7 +36,7 @@ static void setupTestDirectory() throws IOException { Files.createDirectories(testDirectory.resolve(d)); add3Files(testDirectory.resolve(d)); } - Awaitility.setDefaultTimeout(1, TimeUnit.SECONDS); + Awaitility.setDefaultTimeout(2, TimeUnit.SECONDS); } private static void add3Files(Path root) throws IOException { From e6fc78ab6819a15d58e3c0f722818ae658abf8f7 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Wed, 27 Dec 2023 13:59:50 +0100 Subject: [PATCH 05/89] Simplified recursive watcher --- .../java/engineering/swat/watch/Watcher.java | 5 +- .../swat/watch/impl/JDKDirectoryWatcher.java | 24 +--- .../swat/watch/impl/JDKPoller.java | 5 +- .../impl/JDKRecursiveDirectoryWatcher.java | 111 ++++++------------ 4 files changed, 42 insertions(+), 103 deletions(-) diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index df373746..eb923e7f 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import engineering.swat.watch.WatchEvent.Kind; import engineering.swat.watch.impl.JDKDirectoryWatcher; import engineering.swat.watch.impl.JDKRecursiveDirectoryWatcher; @@ -84,12 +83,12 @@ public Closeable start() throws IOException { switch (kind) { case DIRECTORY: { var result = new JDKDirectoryWatcher(path, executor, this::handleEvent); - result.start(Kind.CREATED, Kind.MODIFIED, Kind.DELETED); + result.start(); return result; } case RECURSIVE_DIRECTORY: { var result = new JDKRecursiveDirectoryWatcher(path, executor, this::handleEvent); - result.start(Kind.CREATED, Kind.MODIFIED, Kind.DELETED); + result.start(); return result; } case FILE: diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java index 3331f008..163a00a5 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -1,15 +1,9 @@ package engineering.swat.watch.impl; -import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; -import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; -import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; - import java.io.Closeable; import java.io.IOException; import java.nio.file.Path; import java.nio.file.StandardWatchEventKinds; -import java.nio.file.WatchEvent.Kind; -import java.util.Arrays; import java.util.List; import java.util.concurrent.Executor; import java.util.function.Consumer; @@ -34,34 +28,20 @@ public JDKDirectoryWatcher(Path directory, Executor exec, Consumer e this.eventHandler = eventHandler; } - public void start(WatchEvent.Kind... eventKinds) throws IOException { + public void start() throws IOException { try { if (activeWatch != null) { // TODO make sure there is no cross thread race possible here. throw new IOException("Cannot start a watcher twice"); } - var kinds = Arrays.stream(eventKinds).map(JDKDirectoryWatcher::convertKind) - .toArray(Kind[]::new); - activeWatch = JDKPoller.INSTANCE.register(directory, this::handleChanges, kinds); + activeWatch = JDKPoller.INSTANCE.register(directory, this::handleChanges); logger.debug("Started watch for: {}", directory); } catch (IOException e) { throw new IOException("Could not register directory watcher for: " + directory, e); } } - private static java.nio.file.WatchEvent.Kind convertKind(engineering.swat.watch.WatchEvent.Kind k) { - switch (k) { - case CREATED: - return ENTRY_CREATE; - case DELETED: - return ENTRY_DELETE; - case MODIFIED: - return ENTRY_MODIFY; - default: throw new IllegalArgumentException("Missing case for: " + k); - } - } - private void handleChanges(List> events) { exec.execute(() -> events.forEach(ev -> eventHandler.accept(translate(ev))) diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index 6b26255d..277339ee 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -10,7 +10,6 @@ import java.nio.file.WatchEvent; import java.nio.file.WatchKey; import java.nio.file.WatchService; -import java.nio.file.WatchEvent.Kind; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -64,9 +63,9 @@ private void poller() { } } - public Closeable register(Path path, Consumer>> changes, Kind[] kinds) throws IOException { + public Closeable register(Path path, Consumer>> changes) throws IOException { logger.debug("Register watch for: {}", path); - var key = path.register(service, kinds); + var key = path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY); logger.trace("Got watch key: {}", key); watchers.put(key, changes); return new Closeable() { diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index 2a8c4cf0..34171190 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -3,17 +3,13 @@ import java.io.Closeable; import java.io.IOException; import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import org.apache.logging.log4j.LogManager; @@ -35,54 +31,39 @@ public JDKRecursiveDirectoryWatcher(Path directory, Executor exec, Consumer wrappedHandler(boolean hadCreate, boolean hadModify, boolean hadDelete, Kind[] kinds) { - return ev -> { - logger.trace("Unwrapping event: {}", ev); - switch (ev.getKind()) { - case CREATED: - addNewDirectoryWatch(hadCreate, hadModify, hadDelete, kinds, ev); - if (hadCreate) { - eventHandler.accept(ev); - } - break; - case DELETED: - handleDeleteDirectory(ev); - if (hadDelete) { - eventHandler.accept(ev); - } - break; - case MODIFIED: - if (hadModify) { - eventHandler.accept(ev); - } - break; + private void wrappedHandler(WatchEvent ev) { + logger.trace("Unwrapping event: {}", ev); + try { + if (ev.getKind() == Kind.CREATED) { + // between the event and the current state of the file system + // we might have some nested directories we missed + // so if we have a new directory, we have to go in and iterate over it + try { + startRecursive(ev.calculateFullPath()); + } catch (IOException e) { + logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), e); + } + } + else if (ev.getKind() == Kind.DELETED) { + handleDeleteDirectory(ev.calculateFullPath()); } - }; + } finally { + eventHandler.accept(ev); + } } - private void handleDeleteDirectory(WatchEvent ev) { - var removedPath = ev.calculateFullPath(); - var existingWatch = activeWatches.get(removedPath); + private void handleDeleteDirectory(Path removedPath) { try { + var existingWatch = activeWatches.remove(removedPath); if (existingWatch != null) { logger.debug("Clearing watch on removed directory: {}", removedPath); existingWatch.close(); @@ -92,20 +73,8 @@ private void handleDeleteDirectory(WatchEvent ev) { } } - private void addNewDirectoryWatch(boolean hadCreate, boolean hadModify, boolean hadDelete, Kind[] kinds, WatchEvent ev) { - var newPath = ev.calculateFullPath(); - if (Files.isDirectory(newPath)) { - try { - logger.debug("New directory found, adding a watch for it: {}", newPath); - startRecursive(newPath, wrappedHandler(hadCreate, hadModify, hadDelete, kinds), kinds); - } catch (IOException ex) { - logger.error("Error adding watch for: {} {}", newPath, ex); - } - } - } - - private void startRecursive(Path root, Consumer handler, Kind[] modifiedKinds) throws IOException { - Files.walkFileTree(root, new SimpleFileVisitor() { + private void startRecursive(Path dir) throws IOException { + Files.walkFileTree(dir, new SimpleFileVisitor() { @Override public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { logger.error("We could not visit {} to schedule recursive file watches: {}", file, exc); @@ -114,18 +83,10 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOExce @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - var watcher = new JDKDirectoryWatcher(dir, exec, handler); - activeWatches.put(dir, watcher); - try { - watcher.start(modifiedKinds); - return FileVisitResult.CONTINUE; - } catch (IOException ex) { - activeWatches.remove(dir); - throw ex; - } + addNewDirectory(dir); + return FileVisitResult.CONTINUE; } - @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { if (exc != null) { @@ -136,16 +97,16 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx }); } - - - - private boolean contains(Kind[] eventKinds, Kind needle) { - for (var k : eventKinds) { - if (k == needle) { - return true; - } + private void addNewDirectory(Path dir) throws IOException { + var watcher = new JDKDirectoryWatcher(dir, exec, this::wrappedHandler); + activeWatches.put(dir, watcher); + try { + watcher.start(); + } catch (IOException ex) { + activeWatches.remove(dir); + logger.error("Could not register a watch for: {} ({})", dir, ex); + throw ex; } - return false; } @Override From 4313b3e8186e1246d8b44fca4fe8a3736e17759e Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Wed, 27 Dec 2023 14:07:37 +0100 Subject: [PATCH 06/89] Simplified JDKPoller singleton and avoided extra thread --- .../swat/watch/impl/JDKDirectoryWatcher.java | 2 +- .../swat/watch/impl/JDKPoller.java | 64 ++++++++++--------- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java index 163a00a5..62f19616 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -35,7 +35,7 @@ public void start() throws IOException { throw new IOException("Cannot start a watcher twice"); } - activeWatch = JDKPoller.INSTANCE.register(directory, this::handleChanges); + activeWatch = JDKPoller.register(directory, this::handleChanges); logger.debug("Started watch for: {}", directory); } catch (IOException e) { throw new IOException("Could not register directory watcher for: " + directory, e); diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index 277339ee..bb806186 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -12,6 +12,7 @@ import java.nio.file.WatchService; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -19,51 +20,54 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -enum JDKPoller { - INSTANCE; +class JDKPoller { + private JDKPoller() {} - private final Logger logger = LogManager.getLogger(); - private final WatchService service; - private final Map>>> watchers = new ConcurrentHashMap<>(); - private final Thread pollThread; + private static final Logger logger = LogManager.getLogger(); + private static final Map>>> watchers = new ConcurrentHashMap<>(); + private static final WatchService service; - private JDKPoller() { + static { try { service = FileSystems.getDefault().newWatchService(); } catch (IOException e) { throw new RuntimeException("Could not start watcher", e); } - pollThread = new Thread(this::poller); - pollThread.setDaemon(true); - pollThread.setName("file-watcher poll events thread"); - pollThread.start(); + // kick off the poll loop + poll(); } - private void poller() { - while (true) { - try { - WatchKey hit; - if ((hit = service.poll(1, TimeUnit.MILLISECONDS)) != null) { - logger.trace("Got hit: {}", hit); - try { - var watchHandler = watchers.get(hit); - if (watchHandler != null) { - var events = hit.pollEvents(); - logger.trace("Found watcher for hit: {}, sending: {} (size: {})", watchHandler, events, events.size()); - watchHandler.accept(events); - } - } - finally{ - hit.reset(); + private static void poll() { + try { + WatchKey hit; + while ((hit = service.poll()) != null) { + logger.trace("Got hit: {}", hit); + try { + var watchHandler = watchers.get(hit); + if (watchHandler != null) { + var events = hit.pollEvents(); + logger.trace("Found watcher for hit: {}, sending: {} (size: {})", watchHandler, events, events.size()); + watchHandler.accept(events); } } - } catch (InterruptedException e) { - return; + finally{ + hit.reset(); + } } + + } + finally { + // schedule next run + // note we don't want to have multiple polls running in parallel + // so that is why we only schedule the next one after we're done + // processing all messages + CompletableFuture + .delayedExecutor(1, TimeUnit.MILLISECONDS) + .execute(JDKPoller::poll); } } - public Closeable register(Path path, Consumer>> changes) throws IOException { + public static Closeable register(Path path, Consumer>> changes) throws IOException { logger.debug("Register watch for: {}", path); var key = path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY); logger.trace("Got watch key: {}", key); From 82615a3cf2754debb4cd34e6cb65d5a304721496 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 24 Feb 2024 22:10:18 +0100 Subject: [PATCH 07/89] Added recursive tests --- .../swat/watch/RecursiveWatchTests.java | 55 +++++++++++++++++++ .../engineering/swat/watch/SmokeTests.java | 52 +++++------------- .../engineering/swat/watch/TestDirectory.java | 50 +++++++++++++++++ 3 files changed, 119 insertions(+), 38 deletions(-) create mode 100644 src/test/java/engineering/swat/watch/RecursiveWatchTests.java create mode 100644 src/test/java/engineering/swat/watch/TestDirectory.java diff --git a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java new file mode 100644 index 00000000..99787662 --- /dev/null +++ b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java @@ -0,0 +1,55 @@ +package engineering.swat.watch; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import org.awaitility.Awaitility; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import static org.awaitility.Awaitility.await; + +class RecursiveWatchTests { + + static @MonotonicNonNull TestDirectory testDir; + + @BeforeAll + static void setupEverything() throws IOException { + testDir = new TestDirectory(); + Awaitility.setDefaultTimeout(2, TimeUnit.SECONDS); + } + + @AfterAll + static void cleanupDirectory() throws IOException { + if (testDir != null) { + testDir.close(); + } + } + + @Test + void newDirectoryWithFilesChangesDetected() throws IOException { + var target = new AtomicReference(); + var created = new AtomicBoolean(false); + var changed = new AtomicBoolean(false); + var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) + .onCreate(p -> {if (p.equals(target.get())) { created.set(true); }}) + .onModified(p -> {if (p.equals(target.get())) { changed.set(true); }}) + ; + + try (var activeWatch = watchConfig.start() ) { + var freshFile = Files.createTempDirectory(testDir.getTestDirectory(), "new-dir").resolve("test-file.txt"); + target.set(freshFile); + Files.writeString(freshFile, "Hello world"); + await().alias("New files should have been seen").until(created::get); + Files.writeString(freshFile, "Hello world 2"); + await().alias("Fresh file change have been detected").until(changed::get); + } + } + +} diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index 766b7ad8..79c9df09 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -1,65 +1,41 @@ package engineering.swat.watch; -import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.File; +import static org.awaitility.Awaitility.await; + import java.io.IOException; -import java.net.URL; import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.awaitility.Awaitility; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import static org.awaitility.Awaitility.*; -import static java.time.Duration.*; - - -public class SmokeTests { - static Path testDirectory; - static List testFiles = new ArrayList<>(); +class SmokeTests { + static @MonotonicNonNull TestDirectory testDir; @BeforeAll - static void setupTestDirectory() throws IOException { - testDirectory = Files.createTempDirectory("smoke-test"); - add3Files(testDirectory); - for (var d: Arrays.asList("d1", "d2", "d3")) { - Files.createDirectories(testDirectory.resolve(d)); - add3Files(testDirectory.resolve(d)); - } + static void setupEverything() throws IOException { + testDir = new TestDirectory(); Awaitility.setDefaultTimeout(2, TimeUnit.SECONDS); } - private static void add3Files(Path root) throws IOException { - for (var f : Arrays.asList("a.txt", "b.txt", "c.txt")) { - testFiles.add(Files.createFile(root.resolve(f))); - } - } - @AfterAll static void cleanupDirectory() throws IOException { - if (testDirectory != null) { - Files.walk(testDirectory) - .sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .forEach(File::delete); + if (testDir != null) { + testDir.close(); } } @Test void watchDirectory() throws IOException, InterruptedException { var changed = new AtomicBoolean(false); - var target = testFiles.get(0); - var watchConfig = Watcher.singleDirectory(testDirectory) + var target = testDir.getTestFiles().get(0); + var watchConfig = Watcher.singleDirectory(testDir.getTestDirectory()) .onModified(p -> {if (p.equals(target)) { changed.set(true); }}) ; @@ -72,11 +48,11 @@ void watchDirectory() throws IOException, InterruptedException { @Test void watchRecursiveDirectory() throws IOException, InterruptedException { var changed = new AtomicBoolean(false); - var target = testFiles.stream() - .filter(p -> !p.getParent().equals(testDirectory)) + var target = testDir.getTestFiles().stream() + .filter(p -> !p.getParent().equals(testDir.getTestDirectory())) .findFirst() .orElseThrow(); - var watchConfig = Watcher.recursiveDirectory(testDirectory) + var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) .onModified(p -> {if (p.equals(target)) { changed.set(true); }}) ; diff --git a/src/test/java/engineering/swat/watch/TestDirectory.java b/src/test/java/engineering/swat/watch/TestDirectory.java new file mode 100644 index 00000000..27a955c8 --- /dev/null +++ b/src/test/java/engineering/swat/watch/TestDirectory.java @@ -0,0 +1,50 @@ +package engineering.swat.watch; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +class TestDirectory implements Closeable { + private final Path testDirectory; + private final List testFiles; + + + TestDirectory() throws IOException { + testDirectory = Files.createTempDirectory("smoke-test"); + List testFiles = new ArrayList<>(); + add3Files(testFiles, testDirectory); + for (var d: Arrays.asList("d1", "d2", "d3")) { + Files.createDirectories(testDirectory.resolve(d)); + add3Files(testFiles, testDirectory.resolve(d)); + } + this.testFiles = Collections.unmodifiableList(testFiles); + } + + private static void add3Files(List testFiles, Path root) throws IOException { + for (var f : Arrays.asList("a.txt", "b.txt", "c.txt")) { + testFiles.add(Files.createFile(root.resolve(f))); + } + } + + @Override + public void close() throws IOException { + Files.walk(testDirectory) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + + public Path getTestDirectory() { + return testDirectory; + } + public List getTestFiles() { + return testFiles; + } +} From a0ea7db3dd97d09009caa218a0129046be3157fe Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 24 Feb 2024 22:11:01 +0100 Subject: [PATCH 08/89] Implemented virtual events for new nested directories and files --- .../engineering/swat/watch/WatchEvent.java | 5 +++ .../impl/JDKRecursiveDirectoryWatcher.java | 33 +++++++++++++++---- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/main/java/engineering/swat/watch/WatchEvent.java b/src/main/java/engineering/swat/watch/WatchEvent.java index e753462b..c2412ba9 100644 --- a/src/main/java/engineering/swat/watch/WatchEvent.java +++ b/src/main/java/engineering/swat/watch/WatchEvent.java @@ -36,4 +36,9 @@ public Path calculateFullPath() { return rootPath.resolve(relativePath); } + @Override + public String toString() { + return String.format("WatchEvent[%s, %s, %s]", this.rootPath, this.kind, this.relativePath); + } + } diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index 34171190..e424a7b2 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -7,6 +7,8 @@ import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; @@ -34,7 +36,7 @@ public JDKRecursiveDirectoryWatcher(Path directory, Executor exec, Consumer newEvents.forEach(eventHandler)); } catch (IOException e) { logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), e); } @@ -73,7 +79,8 @@ private void handleDeleteDirectory(Path removedPath) { } } - private void startRecursive(Path dir) throws IOException { + private List startRecursive(Path dir, boolean collectCreates) throws IOException { + var events = new ArrayList(); Files.walkFileTree(dir, new SimpleFileVisitor() { @Override public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { @@ -82,19 +89,31 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOExce } @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - addNewDirectory(dir); + public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs) throws IOException { + addNewDirectory(subdir); + if (collectCreates && !dir.equals(subdir)) { + events.add(new WatchEvent(WatchEvent.Kind.CREATED, directory, directory.relativize(subdir))); + } return FileVisitResult.CONTINUE; } @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + public FileVisitResult postVisitDirectory(Path subdir, IOException exc) throws IOException { if (exc != null) { - logger.error("Error during directory iteration: {} = {}", dir, exc); + logger.error("Error during directory iteration: {} = {}", subdir, exc); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (collectCreates) { + events.add(new WatchEvent(WatchEvent.Kind.CREATED, directory, directory.relativize(file))); } return FileVisitResult.CONTINUE; } }); + return events; } private void addNewDirectory(Path dir) throws IOException { From a12b50ebc6890145b4f19b05606025bd94109a9c Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 24 Feb 2024 23:22:57 +0100 Subject: [PATCH 09/89] Got checker framework to properly work --- pom.xml | 34 +- src/main/checkerframework/log4j2.astub | 882 +++++++++++++++++++++++++ 2 files changed, 902 insertions(+), 14 deletions(-) create mode 100644 src/main/checkerframework/log4j2.astub diff --git a/pom.xml b/pom.xml index 54a91c26..995c88cb 100644 --- a/pom.xml +++ b/pom.xml @@ -10,9 +10,9 @@ UTF-8 - 3.40.0 - 5.10.1 - 2.22.0 + 3.42.0 + 5.10.2 + 2.23.0 11 11 @@ -143,10 +143,23 @@ true 11 - - 1000 - 1000 - + + -Xmaxerrs + 10000 + -Xmaxwarns + 10000 + + -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED + -Astubs=src/main/checkerframework + org.checkerframework @@ -172,13 +185,6 @@ ${checkerframework.version} provided - diff --git a/src/main/checkerframework/log4j2.astub b/src/main/checkerframework/log4j2.astub new file mode 100644 index 00000000..17b7dba6 --- /dev/null +++ b/src/main/checkerframework/log4j2.astub @@ -0,0 +1,882 @@ +package org.apache.logging.log4j; + +import org.apache.logging.log4j.message.EntryMessage; +import org.apache.logging.log4j.message.FlowMessageFactory; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.MessageFactory; +import org.apache.logging.log4j.message.MessageFactory2; +import org.apache.logging.log4j.util.MessageSupplier; +import org.apache.logging.log4j.util.Supplier; +import org.checkerframework.checker.nullness.qual.Nullable; + +public interface Logger { + void catching(Level level, @Nullable Throwable throwable); + void catching(@Nullable Throwable throwable); + void debug(Marker marker, Message message); + void debug(Marker marker, Message message, @Nullable Throwable throwable); + void debug(Marker marker, MessageSupplier messageSupplier); + void debug(Marker marker, MessageSupplier messageSupplier, @Nullable Throwable throwable); + void debug(Marker marker, CharSequence message); + void debug(Marker marker, CharSequence message, @Nullable Throwable throwable); + void debug(Marker marker, Object message); + void debug(Marker marker, Object message, @Nullable Throwable throwable); + void debug(Marker marker, String message); + void debug(Marker marker, String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void debug(Marker marker, String message, Supplier... paramSuppliers); + void debug(Marker marker, String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void debug(Marker marker, Supplier messageSupplier); + @SuppressWarnings("deprecation") + void debug(Marker marker, Supplier messageSupplier, @Nullable Throwable throwable); + void debug(Message message); + void debug(Message message, @Nullable Throwable throwable); + void debug(MessageSupplier messageSupplier); + void debug(MessageSupplier messageSupplier, @Nullable Throwable throwable); + void debug(CharSequence message); + void debug(CharSequence message, @Nullable Throwable throwable); + void debug(Object message); + void debug(Object message, @Nullable Throwable throwable); + void debug(String message); + void debug(String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void debug(String message, Supplier... paramSuppliers); + void debug(String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void debug(Supplier messageSupplier); + @SuppressWarnings("deprecation") + void debug(Supplier messageSupplier, @Nullable Throwable throwable); + void debug(Marker marker, String message, @Nullable Object p0); + void debug(Marker marker, String message, @Nullable Object p0, @Nullable Object p1); + void debug(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void debug(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void debug(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void debug(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void debug( + Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void debug( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7); + void debug( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void debug( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + void debug(String message, @Nullable Object p0); + void debug(String message, @Nullable Object p0, @Nullable Object p1); + void debug(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void debug(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void debug(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void debug(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void debug(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void debug(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6, @Nullable Object p7); + void debug( + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void debug( + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + @Deprecated + void entry(); + @Deprecated + void entry(@Nullable Object... params); + void error(Marker marker, Message message); + void error(Marker marker, Message message, @Nullable Throwable throwable); + void error(Marker marker, MessageSupplier messageSupplier); + void error(Marker marker, MessageSupplier messageSupplier, @Nullable Throwable throwable); + void error(Marker marker, CharSequence message); + void error(Marker marker, CharSequence message, @Nullable Throwable throwable); + void error(Marker marker, Object message); + void error(Marker marker, Object message, @Nullable Throwable throwable); + void error(Marker marker, String message); + void error(Marker marker, String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void error(Marker marker, String message, Supplier... paramSuppliers); + void error(Marker marker, String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void error(Marker marker, Supplier messageSupplier); + @SuppressWarnings("deprecation") + void error(Marker marker, Supplier messageSupplier, @Nullable Throwable throwable); + void error(Message message); + void error(Message message, @Nullable Throwable throwable); + void error(MessageSupplier messageSupplier); + void error(MessageSupplier messageSupplier, @Nullable Throwable throwable); + void error(CharSequence message); + void error(CharSequence message, @Nullable Throwable throwable); + void error(Object message); + void error(Object message, @Nullable Throwable throwable); + void error(String message); + void error(String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void error(String message, Supplier... paramSuppliers); + void error(String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void error(Supplier messageSupplier); + @SuppressWarnings("deprecation") + void error(Supplier messageSupplier, @Nullable Throwable throwable); + void error(Marker marker, String message, @Nullable Object p0); + void error(Marker marker, String message, @Nullable Object p0, @Nullable Object p1); + void error(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void error(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void error(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void error(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void error( + Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void error( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7); + void error( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void error( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + void error(String message, @Nullable Object p0); + void error(String message, @Nullable Object p0, @Nullable Object p1); + void error(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void error(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void error(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void error(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void error(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void error(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6, @Nullable Object p7); + void error( + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void error( + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + @Deprecated + void exit(); + @Deprecated + R exit(R result); + void fatal(Marker marker, Message message); + void fatal(Marker marker, Message message, @Nullable Throwable throwable); + void fatal(Marker marker, MessageSupplier messageSupplier); + void fatal(Marker marker, MessageSupplier messageSupplier, @Nullable Throwable throwable); + void fatal(Marker marker, CharSequence message); + void fatal(Marker marker, CharSequence message, @Nullable Throwable throwable); + void fatal(Marker marker, Object message); + void fatal(Marker marker, Object message, @Nullable Throwable throwable); + void fatal(Marker marker, String message); + void fatal(Marker marker, String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void fatal(Marker marker, String message, Supplier... paramSuppliers); + void fatal(Marker marker, String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void fatal(Marker marker, Supplier messageSupplier); + @SuppressWarnings("deprecation") + void fatal(Marker marker, Supplier messageSupplier, @Nullable Throwable throwable); + void fatal(Message message); + void fatal(Message message, @Nullable Throwable throwable); + void fatal(MessageSupplier messageSupplier); + void fatal(MessageSupplier messageSupplier, @Nullable Throwable throwable); + void fatal(CharSequence message); + void fatal(CharSequence message, @Nullable Throwable throwable); + void fatal(Object message); + void fatal(Object message, @Nullable Throwable throwable); + void fatal(String message); + void fatal(String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void fatal(String message, Supplier... paramSuppliers); + void fatal(String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void fatal(Supplier messageSupplier); + @SuppressWarnings("deprecation") + void fatal(Supplier messageSupplier, @Nullable Throwable throwable); + void fatal(Marker marker, String message, @Nullable Object p0); + void fatal(Marker marker, String message, @Nullable Object p0, @Nullable Object p1); + void fatal(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void fatal(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void fatal(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void fatal(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void fatal( + Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void fatal( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7); + void fatal( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void fatal( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + void fatal(String message, @Nullable Object p0); + void fatal(String message, @Nullable Object p0, @Nullable Object p1); + void fatal(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void fatal(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void fatal(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void fatal(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void fatal(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void fatal(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6, @Nullable Object p7); + void fatal( + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void fatal( + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + Level getLevel(); + MF getMessageFactory(); + FlowMessageFactory getFlowMessageFactory(); + String getName(); + void info(Marker marker, Message message); + void info(Marker marker, Message message, @Nullable Throwable throwable); + void info(Marker marker, MessageSupplier messageSupplier); + void info(Marker marker, MessageSupplier messageSupplier, @Nullable Throwable throwable); + void info(Marker marker, CharSequence message); + void info(Marker marker, CharSequence message, @Nullable Throwable throwable); + void info(Marker marker, Object message); + void info(Marker marker, Object message, @Nullable Throwable throwable); + void info(Marker marker, String message); + void info(Marker marker, String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void info(Marker marker, String message, Supplier... paramSuppliers); + void info(Marker marker, String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void info(Marker marker, Supplier messageSupplier); + @SuppressWarnings("deprecation") + void info(Marker marker, Supplier messageSupplier, @Nullable Throwable throwable); + void info(Message message); + void info(Message message, @Nullable Throwable throwable); + void info(MessageSupplier messageSupplier); + void info(MessageSupplier messageSupplier, @Nullable Throwable throwable); + void info(CharSequence message); + void info(CharSequence message, @Nullable Throwable throwable); + void info(Object message); + void info(Object message, @Nullable Throwable throwable); + void info(String message); + void info(String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void info(String message, Supplier... paramSuppliers); + void info(String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void info(Supplier messageSupplier); + @SuppressWarnings("deprecation") + void info(Supplier messageSupplier, @Nullable Throwable throwable); + void info(Marker marker, String message, @Nullable Object p0); + void info(Marker marker, String message, @Nullable Object p0, @Nullable Object p1); + void info(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void info(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void info(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void info(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void info( + Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void info( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7); + void info( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void info( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + void info(String message, @Nullable Object p0); + void info(String message, @Nullable Object p0, @Nullable Object p1); + void info(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void info(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void info(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void info(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void info(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void info(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6, @Nullable Object p7); + void info( + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void info( + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + boolean isDebugEnabled(); + boolean isDebugEnabled(Marker marker); + boolean isEnabled(Level level); + boolean isEnabled(Level level, Marker marker); + boolean isErrorEnabled(); + boolean isErrorEnabled(Marker marker); + boolean isFatalEnabled(); + boolean isFatalEnabled(Marker marker); + boolean isInfoEnabled(); + boolean isInfoEnabled(Marker marker); + boolean isTraceEnabled(); + boolean isTraceEnabled(Marker marker); + boolean isWarnEnabled(); + boolean isWarnEnabled(Marker marker); + void log(Level level, Marker marker, Message message); + void log(Level level, Marker marker, Message message, @Nullable Throwable throwable); + void log(Level level, Marker marker, MessageSupplier messageSupplier); + void log(Level level, Marker marker, MessageSupplier messageSupplier, @Nullable Throwable throwable); + void log(Level level, Marker marker, CharSequence message); + void log(Level level, Marker marker, CharSequence message, @Nullable Throwable throwable); + void log(Level level, Marker marker, Object message); + void log(Level level, Marker marker, Object message, @Nullable Throwable throwable); + void log(Level level, Marker marker, String message); + void log(Level level, Marker marker, String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void log(Level level, Marker marker, String message, Supplier... paramSuppliers); + void log(Level level, Marker marker, String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void log(Level level, Marker marker, Supplier messageSupplier); + @SuppressWarnings("deprecation") + void log(Level level, Marker marker, Supplier messageSupplier, @Nullable Throwable throwable); + void log(Level level, Message message); + void log(Level level, Message message, @Nullable Throwable throwable); + void log(Level level, MessageSupplier messageSupplier); + void log(Level level, MessageSupplier messageSupplier, @Nullable Throwable throwable); + void log(Level level, CharSequence message); + void log(Level level, CharSequence message, @Nullable Throwable throwable); + void log(Level level, Object message); + void log(Level level, Object message, @Nullable Throwable throwable); + void log(Level level, String message); + void log(Level level, String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void log(Level level, String message, Supplier... paramSuppliers); + void log(Level level, String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void log(Level level, Supplier messageSupplier); + @SuppressWarnings("deprecation") + void log(Level level, Supplier messageSupplier, @Nullable Throwable throwable); + void log(Level level, Marker marker, String message, @Nullable Object p0); + void log(Level level, Marker marker, String message, @Nullable Object p0, @Nullable Object p1); + void log(Level level, Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void log(Level level, Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void log(Level level, Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void log( + Level level, + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5); + void log( + Level level, + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6); + void log( + Level level, + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7); + void log( + Level level, + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void log( + Level level, + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + void log(Level level, String message, @Nullable Object p0); + void log(Level level, String message, @Nullable Object p0, @Nullable Object p1); + void log(Level level, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void log(Level level, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void log(Level level, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void log(Level level, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void log(Level level, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void log( + Level level, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7); + void log( + Level level, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void log( + Level level, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + void printf(Level level, Marker marker, String format, @Nullable Object... params); + void printf(Level level, String format, @Nullable Object... params); + T throwing(Level level, T throwable); + T throwing(T throwable); + void trace(Marker marker, Message message); + void trace(Marker marker, Message message, @Nullable Throwable throwable); + void trace(Marker marker, MessageSupplier messageSupplier); + void trace(Marker marker, MessageSupplier messageSupplier, @Nullable Throwable throwable); + void trace(Marker marker, CharSequence message); + void trace(Marker marker, CharSequence message, @Nullable Throwable throwable); + void trace(Marker marker, Object message); + void trace(Marker marker, Object message, @Nullable Throwable throwable); + void trace(Marker marker, String message); + void trace(Marker marker, String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void trace(Marker marker, String message, Supplier... paramSuppliers); + void trace(Marker marker, String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void trace(Marker marker, Supplier messageSupplier); + @SuppressWarnings("deprecation") + void trace(Marker marker, Supplier messageSupplier, @Nullable Throwable throwable); + void trace(Message message); + void trace(Message message, @Nullable Throwable throwable); + void trace(MessageSupplier messageSupplier); + void trace(MessageSupplier messageSupplier, @Nullable Throwable throwable); + void trace(CharSequence message); + void trace(CharSequence message, @Nullable Throwable throwable); + void trace(Object message); + void trace(Object message, @Nullable Throwable throwable); + void trace(String message); + void trace(String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void trace(String message, Supplier... paramSuppliers); + void trace(String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void trace(Supplier messageSupplier); + @SuppressWarnings("deprecation") + void trace(Supplier messageSupplier, @Nullable Throwable throwable); + void trace(Marker marker, String message, @Nullable Object p0); + void trace(Marker marker, String message, @Nullable Object p0, @Nullable Object p1); + void trace(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void trace(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void trace(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void trace(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void trace( + Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void trace( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7); + void trace( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void trace( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + void trace(String message, @Nullable Object p0); + void trace(String message, @Nullable Object p0, @Nullable Object p1); + void trace(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void trace(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void trace(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void trace(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void trace(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void trace(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6, @Nullable Object p7); + void trace( + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void trace( + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + EntryMessage traceEntry(); + EntryMessage traceEntry(String format, @Nullable Object... params); + @SuppressWarnings("deprecation") + EntryMessage traceEntry(Supplier... paramSuppliers); + @SuppressWarnings("deprecation") + EntryMessage traceEntry(String format, Supplier... paramSuppliers); + EntryMessage traceEntry(Message message); + void traceExit(); + R traceExit(R result); + R traceExit(String format, R result); + void traceExit(EntryMessage message); + R traceExit(EntryMessage message, R result); + R traceExit(Message message, R result); + void warn(Marker marker, Message message); + void warn(Marker marker, Message message, @Nullable Throwable throwable); + void warn(Marker marker, MessageSupplier messageSupplier); + void warn(Marker marker, MessageSupplier messageSupplier, @Nullable Throwable throwable); + void warn(Marker marker, CharSequence message); + void warn(Marker marker, CharSequence message, @Nullable Throwable throwable); + void warn(Marker marker, Object message); + void warn(Marker marker, Object message, @Nullable Throwable throwable); + void warn(Marker marker, String message); + void warn(Marker marker, String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void warn(Marker marker, String message, Supplier... paramSuppliers); + void warn(Marker marker, String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void warn(Marker marker, Supplier messageSupplier); + @SuppressWarnings("deprecation") + void warn(Marker marker, Supplier messageSupplier, @Nullable Throwable throwable); + void warn(Message message); + void warn(Message message, @Nullable Throwable throwable); + void warn(MessageSupplier messageSupplier); + void warn(MessageSupplier messageSupplier, @Nullable Throwable throwable); + void warn(CharSequence message); + void warn(CharSequence message, @Nullable Throwable throwable); + void warn(Object message); + void warn(Object message, @Nullable Throwable throwable); + void warn(String message); + void warn(String message, @Nullable Object... params); + @SuppressWarnings("deprecation") + void warn(String message, Supplier... paramSuppliers); + void warn(String message, @Nullable Throwable throwable); + @SuppressWarnings("deprecation") + void warn(Supplier messageSupplier); + @SuppressWarnings("deprecation") + void warn(Supplier messageSupplier, @Nullable Throwable throwable); + void warn(Marker marker, String message, @Nullable Object p0); + void warn(Marker marker, String message, @Nullable Object p0, @Nullable Object p1); + void warn(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void warn(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void warn(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void warn(Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void warn( + Marker marker, String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void warn( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7); + void warn( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void warn( + Marker marker, + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + void warn(String message, @Nullable Object p0); + void warn(String message, @Nullable Object p0, @Nullable Object p1); + void warn(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2); + void warn(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3); + void warn(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4); + void warn(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5); + void warn(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6); + void warn(String message, @Nullable Object p0, @Nullable Object p1, @Nullable Object p2, @Nullable Object p3, @Nullable Object p4, @Nullable Object p5, @Nullable Object p6, @Nullable Object p7); + void warn( + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8); + void warn( + String message, + @Nullable Object p0, + @Nullable Object p1, + @Nullable Object p2, + @Nullable Object p3, + @Nullable Object p4, + @Nullable Object p5, + @Nullable Object p6, + @Nullable Object p7, + @Nullable Object p8, + @Nullable Object p9); + default void logMessage( + Level level, Marker marker, String fqcn, StackTraceElement location, Message message, @Nullable Throwable throwable) { + // noop + } + default LogBuilder atTrace() { + return LogBuilder.NOOP; + } + default LogBuilder atDebug() { + return LogBuilder.NOOP; + } + default LogBuilder atInfo() { + return LogBuilder.NOOP; + } + default LogBuilder atWarn() { + return LogBuilder.NOOP; + } + default LogBuilder atError() { + return LogBuilder.NOOP; + } + default LogBuilder atFatal() { + return LogBuilder.NOOP; + } + default LogBuilder always() { + return LogBuilder.NOOP; + } + default LogBuilder atLevel(Level level) { + return LogBuilder.NOOP; + } +} From d58eff4e8fb75e9466d3f649ebd7284464d47640 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sun, 25 Feb 2024 08:52:00 +0100 Subject: [PATCH 10/89] Added initial support for overflow event --- src/main/java/engineering/swat/watch/WatchEvent.java | 2 +- src/main/java/engineering/swat/watch/Watcher.java | 9 +++++++++ .../engineering/swat/watch/impl/JDKDirectoryWatcher.java | 6 +++++- src/test/java/engineering/swat/watch/OverflowTests.java | 6 ++++++ 4 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 src/test/java/engineering/swat/watch/OverflowTests.java diff --git a/src/main/java/engineering/swat/watch/WatchEvent.java b/src/main/java/engineering/swat/watch/WatchEvent.java index c2412ba9..a1329480 100644 --- a/src/main/java/engineering/swat/watch/WatchEvent.java +++ b/src/main/java/engineering/swat/watch/WatchEvent.java @@ -7,7 +7,7 @@ public class WatchEvent { public enum Kind { - CREATED, MODIFIED, DELETED + CREATED, MODIFIED, DELETED, OVERFLOW } private final Kind kind; diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index eb923e7f..0a62af33 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -24,6 +24,7 @@ public class Watcher { private Consumer createHandler = NO_OP; private Consumer modifiedHandler = NO_OP; private Consumer deletedHandler = NO_OP; + private Consumer overflowHandler = NO_OP; private Watcher(WatcherKind kind, Path path) { @@ -74,6 +75,11 @@ public Watcher onDeleted(Consumer removeHandler) { return this; } + public Watcher onOverflow(Consumer overflowHandler) { + this.overflowHandler = overflowHandler; + return this; + } + public Watcher withExecutor(Executor callbackHandler) { this.executor = callbackHandler; return this; @@ -108,6 +114,9 @@ private void handleEvent(WatchEvent ev) { case MODIFIED: callIfDefined(modifiedHandler, ev); break; + case OVERFLOW: + callIfDefined(overflowHandler, ev); + break; } } diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java index 62f19616..d362ce4b 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.async.JCToolsBlockingQueueFactory.WaitStrategy; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -59,10 +60,13 @@ else if (ev.kind() == StandardWatchEventKinds.ENTRY_MODIFY) { else if (ev.kind() == StandardWatchEventKinds.ENTRY_DELETE) { kind = WatchEvent.Kind.DELETED; } + else if (ev.kind() == StandardWatchEventKinds.OVERFLOW) { + kind = WatchEvent.Kind.OVERFLOW; + } else { throw new IllegalArgumentException("Unexpected watch event: " + ev); } - var path = (@Nullable Path)ev.context(); + var path = kind == WatchEvent.Kind.OVERFLOW ? this.directory : (@Nullable Path)ev.context(); logger.trace("Translated: {} to {} at {}", ev, kind, path); return new WatchEvent(kind, directory, path); } diff --git a/src/test/java/engineering/swat/watch/OverflowTests.java b/src/test/java/engineering/swat/watch/OverflowTests.java new file mode 100644 index 00000000..993a9977 --- /dev/null +++ b/src/test/java/engineering/swat/watch/OverflowTests.java @@ -0,0 +1,6 @@ +package engineering.swat.watch; + +class OverflowTests { + // TODO: add test for overflow behavior (recursive should for example manually scan for newly missed directories) + +} From 4f227957317431218ce2c322a700c8db0ad09539 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 26 Feb 2024 09:38:34 +0100 Subject: [PATCH 11/89] Rewrote tests to start with a fresh directory --- .vscode/settings.json | 2 +- .../java/engineering/swat/watch/Watcher.java | 1 + .../swat/watch/RecursiveWatchTests.java | 25 +++++++++++-------- .../engineering/swat/watch/SmokeTests.java | 20 +++++++++------ .../engineering/swat/watch/TestDirectory.java | 3 ++- 5 files changed, 31 insertions(+), 20 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 050505ce..04cd6188 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,3 @@ { - "java.configuration.updateBuildConfiguration": "interactive" + "java.configuration.updateBuildConfiguration": "automatic" } diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 0a62af33..aacffb63 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -21,6 +21,7 @@ public class Watcher { private static final Consumer NO_OP = p -> {}; + // TODO: reconsider interface, not `Path` but `WatchEvent`. private Consumer createHandler = NO_OP; private Consumer modifiedHandler = NO_OP; private Consumer deletedHandler = NO_OP; diff --git a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java index 99787662..7a2d06c5 100644 --- a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java +++ b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java @@ -1,5 +1,7 @@ package engineering.swat.watch; +import static org.awaitility.Awaitility.await; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -8,30 +10,33 @@ import java.util.concurrent.atomic.AtomicReference; import org.awaitility.Awaitility; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.awaitility.Awaitility.await; - class RecursiveWatchTests { - static @MonotonicNonNull TestDirectory testDir; + private TestDirectory testDir; - @BeforeAll - static void setupEverything() throws IOException { + @BeforeEach + void setup() throws IOException { testDir = new TestDirectory(); - Awaitility.setDefaultTimeout(2, TimeUnit.SECONDS); } - @AfterAll - static void cleanupDirectory() throws IOException { + @AfterEach + void cleanup() throws IOException { if (testDir != null) { testDir.close(); } } + @BeforeAll + static void setupEverything() throws IOException { + Awaitility.setDefaultTimeout(2, TimeUnit.SECONDS); + } + + @Test void newDirectoryWithFilesChangesDetected() throws IOException { var target = new AtomicReference(); diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index 79c9df09..b8545167 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -9,28 +9,32 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.awaitility.Awaitility; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class SmokeTests { - static @MonotonicNonNull TestDirectory testDir; + private TestDirectory testDir; - @BeforeAll - static void setupEverything() throws IOException { + @BeforeEach + void setup() throws IOException { testDir = new TestDirectory(); - Awaitility.setDefaultTimeout(2, TimeUnit.SECONDS); } - @AfterAll - static void cleanupDirectory() throws IOException { + @AfterEach + void cleanup() throws IOException { if (testDir != null) { testDir.close(); } } + @BeforeAll + static void setupEverything() { + Awaitility.setDefaultTimeout(2, TimeUnit.SECONDS); + } + @Test void watchDirectory() throws IOException, InterruptedException { var changed = new AtomicBoolean(false); diff --git a/src/test/java/engineering/swat/watch/TestDirectory.java b/src/test/java/engineering/swat/watch/TestDirectory.java index 27a955c8..10b17b26 100644 --- a/src/test/java/engineering/swat/watch/TestDirectory.java +++ b/src/test/java/engineering/swat/watch/TestDirectory.java @@ -17,7 +17,7 @@ class TestDirectory implements Closeable { TestDirectory() throws IOException { - testDirectory = Files.createTempDirectory("smoke-test"); + testDirectory = Files.createTempDirectory("java-watch-test"); List testFiles = new ArrayList<>(); add3Files(testFiles, testDirectory); for (var d: Arrays.asList("d1", "d2", "d3")) { @@ -44,6 +44,7 @@ public void close() throws IOException { public Path getTestDirectory() { return testDirectory; } + public List getTestFiles() { return testFiles; } From 4b3c16792c4a383f757996f5dbee68388b9e7f1a Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 26 Feb 2024 17:20:40 +0100 Subject: [PATCH 12/89] Added full event support to the watcher interface --- .../java/engineering/swat/watch/Watcher.java | 23 ++++++++++--------- .../swat/watch/impl/JDKDirectoryWatcher.java | 1 - .../swat/watch/RecursiveWatchTests.java | 6 ++++- src/test/resources/log4j2-test.xml | 2 +- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index aacffb63..052ce20d 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -20,12 +20,13 @@ public class Watcher { private Executor executor = CompletableFuture::runAsync; private static final Consumer NO_OP = p -> {}; + private static final Consumer NO_OP_WE = p -> {}; - // TODO: reconsider interface, not `Path` but `WatchEvent`. private Consumer createHandler = NO_OP; private Consumer modifiedHandler = NO_OP; private Consumer deletedHandler = NO_OP; private Consumer overflowHandler = NO_OP; + private Consumer eventHandler = NO_OP_WE; private Watcher(WatcherKind kind, Path path) { @@ -81,6 +82,11 @@ public Watcher onOverflow(Consumer overflowHandler) { return this; } + public Watcher onEvent(Consumer eventHandler) { + this.eventHandler = eventHandler; + return this; + } + public Watcher withExecutor(Executor callbackHandler) { this.executor = callbackHandler; return this; @@ -107,23 +113,18 @@ public Closeable start() throws IOException { private void handleEvent(WatchEvent ev) { switch (ev.getKind()) { case CREATED: - callIfDefined(createHandler, ev); + createHandler.accept(ev.calculateFullPath()); break; case DELETED: - callIfDefined(deletedHandler, ev); + deletedHandler.accept(ev.calculateFullPath()); break; case MODIFIED: - callIfDefined(modifiedHandler, ev); + modifiedHandler.accept(ev.calculateFullPath()); break; case OVERFLOW: - callIfDefined(overflowHandler, ev); + overflowHandler.accept(ev.calculateFullPath()); break; } - } - - private void callIfDefined(Consumer target, WatchEvent ev) { - if (target != NO_OP) { - target.accept(ev.calculateFullPath()); - } + eventHandler.accept(ev); } } diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java index d362ce4b..91d2c0fa 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.core.async.JCToolsBlockingQueueFactory.WaitStrategy; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; diff --git a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java index 7a2d06c5..e47f588b 100644 --- a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java +++ b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java @@ -9,6 +9,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -16,6 +18,7 @@ import org.junit.jupiter.api.Test; class RecursiveWatchTests { + private final Logger logger = LogManager.getLogger(); private TestDirectory testDir; @@ -33,7 +36,7 @@ void cleanup() throws IOException { @BeforeAll static void setupEverything() throws IOException { - Awaitility.setDefaultTimeout(2, TimeUnit.SECONDS); + Awaitility.setDefaultTimeout(4, TimeUnit.SECONDS); } @@ -45,6 +48,7 @@ void newDirectoryWithFilesChangesDetected() throws IOException { var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) .onCreate(p -> {if (p.equals(target.get())) { created.set(true); }}) .onModified(p -> {if (p.equals(target.get())) { changed.set(true); }}) + .onEvent(e -> logger.debug("Event received: {}", e)) ; try (var activeWatch = watchConfig.start() ) { diff --git a/src/test/resources/log4j2-test.xml b/src/test/resources/log4j2-test.xml index d341ee55..d417041b 100644 --- a/src/test/resources/log4j2-test.xml +++ b/src/test/resources/log4j2-test.xml @@ -6,7 +6,7 @@ - + From 2194412a2874b2bab39ecc25076acefa41a89293 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 27 Feb 2024 15:32:13 +0100 Subject: [PATCH 13/89] Making sure not to leak watches --- .../watch/impl/JDKRecursiveDirectoryWatcher.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index e424a7b2..bb5d41f3 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -16,6 +16,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.checkerframework.checker.nullness.qual.Nullable; import engineering.swat.watch.WatchEvent; import engineering.swat.watch.WatchEvent.Kind; @@ -118,7 +119,8 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO private void addNewDirectory(Path dir) throws IOException { var watcher = new JDKDirectoryWatcher(dir, exec, this::wrappedHandler); - activeWatches.put(dir, watcher); + var oldEntry = activeWatches.put(dir, watcher); + cleanupOld(dir, oldEntry); try { watcher.start(); } catch (IOException ex) { @@ -128,6 +130,17 @@ private void addNewDirectory(Path dir) throws IOException { } } + private void cleanupOld(Path dir, @Nullable Closeable oldEntry) { + if (oldEntry != null) { + logger.error("Registered a watch for a directory that was already watched: {}", dir); + try { + oldEntry.close(); + } catch (IOException ex) { + logger.error("Could not close old watch for: {} ({})", dir, ex); + } + } + } + @Override public void close() throws IOException { IOException firstFail = null; From 515a56aab5b1a5b00de32b615cfde49cf14395a3 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sun, 17 Mar 2024 11:25:36 +0100 Subject: [PATCH 14/89] Added overflow support for the recursive watcher --- .../impl/JDKRecursiveDirectoryWatcher.java | 170 +++++++++++++----- 1 file changed, 122 insertions(+), 48 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index bb5d41f3..c98622bc 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -7,8 +7,11 @@ import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; import java.util.List; +import java.util.Stack; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; @@ -37,7 +40,7 @@ public JDKRecursiveDirectoryWatcher(Path directory, Executor exec, Consumer newEvents.forEach(eventHandler)); } catch (IOException e) { @@ -63,6 +66,17 @@ private void wrappedHandler(WatchEvent ev) { else if (ev.getKind() == Kind.DELETED) { handleDeleteDirectory(ev.calculateFullPath()); } + else if (ev.getKind() == Kind.OVERFLOW) { + try { + logger.debug("Overflow detected, rescanning to find missed entries in {}", directory); + // we have to rescan everything, and at least make sure to add new entries to that recursive watcher + var newEntries = syncAfterOverflow(directory); + logger.trace("Reporting new nested directories & files: {}", newEntries); + exec.execute(() -> newEntries.forEach(eventHandler)); + } catch (IOException e) { + logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), e); + } + } } finally { eventHandler.accept(ev); } @@ -80,67 +94,127 @@ private void handleDeleteDirectory(Path removedPath) { } } - private List startRecursive(Path dir, boolean collectCreates) throws IOException { - var events = new ArrayList(); - Files.walkFileTree(dir, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { - logger.error("We could not visit {} to schedule recursive file watches: {}", file, exc); - return FileVisitResult.CONTINUE; - } + /** Only register a watched for every sub directory */ + private class InitialDirectoryScan extends SimpleFileVisitor { + protected final Path root; - @Override - public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs) throws IOException { - addNewDirectory(subdir); - if (collectCreates && !dir.equals(subdir)) { - events.add(new WatchEvent(WatchEvent.Kind.CREATED, directory, directory.relativize(subdir))); - } - return FileVisitResult.CONTINUE; - } + public InitialDirectoryScan(Path root) { + this.root = root; + } + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + logger.error("We could not visit {} to schedule recursive file watches: {}", file, exc); + return FileVisitResult.CONTINUE; + } - @Override - public FileVisitResult postVisitDirectory(Path subdir, IOException exc) throws IOException { - if (exc != null) { - logger.error("Error during directory iteration: {} = {}", subdir, exc); - } - return FileVisitResult.CONTINUE; + @Override + public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs) throws IOException { + addNewDirectory(subdir); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path subdir, IOException exc) throws IOException { + if (exc != null) { + logger.error("Error during directory iteration: {} = {}", subdir, exc); + } + return FileVisitResult.CONTINUE; + } + private void addNewDirectory(Path dir) throws IOException { + var watcher = new JDKDirectoryWatcher(dir, exec, JDKRecursiveDirectoryWatcher.this::wrappedHandler); + var oldEntry = activeWatches.put(dir, watcher); + cleanupOld(dir, oldEntry); + try { + watcher.start(); + } catch (IOException ex) { + activeWatches.remove(dir); + logger.error("Could not register a watch for: {} ({})", dir, ex); + throw ex; } + } - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (collectCreates) { - events.add(new WatchEvent(WatchEvent.Kind.CREATED, directory, directory.relativize(file))); + private void cleanupOld(Path dir, @Nullable Closeable oldEntry) { + if (oldEntry != null) { + logger.error("Registered a watch for a directory that was already watched: {}", dir); + try { + oldEntry.close(); + } catch (IOException ex) { + logger.error("Could not close old watch for: {} ({})", dir, ex); } - return FileVisitResult.CONTINUE; } - }); - return events; + } } - private void addNewDirectory(Path dir) throws IOException { - var watcher = new JDKDirectoryWatcher(dir, exec, this::wrappedHandler); - var oldEntry = activeWatches.put(dir, watcher); - cleanupOld(dir, oldEntry); - try { - watcher.start(); - } catch (IOException ex) { - activeWatches.remove(dir); - logger.error("Could not register a watch for: {} ({})", dir, ex); - throw ex; + /** register watch for new sub-dir, but also simulate event for every file & subdir found */ + private class NewDirectoryScan extends InitialDirectoryScan { + protected final List events; + public NewDirectoryScan(Path root, List events) { + super(root); + this.events = events; + } + + @Override + public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs) throws IOException { + if (!subdir.equals(root)) { + events.add(new WatchEvent(WatchEvent.Kind.CREATED, root, root.relativize(subdir))); + } + return super.preVisitDirectory(subdir, attrs); + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + events.add(new WatchEvent(WatchEvent.Kind.CREATED, root, root.relativize(file))); + return FileVisitResult.CONTINUE; } } - private void cleanupOld(Path dir, @Nullable Closeable oldEntry) { - if (oldEntry != null) { - logger.error("Registered a watch for a directory that was already watched: {}", dir); - try { - oldEntry.close(); - } catch (IOException ex) { - logger.error("Could not close old watch for: {} ({})", dir, ex); + /** detect directories that aren't tracked yet, and generate events only for new entries */ + private class OverflowSyncScan extends NewDirectoryScan { + private final Deque isNewDirectory = new ArrayDeque<>(); + public OverflowSyncScan(Path root, List events) { + super(root, events); + } + @Override + public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs) throws IOException { + if (!activeWatches.containsKey(subdir)) { + isNewDirectory.addLast(true); + return super.preVisitDirectory(subdir, attrs); + } + isNewDirectory.addLast(false); + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult postVisitDirectory(Path subdir, IOException exc) throws IOException { + isNewDirectory.removeLast(); + return super.postVisitDirectory(subdir, exc); + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (isNewDirectory.peekLast() == Boolean.TRUE) { + return super.visitFile(file, attrs); } + return FileVisitResult.CONTINUE; } } + private void registerInitialWatches(Path dir) throws IOException { + Files.walkFileTree(dir, new InitialDirectoryScan(dir)); + } + + private List registerForNewDirectory(Path dir) throws IOException { + var events = new ArrayList(); + Files.walkFileTree(dir, new NewDirectoryScan(dir, events)); + return events; + } + + private List syncAfterOverflow(Path dir) throws IOException { + var events = new ArrayList(); + Files.walkFileTree(dir, new OverflowSyncScan(dir, events)); + return events; + } + + + @Override public void close() throws IOException { IOException firstFail = null; From 6812d78c2de0b355c90ea0fb73c2a836435812e9 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sun, 17 Mar 2024 20:39:40 +0100 Subject: [PATCH 15/89] Refactoring the functions to make them smaller --- .../impl/JDKRecursiveDirectoryWatcher.java | 69 +++++++++++-------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index c98622bc..e218b6e5 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -11,7 +11,6 @@ import java.util.ArrayList; import java.util.Deque; import java.util.List; -import java.util.Stack; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; @@ -49,40 +48,46 @@ public void start() throws IOException { private void wrappedHandler(WatchEvent ev) { logger.trace("Unwrapping event: {}", ev); try { - if (ev.getKind() == Kind.CREATED) { - // between the event and the current state of the file system - // we might have some nested directories we missed - // so if we have a new directory, we have to go in and iterate over it - // we also have to report all nested files & dirs as created paths - // but we don't want to burden ourselves with those events - try { - var newEvents = registerForNewDirectory(ev.calculateFullPath()); - logger.trace("Reporting new nested directories & files: {}", newEvents); - exec.execute(() -> newEvents.forEach(eventHandler)); - } catch (IOException e) { - logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), e); - } - } - else if (ev.getKind() == Kind.DELETED) { - handleDeleteDirectory(ev.calculateFullPath()); - } - else if (ev.getKind() == Kind.OVERFLOW) { - try { - logger.debug("Overflow detected, rescanning to find missed entries in {}", directory); - // we have to rescan everything, and at least make sure to add new entries to that recursive watcher - var newEntries = syncAfterOverflow(directory); - logger.trace("Reporting new nested directories & files: {}", newEntries); - exec.execute(() -> newEntries.forEach(eventHandler)); - } catch (IOException e) { - logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), e); - } + switch (ev.getKind()) { + case CREATED: handleCreate(ev); break; + case DELETED: handleDeleteDirectory(ev); break; + case OVERFLOW: handleOverflow(ev); break; + case MODIFIED: break; } } finally { eventHandler.accept(ev); } } - private void handleDeleteDirectory(Path removedPath) { + private void handleCreate(WatchEvent ev) { + // between the event and the current state of the file system + // we might have some nested directories we missed + // so if we have a new directory, we have to go in and iterate over it + // we also have to report all nested files & dirs as created paths + // but we don't want to burden ourselves with those events + try { + var newEvents = registerForNewDirectory(ev.calculateFullPath()); + logger.trace("Reporting new nested directories & files: {}", newEvents); + exec.execute(() -> newEvents.forEach(eventHandler)); + } catch (IOException e) { + logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), e); + } + } + + private void handleOverflow(WatchEvent ev) { + try { + logger.debug("Overflow detected, rescanning to find missed entries in {}", directory); + // we have to rescan everything, and at least make sure to add new entries to that recursive watcher + var newEntries = syncAfterOverflow(directory); + logger.trace("Reporting new nested directories & files: {}", newEntries); + exec.execute(() -> newEntries.forEach(eventHandler)); + } catch (IOException e) { + logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), e); + } + } + + private void handleDeleteDirectory(WatchEvent ev) { + var removedPath = ev.calculateFullPath(); try { var existingWatch = activeWatches.remove(removedPath); if (existingWatch != null) { @@ -227,6 +232,12 @@ public void close() throws IOException { firstFail = ex; } } + catch (Exception ex) { + logger.error("Could not close watch", ex); + if (firstFail == null) { + firstFail = new IOException("Unexpected exception when closing", ex); + } + } } if (firstFail != null) { throw firstFail; From aff2b729dac94bda26a285b2675d7528927191f1 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 30 Mar 2024 10:54:10 +0100 Subject: [PATCH 16/89] Rewrote recursive watcher to report to relative root This better reflects other recursive implementations --- .../java/engineering/swat/watch/Watcher.java | 50 ++----------------- .../swat/watch/impl/JDKDirectoryWatcher.java | 10 ++-- .../swat/watch/impl/JDKPoller.java | 1 + .../impl/JDKRecursiveDirectoryWatcher.java | 44 ++++++++++------ .../swat/watch/RecursiveWatchTests.java | 41 +++++++++++++-- .../engineering/swat/watch/SmokeTests.java | 5 +- 6 files changed, 78 insertions(+), 73 deletions(-) diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 052ce20d..28ee5730 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -19,14 +19,7 @@ public class Watcher { private final Path path; private Executor executor = CompletableFuture::runAsync; - private static final Consumer NO_OP = p -> {}; - private static final Consumer NO_OP_WE = p -> {}; - - private Consumer createHandler = NO_OP; - private Consumer modifiedHandler = NO_OP; - private Consumer deletedHandler = NO_OP; - private Consumer overflowHandler = NO_OP; - private Consumer eventHandler = NO_OP_WE; + private Consumer eventHandler = p -> {}; private Watcher(WatcherKind kind, Path path) { @@ -62,26 +55,6 @@ public static Watcher recursiveDirectory(Path path) throws IOException { return new Watcher(WatcherKind.RECURSIVE_DIRECTORY, path); } - public Watcher onCreate(Consumer createHandler) { - this.createHandler = createHandler; - return this; - } - - public Watcher onModified(Consumer changeHandler) { - this.modifiedHandler = changeHandler; - return this; - } - - public Watcher onDeleted(Consumer removeHandler) { - this.deletedHandler = removeHandler; - return this; - } - - public Watcher onOverflow(Consumer overflowHandler) { - this.overflowHandler = overflowHandler; - return this; - } - public Watcher onEvent(Consumer eventHandler) { this.eventHandler = eventHandler; return this; @@ -95,12 +68,12 @@ public Watcher withExecutor(Executor callbackHandler) { public Closeable start() throws IOException { switch (kind) { case DIRECTORY: { - var result = new JDKDirectoryWatcher(path, executor, this::handleEvent); + var result = new JDKDirectoryWatcher(path, executor, this.eventHandler); result.start(); return result; } case RECURSIVE_DIRECTORY: { - var result = new JDKRecursiveDirectoryWatcher(path, executor, this::handleEvent); + var result = new JDKRecursiveDirectoryWatcher(path, executor, this.eventHandler); result.start(); return result; } @@ -110,21 +83,4 @@ public Closeable start() throws IOException { } } - private void handleEvent(WatchEvent ev) { - switch (ev.getKind()) { - case CREATED: - createHandler.accept(ev.calculateFullPath()); - break; - case DELETED: - deletedHandler.accept(ev.calculateFullPath()); - break; - case MODIFIED: - modifiedHandler.accept(ev.calculateFullPath()); - break; - case OVERFLOW: - overflowHandler.accept(ev.calculateFullPath()); - break; - } - eventHandler.accept(ev); - } } diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java index 91d2c0fa..ef0ffc68 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -30,12 +30,14 @@ public JDKDirectoryWatcher(Path directory, Executor exec, Consumer e public void start() throws IOException { try { - if (activeWatch != null) { - // TODO make sure there is no cross thread race possible here. - throw new IOException("Cannot start a watcher twice"); + synchronized(this) { + if (activeWatch != null) { + throw new IOException("Cannot start a watcher twice"); + } + + activeWatch = JDKPoller.register(directory, this::handleChanges); } - activeWatch = JDKPoller.register(directory, this::handleChanges); logger.debug("Started watch for: {}", directory); } catch (IOException e) { throw new IOException("Could not register directory watcher for: " + directory, e); diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index bb806186..84f33073 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -69,6 +69,7 @@ private static void poll() { public static Closeable register(Path path, Consumer>> changes) throws IOException { logger.debug("Register watch for: {}", path); + // TODO: consider upgrading the events the moment we actually get a request for all of it var key = path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY); logger.trace("Got watch key: {}", key); watchers.put(key, changes); diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index e218b6e5..9bcbfee6 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -15,6 +15,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; import java.util.function.Consumer; +import java.util.function.Function; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -25,27 +26,27 @@ public class JDKRecursiveDirectoryWatcher implements Closeable { private final Logger logger = LogManager.getLogger(); - private final Path directory; + private final Path root; private final Executor exec; private final Consumer eventHandler; private final ConcurrentMap activeWatches = new ConcurrentHashMap<>(); public JDKRecursiveDirectoryWatcher(Path directory, Executor exec, Consumer eventHandler) { - this.directory = directory; + this.root = directory; this.exec = exec; this.eventHandler = eventHandler; } public void start() throws IOException { try { - logger.debug("Starting recursive watch for: {}", directory); - registerInitialWatches(directory); + logger.debug("Starting recursive watch for: {}", root); + registerInitialWatches(root); } catch (IOException e) { - throw new IOException("Could not register directory watcher for: " + directory, e); + throw new IOException("Could not register directory watcher for: " + root, e); } } - private void wrappedHandler(WatchEvent ev) { + private void processEvents(WatchEvent ev) { logger.trace("Unwrapping event: {}", ev); try { switch (ev.getKind()) { @@ -59,6 +60,7 @@ private void wrappedHandler(WatchEvent ev) { } } + private void handleCreate(WatchEvent ev) { // between the event and the current state of the file system // we might have some nested directories we missed @@ -76,9 +78,9 @@ private void handleCreate(WatchEvent ev) { private void handleOverflow(WatchEvent ev) { try { - logger.debug("Overflow detected, rescanning to find missed entries in {}", directory); + logger.debug("Overflow detected, rescanning to find missed entries in {}", root); // we have to rescan everything, and at least make sure to add new entries to that recursive watcher - var newEntries = syncAfterOverflow(directory); + var newEntries = syncAfterOverflow(ev.calculateFullPath()); logger.trace("Reporting new nested directories & files: {}", newEntries); exec.execute(() -> newEntries.forEach(eventHandler)); } catch (IOException e) { @@ -101,10 +103,10 @@ private void handleDeleteDirectory(WatchEvent ev) { /** Only register a watched for every sub directory */ private class InitialDirectoryScan extends SimpleFileVisitor { - protected final Path root; + protected final Path subRoot; public InitialDirectoryScan(Path root) { - this.root = root; + this.subRoot = root; } @Override public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { @@ -125,8 +127,9 @@ public FileVisitResult postVisitDirectory(Path subdir, IOException exc) throws I } return FileVisitResult.CONTINUE; } + private void addNewDirectory(Path dir) throws IOException { - var watcher = new JDKDirectoryWatcher(dir, exec, JDKRecursiveDirectoryWatcher.this::wrappedHandler); + var watcher = new JDKDirectoryWatcher(dir, exec, relocater(dir)); var oldEntry = activeWatches.put(dir, watcher); cleanupOld(dir, oldEntry); try { @@ -138,6 +141,15 @@ private void addNewDirectory(Path dir) throws IOException { } } + /** Make sure that the events are relative to the actual root of the recursive watcher */ + private Consumer relocater(Path subRoot) { + final Path newRelative = root.relativize(subRoot); + return ev -> { + var rewritten = new WatchEvent(ev.getKind(), root, newRelative.resolve(ev.getRelativePath())); + processEvents(rewritten); + }; + } + private void cleanupOld(Path dir, @Nullable Closeable oldEntry) { if (oldEntry != null) { logger.error("Registered a watch for a directory that was already watched: {}", dir); @@ -153,14 +165,14 @@ private void cleanupOld(Path dir, @Nullable Closeable oldEntry) { /** register watch for new sub-dir, but also simulate event for every file & subdir found */ private class NewDirectoryScan extends InitialDirectoryScan { protected final List events; - public NewDirectoryScan(Path root, List events) { - super(root); + public NewDirectoryScan(Path subRoot, List events) { + super(subRoot); this.events = events; } @Override public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs) throws IOException { - if (!subdir.equals(root)) { + if (!subdir.equals(subRoot)) { events.add(new WatchEvent(WatchEvent.Kind.CREATED, root, root.relativize(subdir))); } return super.preVisitDirectory(subdir, attrs); @@ -176,8 +188,8 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO /** detect directories that aren't tracked yet, and generate events only for new entries */ private class OverflowSyncScan extends NewDirectoryScan { private final Deque isNewDirectory = new ArrayDeque<>(); - public OverflowSyncScan(Path root, List events) { - super(root, events); + public OverflowSyncScan(Path subRoot, List events) { + super(subRoot, events); } @Override public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs) throws IOException { diff --git a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java index e47f588b..4f0a79ee 100644 --- a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java +++ b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java @@ -46,14 +46,26 @@ void newDirectoryWithFilesChangesDetected() throws IOException { var created = new AtomicBoolean(false); var changed = new AtomicBoolean(false); var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) - .onCreate(p -> {if (p.equals(target.get())) { created.set(true); }}) - .onModified(p -> {if (p.equals(target.get())) { changed.set(true); }}) - .onEvent(e -> logger.debug("Event received: {}", e)) - ; + .onEvent(ev -> { + logger.debug("Event received: {}", ev); + if (ev.calculateFullPath().equals(target.get())) { + switch (ev.getKind()) { + case CREATED: + created.set(true); + break; + case MODIFIED: + changed.set(true); + break; + default: + break; + } + } + }); try (var activeWatch = watchConfig.start() ) { var freshFile = Files.createTempDirectory(testDir.getTestDirectory(), "new-dir").resolve("test-file.txt"); target.set(freshFile); + logger.debug("Interested in: {}", freshFile); Files.writeString(freshFile, "Hello world"); await().alias("New files should have been seen").until(created::get); Files.writeString(freshFile, "Hello world 2"); @@ -61,4 +73,25 @@ void newDirectoryWithFilesChangesDetected() throws IOException { } } + @Test + void correctRelativePathIsReported() throws IOException { + Path relative = Path.of("a","b", "c", "d.txt"); + var seen = new AtomicBoolean(false); + var watcher = Watcher.recursiveDirectory(testDir.getTestDirectory()) + .onEvent(ev -> { + logger.debug("Seen event: {}", ev); + if (ev.getRelativePath().equals(relative)) { + seen.set(true); + } + }); + + try (var w = watcher.start()) { + var targetFile = testDir.getTestDirectory().resolve(relative); + Files.createDirectories(targetFile.getParent()); + Files.writeString(targetFile, "Hello World"); + await().alias("Nested path is seen").until(seen::get); + } + + } + } diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index b8545167..96d82fe4 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -1,6 +1,7 @@ package engineering.swat.watch; +import static engineering.swat.watch.WatchEvent.Kind.*; import static org.awaitility.Awaitility.await; import java.io.IOException; @@ -40,7 +41,7 @@ void watchDirectory() throws IOException, InterruptedException { var changed = new AtomicBoolean(false); var target = testDir.getTestFiles().get(0); var watchConfig = Watcher.singleDirectory(testDir.getTestDirectory()) - .onModified(p -> {if (p.equals(target)) { changed.set(true); }}) + .onEvent(ev -> {if (ev.getKind() == MODIFIED && ev.calculateFullPath().equals(target)) { changed.set(true); }}) ; try (var activeWatch = watchConfig.start() ) { @@ -57,7 +58,7 @@ void watchRecursiveDirectory() throws IOException, InterruptedException { .findFirst() .orElseThrow(); var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) - .onModified(p -> {if (p.equals(target)) { changed.set(true); }}) + .onEvent(ev -> { if (ev.getKind() == MODIFIED && ev.calculateFullPath().equals(target)) { changed.set(true);}}) ; try (var activeWatch = watchConfig.start() ) { From 829e5068bfa79432f16c60d97904fa6a2aa744df Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sun, 31 Mar 2024 15:38:02 +0200 Subject: [PATCH 17/89] Execute syntatic events in the correct order --- .../impl/JDKRecursiveDirectoryWatcher.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index 9bcbfee6..315462fa 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -9,6 +9,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collections; import java.util.Deque; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -48,20 +49,24 @@ public void start() throws IOException { private void processEvents(WatchEvent ev) { logger.trace("Unwrapping event: {}", ev); + List extraEvents = null; try { switch (ev.getKind()) { - case CREATED: handleCreate(ev); break; + case CREATED: extraEvents = handleCreate(ev); break; case DELETED: handleDeleteDirectory(ev); break; - case OVERFLOW: handleOverflow(ev); break; + case OVERFLOW: extraEvents = handleOverflow(ev); break; case MODIFIED: break; } } finally { eventHandler.accept(ev); + if (extraEvents != null) { + extraEvents.forEach(eventHandler); + } } } - private void handleCreate(WatchEvent ev) { + private List handleCreate(WatchEvent ev) { // between the event and the current state of the file system // we might have some nested directories we missed // so if we have a new directory, we have to go in and iterate over it @@ -70,21 +75,23 @@ private void handleCreate(WatchEvent ev) { try { var newEvents = registerForNewDirectory(ev.calculateFullPath()); logger.trace("Reporting new nested directories & files: {}", newEvents); - exec.execute(() -> newEvents.forEach(eventHandler)); + return newEvents; } catch (IOException e) { logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), e); + return Collections.emptyList(); } } - private void handleOverflow(WatchEvent ev) { + private List handleOverflow(WatchEvent ev) { try { logger.debug("Overflow detected, rescanning to find missed entries in {}", root); // we have to rescan everything, and at least make sure to add new entries to that recursive watcher var newEntries = syncAfterOverflow(ev.calculateFullPath()); logger.trace("Reporting new nested directories & files: {}", newEntries); - exec.execute(() -> newEntries.forEach(eventHandler)); + return newEntries; } catch (IOException e) { logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), e); + return Collections.emptyList(); } } From c0a7f3f21d121184ca97d6ed3debdcd94fdf1361 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Fri, 19 Jul 2024 10:22:19 +0200 Subject: [PATCH 18/89] Added torture tests to make sure everything works, even in very busy IO --- .../impl/JDKRecursiveDirectoryWatcher.java | 12 +- .../engineering/swat/watch/OverflowTests.java | 6 - .../engineering/swat/watch/TortureTests.java | 147 ++++++++++++++++++ src/test/resources/log4j2-test.xml | 2 +- 4 files changed, 157 insertions(+), 10 deletions(-) delete mode 100644 src/test/java/engineering/swat/watch/OverflowTests.java create mode 100644 src/test/java/engineering/swat/watch/TortureTests.java diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index 315462fa..33148d7c 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -73,9 +73,15 @@ private List handleCreate(WatchEvent ev) { // we also have to report all nested files & dirs as created paths // but we don't want to burden ourselves with those events try { - var newEvents = registerForNewDirectory(ev.calculateFullPath()); - logger.trace("Reporting new nested directories & files: {}", newEvents); - return newEvents; + var fullPath = ev.calculateFullPath(); + if (!activeWatches.containsKey(fullPath)) { + var newEvents = registerForNewDirectory(ev.calculateFullPath()); + logger.trace("Reporting new nested directories & files: {}", newEvents); + return newEvents; + } + else { + return Collections.emptyList(); + } } catch (IOException e) { logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), e); return Collections.emptyList(); diff --git a/src/test/java/engineering/swat/watch/OverflowTests.java b/src/test/java/engineering/swat/watch/OverflowTests.java deleted file mode 100644 index 993a9977..00000000 --- a/src/test/java/engineering/swat/watch/OverflowTests.java +++ /dev/null @@ -1,6 +0,0 @@ -package engineering.swat.watch; - -class OverflowTests { - // TODO: add test for overflow behavior (recursive should for example manually scan for newly missed directories) - -} diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java new file mode 100644 index 00000000..e71d5ef7 --- /dev/null +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -0,0 +1,147 @@ +package engineering.swat.watch; + +import static engineering.swat.watch.WatchEvent.Kind.OVERFLOW; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.LocalTime; +import java.util.ArrayList; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TortureTests { + + private final Logger logger = LogManager.getLogger(); + + private TestDirectory testDir; + + @BeforeEach + void setup() throws IOException { + testDir = new TestDirectory(); + } + + @AfterEach + void cleanup() throws IOException { + if (testDir != null) { + testDir.close(); + } + } + + @BeforeAll + static void setupEverything() throws IOException { + Awaitility.setDefaultTimeout(4, TimeUnit.SECONDS); + } + + private final static int THREADS = 4; + private final static int BURST_SIZE = 1000; + private final static Duration STOP_AFTER = Duration.ofSeconds(4); + @Test + void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOException { + final var root = testDir.getTestDirectory(); + final var pathWritten = ConcurrentHashMap.newKeySet(); + final var stopRunning = new Semaphore(0); + final var done = new Semaphore(0); + final var jobs = new ArrayList>(); + + for (int j = 0; j < THREADS; j++) { + var r = new Random(j); + jobs.add(() -> { + try { + var end = LocalTime.now().plus(STOP_AFTER); + while (!stopRunning.tryAcquire(100, TimeUnit.MICROSECONDS)) { + if (LocalTime.now().isAfter(end)) { + break; + } + try { + // burst a bunch of creates creates and then sleep a bit + for (int i = 0; i< BURST_SIZE; i++) { + var file = root.resolve("l1" + r.nextInt(1000)) + .resolve("l2" + r.nextInt() + ".txt"); + Files.createDirectories(file.getParent()); + Files.writeString(file, "Hello world"); + pathWritten.add(file); + } + } catch (IOException e) { + } + Thread.yield(); + } + return null; + } catch (InterruptedException e) { + return null; + } + finally { + done.release(); + } + }); + } + + var pool = Executors.newCachedThreadPool(); + + final var events = new AtomicInteger(0); + var seenPaths = ConcurrentHashMap.newKeySet(); + var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) + .withExecutor(pool) + .onEvent(ev -> { + events.getAndIncrement(); + seenPaths.add(ev.calculateFullPath()); + }); + + try (var activeWatch = watchConfig.start() ) { + logger.info("Starting {} jobs", THREADS); + pool.invokeAll(jobs); + // now we generate a whole bunch of events + Thread.sleep(STOP_AFTER.toMillis()); + logger.info("Stopping jobs"); + stopRunning.release(THREADS); + assertTrue(done.tryAcquire(THREADS, STOP_AFTER.toMillis(), TimeUnit.MILLISECONDS), "The runners should have stopped running"); + logger.info("Generated: {} files", pathWritten.size()); + + logger.info("Waiting for the events processing to settle down"); + int lastEventCount = events.get(); + do { + Thread.sleep(STOP_AFTER.toMillis() * 2); + int currentEventCounts = events.get(); + if (currentEventCounts == lastEventCount) { + logger.info("Stable after: {} events", currentEventCounts); + break; + } + lastEventCount = currentEventCounts; + } while (true); + } + finally { + stopRunning.release(THREADS); + // shutdown the pool (so no new events are registered) + pool.shutdown(); + } + + // but wait till all scheduled tasks have been completed + pool.awaitTermination(10, TimeUnit.SECONDS); + + logger.info("Comparing events and files seen"); + // now make sure that the two sets are the same + for (var f : pathWritten) { + assertTrue(seenPaths.contains(f), "We should have seen all paths"); + } + } +} diff --git a/src/test/resources/log4j2-test.xml b/src/test/resources/log4j2-test.xml index d417041b..a6df566b 100644 --- a/src/test/resources/log4j2-test.xml +++ b/src/test/resources/log4j2-test.xml @@ -6,7 +6,7 @@ - + From 0858ca2cd06f359065cc9a5cf9326bdd2dc762a3 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Fri, 19 Jul 2024 10:27:39 +0200 Subject: [PATCH 19/89] Cleanup of code --- .../engineering/swat/watch/WatchSubscription.java | 14 -------------- .../watch/impl/JDKRecursiveDirectoryWatcher.java | 2 -- 2 files changed, 16 deletions(-) delete mode 100644 src/main/java/engineering/swat/watch/WatchSubscription.java diff --git a/src/main/java/engineering/swat/watch/WatchSubscription.java b/src/main/java/engineering/swat/watch/WatchSubscription.java deleted file mode 100644 index ad9bc3f5..00000000 --- a/src/main/java/engineering/swat/watch/WatchSubscription.java +++ /dev/null @@ -1,14 +0,0 @@ -package engineering.swat.watch; - -import java.io.Closeable; -import java.io.IOException; - -public class WatchSubscription implements Closeable { - - @Override - public void close() throws IOException { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'close'"); - } - -} diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index 33148d7c..0ff893c8 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -16,14 +16,12 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; import java.util.function.Consumer; -import java.util.function.Function; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.checkerframework.checker.nullness.qual.Nullable; import engineering.swat.watch.WatchEvent; -import engineering.swat.watch.WatchEvent.Kind; public class JDKRecursiveDirectoryWatcher implements Closeable { private final Logger logger = LogManager.getLogger(); From 46acd38dd65b94df8f627027f7eac577e75f339d Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Thu, 5 Sep 2024 10:21:02 +0200 Subject: [PATCH 20/89] Bundle registered watches to avoid duplicate FS watches --- .../swat/watch/impl/BundledSubscription.java | 95 +++++++++++++++++++ .../swat/watch/impl/ISubscribable.java | 9 ++ .../swat/watch/impl/JDKDirectoryWatcher.java | 5 +- .../swat/watch/impl/JDKFileWatcher.java | 0 .../impl/JDKRecursiveDirectoryWatcher.java | 2 +- .../engineering/swat/watch/TortureTests.java | 13 ++- 6 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 src/main/java/engineering/swat/watch/impl/BundledSubscription.java create mode 100644 src/main/java/engineering/swat/watch/impl/ISubscribable.java create mode 100644 src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java diff --git a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java new file mode 100644 index 00000000..854d5ab0 --- /dev/null +++ b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java @@ -0,0 +1,95 @@ +package engineering.swat.watch.impl; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; + +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +public class BundledSubscription implements ISubscribable { + private final ISubscribable around; + private final Map> subscriptions = new ConcurrentHashMap<>(); + + public BundledSubscription(ISubscribable around) { + this.around = around; + + } + + private static class Subscription implements Consumer { + private final List> consumers = new CopyOnWriteArrayList<>(); + private volatile @MonotonicNonNull Closeable closer; + Subscription(Consumer initialConsumer) { + consumers.add(initialConsumer); + } + + public void setCloser(Closeable closer) { + this.closer = closer; + } + + void add(Consumer newConsumer) { + consumers.add(newConsumer); + } + + synchronized boolean remove(Consumer existingConsumer) { + consumers.remove(existingConsumer); + return consumers.isEmpty(); + } + + @Override + public void accept(R t) { + for (var child: consumers) { + child.accept(t); + } + } + + boolean hasActiveConsumers() { + return !consumers.isEmpty(); + } + + + } + + @Override + public Closeable subscribe(A target, Consumer eventListener) throws IOException { + var active = this.subscriptions.get(target); + if (active == null) { + active = new Subscription<>(eventListener); + var newSubscriptions = around.subscribe(target, active); + active.setCloser(newSubscriptions); + var lostRace = this.subscriptions.putIfAbsent(target, active); + if (lostRace != null) { + try { + newSubscriptions.close(); + } catch (IOException _ignore) { + // ignore + } + lostRace.add(eventListener); + active = lostRace; + } + } + else { + active.add(eventListener); + } + var finalActive = active; + return () -> { + if (finalActive.remove(eventListener)) { + subscriptions.remove(target); + if (finalActive.hasActiveConsumers()) { + // we lost the race, someone else added something again + // so we put it back in the list + subscriptions.put(target, finalActive); + } + else { + finalActive.closer.close(); + } + } + }; + + } + + +} diff --git a/src/main/java/engineering/swat/watch/impl/ISubscribable.java b/src/main/java/engineering/swat/watch/impl/ISubscribable.java new file mode 100644 index 00000000..7f32a27b --- /dev/null +++ b/src/main/java/engineering/swat/watch/impl/ISubscribable.java @@ -0,0 +1,9 @@ +package engineering.swat.watch.impl; + +import java.io.Closeable; +import java.io.IOException; +import java.util.function.Consumer; + +public interface ISubscribable { + Closeable subscribe(A target, Consumer eventListener) throws IOException; +} diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java index ef0ffc68..4b8e65f4 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -22,6 +22,9 @@ public class JDKDirectoryWatcher implements Closeable { private final Consumer eventHandler; private volatile @MonotonicNonNull Closeable activeWatch; + private static final BundledSubscription>> + BUNDLED_JDK_WATCHERS = new BundledSubscription<>(JDKPoller::register); + public JDKDirectoryWatcher(Path directory, Executor exec, Consumer eventHandler) { this.directory = directory; this.exec = exec; @@ -35,7 +38,7 @@ public void start() throws IOException { throw new IOException("Cannot start a watcher twice"); } - activeWatch = JDKPoller.register(directory, this::handleChanges); + activeWatch = BUNDLED_JDK_WATCHERS.subscribe(directory, this::handleChanges); } logger.debug("Started watch for: {}", directory); diff --git a/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java new file mode 100644 index 00000000..e69de29b diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index 0ff893c8..842f5976 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -88,7 +88,7 @@ private List handleCreate(WatchEvent ev) { private List handleOverflow(WatchEvent ev) { try { - logger.debug("Overflow detected, rescanning to find missed entries in {}", root); + logger.info("Overflow detected, rescanning to find missed entries in {}", root); // we have to rescan everything, and at least make sure to add new entries to that recursive watcher var newEntries = syncAfterOverflow(ev.calculateFullPath()); logger.trace("Reporting new nested directories & files: {}", newEntries); diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index e71d5ef7..c3287785 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -119,12 +119,21 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio logger.info("Waiting for the events processing to settle down"); int lastEventCount = events.get(); + int stableCount = 0; do { Thread.sleep(STOP_AFTER.toMillis() * 2); int currentEventCounts = events.get(); if (currentEventCounts == lastEventCount) { - logger.info("Stable after: {} events", currentEventCounts); - break; + if (stableCount == 2) { + logger.info("Stable after: {} events", currentEventCounts); + break; + } + else { + stableCount++; + } + } + else { + stableCount = 0; } lastEventCount = currentEventCounts; } while (true); From de3ea3326c8774b9649d7f99eb04f523f350d254 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Thu, 5 Sep 2024 13:13:23 +0200 Subject: [PATCH 21/89] Implemented single file support --- .../java/engineering/swat/watch/Watcher.java | 8 ++- .../swat/watch/impl/JDKFileWatcher.java | 64 +++++++++++++++++++ .../swat/watch/SingleFileTests.java | 64 +++++++++++++++++++ .../engineering/swat/watch/SmokeTests.java | 21 ++++++ 4 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 src/test/java/engineering/swat/watch/SingleFileTests.java diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 28ee5730..39678b45 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.Logger; import engineering.swat.watch.impl.JDKDirectoryWatcher; +import engineering.swat.watch.impl.JDKFileWatcher; import engineering.swat.watch.impl.JDKRecursiveDirectoryWatcher; public class Watcher { @@ -77,7 +78,12 @@ public Closeable start() throws IOException { result.start(); return result; } - case FILE: + case FILE: { + var result = new JDKFileWatcher(path, executor, this.eventHandler); + result.start(); + return result; + } + default: throw new IllegalArgumentException("Not supported yet"); } diff --git a/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java index e69de29b..a99a692b 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java @@ -0,0 +1,64 @@ +package engineering.swat.watch.impl; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Path; +import java.util.concurrent.Executor; +import java.util.function.Consumer; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +import engineering.swat.watch.WatchEvent; + +/** + * It's not possible to monitor a single file, so we have to find a directory watcher, and connect to that + */ +public class JDKFileWatcher implements Closeable { + private final Logger logger = LogManager.getLogger(); + private final Path file; + private final Path fileName; + private final Executor exec; + private final Consumer eventHandler; + private volatile @MonotonicNonNull Closeable activeWatch; + + public JDKFileWatcher(Path file, Executor exec, Consumer eventHandler) { + this.file = file; + this.fileName = file.getFileName(); + this.exec = exec; + this.eventHandler = eventHandler; + } + + public void start() throws IOException { + try { + synchronized(this) { + if (activeWatch != null) { + throw new IOException("Cannot start an already started watch"); + } + var dir = file.getParent(); + assert !dir.equals(file); + var parentWatch = new JDKDirectoryWatcher(dir, exec, this::filter); + activeWatch = parentWatch; + parentWatch.start(); + logger.debug("Started file watch for {} (in reality a watch on {}): {}", file, dir, parentWatch); + } + + } catch (IOException e) { + throw new IOException("Could not register file watcher for: " + file, e); + } + } + + private void filter(WatchEvent event) { + if (fileName.equals(event.getRelativePath())) { + exec.execute(() -> eventHandler.accept(event)); + } + } + + @Override + public void close() throws IOException { + if (activeWatch != null) { + activeWatch.close(); + } + } +} diff --git a/src/test/java/engineering/swat/watch/SingleFileTests.java b/src/test/java/engineering/swat/watch/SingleFileTests.java new file mode 100644 index 00000000..ab387ce1 --- /dev/null +++ b/src/test/java/engineering/swat/watch/SingleFileTests.java @@ -0,0 +1,64 @@ +package engineering.swat.watch; + +import static org.awaitility.Awaitility.await; + +import java.io.IOException; +import java.nio.file.Files; +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class SingleFileTests { + private TestDirectory testDir; + + @BeforeEach + void setup() throws IOException { + testDir = new TestDirectory(); + } + + @AfterEach + void cleanup() throws IOException { + if (testDir != null) { + testDir.close(); + } + } + + @BeforeAll + static void setupEverything() { + Awaitility.setDefaultTimeout(2, TimeUnit.SECONDS); + } + + @Test + void singleFileShouldNotTriggerOnOtherFilesInSameDir() throws IOException, InterruptedException { + var target = testDir.getTestFiles().get(0); + var seen = new AtomicBoolean(false); + var others = new AtomicBoolean(false); + var watchConfig = Watcher.singleFile(target) + .onEvent(ev -> { + if (ev.calculateFullPath().equals(target)) { + seen.set(true); + } + else { + others.set(true); + } + }); + try (var watch = watchConfig.start()) { + for (var f : testDir.getTestFiles()) { + if (!f.equals(target)) { + Files.writeString(f, "Hello"); + } + } + Thread.sleep(1000); + Files.writeString(target, "Hello world"); + await("Single file does trigger") + .failFast("No others should be notified", others::get) + .untilTrue(seen); + } + } +} diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index 96d82fe4..94555720 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -67,5 +67,26 @@ void watchRecursiveDirectory() throws IOException, InterruptedException { } } + @Test + void watchSingleFile() throws IOException { + var changed = new AtomicBoolean(false); + var target = testDir.getTestFiles().stream() + .filter(p -> p.getParent().equals(testDir.getTestDirectory())) + .findFirst() + .orElseThrow(); + + var watchConfig = Watcher.singleFile(target) + .onEvent(ev -> { + if (ev.calculateFullPath().equals(target)) { + changed.set(true); + } + }); + + try (var watch = watchConfig.start()) { + Files.writeString(target, "Hello world"); + await().alias("Single file change").until(changed::get); + } + } + } From 939e57bbd097c2428708972a45fa9a6468e2a9af Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 7 Sep 2024 21:01:51 +0200 Subject: [PATCH 22/89] Improve how long we wait for failure condition --- src/test/java/engineering/swat/watch/SingleFileTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/engineering/swat/watch/SingleFileTests.java b/src/test/java/engineering/swat/watch/SingleFileTests.java index ab387ce1..24f70c2f 100644 --- a/src/test/java/engineering/swat/watch/SingleFileTests.java +++ b/src/test/java/engineering/swat/watch/SingleFileTests.java @@ -31,7 +31,7 @@ void cleanup() throws IOException { @BeforeAll static void setupEverything() { - Awaitility.setDefaultTimeout(2, TimeUnit.SECONDS); + Awaitility.setDefaultTimeout(3, TimeUnit.SECONDS); } @Test @@ -57,6 +57,7 @@ void singleFileShouldNotTriggerOnOtherFilesInSameDir() throws IOException, Inter Thread.sleep(1000); Files.writeString(target, "Hello world"); await("Single file does trigger") + .during(Duration.ofSeconds(2)) .failFast("No others should be notified", others::get) .untilTrue(seen); } From 03ab088af5e24945e910ae083db05e3674cd7d43 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Thu, 12 Sep 2024 14:22:41 +0200 Subject: [PATCH 23/89] Made the poller loop a bit more resistant to misbehaving callbacks --- src/main/java/engineering/swat/watch/impl/JDKPoller.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index 84f33073..26c4d20a 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -17,6 +17,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -50,6 +51,10 @@ private static void poll() { watchHandler.accept(events); } } + catch (Throwable t) { + logger.catching(Level.INFO, t); + // one exception shouldn't stop all the processing + } finally{ hit.reset(); } From 54d4d8845b83ff309724cd6b45612ce9ac233453 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Fri, 13 Sep 2024 20:27:14 +0200 Subject: [PATCH 24/89] Added test for delete behavior --- .../swat/watch/DeleteLockTests.java | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 src/test/java/engineering/swat/watch/DeleteLockTests.java diff --git a/src/test/java/engineering/swat/watch/DeleteLockTests.java b/src/test/java/engineering/swat/watch/DeleteLockTests.java new file mode 100644 index 00000000..d1acac53 --- /dev/null +++ b/src/test/java/engineering/swat/watch/DeleteLockTests.java @@ -0,0 +1,105 @@ +package engineering.swat.watch; + + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Comparator; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import engineering.swat.watch.WatchEvent.Kind; + +class DeleteLockTests { + + private TestDirectory testDir; + + @BeforeEach + void setup() throws IOException { + testDir = new TestDirectory(); + } + + @AfterEach + void cleanup() throws IOException { + if (testDir != null) { + testDir.close(); + } + } + + @BeforeAll + static void setupEverything() { + Awaitility.setDefaultTimeout(Duration.ofSeconds(3)); + } + + + @FunctionalInterface + private interface Deleter { + void run(Path target) throws IOException; + } + + @FunctionalInterface + private interface Builder { + Watcher build(Path target) throws IOException; + } + + private static void recursiveDelete(Path target) throws IOException { + try (var paths = Files.walk(target)) { + paths.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } + + private void deleteAndVerify(Path target, Builder setup) throws IOException { + var seen = new AtomicBoolean(false); + var watchConfig = setup.build(target) + .onEvent(ev -> { + if (ev.getKind() == Kind.DELETED + && ev.calculateFullPath().equals(target)) { + seen.set(true); + } + }); + try (var watch = watchConfig.start()) { + recursiveDelete(target); + assertFalse(Files.exists(target), "The file/directory shouldn't exist anymore"); + await("Watched object can be deleted, and we should see an event of it") + .untilTrue(seen); + } + } + + @Test + void watchedFileCanBeDeleted() throws IOException { + deleteAndVerify( + testDir.getTestFiles().get(0), + Watcher::singleFile + ); + } + + + @Test + void watchedDirectoryCanBeDeleted() throws IOException { + deleteAndVerify( + testDir.getTestDirectory(), + Watcher::singleDirectory + ); + } + + + @Test + void watchedRecursiveDirectoryCanBeDeleted() throws IOException { + deleteAndVerify( + testDir.getTestDirectory(), + Watcher::recursiveDirectory + ); + } +} From 8cc2c39e735dbf94893bc2c7657264ab3c0eb21c Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Fri, 13 Sep 2024 21:32:28 +0200 Subject: [PATCH 25/89] Wrote javadocs --- .../engineering/swat/watch/WatchEvent.java | 41 +++++++++++- .../java/engineering/swat/watch/Watcher.java | 65 +++++++++++++++++-- 2 files changed, 101 insertions(+), 5 deletions(-) diff --git a/src/main/java/engineering/swat/watch/WatchEvent.java b/src/main/java/engineering/swat/watch/WatchEvent.java index a1329480..5c8cc54a 100644 --- a/src/main/java/engineering/swat/watch/WatchEvent.java +++ b/src/main/java/engineering/swat/watch/WatchEvent.java @@ -4,10 +4,38 @@ import org.checkerframework.checker.nullness.qual.Nullable; +/** + * The library publishes these events to all subscribers, they are immutable and safe to share around. + */ public class WatchEvent { + /** + * What happened with the file or directory + */ public enum Kind { - CREATED, MODIFIED, DELETED, OVERFLOW + /** + * A path entry was created. Be careful not to assume that when the event arrives, the path still exists. + **/ + CREATED, + /** + * The path entry was saved. It is platform specific if this relates to flushes or other events. + * a single user action can generate multiple of these events. + */ + MODIFIED, + /** + * The path entry was deleted. + * Note that if the path entry was the watched item (aka the root of the watch), + * there is no guarantee if you will receive this event (depending on the level and on the platform). + * The watch will be invalid after that, even if a new item is created afterwards with the same name. + * In some cases this can be fixed/detected by also watching the parent, but that is only valid if they are on the same mountpoint. + */ + DELETED, + /** + * Rare event where there were so many file events, that the kernel lost a few. + * In that case you'll have to consider the whole directory (and it's sub directories) as modified. + * The library will try and send events for new and deleted files, but it won't be able to detect modified files. + */ + OVERFLOW } private final Kind kind; @@ -24,14 +52,25 @@ public Kind getKind() { return this.kind; } + /** + * + * @return the path relative to the monitored root, it can be empty path if it's the root. + */ public Path getRelativePath() { return relativePath; } + /** + * + * @return A copy of the root path that this event belongs to. + */ public Path getRootPath() { return rootPath; } + /** + * @return utility function that resolves the relative path to the full path. + */ public Path calculateFullPath() { return rootPath.resolve(relativePath); } diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 39678b45..81cc9deb 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -2,6 +2,8 @@ import java.io.Closeable; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.Path; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; @@ -14,13 +16,20 @@ import engineering.swat.watch.impl.JDKFileWatcher; import engineering.swat.watch.impl.JDKRecursiveDirectoryWatcher; +/** + * Watch a path for changes. + * + * It will avoid common errors using the raw apis, and will try to use the most native api where possible. + * Note, there are differences per platform that cannot be avoided, please review the readme of the library. + */ public class Watcher { private final Logger logger = LogManager.getLogger(); private final WatcherKind kind; private final Path path; private Executor executor = CompletableFuture::runAsync; - private Consumer eventHandler = p -> {}; + private static final Consumer NULL_HANDLER = p -> {}; + private Consumer eventHandler = NULL_HANDLER; private Watcher(WatcherKind kind, Path path) { @@ -35,38 +44,86 @@ private enum WatcherKind { RECURSIVE_DIRECTORY } - public static Watcher singleFile(Path path) throws IOException { + /** + * Request a watcher for a single path (file or directory). + * If it's a file, depending on the platform this will watch the whole directory and filter the results, or only watch a single file. + * @param path a single path entry, either a file or a directory + * @return a watcher that only fires events related to the requested path + * @throws IOException in case the path is not absolute + */ + public static Watcher single(Path path) throws IOException { if (!path.isAbsolute()) { throw new IOException("We can only watch absolute paths"); } return new Watcher(WatcherKind.FILE, path); } + /** + * Request a watcher for a directory, getting events for its direct children. + * @param path a directory to monitor for changes + * @return a watcher that fires events for any of the direct children (and its self). + * @throws IOException in cas the path is not absolute or it's not an directory + */ public static Watcher singleDirectory(Path path) throws IOException { if (!path.isAbsolute()) { throw new IOException("We can only watch absolute paths"); } + if (!Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Only directories are supported"); + } return new Watcher(WatcherKind.DIRECTORY, path); } + /** + * Request a watcher for a directory, getting events for all of its children. Even those added after the watch started. + * On some platforms, this can be quite expansive, so be sure you want this. + * @param path a directory to monitor for changes + * @return a watcher that fires events for any of its children (and its self). + * @throws IOException in case the path is not absolute or it's not an directory + */ public static Watcher recursiveDirectory(Path path) throws IOException { if (!path.isAbsolute()) { throw new IOException("We can only watch absolute paths"); } + if (!Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Only directories are supported"); + } return new Watcher(WatcherKind.RECURSIVE_DIRECTORY, path); } + /** + * Callback that gets executed for every event. Can get called quite a bit, so be careful what happens here. + * Use the {@link #withExecutor(Executor)} function to influence the sequencing of these events. + * By default they can arrive in parallel. + * @param eventHandler a callback that handles the watch event, will be called once per event. + * @return this for optional method chaining + */ public Watcher onEvent(Consumer eventHandler) { this.eventHandler = eventHandler; return this; } + /** + * Optionally configure the executor in which the {@link #onEvent(Consumer)} callbacks are scheduled. + * If not defined, every task will be scheduled on the {@link java.util.concurrent.ForkJoinPool#commonPool()}. + * @param callbackHandler worker pool to use + * @return this for optional method chaining + */ public Watcher withExecutor(Executor callbackHandler) { this.executor = callbackHandler; return this; } - public Closeable start() throws IOException { + /** + * Start watch the path for events. + * @return a subscription for the watch, when closed, new events will stop being registered to the worker pool. + * @throws IOException in case the starting of the watcher caused an underlying IO exception + * @throws IllegalStateException the watchers is not configured correctly (for example, missing {@link #onEvent(Consumer)}) + */ + public Closeable start() throws IOException, IllegalStateException { + if (this.eventHandler == NULL_HANDLER) { + throw new IllegalStateException("There is no onEvent handler defined"); + } switch (kind) { case DIRECTORY: { var result = new JDKDirectoryWatcher(path, executor, this.eventHandler); @@ -85,7 +142,7 @@ public Closeable start() throws IOException { } default: - throw new IllegalArgumentException("Not supported yet"); + throw new IllegalStateException("Not supported yet"); } } From dd18b9b8958e1b44e5959280f7da749f6e6df7ee Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Fri, 13 Sep 2024 21:32:47 +0200 Subject: [PATCH 26/89] Improved handling of deletes --- .../swat/watch/impl/JDKPoller.java | 3 +- .../swat/watch/DeleteLockTests.java | 14 +---- .../swat/watch/RecursiveWatchTests.java | 23 ++++++++ .../swat/watch/SingleDirectoryTests.java | 55 +++++++++++++++++++ .../swat/watch/SingleFileTests.java | 33 ++++++++++- .../engineering/swat/watch/SmokeTests.java | 2 +- 6 files changed, 115 insertions(+), 15 deletions(-) create mode 100644 src/test/java/engineering/swat/watch/SingleDirectoryTests.java diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index 26c4d20a..a264b4c0 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -1,6 +1,7 @@ package engineering.swat.watch.impl; import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; import java.io.Closeable; @@ -75,7 +76,7 @@ private static void poll() { public static Closeable register(Path path, Consumer>> changes) throws IOException { logger.debug("Register watch for: {}", path); // TODO: consider upgrading the events the moment we actually get a request for all of it - var key = path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY); + var key = path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY, ENTRY_DELETE); logger.trace("Got watch key: {}", key); watchers.put(key, changes); return new Closeable() { diff --git a/src/test/java/engineering/swat/watch/DeleteLockTests.java b/src/test/java/engineering/swat/watch/DeleteLockTests.java index d1acac53..8c86bc16 100644 --- a/src/test/java/engineering/swat/watch/DeleteLockTests.java +++ b/src/test/java/engineering/swat/watch/DeleteLockTests.java @@ -61,19 +61,9 @@ private static void recursiveDelete(Path target) throws IOException { } private void deleteAndVerify(Path target, Builder setup) throws IOException { - var seen = new AtomicBoolean(false); - var watchConfig = setup.build(target) - .onEvent(ev -> { - if (ev.getKind() == Kind.DELETED - && ev.calculateFullPath().equals(target)) { - seen.set(true); - } - }); - try (var watch = watchConfig.start()) { + try (var watch = setup.build(target).start()) { recursiveDelete(target); assertFalse(Files.exists(target), "The file/directory shouldn't exist anymore"); - await("Watched object can be deleted, and we should see an event of it") - .untilTrue(seen); } } @@ -81,7 +71,7 @@ private void deleteAndVerify(Path target, Builder setup) throws IOException { void watchedFileCanBeDeleted() throws IOException { deleteAndVerify( testDir.getTestFiles().get(0), - Watcher::singleFile + Watcher::single ); } diff --git a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java index 4f0a79ee..d289f354 100644 --- a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java +++ b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java @@ -17,6 +17,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import engineering.swat.watch.WatchEvent.Kind; + class RecursiveWatchTests { private final Logger logger = LogManager.getLogger(); @@ -94,4 +96,25 @@ void correctRelativePathIsReported() throws IOException { } + @Test + void deleteOfFileInDirectoryShouldBeVisible() throws IOException, InterruptedException { + var target = testDir.getTestFiles() + .stream() + .filter(p -> !p.getParent().equals(testDir.getTestDirectory())) + .findAny() + .orElseThrow(); + var seen = new AtomicBoolean(false); + var watchConfig = Watcher.singleDirectory(target.getParent()) + .onEvent(ev -> { + if (ev.getKind() == Kind.DELETED && ev.calculateFullPath().equals(target)) { + seen.set(true); + } + }); + try (var watch = watchConfig.start()) { + Files.delete(target); + await("File deletion should generate delete event") + .untilTrue(seen); + } + } + } diff --git a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java new file mode 100644 index 00000000..7775314e --- /dev/null +++ b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java @@ -0,0 +1,55 @@ +package engineering.swat.watch; + +import static org.awaitility.Awaitility.await; + +import java.io.IOException; +import java.nio.file.Files; +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import engineering.swat.watch.WatchEvent.Kind; + +public class SingleDirectoryTests { + private TestDirectory testDir; + + @BeforeEach + void setup() throws IOException { + testDir = new TestDirectory(); + } + + @AfterEach + void cleanup() throws IOException { + if (testDir != null) { + testDir.close(); + } + } + + @BeforeAll + static void setupEverything() { + Awaitility.setDefaultTimeout(3, TimeUnit.SECONDS); + } + + @Test + void deleteOfFileInDirectoryShouldBeVisible() throws IOException, InterruptedException { + var target = testDir.getTestFiles().get(0); + var seen = new AtomicBoolean(false); + var watchConfig = Watcher.singleDirectory(target.getParent()) + .onEvent(ev -> { + if (ev.getKind() == Kind.DELETED && ev.calculateFullPath().equals(target)) { + seen.set(true); + } + }); + try (var watch = watchConfig.start()) { + Files.delete(target); + await("File deletion should generate delete event") + .untilTrue(seen); + } + } +} diff --git a/src/test/java/engineering/swat/watch/SingleFileTests.java b/src/test/java/engineering/swat/watch/SingleFileTests.java index 24f70c2f..3d5719bf 100644 --- a/src/test/java/engineering/swat/watch/SingleFileTests.java +++ b/src/test/java/engineering/swat/watch/SingleFileTests.java @@ -4,7 +4,9 @@ import java.io.IOException; import java.nio.file.Files; +import java.nio.file.attribute.FileTime; import java.time.Duration; +import java.time.Instant; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -39,7 +41,7 @@ void singleFileShouldNotTriggerOnOtherFilesInSameDir() throws IOException, Inter var target = testDir.getTestFiles().get(0); var seen = new AtomicBoolean(false); var others = new AtomicBoolean(false); - var watchConfig = Watcher.singleFile(target) + var watchConfig = Watcher.single(target) .onEvent(ev -> { if (ev.calculateFullPath().equals(target)) { seen.set(true); @@ -62,4 +64,33 @@ void singleFileShouldNotTriggerOnOtherFilesInSameDir() throws IOException, Inter .untilTrue(seen); } } + + @Test + void singleFileThatMonitorsOnlyADirectory() throws IOException, InterruptedException { + var target = testDir.getTestDirectory(); + var seen = new AtomicBoolean(false); + var others = new AtomicBoolean(false); + var watchConfig = Watcher.single(target) + .onEvent(ev -> { + if (ev.calculateFullPath().equals(target)) { + seen.set(true); + } + else { + others.set(true); + } + }); + try (var watch = watchConfig.start()) { + for (var f : testDir.getTestFiles()) { + if (!f.equals(target)) { + Files.writeString(f, "Hello"); + } + } + Thread.sleep(1000); + Files.setLastModifiedTime(target, FileTime.from(Instant.now())); + await("Single file does trigger") + .during(Duration.ofSeconds(2)) + .failFast("No others should be notified", others::get) + .untilTrue(seen); + } + } } diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index 94555720..5ea14c9a 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -75,7 +75,7 @@ void watchSingleFile() throws IOException { .findFirst() .orElseThrow(); - var watchConfig = Watcher.singleFile(target) + var watchConfig = Watcher.single(target) .onEvent(ev -> { if (ev.calculateFullPath().equals(target)) { changed.set(true); From 6a41f57d7145f28cc1170e769539429da13219d2 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Fri, 13 Sep 2024 21:38:59 +0200 Subject: [PATCH 27/89] [ci] starting with ci --- .github/workflows/build.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/build.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 00000000..b84b812f --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,28 @@ +name: Build and Test +on: + push: + branches: + - main + tags: + - 'v[0-9]+.*' + pull_request: + branches: + - main + +jobs: + test: + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: 'temurin' + cache: 'maven' + + - name: test + run: mvn -B clean test From 1d4ced38d3060f15f5b8bdf26c513136f25400ad Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Fri, 13 Sep 2024 21:41:16 +0200 Subject: [PATCH 28/89] [ci] also run checker-framework --- .github/workflows/build.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b84b812f..a8473cab 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -26,3 +26,16 @@ jobs: - name: test run: mvn -B clean test + + checker-framework: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: 'temurin' + cache: 'maven' + + - run: mvn -B -Pchecker-framework clean compile From 0ccca4cdbb5ecf743e542b63efa43efa98c72205 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 14 Sep 2024 00:03:23 +0200 Subject: [PATCH 29/89] Fixed broken tests --- .../java/engineering/swat/watch/DeleteLockTests.java | 2 +- .../java/engineering/swat/watch/TestDirectory.java | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/test/java/engineering/swat/watch/DeleteLockTests.java b/src/test/java/engineering/swat/watch/DeleteLockTests.java index 8c86bc16..6c67602d 100644 --- a/src/test/java/engineering/swat/watch/DeleteLockTests.java +++ b/src/test/java/engineering/swat/watch/DeleteLockTests.java @@ -61,7 +61,7 @@ private static void recursiveDelete(Path target) throws IOException { } private void deleteAndVerify(Path target, Builder setup) throws IOException { - try (var watch = setup.build(target).start()) { + try (var watch = setup.build(target).onEvent(ev -> {}).start()) { recursiveDelete(target); assertFalse(Files.exists(target), "The file/directory shouldn't exist anymore"); } diff --git a/src/test/java/engineering/swat/watch/TestDirectory.java b/src/test/java/engineering/swat/watch/TestDirectory.java index 10b17b26..b0630f6c 100644 --- a/src/test/java/engineering/swat/watch/TestDirectory.java +++ b/src/test/java/engineering/swat/watch/TestDirectory.java @@ -34,11 +34,13 @@ private static void add3Files(List testFiles, Path root) throws IOExceptio } @Override - public void close() throws IOException { - Files.walk(testDirectory) - .sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .forEach(File::delete); + public void close() { + try { + Files.walk(testDirectory) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } catch (IOException _ignored) { } } public Path getTestDirectory() { From 800d1b09f8d162e9360086d4440cba18d1ec9c63 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 14 Sep 2024 20:38:59 +0200 Subject: [PATCH 30/89] [ci] also test under different jdks --- .github/workflows/build.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a8473cab..b5ce8eb9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -14,13 +14,15 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] + jdk: [11, 17, 21] + fail-fast: false runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - - name: Set up JDK 11 + - name: Set up JDK uses: actions/setup-java@v4 with: - java-version: 11 + java-version: ${{ matrix.jdk }} distribution: 'temurin' cache: 'maven' From 9cf1997f0959982b461ee70fb52828f171bb6652 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 14 Sep 2024 20:39:28 +0200 Subject: [PATCH 31/89] [ci] make sure deletes are also torture tested --- .../engineering/swat/watch/TestDirectory.java | 13 +++++++++---- .../java/engineering/swat/watch/TortureTests.java | 15 +++++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/test/java/engineering/swat/watch/TestDirectory.java b/src/test/java/engineering/swat/watch/TestDirectory.java index b0630f6c..ebd8beb8 100644 --- a/src/test/java/engineering/swat/watch/TestDirectory.java +++ b/src/test/java/engineering/swat/watch/TestDirectory.java @@ -33,13 +33,18 @@ private static void add3Files(List testFiles, Path root) throws IOExceptio } } + public void deleteAllFiles() throws IOException { + try (var files = Files.walk(testDirectory)) { + files.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } + @Override public void close() { try { - Files.walk(testDirectory) - .sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .forEach(File::delete); + deleteAllFiles(); } catch (IOException _ignored) { } } diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index c3287785..a069b58d 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -30,6 +30,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import engineering.swat.watch.WatchEvent.Kind; + class TortureTests { private final Logger logger = LogManager.getLogger(); @@ -100,11 +102,18 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio final var events = new AtomicInteger(0); var seenPaths = ConcurrentHashMap.newKeySet(); + var seenDeletes = ConcurrentHashMap.newKeySet(); var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) .withExecutor(pool) .onEvent(ev -> { events.getAndIncrement(); - seenPaths.add(ev.calculateFullPath()); + Path fullPath = ev.calculateFullPath(); + if (ev.getKind() == Kind.DELETED) { + seenDeletes.add(fullPath); + } + else { + seenPaths.add(fullPath); + } }); try (var activeWatch = watchConfig.start() ) { @@ -116,6 +125,7 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio stopRunning.release(THREADS); assertTrue(done.tryAcquire(THREADS, STOP_AFTER.toMillis(), TimeUnit.MILLISECONDS), "The runners should have stopped running"); logger.info("Generated: {} files", pathWritten.size()); + testDir.deleteAllFiles(); logger.info("Waiting for the events processing to settle down"); int lastEventCount = events.get(); @@ -150,7 +160,8 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio logger.info("Comparing events and files seen"); // now make sure that the two sets are the same for (var f : pathWritten) { - assertTrue(seenPaths.contains(f), "We should have seen all paths"); + assertTrue(seenPaths.contains(f), () -> "Missing event for: " + f); + assertTrue(seenDeletes.contains(f), () -> "Missing delete for: " + f); } } } From e0281819aaa943015cc6ec1417c5ed0bd12a9d90 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 14 Sep 2024 20:40:02 +0200 Subject: [PATCH 32/89] Fixed null errors found by CF --- .../swat/watch/impl/BundledSubscription.java | 7 +++++-- .../engineering/swat/watch/impl/JDKFileWatcher.java | 10 +++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java index 854d5ab0..1abab350 100644 --- a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java +++ b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java @@ -9,8 +9,9 @@ import java.util.function.Consumer; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.NonNull; -public class BundledSubscription implements ISubscribable { +public class BundledSubscription implements ISubscribable { private final ISubscribable around; private final Map> subscriptions = new ConcurrentHashMap<>(); @@ -84,7 +85,9 @@ public Closeable subscribe(A target, Consumer eventListener) throws IOExcepti subscriptions.put(target, finalActive); } else { - finalActive.closer.close(); + if (finalActive.closer != null) { + finalActive.closer.close(); + } } } }; diff --git a/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java index a99a692b..45246d54 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java @@ -25,7 +25,11 @@ public class JDKFileWatcher implements Closeable { public JDKFileWatcher(Path file, Executor exec, Consumer eventHandler) { this.file = file; - this.fileName = file.getFileName(); + Path filename= file.getFileName(); + if (filename == null) { + throw new IllegalArgumentException("Cannot pass in a root path"); + } + this.fileName = filename; this.exec = exec; this.eventHandler = eventHandler; } @@ -37,6 +41,10 @@ public void start() throws IOException { throw new IOException("Cannot start an already started watch"); } var dir = file.getParent(); + if (dir == null) { + throw new IllegalArgumentException("cannot watch a single entry that is on the root"); + + } assert !dir.equals(file); var parentWatch = new JDKDirectoryWatcher(dir, exec, this::filter); activeWatch = parentWatch; From 174f4d60e7e0f96ef0642f5301ce9af280341f1e Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 14 Sep 2024 20:40:12 +0200 Subject: [PATCH 33/89] Slight tweak to the documentation --- src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java index 45246d54..6306ae95 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java @@ -13,7 +13,7 @@ import engineering.swat.watch.WatchEvent; /** - * It's not possible to monitor a single file, so we have to find a directory watcher, and connect to that + * It's not possible to monitor a single file (or directory), so we have to find a directory watcher, and connect to that */ public class JDKFileWatcher implements Closeable { private final Logger logger = LogManager.getLogger(); From fde2d6e2365bca5e9069986e1f962e24022f6627 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 14 Sep 2024 21:45:35 +0200 Subject: [PATCH 34/89] Improving the torture test to wait a bit long before events have stabilized --- .../engineering/swat/watch/TortureTests.java | 49 +++++++++++-------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index a069b58d..49586bf7 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -125,28 +125,15 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio stopRunning.release(THREADS); assertTrue(done.tryAcquire(THREADS, STOP_AFTER.toMillis(), TimeUnit.MILLISECONDS), "The runners should have stopped running"); logger.info("Generated: {} files", pathWritten.size()); - testDir.deleteAllFiles(); logger.info("Waiting for the events processing to settle down"); - int lastEventCount = events.get(); - int stableCount = 0; - do { - Thread.sleep(STOP_AFTER.toMillis() * 2); - int currentEventCounts = events.get(); - if (currentEventCounts == lastEventCount) { - if (stableCount == 2) { - logger.info("Stable after: {} events", currentEventCounts); - break; - } - else { - stableCount++; - } - } - else { - stableCount = 0; - } - lastEventCount = currentEventCounts; - } while (true); + waitForStable(events); + + logger.info("Now deleting everything"); + testDir.deleteAllFiles(); + logger.info("Waiting for the events processing to settle down"); + Thread.sleep(STOP_AFTER.toMillis()); + waitForStable(events); } finally { stopRunning.release(THREADS); @@ -164,4 +151,26 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio assertTrue(seenDeletes.contains(f), () -> "Missing delete for: " + f); } } + + private void waitForStable(final AtomicInteger events) throws InterruptedException { + int lastEventCount = events.get(); + int stableCount = 0; + do { + Thread.sleep(STOP_AFTER.toMillis() * 2); + int currentEventCounts = events.get(); + if (currentEventCounts == lastEventCount) { + if (stableCount == 2) { + logger.info("Stable after: {} events", currentEventCounts); + break; + } + else { + stableCount++; + } + } + else { + stableCount = 0; + } + lastEventCount = currentEventCounts; + } while (true); + } } From 2bf0c7594969c9514192cc972c08d96c571bb85b Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 14 Sep 2024 21:47:52 +0200 Subject: [PATCH 35/89] Cleanup of test api usage --- .../swat/watch/RecursiveWatchTests.java | 6 +++--- .../java/engineering/swat/watch/SmokeTests.java | 6 +++--- .../java/engineering/swat/watch/TortureTests.java | 15 +-------------- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java index d289f354..cc42bd47 100644 --- a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java +++ b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java @@ -69,9 +69,9 @@ void newDirectoryWithFilesChangesDetected() throws IOException { target.set(freshFile); logger.debug("Interested in: {}", freshFile); Files.writeString(freshFile, "Hello world"); - await().alias("New files should have been seen").until(created::get); + await("New files should have been seen").untilTrue(created); Files.writeString(freshFile, "Hello world 2"); - await().alias("Fresh file change have been detected").until(changed::get); + await("Fresh file change have been detected").untilTrue(changed); } } @@ -91,7 +91,7 @@ void correctRelativePathIsReported() throws IOException { var targetFile = testDir.getTestDirectory().resolve(relative); Files.createDirectories(targetFile.getParent()); Files.writeString(targetFile, "Hello World"); - await().alias("Nested path is seen").until(seen::get); + await("Nested path is seen").untilTrue(seen); } } diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index 5ea14c9a..63022acc 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -46,7 +46,7 @@ void watchDirectory() throws IOException, InterruptedException { try (var activeWatch = watchConfig.start() ) { Files.writeString(target, "Hello world"); - await().alias("Target file change").until(changed::get); + await("Target file change").untilTrue(changed); } } @@ -63,7 +63,7 @@ void watchRecursiveDirectory() throws IOException, InterruptedException { try (var activeWatch = watchConfig.start() ) { Files.writeString(target, "Hello world"); - await().alias("Nested file change").until(changed::get); + await("Nested file change").untilTrue(changed); } } @@ -84,7 +84,7 @@ void watchSingleFile() throws IOException { try (var watch = watchConfig.start()) { Files.writeString(target, "Hello world"); - await().alias("Single file change").until(changed::get); + await("Single file change").untilTrue(changed); } } diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 49586bf7..15a74cbb 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -1,8 +1,5 @@ package engineering.swat.watch; -import static engineering.swat.watch.WatchEvent.Kind.OVERFLOW; -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertIterableEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; @@ -13,20 +10,15 @@ import java.util.ArrayList; import java.util.Random; import java.util.concurrent.Callable; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -50,14 +42,9 @@ void cleanup() throws IOException { } } - @BeforeAll - static void setupEverything() throws IOException { - Awaitility.setDefaultTimeout(4, TimeUnit.SECONDS); - } - + private final static Duration STOP_AFTER = Duration.ofSeconds(4); private final static int THREADS = 4; private final static int BURST_SIZE = 1000; - private final static Duration STOP_AFTER = Duration.ofSeconds(4); @Test void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOException { final var root = testDir.getTestDirectory(); From 66d588f40a88c34078aa3c2809843f7f67dcb672 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Sat, 14 Sep 2024 22:04:21 +0200 Subject: [PATCH 36/89] Trying to make the test run better on ci --- .../swat/watch/DeleteLockTests.java | 2 +- .../swat/watch/RecursiveWatchTests.java | 2 +- .../swat/watch/SingleDirectoryTests.java | 2 +- .../swat/watch/SingleFileTests.java | 14 +++++------ .../engineering/swat/watch/TestHelper.java | 23 +++++++++++++++++++ .../engineering/swat/watch/TortureTests.java | 16 ++++++------- 6 files changed, 39 insertions(+), 20 deletions(-) create mode 100644 src/test/java/engineering/swat/watch/TestHelper.java diff --git a/src/test/java/engineering/swat/watch/DeleteLockTests.java b/src/test/java/engineering/swat/watch/DeleteLockTests.java index 6c67602d..294b52ba 100644 --- a/src/test/java/engineering/swat/watch/DeleteLockTests.java +++ b/src/test/java/engineering/swat/watch/DeleteLockTests.java @@ -38,7 +38,7 @@ void cleanup() throws IOException { @BeforeAll static void setupEverything() { - Awaitility.setDefaultTimeout(Duration.ofSeconds(3)); + Awaitility.setDefaultTimeout(TestHelper.NORMAL_WAIT); } diff --git a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java index cc42bd47..f5f9439a 100644 --- a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java +++ b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java @@ -38,7 +38,7 @@ void cleanup() throws IOException { @BeforeAll static void setupEverything() throws IOException { - Awaitility.setDefaultTimeout(4, TimeUnit.SECONDS); + Awaitility.setDefaultTimeout(TestHelper.NORMAL_WAIT); } diff --git a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java index 7775314e..594db8af 100644 --- a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java +++ b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java @@ -33,7 +33,7 @@ void cleanup() throws IOException { @BeforeAll static void setupEverything() { - Awaitility.setDefaultTimeout(3, TimeUnit.SECONDS); + Awaitility.setDefaultTimeout(TestHelper.NORMAL_WAIT); } @Test diff --git a/src/test/java/engineering/swat/watch/SingleFileTests.java b/src/test/java/engineering/swat/watch/SingleFileTests.java index 3d5719bf..b809318d 100644 --- a/src/test/java/engineering/swat/watch/SingleFileTests.java +++ b/src/test/java/engineering/swat/watch/SingleFileTests.java @@ -5,9 +5,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.attribute.FileTime; -import java.time.Duration; import java.time.Instant; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.awaitility.Awaitility; @@ -33,7 +31,7 @@ void cleanup() throws IOException { @BeforeAll static void setupEverything() { - Awaitility.setDefaultTimeout(3, TimeUnit.SECONDS); + Awaitility.setDefaultTimeout(TestHelper.NORMAL_WAIT); } @Test @@ -56,10 +54,10 @@ void singleFileShouldNotTriggerOnOtherFilesInSameDir() throws IOException, Inter Files.writeString(f, "Hello"); } } - Thread.sleep(1000); + Thread.sleep(TestHelper.SHORT_WAIT.toMillis()); Files.writeString(target, "Hello world"); await("Single file does trigger") - .during(Duration.ofSeconds(2)) + .during(TestHelper.NORMAL_WAIT) .failFast("No others should be notified", others::get) .untilTrue(seen); } @@ -85,10 +83,10 @@ void singleFileThatMonitorsOnlyADirectory() throws IOException, InterruptedExcep Files.writeString(f, "Hello"); } } - Thread.sleep(1000); + Thread.sleep(TestHelper.SHORT_WAIT.toMillis()); Files.setLastModifiedTime(target, FileTime.from(Instant.now())); - await("Single file does trigger") - .during(Duration.ofSeconds(2)) + await("Single directory does trigger") + .during(TestHelper.NORMAL_WAIT) .failFast("No others should be notified", others::get) .untilTrue(seen); } diff --git a/src/test/java/engineering/swat/watch/TestHelper.java b/src/test/java/engineering/swat/watch/TestHelper.java new file mode 100644 index 00000000..702858db --- /dev/null +++ b/src/test/java/engineering/swat/watch/TestHelper.java @@ -0,0 +1,23 @@ +package engineering.swat.watch; + +import java.time.Duration; + +public class TestHelper { + + public static final Duration SHORT_WAIT; + public static final Duration NORMAL_WAIT; + public final static Duration LONG_WAIT; + + static { + var delayFactorConfig = System.getenv("DELAY_FACTOR"); + int delayFactor = delayFactorConfig == null ? 1 : Integer.parseInt(delayFactorConfig); + if (System.getProperty("os", "?").toLowerCase().contains("mac")) { + // OSX is SLOW on it's watches + delayFactor *= 2; + } + SHORT_WAIT = Duration.ofSeconds(1 * delayFactor); + NORMAL_WAIT = Duration.ofSeconds(4 * delayFactor); + LONG_WAIT = Duration.ofSeconds(8 * delayFactor); + } + +} diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 15a74cbb..2069b2de 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Duration; import java.time.LocalTime; import java.util.ArrayList; import java.util.Random; @@ -42,7 +41,6 @@ void cleanup() throws IOException { } } - private final static Duration STOP_AFTER = Duration.ofSeconds(4); private final static int THREADS = 4; private final static int BURST_SIZE = 1000; @Test @@ -57,7 +55,7 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio var r = new Random(j); jobs.add(() -> { try { - var end = LocalTime.now().plus(STOP_AFTER); + var end = LocalTime.now().plus(TestHelper.NORMAL_WAIT); while (!stopRunning.tryAcquire(100, TimeUnit.MICROSECONDS)) { if (LocalTime.now().isAfter(end)) { break; @@ -78,7 +76,7 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio return null; } catch (InterruptedException e) { return null; - } + } finally { done.release(); } @@ -107,10 +105,10 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio logger.info("Starting {} jobs", THREADS); pool.invokeAll(jobs); // now we generate a whole bunch of events - Thread.sleep(STOP_AFTER.toMillis()); + Thread.sleep(TestHelper.NORMAL_WAIT.toMillis()); logger.info("Stopping jobs"); stopRunning.release(THREADS); - assertTrue(done.tryAcquire(THREADS, STOP_AFTER.toMillis(), TimeUnit.MILLISECONDS), "The runners should have stopped running"); + assertTrue(done.tryAcquire(THREADS, TestHelper.NORMAL_WAIT.toMillis(), TimeUnit.MILLISECONDS), "The runners should have stopped running"); logger.info("Generated: {} files", pathWritten.size()); logger.info("Waiting for the events processing to settle down"); @@ -119,7 +117,7 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio logger.info("Now deleting everything"); testDir.deleteAllFiles(); logger.info("Waiting for the events processing to settle down"); - Thread.sleep(STOP_AFTER.toMillis()); + Thread.sleep(TestHelper.NORMAL_WAIT.toMillis()); waitForStable(events); } finally { @@ -143,10 +141,10 @@ private void waitForStable(final AtomicInteger events) throws InterruptedExcepti int lastEventCount = events.get(); int stableCount = 0; do { - Thread.sleep(STOP_AFTER.toMillis() * 2); + Thread.sleep(TestHelper.SHORT_WAIT.toMillis()); int currentEventCounts = events.get(); if (currentEventCounts == lastEventCount) { - if (stableCount == 2) { + if (stableCount == 10) { logger.info("Stable after: {} events", currentEventCounts); break; } From bc777ef3d80f29ac535f18bc8c7132cbe45e708a Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 09:46:51 +0200 Subject: [PATCH 37/89] Fixing broken test --- pom.xml | 3 +-- src/test/java/engineering/swat/watch/SingleFileTests.java | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 995c88cb..88b6228b 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ org.awaitility awaitility - 4.2.0 + 4.2.2 test @@ -190,4 +190,3 @@ - diff --git a/src/test/java/engineering/swat/watch/SingleFileTests.java b/src/test/java/engineering/swat/watch/SingleFileTests.java index b809318d..f6d094a0 100644 --- a/src/test/java/engineering/swat/watch/SingleFileTests.java +++ b/src/test/java/engineering/swat/watch/SingleFileTests.java @@ -57,7 +57,7 @@ void singleFileShouldNotTriggerOnOtherFilesInSameDir() throws IOException, Inter Thread.sleep(TestHelper.SHORT_WAIT.toMillis()); Files.writeString(target, "Hello world"); await("Single file does trigger") - .during(TestHelper.NORMAL_WAIT) + .pollDelay(TestHelper.NORMAL_WAIT.minusMillis(10)) .failFast("No others should be notified", others::get) .untilTrue(seen); } @@ -86,7 +86,7 @@ void singleFileThatMonitorsOnlyADirectory() throws IOException, InterruptedExcep Thread.sleep(TestHelper.SHORT_WAIT.toMillis()); Files.setLastModifiedTime(target, FileTime.from(Instant.now())); await("Single directory does trigger") - .during(TestHelper.NORMAL_WAIT) + .pollDelay(TestHelper.NORMAL_WAIT.minusMillis(10)) .failFast("No others should be notified", others::get) .untilTrue(seen); } From c8d4fd5f4ed2d9fc6db43d68d666ced570e2267c Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 09:55:39 +0200 Subject: [PATCH 38/89] Trying to make a better stabilizing torture test detection --- .../java/engineering/swat/watch/TortureTests.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 2069b2de..60f0faf8 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -86,12 +86,14 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio var pool = Executors.newCachedThreadPool(); final var events = new AtomicInteger(0); + final var happened = new Semaphore(0); var seenPaths = ConcurrentHashMap.newKeySet(); var seenDeletes = ConcurrentHashMap.newKeySet(); var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) .withExecutor(pool) .onEvent(ev -> { events.getAndIncrement(); + happened.release(); Path fullPath = ev.calculateFullPath(); if (ev.getKind() == Kind.DELETED) { seenDeletes.add(fullPath); @@ -112,13 +114,13 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio logger.info("Generated: {} files", pathWritten.size()); logger.info("Waiting for the events processing to settle down"); - waitForStable(events); + waitForStable(events, happened); logger.info("Now deleting everything"); testDir.deleteAllFiles(); logger.info("Waiting for the events processing to settle down"); Thread.sleep(TestHelper.NORMAL_WAIT.toMillis()); - waitForStable(events); + waitForStable(events, happened); } finally { stopRunning.release(THREADS); @@ -137,14 +139,16 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio } } - private void waitForStable(final AtomicInteger events) throws InterruptedException { + private void waitForStable(final AtomicInteger events, final Semaphore happened) throws InterruptedException { int lastEventCount = events.get(); int stableCount = 0; do { - Thread.sleep(TestHelper.SHORT_WAIT.toMillis()); + while (happened.tryAcquire(TestHelper.SHORT_WAIT.toMillis() / 4, TimeUnit.MILLISECONDS)) { + happened.drainPermits(); + } int currentEventCounts = events.get(); if (currentEventCounts == lastEventCount) { - if (stableCount == 10) { + if (stableCount == 20) { logger.info("Stable after: {} events", currentEventCounts); break; } From 524d690f61f409b6b736546b785b070ccacd103a Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 09:58:02 +0200 Subject: [PATCH 39/89] Change smoke test to wait for appropriate time --- src/test/java/engineering/swat/watch/SmokeTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index 63022acc..5ed974e8 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -33,7 +33,7 @@ void cleanup() throws IOException { @BeforeAll static void setupEverything() { - Awaitility.setDefaultTimeout(2, TimeUnit.SECONDS); + Awaitility.setDefaultTimeout(TestHelper.NORMAL_WAIT); } @Test From d8170f82889bae9bae17898f3952917b8da2f7f3 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 11:50:58 +0200 Subject: [PATCH 40/89] Trying to make give the tests a bit more time --- .github/workflows/build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b5ce8eb9..e261ac46 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,6 +28,8 @@ jobs: - name: test run: mvn -B clean test + env: + DELAY_FACTOR: 3 checker-framework: runs-on: ubuntu-latest From b26a1a34cf076a0f7a63327fd1b6b9f3ef70a5ab Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 13:42:50 +0200 Subject: [PATCH 41/89] Improved the tests by splitting up the 2 torture tests --- .../engineering/swat/watch/TortureTests.java | 177 ++++++++++++++---- 1 file changed, 136 insertions(+), 41 deletions(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 60f0faf8..cc0641df 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -1,15 +1,17 @@ package engineering.swat.watch; +import static org.awaitility.Awaitility.doNotCatchUncaughtExceptionsByDefault; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalTime; -import java.util.ArrayList; import java.util.Random; -import java.util.concurrent.Callable; +import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; @@ -41,21 +43,27 @@ void cleanup() throws IOException { } } - private final static int THREADS = 4; - private final static int BURST_SIZE = 1000; - @Test - void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOException { - final var root = testDir.getTestDirectory(); - final var pathWritten = ConcurrentHashMap.newKeySet(); - final var stopRunning = new Semaphore(0); - final var done = new Semaphore(0); - final var jobs = new ArrayList>(); - - for (int j = 0; j < THREADS; j++) { - var r = new Random(j); - jobs.add(() -> { + private final class IOGenerator { + private final Set pathsWritten = ConcurrentHashMap.newKeySet(); + private final Semaphore startRunning = new Semaphore(0); + private final Semaphore stopRunning = new Semaphore(0); + private final Semaphore done = new Semaphore(0); + private final int jobs; + + IOGenerator(int jobs, Path root, Executor exec) { + this.jobs = jobs; + for (int j = 0; j < jobs; j++) { + startJob(root.resolve("run" + j), new Random(j), exec); + } + } + + private final static int BURST_SIZE = 1000; + + private void startJob(final Path root, Random r, Executor exec) { + exec.execute(() -> { try { - var end = LocalTime.now().plus(TestHelper.NORMAL_WAIT); + startRunning.acquire(); + var end = LocalTime.now().plus(TestHelper.NORMAL_WAIT.multipliedBy(2)); while (!stopRunning.tryAcquire(100, TimeUnit.MICROSECONDS)) { if (LocalTime.now().isAfter(end)) { break; @@ -63,19 +71,18 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio try { // burst a bunch of creates creates and then sleep a bit for (int i = 0; i< BURST_SIZE; i++) { - var file = root.resolve("l1" + r.nextInt(1000)) - .resolve("l2" + r.nextInt() + ".txt"); + var file = root.resolve("l1-" + r.nextInt(1000)) + .resolve("l2-" + r.nextInt(1000)) + .resolve("l3-" + r.nextInt() + ".txt"); Files.createDirectories(file.getParent()); Files.writeString(file, "Hello world"); - pathWritten.add(file); + pathsWritten.add(file); } } catch (IOException e) { } Thread.yield(); } - return null; } catch (InterruptedException e) { - return null; } finally { done.release(); @@ -83,47 +90,134 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio }); } + void start() { + startRunning.release(jobs); + } + + Set stop() throws InterruptedException { + startRunning.release(jobs); + stopRunning.release(jobs); + assertTrue(done.tryAcquire(jobs, TestHelper.NORMAL_WAIT.toMillis(), TimeUnit.MILLISECONDS), "IO workers should stop in a reasonable time"); + return pathsWritten; + } + } + + private static final int THREADS = 4; + + @Test + void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IOException { + final var root = testDir.getTestDirectory(); var pool = Executors.newCachedThreadPool(); + var io = new IOGenerator(THREADS, root, pool); + + final var events = new AtomicInteger(0); final var happened = new Semaphore(0); - var seenPaths = ConcurrentHashMap.newKeySet(); - var seenDeletes = ConcurrentHashMap.newKeySet(); + var seenCreates = ConcurrentHashMap.newKeySet(); + var seenWrites = ConcurrentHashMap.newKeySet(); var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) .withExecutor(pool) .onEvent(ev -> { events.getAndIncrement(); happened.release(); - Path fullPath = ev.calculateFullPath(); - if (ev.getKind() == Kind.DELETED) { - seenDeletes.add(fullPath); - } - else { - seenPaths.add(fullPath); + var fullPath = ev.calculateFullPath(); + switch (ev.getKind()) { + case CREATED: + seenCreates.add(fullPath); + break; + case MODIFIED: + seenWrites.add(fullPath); + break; + default: + logger.error("Unexpected event: {}", ev); + break; } }); + Set pathsWritten; + try (var activeWatch = watchConfig.start() ) { logger.info("Starting {} jobs", THREADS); - pool.invokeAll(jobs); + io.start(); // now we generate a whole bunch of events Thread.sleep(TestHelper.NORMAL_WAIT.toMillis()); logger.info("Stopping jobs"); - stopRunning.release(THREADS); - assertTrue(done.tryAcquire(THREADS, TestHelper.NORMAL_WAIT.toMillis(), TimeUnit.MILLISECONDS), "The runners should have stopped running"); - logger.info("Generated: {} files", pathWritten.size()); + pathsWritten = io.stop(); + logger.info("Generated: {} files", pathsWritten.size()); logger.info("Waiting for the events processing to settle down"); waitForStable(events, happened); - logger.info("Now deleting everything"); - testDir.deleteAllFiles(); - logger.info("Waiting for the events processing to settle down"); + } + finally { + try { + io.stop(); + } + catch (Throwable _ignored) {} + // shutdown the pool (so no new events are registered) + pool.shutdown(); + } + + // but wait till all scheduled tasks have been completed + pool.awaitTermination(10, TimeUnit.SECONDS); + + logger.info("Comparing events and files seen"); + // now make sure that the two sets are the same + for (var f : pathsWritten) { + assertTrue(seenCreates.contains(f), () -> "Missing create event for: " + f); + assertTrue(seenWrites.contains(f), () -> "Missing modify event for: " + f); + } + } + + + + @Test + void pressureOnFSShouldNotMissDeletes() throws InterruptedException, IOException { + final var root = testDir.getTestDirectory(); + var pool = Executors.newCachedThreadPool(); + + Set pathsWritten; + var seenDeletes = ConcurrentHashMap.newKeySet(); + var io = new IOGenerator(THREADS, root, pool); + try { + io.start(); Thread.sleep(TestHelper.NORMAL_WAIT.toMillis()); - waitForStable(events, happened); + pathsWritten = io.stop(); + + final var events = new AtomicInteger(0); + final var happened = new Semaphore(0); + var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) + .withExecutor(pool) + .onEvent(ev -> { + events.getAndIncrement(); + happened.release(); + var fullPath = ev.calculateFullPath(); + switch (ev.getKind()) { + case DELETED: + seenDeletes.add(fullPath); + break; + case MODIFIED: + // happens on dir level, as the files are getting removed + break; + default: + logger.error("Unexpected event: {}", ev); + break; + } + }); + + try (var activeWatch = watchConfig.start() ) { + logger.info("Deleting files now", THREADS); + testDir.deleteAllFiles(); + logger.info("Waiting for the events processing to settle down"); + waitForStable(events, happened); + } } finally { - stopRunning.release(THREADS); + try { + io.stop(); + } + catch (Throwable _ignored) {} // shutdown the pool (so no new events are registered) pool.shutdown(); } @@ -133,12 +227,13 @@ void pressureOnFSShouldNotMissAnything() throws InterruptedException, IOExceptio logger.info("Comparing events and files seen"); // now make sure that the two sets are the same - for (var f : pathWritten) { - assertTrue(seenPaths.contains(f), () -> "Missing event for: " + f); - assertTrue(seenDeletes.contains(f), () -> "Missing delete for: " + f); + for (var f : pathsWritten) { + assertTrue(seenDeletes.contains(f), () -> "Missing delete event for: " + f); } } + + private void waitForStable(final AtomicInteger events, final Semaphore happened) throws InterruptedException { int lastEventCount = events.get(); int stableCount = 0; From 66983327254297486f6984e27b2149b8601d1eb5 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 14:09:06 +0200 Subject: [PATCH 42/89] Lets make sure sync events are generated more correctly --- .../impl/JDKRecursiveDirectoryWatcher.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index 842f5976..3bd22b42 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -176,6 +176,7 @@ private void cleanupOld(Path dir, @Nullable Closeable oldEntry) { /** register watch for new sub-dir, but also simulate event for every file & subdir found */ private class NewDirectoryScan extends InitialDirectoryScan { protected final List events; + private boolean hasFiles = false; public NewDirectoryScan(Path subRoot, List events) { super(subRoot); this.events = events; @@ -186,12 +187,26 @@ public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs) if (!subdir.equals(subRoot)) { events.add(new WatchEvent(WatchEvent.Kind.CREATED, root, root.relativize(subdir))); } + hasFiles = false; return super.preVisitDirectory(subdir, attrs); } + @Override + public FileVisitResult postVisitDirectory(Path subdir, IOException exc) throws IOException { + if (hasFiles) { + events.add(new WatchEvent(WatchEvent.Kind.MODIFIED, root, root.relativize(subdir))); + } + return super.postVisitDirectory(subdir, exc); + } + @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - events.add(new WatchEvent(WatchEvent.Kind.CREATED, root, root.relativize(file))); + hasFiles = true; + var relative = root.relativize(file); + events.add(new WatchEvent(WatchEvent.Kind.CREATED, root, relative)); + if (attrs.size() > 0) { + events.add(new WatchEvent(WatchEvent.Kind.MODIFIED, root, relative)); + } return FileVisitResult.CONTINUE; } } From 3157d6256259e55f223dcea13539ee186b595d77 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 14:40:04 +0200 Subject: [PATCH 43/89] Better print for torture test --- src/test/java/engineering/swat/watch/TortureTests.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index cc0641df..a5397dc1 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -115,7 +115,6 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO final var events = new AtomicInteger(0); final var happened = new Semaphore(0); var seenCreates = ConcurrentHashMap.newKeySet(); - var seenWrites = ConcurrentHashMap.newKeySet(); var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) .withExecutor(pool) .onEvent(ev -> { @@ -127,7 +126,7 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO seenCreates.add(fullPath); break; case MODIFIED: - seenWrites.add(fullPath); + // platform specific if this comes by or not break; default: logger.error("Unexpected event: {}", ev); @@ -162,11 +161,14 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO // but wait till all scheduled tasks have been completed pool.awaitTermination(10, TimeUnit.SECONDS); - logger.info("Comparing events and files seen"); + var totalFileCreated = seenCreates.stream() + .filter(p -> !Files.isDirectory(p)) + .count(); + + logger.info("Comparing events ({} events for {} files) and files (total {}) created", events.get(), totalFileCreated, pathsWritten.size()); // now make sure that the two sets are the same for (var f : pathsWritten) { assertTrue(seenCreates.contains(f), () -> "Missing create event for: " + f); - assertTrue(seenWrites.contains(f), () -> "Missing modify event for: " + f); } } From 44ed90c57c6f7e34c7f65bf9f73c6420948f53ee Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 15:23:22 +0200 Subject: [PATCH 44/89] Trying to unbreak the test suite --- src/test/java/engineering/swat/watch/TortureTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index a5397dc1..1a216d09 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -159,7 +159,7 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO } // but wait till all scheduled tasks have been completed - pool.awaitTermination(10, TimeUnit.SECONDS); + // pool.awaitTermination(10, TimeUnit.SECONDS); var totalFileCreated = seenCreates.stream() .filter(p -> !Files.isDirectory(p)) @@ -240,7 +240,7 @@ private void waitForStable(final AtomicInteger events, final Semaphore happened) int lastEventCount = events.get(); int stableCount = 0; do { - while (happened.tryAcquire(TestHelper.SHORT_WAIT.toMillis() / 4, TimeUnit.MILLISECONDS)) { + while (happened.tryAcquire(TestHelper.SHORT_WAIT.toMillis(), TimeUnit.MILLISECONDS)) { happened.drainPermits(); } int currentEventCounts = events.get(); From 51a6930bcba75204a895a5ef13c915c8f920237b Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 15:41:11 +0200 Subject: [PATCH 45/89] Only start watch after the directory has been processed, to avoid a race on the same directory --- .../swat/watch/impl/JDKRecursiveDirectoryWatcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index 3bd22b42..811de190 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -127,7 +127,6 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOExce @Override public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs) throws IOException { - addNewDirectory(subdir); return FileVisitResult.CONTINUE; } @@ -136,6 +135,7 @@ public FileVisitResult postVisitDirectory(Path subdir, IOException exc) throws I if (exc != null) { logger.error("Error during directory iteration: {} = {}", subdir, exc); } + addNewDirectory(subdir); return FileVisitResult.CONTINUE; } From c9cee56b19a76bdb4c7886150a3a0e45c698bb15 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 15:46:02 +0200 Subject: [PATCH 46/89] Disable delete test, which will mostly fail, as it can be a race with on-disk events vs kernel --- src/test/java/engineering/swat/watch/TortureTests.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 1a216d09..edf6c583 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -21,7 +21,11 @@ import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable; +import org.junit.jupiter.api.condition.EnabledIf; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import engineering.swat.watch.WatchEvent.Kind; @@ -175,6 +179,8 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO @Test + //Deletes can race the filesystem, so you might miss a few files in a dir, if that dir is already deleted + @EnabledIfEnvironmentVariable(named="TORTURE_DELETE", matches="true") void pressureOnFSShouldNotMissDeletes() throws InterruptedException, IOException { final var root = testDir.getTestDirectory(); var pool = Executors.newCachedThreadPool(); @@ -245,7 +251,7 @@ private void waitForStable(final AtomicInteger events, final Semaphore happened) } int currentEventCounts = events.get(); if (currentEventCounts == lastEventCount) { - if (stableCount == 20) { + if (stableCount == 30) { logger.info("Stable after: {} events", currentEventCounts); break; } From 424ad48cd63fa5c3dbc17ca710f5a978ce7c8ef5 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 16:03:25 +0200 Subject: [PATCH 47/89] Fixed race on new directory watches in recursive watch in a better way to avoid missing updates --- .../swat/watch/impl/JDKDirectoryWatcher.java | 17 ++++++++------ .../impl/JDKRecursiveDirectoryWatcher.java | 23 +++++-------------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java index 4b8e65f4..11308bf0 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -31,16 +31,19 @@ public JDKDirectoryWatcher(Path directory, Executor exec, Consumer e this.eventHandler = eventHandler; } + synchronized boolean safeStart() throws IOException { + if (activeWatch != null) { + return false; + } + activeWatch = BUNDLED_JDK_WATCHERS.subscribe(directory, this::handleChanges); + return true; + } + public void start() throws IOException { try { - synchronized(this) { - if (activeWatch != null) { - throw new IOException("Cannot start a watcher twice"); - } - - activeWatch = BUNDLED_JDK_WATCHERS.subscribe(directory, this::handleChanges); + if (!safeStart()) { + throw new IOException("Cannot start a watcher twice"); } - logger.debug("Started watch for: {}", directory); } catch (IOException e) { throw new IOException("Could not register directory watcher for: " + directory, e); diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index 811de190..0b241919 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -28,7 +28,7 @@ public class JDKRecursiveDirectoryWatcher implements Closeable { private final Path root; private final Executor exec; private final Consumer eventHandler; - private final ConcurrentMap activeWatches = new ConcurrentHashMap<>(); + private final ConcurrentMap activeWatches = new ConcurrentHashMap<>(); public JDKRecursiveDirectoryWatcher(Path directory, Executor exec, Consumer eventHandler) { this.root = directory; @@ -127,6 +127,7 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOExce @Override public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs) throws IOException { + addNewDirectory(subdir); return FileVisitResult.CONTINUE; } @@ -135,16 +136,15 @@ public FileVisitResult postVisitDirectory(Path subdir, IOException exc) throws I if (exc != null) { logger.error("Error during directory iteration: {} = {}", subdir, exc); } - addNewDirectory(subdir); return FileVisitResult.CONTINUE; } private void addNewDirectory(Path dir) throws IOException { - var watcher = new JDKDirectoryWatcher(dir, exec, relocater(dir)); - var oldEntry = activeWatches.put(dir, watcher); - cleanupOld(dir, oldEntry); + var watcher = activeWatches.computeIfAbsent(dir, d -> new JDKDirectoryWatcher(d, exec, relocater(dir))); try { - watcher.start(); + if (!watcher.safeStart()) { + logger.info("We lost the race on starting a nested logger, that shouldn't be a problem, but its a very busy, so we might have lost a few events: {}", dir); + } } catch (IOException ex) { activeWatches.remove(dir); logger.error("Could not register a watch for: {} ({})", dir, ex); @@ -160,17 +160,6 @@ private Consumer relocater(Path subRoot) { processEvents(rewritten); }; } - - private void cleanupOld(Path dir, @Nullable Closeable oldEntry) { - if (oldEntry != null) { - logger.error("Registered a watch for a directory that was already watched: {}", dir); - try { - oldEntry.close(); - } catch (IOException ex) { - logger.error("Could not close old watch for: {} ({})", dir, ex); - } - } - } } /** register watch for new sub-dir, but also simulate event for every file & subdir found */ From dae3f5317747cdd2851617bc7318a6d365d87149 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 16:03:54 +0200 Subject: [PATCH 48/89] Trying to get the test to be faster --- src/test/java/engineering/swat/watch/TortureTests.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index edf6c583..a5057ff6 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -149,7 +149,7 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO pathsWritten = io.stop(); logger.info("Generated: {} files", pathsWritten.size()); - logger.info("Waiting for the events processing to settle down"); + logger.info("Waiting for the events processing to stabilize"); waitForStable(events, happened); } @@ -165,11 +165,7 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO // but wait till all scheduled tasks have been completed // pool.awaitTermination(10, TimeUnit.SECONDS); - var totalFileCreated = seenCreates.stream() - .filter(p -> !Files.isDirectory(p)) - .count(); - - logger.info("Comparing events ({} events for {} files) and files (total {}) created", events.get(), totalFileCreated, pathsWritten.size()); + logger.info("Comparing events ({} events for {} paths) and files (total {}) created", events.get(), seenCreates.size(), pathsWritten.size()); // now make sure that the two sets are the same for (var f : pathsWritten) { assertTrue(seenCreates.contains(f), () -> "Missing create event for: " + f); @@ -217,7 +213,7 @@ void pressureOnFSShouldNotMissDeletes() throws InterruptedException, IOException try (var activeWatch = watchConfig.start() ) { logger.info("Deleting files now", THREADS); testDir.deleteAllFiles(); - logger.info("Waiting for the events processing to settle down"); + logger.info("Waiting for the events processing to stabilize"); waitForStable(events, happened); } } From f282d4965a89eaa27b6dbf9338dc074d847be391 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 17:03:29 +0200 Subject: [PATCH 49/89] Adding test to see if on linux the race breaks it --- .../swat/watch/impl/ISubscribable.java | 1 + .../swat/watch/DeleteLockTests.java | 5 - .../engineering/swat/watch/TortureTests.java | 4 + .../swat/watch/impl/BundlingTests.java | 104 ++++++++++++++++++ 4 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 src/test/java/engineering/swat/watch/impl/BundlingTests.java diff --git a/src/main/java/engineering/swat/watch/impl/ISubscribable.java b/src/main/java/engineering/swat/watch/impl/ISubscribable.java index 7f32a27b..01646ae5 100644 --- a/src/main/java/engineering/swat/watch/impl/ISubscribable.java +++ b/src/main/java/engineering/swat/watch/impl/ISubscribable.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.util.function.Consumer; +@FunctionalInterface public interface ISubscribable { Closeable subscribe(A target, Consumer eventListener) throws IOException; } diff --git a/src/test/java/engineering/swat/watch/DeleteLockTests.java b/src/test/java/engineering/swat/watch/DeleteLockTests.java index 294b52ba..c4c99bc2 100644 --- a/src/test/java/engineering/swat/watch/DeleteLockTests.java +++ b/src/test/java/engineering/swat/watch/DeleteLockTests.java @@ -1,16 +1,13 @@ package engineering.swat.watch; -import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertFalse; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Duration; import java.util.Comparator; -import java.util.concurrent.atomic.AtomicBoolean; import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; @@ -18,8 +15,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import engineering.swat.watch.WatchEvent.Kind; - class DeleteLockTests { private TestDirectory testDir; diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index a5057ff6..d43dbf76 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -155,9 +155,11 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO } finally { try { + logger.info("stopping IOGenerator"); io.stop(); } catch (Throwable _ignored) {} + logger.info("Shutting down pool"); // shutdown the pool (so no new events are registered) pool.shutdown(); } @@ -165,7 +167,9 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO // but wait till all scheduled tasks have been completed // pool.awaitTermination(10, TimeUnit.SECONDS); + logger.info("Calculating sizes"); logger.info("Comparing events ({} events for {} paths) and files (total {}) created", events.get(), seenCreates.size(), pathsWritten.size()); + logger.info("Comparing paths"); // now make sure that the two sets are the same for (var f : pathsWritten) { assertTrue(seenCreates.contains(f), () -> "Missing create event for: " + f); diff --git a/src/test/java/engineering/swat/watch/impl/BundlingTests.java b/src/test/java/engineering/swat/watch/impl/BundlingTests.java new file mode 100644 index 00000000..b7378195 --- /dev/null +++ b/src/test/java/engineering/swat/watch/impl/BundlingTests.java @@ -0,0 +1,104 @@ +package engineering.swat.watch.impl; + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import engineering.swat.watch.TestHelper; + +public class BundlingTests { + + private final Logger logger = LogManager.getLogger(); + private BundledSubscription target; + private FakeSubscribable fakeSubs; + + private static class FakeSubscribable implements ISubscribable { + private final Map> subs = new ConcurrentHashMap<>(); + + @Override + public Closeable subscribe(Long target, Consumer eventListener) throws IOException { + subs.put(target, eventListener); + return () -> { + subs.remove(target); + }; + } + + void publish(Long x) { + var s = subs.get(x); + if (s != null) { + s.accept(true); + } + } + }; + + + @BeforeEach + void setup() { + fakeSubs = new FakeSubscribable(); + target = new BundledSubscription<>(fakeSubs); + } + + @BeforeAll + static void setupEverything() { + Awaitility.setDefaultTimeout(TestHelper.LONG_WAIT.getSeconds(), TimeUnit.SECONDS); + } + + private static final long SUBs = 100; + private static final long MSGs = 100_000; + + @Test + void manySubscriptions() throws IOException { + AtomicInteger hits = new AtomicInteger(); + List closers = new ArrayList<>(); + + for (int i = 0; i < MSGs; i++) { + for (int j = 0; j < SUBs; j++) { + closers.add(target.subscribe(Long.valueOf(i), b -> hits.incrementAndGet())); + } + } + + logger.info("Sending single message"); + fakeSubs.publish(Long.valueOf(0)); + assertEquals(SUBs, hits.get()); + logger.info("Sending all messages"); + hits.set(0); + for (int i = 0; i < MSGs; i++) { + fakeSubs.publish(Long.valueOf(i)); + } + assertEquals(SUBs * MSGs, hits.get()); + + logger.info("Clearing subs in parallel"); + for (var clos : closers) { + CompletableFuture.runAsync(() -> { + try { + clos.close(); + } catch (IOException e) { + logger.catching(e); + } + }); + } + + await("Closing should finish") + .until(fakeSubs.subs::isEmpty); + logger.info("Done clearing"); + + + } +} From a5a76d63b13e3c614f5f3988abcd0bfb5c44a34b Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 17:13:34 +0200 Subject: [PATCH 50/89] Checking that we log what is thrown --- src/test/java/engineering/swat/watch/TortureTests.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index d43dbf76..ed4a976b 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -152,6 +152,11 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO logger.info("Waiting for the events processing to stabilize"); waitForStable(events, happened); + } + catch (Exception ex) { + logger.catching(ex); + throw ex; + } finally { try { From 6e9ca1b167f8eca8b066bd80716e484a5724d7e5 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 17:43:19 +0200 Subject: [PATCH 51/89] Trying to give windows a bit more time to work through the tests --- .../engineering/swat/watch/impl/BundledSubscription.java | 4 ++-- src/test/java/engineering/swat/watch/TortureTests.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java index 1abab350..2680ccc9 100644 --- a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java +++ b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java @@ -3,8 +3,8 @@ import java.io.Closeable; import java.io.IOException; import java.util.List; -import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.Consumer; @@ -13,7 +13,7 @@ public class BundledSubscription implements ISubscribable { private final ISubscribable around; - private final Map> subscriptions = new ConcurrentHashMap<>(); + private final ConcurrentMap> subscriptions = new ConcurrentHashMap<>(); public BundledSubscription(ISubscribable around) { this.around = around; diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index ed4a976b..f43f4b51 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -76,7 +76,7 @@ private void startJob(final Path root, Random r, Executor exec) { // burst a bunch of creates creates and then sleep a bit for (int i = 0; i< BURST_SIZE; i++) { var file = root.resolve("l1-" + r.nextInt(1000)) - .resolve("l2-" + r.nextInt(1000)) + .resolve("l2-" + r.nextInt(100)) .resolve("l3-" + r.nextInt() + ".txt"); Files.createDirectories(file.getParent()); Files.writeString(file, "Hello world"); @@ -99,9 +99,9 @@ void start() { } Set stop() throws InterruptedException { - startRunning.release(jobs); stopRunning.release(jobs); - assertTrue(done.tryAcquire(jobs, TestHelper.NORMAL_WAIT.toMillis(), TimeUnit.MILLISECONDS), "IO workers should stop in a reasonable time"); + startRunning.release(jobs); + assertTrue(done.tryAcquire(jobs, TestHelper.NORMAL_WAIT.toMillis() * 2, TimeUnit.MILLISECONDS), "IO workers should stop in a reasonable time"); return pathsWritten; } } From 1666cf37efb80c59355145b5c1dbb8fc8ec72f89 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 16 Sep 2024 18:28:14 +0200 Subject: [PATCH 52/89] Trying to break the deadlock on linux --- .../swat/watch/impl/JDKPoller.java | 50 +++++++++++++++---- .../impl/JDKRecursiveDirectoryWatcher.java | 2 +- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index a264b4c0..21d51526 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -15,6 +15,9 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -28,6 +31,12 @@ private JDKPoller() {} private static final Logger logger = LogManager.getLogger(); private static final Map>>> watchers = new ConcurrentHashMap<>(); private static final WatchService service; + private static final int nCores = Runtime.getRuntime().availableProcessors(); + /** + * We have to be a bit careful with registering too many paths in parallel + * Linux can be thrown into a deadlock if you try to start 1000 threads and then do a register at the same time. + */ + private static final ExecutorService registerPool = Executors.newFixedThreadPool(nCores); static { try { @@ -73,19 +82,40 @@ private static void poll() { } } + public static Closeable register(Path path, Consumer>> changes) throws IOException { logger.debug("Register watch for: {}", path); + + try { // TODO: consider upgrading the events the moment we actually get a request for all of it - var key = path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY, ENTRY_DELETE); - logger.trace("Got watch key: {}", key); - watchers.put(key, changes); - return new Closeable() { - @Override - public void close() throws IOException { - logger.debug("Closing watch for: {}", path); - key.cancel(); - watchers.remove(key); + return CompletableFuture.supplyAsync(() -> { + try { + var key = path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY, ENTRY_DELETE); + logger.trace("Got watch key: {}", key); + watchers.put(key, changes); + return key; + } catch (IOException e) { + throw new RuntimeException(e); + } + }, registerPool) + .thenApplyAsync(key -> new Closeable() { + @Override + public void close() throws IOException { + logger.debug("Closing watch for: {}", path); + key.cancel(); + watchers.remove(key); + } + }) + .get(); + } catch (ExecutionException e) { + if (e.getCause() instanceof IOException) { + throw (IOException)e.getCause(); } - }; + throw new IOException("Could not register path", e.getCause()); + } catch (InterruptedException e) { + // the pool was closing, forward it + Thread.currentThread().interrupt(); + throw new IOException("The registration was canceled"); + } } } diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index 0b241919..d446759c 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -143,7 +143,7 @@ private void addNewDirectory(Path dir) throws IOException { var watcher = activeWatches.computeIfAbsent(dir, d -> new JDKDirectoryWatcher(d, exec, relocater(dir))); try { if (!watcher.safeStart()) { - logger.info("We lost the race on starting a nested logger, that shouldn't be a problem, but its a very busy, so we might have lost a few events: {}", dir); + logger.debug("We lost the race on starting a nested watcher, that shouldn't be a problem, but its a very busy, so we might have lost a few events in {}", dir); } } catch (IOException ex) { activeWatches.remove(dir); From f97d8e021da38f2061fe787090cb2029035448ee Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 17 Sep 2024 09:09:19 +0200 Subject: [PATCH 53/89] Rewrote the register path to reduce the time in a limited thread spacve --- .../swat/watch/impl/JDKPoller.java | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index 21d51526..a75a14b7 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -87,26 +87,25 @@ public static Closeable register(Path path, Consumer>> change logger.debug("Register watch for: {}", path); try { - // TODO: consider upgrading the events the moment we actually get a request for all of it return CompletableFuture.supplyAsync(() -> { try { - var key = path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY, ENTRY_DELETE); - logger.trace("Got watch key: {}", key); - watchers.put(key, changes); - return key; + return path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY, ENTRY_DELETE); } catch (IOException e) { throw new RuntimeException(e); } - }, registerPool) - .thenApplyAsync(key -> new Closeable() { - @Override - public void close() throws IOException { - logger.debug("Closing watch for: {}", path); - key.cancel(); - watchers.remove(key); - } + }, registerPool) // read registerPool why we have to add a limiter here + .thenApplyAsync(key -> { + watchers.put(key, changes); + return new Closeable() { + @Override + public void close() throws IOException { + logger.debug("Closing watch for: {}", path); + key.cancel(); + watchers.remove(key); + } + }; }) - .get(); + .get(); // we have to do a get here, to make sure the `register` function blocks } catch (ExecutionException e) { if (e.getCause() instanceof IOException) { throw (IOException)e.getCause(); From 7b0bfbc8c29c4e467411ad2daf31a1df71cfc25c Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 17 Sep 2024 15:56:28 +0200 Subject: [PATCH 54/89] Added a catchup-loop to make sure we're not missing events in the recursive watch --- .../impl/JDKRecursiveDirectoryWatcher.java | 143 ++++++++++++++++-- 1 file changed, 128 insertions(+), 15 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index d446759c..faf4eb5b 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -9,9 +9,13 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Deque; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; @@ -165,19 +169,31 @@ private Consumer relocater(Path subRoot) { /** register watch for new sub-dir, but also simulate event for every file & subdir found */ private class NewDirectoryScan extends InitialDirectoryScan { protected final List events; + protected final Set seenFiles; + protected final Set seenDirs; private boolean hasFiles = false; - public NewDirectoryScan(Path subRoot, List events) { + public NewDirectoryScan(Path subRoot, List events, Set seenFiles, Set seenDirs) { super(subRoot); this.events = events; + this.seenFiles = seenFiles; + this.seenDirs = seenDirs; } @Override public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs) throws IOException { - if (!subdir.equals(subRoot)) { - events.add(new WatchEvent(WatchEvent.Kind.CREATED, root, root.relativize(subdir))); + try { + hasFiles = false; + if (!seenDirs.contains(subdir)) { + if (!subdir.equals(subRoot)) { + events.add(new WatchEvent(WatchEvent.Kind.CREATED, root, root.relativize(subdir))); + } + return super.preVisitDirectory(subdir, attrs); + } + // our children might have newer results + return FileVisitResult.CONTINUE; + } finally { + seenDirs.add(subdir); } - hasFiles = false; - return super.preVisitDirectory(subdir, attrs); } @Override @@ -190,11 +206,14 @@ public FileVisitResult postVisitDirectory(Path subdir, IOException exc) throws I @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - hasFiles = true; - var relative = root.relativize(file); - events.add(new WatchEvent(WatchEvent.Kind.CREATED, root, relative)); - if (attrs.size() > 0) { - events.add(new WatchEvent(WatchEvent.Kind.MODIFIED, root, relative)); + if (!seenFiles.contains(file)) { + hasFiles = true; + + var relative = root.relativize(file); + events.add(new WatchEvent(WatchEvent.Kind.CREATED, root, relative)); + if (attrs.size() > 0) { + events.add(new WatchEvent(WatchEvent.Kind.MODIFIED, root, relative)); + } } return FileVisitResult.CONTINUE; } @@ -203,8 +222,8 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO /** detect directories that aren't tracked yet, and generate events only for new entries */ private class OverflowSyncScan extends NewDirectoryScan { private final Deque isNewDirectory = new ArrayDeque<>(); - public OverflowSyncScan(Path subRoot, List events) { - super(subRoot, events); + public OverflowSyncScan(Path subRoot, List events, Set seenFiles, Set seenDirs) { + super(subRoot, events, seenFiles, seenDirs); } @Override public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs) throws IOException { @@ -222,7 +241,7 @@ public FileVisitResult postVisitDirectory(Path subdir, IOException exc) throws I } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (isNewDirectory.peekLast() == Boolean.TRUE) { + if (isNewDirectory.peekLast() == Boolean.TRUE || !seenFiles.contains(file)) { return super.visitFile(file, attrs); } return FileVisitResult.CONTINUE; @@ -233,18 +252,112 @@ private void registerInitialWatches(Path dir) throws IOException { Files.walkFileTree(dir, new InitialDirectoryScan(dir)); } + private class FastPutOnly implements Set { + private final Set wrapped; + + FastPutOnly(Set wrapped) { + this.wrapped = wrapped; + } + + @Override + public int size() { + throw new IllegalStateException("Not supported"); + } + + @Override + public boolean isEmpty() { + throw new IllegalStateException("Not supported"); + } + + @Override + public boolean contains(Object o) { + return false; + } + + @Override + public Iterator iterator() { + throw new IllegalStateException("Not supported"); + } + + @Override + public Object[] toArray() { + throw new IllegalStateException("Not supported"); + } + + @Override + public T[] toArray(T[] a) { + throw new IllegalStateException("Not supported"); + } + + @Override + public boolean add(Path e) { + return wrapped.add(e); + } + + @Override + public boolean remove(Object o) { + throw new IllegalStateException("Not supported"); + } + + @Override + public boolean containsAll(Collection c) { + throw new IllegalStateException("Not supported"); + } + + @Override + public boolean addAll(Collection c) { + throw new IllegalStateException("Not supported"); + } + + @Override + public boolean retainAll(Collection c) { + throw new IllegalStateException("Not supported"); + } + + @Override + public boolean removeAll(Collection c) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void clear() { + throw new IllegalStateException("Not supported"); + } + + } + private List registerForNewDirectory(Path dir) throws IOException { var events = new ArrayList(); - Files.walkFileTree(dir, new NewDirectoryScan(dir, events)); + var seen = new HashSet(); + var seenFiles = new HashSet(); + var seenDirectories = new HashSet(); + Files.walkFileTree(dir, new NewDirectoryScan(dir, events, new FastPutOnly(seenFiles), new FastPutOnly(seenDirectories))); + detectedMissingEntries(dir, events, seenFiles, seenDirectories); return events; } + private List syncAfterOverflow(Path dir) throws IOException { var events = new ArrayList(); - Files.walkFileTree(dir, new OverflowSyncScan(dir, events)); + var seenFiles = new HashSet(); + var seenDirectories = new HashSet(); + Files.walkFileTree(dir, new OverflowSyncScan(dir, events, new FastPutOnly(seenFiles), new FastPutOnly(seenDirectories))); + detectedMissingEntries(dir, events, seenFiles, seenDirectories); return events; } + private void detectedMissingEntries(Path dir, ArrayList events, HashSet seenFiles, HashSet seenDirectories) throws IOException { + // why a second round? well there is a race, between iterating the directory (and sending events) + // and when the watches are active. so after we know all the new watches have been registered + // we do a second scan and make sure to find paths that weren't visible the first time + // and emulate events for them (and register new watches) + int directoryCount = seenDirectories.size() - 1; + while (directoryCount != seenDirectories.size()) { + Files.walkFileTree(dir, new OverflowSyncScan(dir, events, seenFiles, seenDirectories)); + directoryCount = seenDirectories.size(); + } + } + @Override From 2090e351c7ef91228446074e4ba1a5dc8a52f1af Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 17 Sep 2024 16:10:35 +0200 Subject: [PATCH 55/89] Removed extra set that was just a premature optimization --- .../impl/JDKRecursiveDirectoryWatcher.java | 82 +------------------ 1 file changed, 2 insertions(+), 80 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index faf4eb5b..e46fd389 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -9,11 +9,9 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Deque; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -23,7 +21,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.checkerframework.checker.nullness.qual.Nullable; import engineering.swat.watch.WatchEvent; @@ -252,86 +249,11 @@ private void registerInitialWatches(Path dir) throws IOException { Files.walkFileTree(dir, new InitialDirectoryScan(dir)); } - private class FastPutOnly implements Set { - private final Set wrapped; - - FastPutOnly(Set wrapped) { - this.wrapped = wrapped; - } - - @Override - public int size() { - throw new IllegalStateException("Not supported"); - } - - @Override - public boolean isEmpty() { - throw new IllegalStateException("Not supported"); - } - - @Override - public boolean contains(Object o) { - return false; - } - - @Override - public Iterator iterator() { - throw new IllegalStateException("Not supported"); - } - - @Override - public Object[] toArray() { - throw new IllegalStateException("Not supported"); - } - - @Override - public T[] toArray(T[] a) { - throw new IllegalStateException("Not supported"); - } - - @Override - public boolean add(Path e) { - return wrapped.add(e); - } - - @Override - public boolean remove(Object o) { - throw new IllegalStateException("Not supported"); - } - - @Override - public boolean containsAll(Collection c) { - throw new IllegalStateException("Not supported"); - } - - @Override - public boolean addAll(Collection c) { - throw new IllegalStateException("Not supported"); - } - - @Override - public boolean retainAll(Collection c) { - throw new IllegalStateException("Not supported"); - } - - @Override - public boolean removeAll(Collection c) { - throw new IllegalStateException("Not supported"); - } - - @Override - public void clear() { - throw new IllegalStateException("Not supported"); - } - - } - private List registerForNewDirectory(Path dir) throws IOException { var events = new ArrayList(); - var seen = new HashSet(); var seenFiles = new HashSet(); var seenDirectories = new HashSet(); - Files.walkFileTree(dir, new NewDirectoryScan(dir, events, new FastPutOnly(seenFiles), new FastPutOnly(seenDirectories))); + Files.walkFileTree(dir, new NewDirectoryScan(dir, events, seenFiles, seenDirectories)); detectedMissingEntries(dir, events, seenFiles, seenDirectories); return events; } @@ -341,7 +263,7 @@ private List syncAfterOverflow(Path dir) throws IOException { var events = new ArrayList(); var seenFiles = new HashSet(); var seenDirectories = new HashSet(); - Files.walkFileTree(dir, new OverflowSyncScan(dir, events, new FastPutOnly(seenFiles), new FastPutOnly(seenDirectories))); + Files.walkFileTree(dir, new OverflowSyncScan(dir, events, seenFiles, seenDirectories)); detectedMissingEntries(dir, events, seenFiles, seenDirectories); return events; } From 51cfb9e3bddb0eaacb45564a5f1d03eaca137b5a Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Wed, 18 Sep 2024 14:00:01 +0200 Subject: [PATCH 56/89] Trying to really wait for all events to have stabilized --- .../java/engineering/swat/watch/TortureTests.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index f43f4b51..fe23b054 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -254,20 +254,16 @@ private void waitForStable(final AtomicInteger events, final Semaphore happened) while (happened.tryAcquire(TestHelper.SHORT_WAIT.toMillis(), TimeUnit.MILLISECONDS)) { happened.drainPermits(); } + int currentEventCounts = events.get(); if (currentEventCounts == lastEventCount) { - if (stableCount == 30) { - logger.info("Stable after: {} events", currentEventCounts); - break; - } - else { - stableCount++; - } + stableCount++; } else { + lastEventCount = currentEventCounts; stableCount = 0; } - lastEventCount = currentEventCounts; - } while (true); + } while (stableCount < 60); + logger.info("Stable after: {} events", lastEventCount); } } From 5ca0fe34f54df9126e253e528b80652cedf42581 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Wed, 18 Sep 2024 14:08:55 +0200 Subject: [PATCH 57/89] Longer wait for stabilization --- src/test/java/engineering/swat/watch/TortureTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index fe23b054..e6893090 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -251,6 +251,7 @@ private void waitForStable(final AtomicInteger events, final Semaphore happened) int lastEventCount = events.get(); int stableCount = 0; do { + Thread.yield(); while (happened.tryAcquire(TestHelper.SHORT_WAIT.toMillis(), TimeUnit.MILLISECONDS)) { happened.drainPermits(); } @@ -263,7 +264,7 @@ private void waitForStable(final AtomicInteger events, final Semaphore happened) lastEventCount = currentEventCounts; stableCount = 0; } - } while (stableCount < 60); + } while (stableCount < 120); logger.info("Stable after: {} events", lastEventCount); } } From 1fbc69bbc66db19449a0f648e7d4204f29089e8d Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Wed, 18 Sep 2024 15:10:31 +0200 Subject: [PATCH 58/89] Running sync in a background thread to get initial events faster --- .../swat/watch/impl/JDKDirectoryWatcher.java | 13 ++- .../impl/JDKRecursiveDirectoryWatcher.java | 89 ++++++++++--------- .../engineering/swat/watch/TortureTests.java | 2 +- 3 files changed, 60 insertions(+), 44 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java index 11308bf0..40ccdebb 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -51,9 +51,16 @@ public void start() throws IOException { } private void handleChanges(List> events) { - exec.execute(() -> - events.forEach(ev -> eventHandler.accept(translate(ev))) - ); + exec.execute(() -> { + for (var ev : events) { + try { + eventHandler.accept(translate(ev)); + } + catch (Throwable ignored) { + logger.error("Ignoring downstream exception:", ignored); + } + } + }); } private WatchEvent translate(java.nio.file.WatchEvent ev) { diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index e46fd389..aea8ae5e 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -14,11 +14,14 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; import java.util.function.Consumer; +import javax.management.RuntimeErrorException; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,57 +50,55 @@ public void start() throws IOException { } private void processEvents(WatchEvent ev) { + logger.trace("Forwarding event: {}", ev); + eventHandler.accept(ev); logger.trace("Unwrapping event: {}", ev); - List extraEvents = null; try { switch (ev.getKind()) { - case CREATED: extraEvents = handleCreate(ev); break; + case CREATED: handleCreate(ev); break; case DELETED: handleDeleteDirectory(ev); break; - case OVERFLOW: extraEvents = handleOverflow(ev); break; + case OVERFLOW: handleOverflow(ev); break; case MODIFIED: break; } } finally { - eventHandler.accept(ev); - if (extraEvents != null) { - extraEvents.forEach(eventHandler); - } } } + private void publishExtraEvents(List ev) { + logger.trace("Reporting new nested directories & files: {}", ev); + ev.forEach(eventHandler); + } - private List handleCreate(WatchEvent ev) { + + private void handleCreate(WatchEvent ev) { // between the event and the current state of the file system // we might have some nested directories we missed // so if we have a new directory, we have to go in and iterate over it // we also have to report all nested files & dirs as created paths // but we don't want to burden ourselves with those events - try { - var fullPath = ev.calculateFullPath(); - if (!activeWatches.containsKey(fullPath)) { - var newEvents = registerForNewDirectory(ev.calculateFullPath()); - logger.trace("Reporting new nested directories & files: {}", newEvents); - return newEvents; - } - else { - return Collections.emptyList(); - } - } catch (IOException e) { - logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), e); - return Collections.emptyList(); + var fullPath = ev.calculateFullPath(); + if (!activeWatches.containsKey(fullPath)) { + CompletableFuture + .completedFuture(fullPath) + .thenApplyAsync(this::registerForNewDirectory, exec) + .thenAcceptAsync(this::publishExtraEvents, exec) + .exceptionally(ex -> { + logger.error("Could not locate new sub directories for: {}", ev.calculateFullPath(), ex); + return null; + }); } } - private List handleOverflow(WatchEvent ev) { - try { - logger.info("Overflow detected, rescanning to find missed entries in {}", root); - // we have to rescan everything, and at least make sure to add new entries to that recursive watcher - var newEntries = syncAfterOverflow(ev.calculateFullPath()); - logger.trace("Reporting new nested directories & files: {}", newEntries); - return newEntries; - } catch (IOException e) { - logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), e); - return Collections.emptyList(); - } + private void handleOverflow(WatchEvent ev) { + logger.info("Overflow detected, rescanning to find missed entries in {}", root); + CompletableFuture + .completedFuture(ev.calculateFullPath()) + .thenApplyAsync(this::syncAfterOverflow, exec) + .thenAcceptAsync(this::publishExtraEvents, exec) + .exceptionally(ex -> { + logger.error("Could not register new watch for: {} ({})", ev.calculateFullPath(), ex); + return null; + }); } private void handleDeleteDirectory(WatchEvent ev) { @@ -249,23 +250,31 @@ private void registerInitialWatches(Path dir) throws IOException { Files.walkFileTree(dir, new InitialDirectoryScan(dir)); } - private List registerForNewDirectory(Path dir) throws IOException { + private List registerForNewDirectory(Path dir) { var events = new ArrayList(); var seenFiles = new HashSet(); var seenDirectories = new HashSet(); - Files.walkFileTree(dir, new NewDirectoryScan(dir, events, seenFiles, seenDirectories)); - detectedMissingEntries(dir, events, seenFiles, seenDirectories); - return events; + try { + Files.walkFileTree(dir, new NewDirectoryScan(dir, events, seenFiles, seenDirectories)); + detectedMissingEntries(dir, events, seenFiles, seenDirectories); + return events; + } catch (IOException ex) { + throw new RuntimeException(ex); + } } - private List syncAfterOverflow(Path dir) throws IOException { + private List syncAfterOverflow(Path dir) { var events = new ArrayList(); var seenFiles = new HashSet(); var seenDirectories = new HashSet(); - Files.walkFileTree(dir, new OverflowSyncScan(dir, events, seenFiles, seenDirectories)); - detectedMissingEntries(dir, events, seenFiles, seenDirectories); - return events; + try { + Files.walkFileTree(dir, new OverflowSyncScan(dir, events, seenFiles, seenDirectories)); + detectedMissingEntries(dir, events, seenFiles, seenDirectories); + return events; + } catch (IOException ex) { + throw new RuntimeException(ex); + } } private void detectedMissingEntries(Path dir, ArrayList events, HashSet seenFiles, HashSet seenDirectories) throws IOException { diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index e6893090..3c1100c5 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -264,7 +264,7 @@ private void waitForStable(final AtomicInteger events, final Semaphore happened) lastEventCount = currentEventCounts; stableCount = 0; } - } while (stableCount < 120); + } while (stableCount < 60); logger.info("Stable after: {} events", lastEventCount); } } From 49da0a9d82b49ddecdc8b6f0e96d38138df65b29 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Wed, 18 Sep 2024 16:01:32 +0200 Subject: [PATCH 59/89] Trying my best to stabilize the torture tests --- src/test/java/engineering/swat/watch/TortureTests.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 3c1100c5..3db7c301 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -111,7 +111,7 @@ Set stop() throws InterruptedException { @Test void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IOException { final var root = testDir.getTestDirectory(); - var pool = Executors.newCachedThreadPool(); + var pool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2); var io = new IOGenerator(THREADS, root, pool); @@ -168,6 +168,8 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO // shutdown the pool (so no new events are registered) pool.shutdown(); } + waitForStable(events, happened); + // but wait till all scheduled tasks have been completed // pool.awaitTermination(10, TimeUnit.SECONDS); @@ -252,7 +254,7 @@ private void waitForStable(final AtomicInteger events, final Semaphore happened) int stableCount = 0; do { Thread.yield(); - while (happened.tryAcquire(TestHelper.SHORT_WAIT.toMillis(), TimeUnit.MILLISECONDS)) { + while (happened.tryAcquire(TestHelper.SHORT_WAIT.toMillis() * 2, TimeUnit.MILLISECONDS)) { happened.drainPermits(); } From bfe3d74ecaeefb65d22da655419baab3e2d69f52 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Wed, 18 Sep 2024 18:25:40 +0200 Subject: [PATCH 60/89] Using await to just wait for the condition we care about --- .../engineering/swat/watch/TortureTests.java | 26 +++++-------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 3db7c301..c0619418 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -1,11 +1,13 @@ package engineering.swat.watch; import static org.awaitility.Awaitility.doNotCatchUncaughtExceptionsByDefault; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.time.LocalTime; import java.util.Random; import java.util.Set; @@ -116,14 +118,10 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO var io = new IOGenerator(THREADS, root, pool); - final var events = new AtomicInteger(0); - final var happened = new Semaphore(0); var seenCreates = ConcurrentHashMap.newKeySet(); var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) .withExecutor(pool) .onEvent(ev -> { - events.getAndIncrement(); - happened.release(); var fullPath = ev.calculateFullPath(); switch (ev.getKind()) { case CREATED: @@ -149,9 +147,10 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO pathsWritten = io.stop(); logger.info("Generated: {} files", pathsWritten.size()); - logger.info("Waiting for the events processing to stabilize"); - waitForStable(events, happened); - + await("After a while we should have seen all the create events") + .timeout(TestHelper.LONG_WAIT.multipliedBy(4)) + .pollInterval(Duration.ofMillis(500)) + .until(() -> seenCreates.containsAll(pathsWritten)); } catch (Exception ex) { logger.catching(ex); @@ -168,19 +167,6 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO // shutdown the pool (so no new events are registered) pool.shutdown(); } - waitForStable(events, happened); - - - // but wait till all scheduled tasks have been completed - // pool.awaitTermination(10, TimeUnit.SECONDS); - - logger.info("Calculating sizes"); - logger.info("Comparing events ({} events for {} paths) and files (total {}) created", events.get(), seenCreates.size(), pathsWritten.size()); - logger.info("Comparing paths"); - // now make sure that the two sets are the same - for (var f : pathsWritten) { - assertTrue(seenCreates.contains(f), () -> "Missing create event for: " + f); - } } From 38f9b9d31f241fec05792ac5138067e22fbbdcf3 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Wed, 18 Sep 2024 18:32:29 +0200 Subject: [PATCH 61/89] Increased timeouts on windows --- src/test/java/engineering/swat/watch/TestHelper.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/test/java/engineering/swat/watch/TestHelper.java b/src/test/java/engineering/swat/watch/TestHelper.java index 702858db..17264b4c 100644 --- a/src/test/java/engineering/swat/watch/TestHelper.java +++ b/src/test/java/engineering/swat/watch/TestHelper.java @@ -11,10 +11,17 @@ public class TestHelper { static { var delayFactorConfig = System.getenv("DELAY_FACTOR"); int delayFactor = delayFactorConfig == null ? 1 : Integer.parseInt(delayFactorConfig); - if (System.getProperty("os", "?").toLowerCase().contains("mac")) { + var os = System.getProperty("os", "?").toLowerCase(); + if (os.contains("mac")) { // OSX is SLOW on it's watches delayFactor *= 2; } + else if (os.contains("win")) { + // windows watches can be slow to get everything + // published + // especially on small core systems + delayFactor *= 4; + } SHORT_WAIT = Duration.ofSeconds(1 * delayFactor); NORMAL_WAIT = Duration.ofSeconds(4 * delayFactor); LONG_WAIT = Duration.ofSeconds(8 * delayFactor); From eb388e8f17698b1148a89e591c971706fc00b3db Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Wed, 18 Sep 2024 20:24:25 +0200 Subject: [PATCH 62/89] Increased timeouts on windows --- src/test/java/engineering/swat/watch/TortureTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index c0619418..58d42570 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -148,7 +148,7 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO logger.info("Generated: {} files", pathsWritten.size()); await("After a while we should have seen all the create events") - .timeout(TestHelper.LONG_WAIT.multipliedBy(4)) + .timeout(TestHelper.LONG_WAIT.multipliedBy(20)) .pollInterval(Duration.ofMillis(500)) .until(() -> seenCreates.containsAll(pathsWritten)); } From 14c501d5b9e4e9766963813cf67b7be42c21a032 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Wed, 18 Sep 2024 20:43:47 +0200 Subject: [PATCH 63/89] Increased timeouts on windows --- src/test/java/engineering/swat/watch/TortureTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 58d42570..63682f51 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -148,7 +148,7 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO logger.info("Generated: {} files", pathsWritten.size()); await("After a while we should have seen all the create events") - .timeout(TestHelper.LONG_WAIT.multipliedBy(20)) + .timeout(TestHelper.LONG_WAIT.multipliedBy(50)) .pollInterval(Duration.ofMillis(500)) .until(() -> seenCreates.containsAll(pathsWritten)); } From bcc19a6bd62057b66b0d24b92a069ed518d51e11 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 24 Sep 2024 20:06:51 +0200 Subject: [PATCH 64/89] Moved towards native file watch support for windows and rewrote the API a bit --- .../engineering/swat/watch/WatchScope.java | 7 ++ .../java/engineering/swat/watch/Watcher.java | 98 ++++++++----------- .../swat/watch/impl/JDKDirectoryWatcher.java | 11 ++- .../swat/watch/impl/JDKPoller.java | 13 ++- .../swat/watch/impl/SubscriptionKey.java | 37 +++++++ .../swat/watch/DeleteLockTests.java | 15 +-- .../swat/watch/RecursiveWatchTests.java | 6 +- .../swat/watch/SingleDirectoryTests.java | 2 +- .../swat/watch/SingleFileTests.java | 4 +- .../engineering/swat/watch/SmokeTests.java | 6 +- .../engineering/swat/watch/TortureTests.java | 4 +- 11 files changed, 120 insertions(+), 83 deletions(-) create mode 100644 src/main/java/engineering/swat/watch/WatchScope.java create mode 100644 src/main/java/engineering/swat/watch/impl/SubscriptionKey.java diff --git a/src/main/java/engineering/swat/watch/WatchScope.java b/src/main/java/engineering/swat/watch/WatchScope.java new file mode 100644 index 00000000..8b38c8d9 --- /dev/null +++ b/src/main/java/engineering/swat/watch/WatchScope.java @@ -0,0 +1,7 @@ +package engineering.swat.watch; + +public enum WatchScope { + SINGLE, + INCLUDING_CHILDREN, + INCLUDING_ALL_DESCENDANTS +} diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 81cc9deb..36158a2a 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -24,7 +24,7 @@ */ public class Watcher { private final Logger logger = LogManager.getLogger(); - private final WatcherKind kind; + private final WatchScope scope; private final Path path; private Executor executor = CompletableFuture::runAsync; @@ -32,63 +32,39 @@ public class Watcher { private Consumer eventHandler = NULL_HANDLER; - private Watcher(WatcherKind kind, Path path) { - this.kind = kind; + private Watcher(WatchScope scope, Path path) { + this.scope = scope; this.path = path; - logger.info("Constructor logger for: {} at {} level", path, kind); - } - - private enum WatcherKind { - FILE, - DIRECTORY, - RECURSIVE_DIRECTORY + logger.info("Constructor logger for: {} at {} level", path, scope); } /** - * Request a watcher for a single path (file or directory). - * If it's a file, depending on the platform this will watch the whole directory and filter the results, or only watch a single file. - * @param path a single path entry, either a file or a directory - * @return a watcher that only fires events related to the requested path - * @throws IOException in case the path is not absolute + * Watch a path for updates, optionally also get events for its children/descendants + * @param path which absolute path to monitor, can be a file or a directory, but has to be absolute + * @param scope for directories you can also choose to monitor it's direct children or all it's descendants + * @throws IllegalArgumentException in case a path is not supported (in relation to the scope) */ - public static Watcher single(Path path) throws IOException { + public static Watcher watch(Path path, WatchScope scope) { if (!path.isAbsolute()) { - throw new IOException("We can only watch absolute paths"); - } - return new Watcher(WatcherKind.FILE, path); - } - - /** - * Request a watcher for a directory, getting events for its direct children. - * @param path a directory to monitor for changes - * @return a watcher that fires events for any of the direct children (and its self). - * @throws IOException in cas the path is not absolute or it's not an directory - */ - public static Watcher singleDirectory(Path path) throws IOException { - if (!path.isAbsolute()) { - throw new IOException("We can only watch absolute paths"); - } - if (!Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { - throw new IOException("Only directories are supported"); + throw new IllegalArgumentException("We can only watch absolute paths"); } - return new Watcher(WatcherKind.DIRECTORY, path); - } + switch (scope) { + case INCLUDING_CHILDREN: // intended fallthrough + case INCLUDING_ALL_DESCENDANTS: + if (!Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { + throw new IllegalArgumentException("Only directories are supported for this scope"); + } + break; + case SINGLE: + if (Files.isSymbolicLink(path)) { + throw new IllegalArgumentException("Symlinks are not supported"); + } + break; + default: + throw new IllegalArgumentException("Unsupported scope: " + scope); - /** - * Request a watcher for a directory, getting events for all of its children. Even those added after the watch started. - * On some platforms, this can be quite expansive, so be sure you want this. - * @param path a directory to monitor for changes - * @return a watcher that fires events for any of its children (and its self). - * @throws IOException in case the path is not absolute or it's not an directory - */ - public static Watcher recursiveDirectory(Path path) throws IOException { - if (!path.isAbsolute()) { - throw new IOException("We can only watch absolute paths"); - } - if (!Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { - throw new IOException("Only directories are supported"); } - return new Watcher(WatcherKind.RECURSIVE_DIRECTORY, path); + return new Watcher(scope, path); } /** @@ -124,18 +100,26 @@ public Closeable start() throws IOException, IllegalStateException { if (this.eventHandler == NULL_HANDLER) { throw new IllegalStateException("There is no onEvent handler defined"); } - switch (kind) { - case DIRECTORY: { - var result = new JDKDirectoryWatcher(path, executor, this.eventHandler); + switch (scope) { + case INCLUDING_CHILDREN: { + var result = new JDKDirectoryWatcher(path, executor, this.eventHandler, false); result.start(); return result; } - case RECURSIVE_DIRECTORY: { - var result = new JDKRecursiveDirectoryWatcher(path, executor, this.eventHandler); - result.start(); - return result; + case INCLUDING_ALL_DESCENDANTS: { + try { + var result = new JDKDirectoryWatcher(path, executor, this.eventHandler, true); + result.start(); + return result; + } catch (Throwable ex) { + // no native support, use the simulation + logger.info("Not possible to register the native watcher for {}", path, ex); + var result = new JDKRecursiveDirectoryWatcher(path, executor, this.eventHandler); + result.start(); + return result; + } } - case FILE: { + case SINGLE: { var result = new JDKFileWatcher(path, executor, this.eventHandler); result.start(); return result; diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java index 40ccdebb..a0b87c27 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -21,21 +21,28 @@ public class JDKDirectoryWatcher implements Closeable { private final Executor exec; private final Consumer eventHandler; private volatile @MonotonicNonNull Closeable activeWatch; + private final boolean nativeRecursive; - private static final BundledSubscription>> + private static final BundledSubscription>> BUNDLED_JDK_WATCHERS = new BundledSubscription<>(JDKPoller::register); public JDKDirectoryWatcher(Path directory, Executor exec, Consumer eventHandler) { + this(directory, exec, eventHandler, false); + } + + public JDKDirectoryWatcher(Path directory, Executor exec, Consumer eventHandler, boolean nativeRecursive) { this.directory = directory; this.exec = exec; this.eventHandler = eventHandler; + this.nativeRecursive = nativeRecursive; } + synchronized boolean safeStart() throws IOException { if (activeWatch != null) { return false; } - activeWatch = BUNDLED_JDK_WATCHERS.subscribe(directory, this::handleChanges); + activeWatch = BUNDLED_JDK_WATCHERS.subscribe(new SubscriptionKey(directory, nativeRecursive), this::handleChanges); return true; } diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index a75a14b7..aacfba5a 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -7,7 +7,6 @@ import java.io.Closeable; import java.io.IOException; import java.nio.file.FileSystems; -import java.nio.file.Path; import java.nio.file.WatchEvent; import java.nio.file.WatchKey; import java.nio.file.WatchService; @@ -25,6 +24,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.sun.nio.file.ExtendedWatchEventModifier; + class JDKPoller { private JDKPoller() {} @@ -83,13 +84,19 @@ private static void poll() { } - public static Closeable register(Path path, Consumer>> changes) throws IOException { + public static Closeable register(SubscriptionKey path, Consumer>> changes) throws IOException { logger.debug("Register watch for: {}", path); try { return CompletableFuture.supplyAsync(() -> { try { - return path.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY, ENTRY_DELETE); + WatchEvent.Kind[] kinds = new WatchEvent.Kind[]{ ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY, ENTRY_DELETE }; + if (path.isRecursive()) { + return path.getPath().register(service, kinds, ExtendedWatchEventModifier.FILE_TREE); + } + else { + return path.getPath().register(service, kinds); + } } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java b/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java new file mode 100644 index 00000000..7a6401b6 --- /dev/null +++ b/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java @@ -0,0 +1,37 @@ +package engineering.swat.watch.impl; + +import java.nio.file.Path; +import java.util.Objects; + +public class SubscriptionKey { + private final Path path; + private final boolean recursive; + + public SubscriptionKey(Path path, boolean recursive) { + this.path = path; + this.recursive = recursive; + } + + public Path getPath() { + return path; + } + + public boolean isRecursive() { + return recursive; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof SubscriptionKey) { + var other = (SubscriptionKey)obj; + return (other.recursive == recursive) + && other.path.equals(path); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(path, recursive); + } +} diff --git a/src/test/java/engineering/swat/watch/DeleteLockTests.java b/src/test/java/engineering/swat/watch/DeleteLockTests.java index c4c99bc2..0d22cba3 100644 --- a/src/test/java/engineering/swat/watch/DeleteLockTests.java +++ b/src/test/java/engineering/swat/watch/DeleteLockTests.java @@ -42,11 +42,6 @@ private interface Deleter { void run(Path target) throws IOException; } - @FunctionalInterface - private interface Builder { - Watcher build(Path target) throws IOException; - } - private static void recursiveDelete(Path target) throws IOException { try (var paths = Files.walk(target)) { paths.sorted(Comparator.reverseOrder()) @@ -55,8 +50,8 @@ private static void recursiveDelete(Path target) throws IOException { } } - private void deleteAndVerify(Path target, Builder setup) throws IOException { - try (var watch = setup.build(target).onEvent(ev -> {}).start()) { + private void deleteAndVerify(Path target, WatchScope scope) throws IOException { + try (var watch = Watcher.watch(target, scope).onEvent(ev -> {}).start()) { recursiveDelete(target); assertFalse(Files.exists(target), "The file/directory shouldn't exist anymore"); } @@ -66,7 +61,7 @@ private void deleteAndVerify(Path target, Builder setup) throws IOException { void watchedFileCanBeDeleted() throws IOException { deleteAndVerify( testDir.getTestFiles().get(0), - Watcher::single + WatchScope.SINGLE ); } @@ -75,7 +70,7 @@ void watchedFileCanBeDeleted() throws IOException { void watchedDirectoryCanBeDeleted() throws IOException { deleteAndVerify( testDir.getTestDirectory(), - Watcher::singleDirectory + WatchScope.INCLUDING_CHILDREN ); } @@ -84,7 +79,7 @@ void watchedDirectoryCanBeDeleted() throws IOException { void watchedRecursiveDirectoryCanBeDeleted() throws IOException { deleteAndVerify( testDir.getTestDirectory(), - Watcher::recursiveDirectory + WatchScope.INCLUDING_ALL_DESCENDANTS ); } } diff --git a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java index f5f9439a..3a9fbbc3 100644 --- a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java +++ b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java @@ -47,7 +47,7 @@ void newDirectoryWithFilesChangesDetected() throws IOException { var target = new AtomicReference(); var created = new AtomicBoolean(false); var changed = new AtomicBoolean(false); - var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) + var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.INCLUDING_ALL_DESCENDANTS) .onEvent(ev -> { logger.debug("Event received: {}", ev); if (ev.calculateFullPath().equals(target.get())) { @@ -79,7 +79,7 @@ void newDirectoryWithFilesChangesDetected() throws IOException { void correctRelativePathIsReported() throws IOException { Path relative = Path.of("a","b", "c", "d.txt"); var seen = new AtomicBoolean(false); - var watcher = Watcher.recursiveDirectory(testDir.getTestDirectory()) + var watcher = Watcher.watch(testDir.getTestDirectory(), WatchScope.INCLUDING_ALL_DESCENDANTS) .onEvent(ev -> { logger.debug("Seen event: {}", ev); if (ev.getRelativePath().equals(relative)) { @@ -104,7 +104,7 @@ void deleteOfFileInDirectoryShouldBeVisible() throws IOException, InterruptedExc .findAny() .orElseThrow(); var seen = new AtomicBoolean(false); - var watchConfig = Watcher.singleDirectory(target.getParent()) + var watchConfig = Watcher.watch(target.getParent(), WatchScope.INCLUDING_CHILDREN) .onEvent(ev -> { if (ev.getKind() == Kind.DELETED && ev.calculateFullPath().equals(target)) { seen.set(true); diff --git a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java index 594db8af..9b92b74f 100644 --- a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java +++ b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java @@ -40,7 +40,7 @@ static void setupEverything() { void deleteOfFileInDirectoryShouldBeVisible() throws IOException, InterruptedException { var target = testDir.getTestFiles().get(0); var seen = new AtomicBoolean(false); - var watchConfig = Watcher.singleDirectory(target.getParent()) + var watchConfig = Watcher.watch(target.getParent(), WatchScope.INCLUDING_CHILDREN) .onEvent(ev -> { if (ev.getKind() == Kind.DELETED && ev.calculateFullPath().equals(target)) { seen.set(true); diff --git a/src/test/java/engineering/swat/watch/SingleFileTests.java b/src/test/java/engineering/swat/watch/SingleFileTests.java index f6d094a0..04aabe25 100644 --- a/src/test/java/engineering/swat/watch/SingleFileTests.java +++ b/src/test/java/engineering/swat/watch/SingleFileTests.java @@ -39,7 +39,7 @@ void singleFileShouldNotTriggerOnOtherFilesInSameDir() throws IOException, Inter var target = testDir.getTestFiles().get(0); var seen = new AtomicBoolean(false); var others = new AtomicBoolean(false); - var watchConfig = Watcher.single(target) + var watchConfig = Watcher.watch(target, WatchScope.SINGLE) .onEvent(ev -> { if (ev.calculateFullPath().equals(target)) { seen.set(true); @@ -68,7 +68,7 @@ void singleFileThatMonitorsOnlyADirectory() throws IOException, InterruptedExcep var target = testDir.getTestDirectory(); var seen = new AtomicBoolean(false); var others = new AtomicBoolean(false); - var watchConfig = Watcher.single(target) + var watchConfig = Watcher.watch(target, WatchScope.SINGLE) .onEvent(ev -> { if (ev.calculateFullPath().equals(target)) { seen.set(true); diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index 5ed974e8..c83a58a0 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -40,7 +40,7 @@ static void setupEverything() { void watchDirectory() throws IOException, InterruptedException { var changed = new AtomicBoolean(false); var target = testDir.getTestFiles().get(0); - var watchConfig = Watcher.singleDirectory(testDir.getTestDirectory()) + var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.INCLUDING_CHILDREN) .onEvent(ev -> {if (ev.getKind() == MODIFIED && ev.calculateFullPath().equals(target)) { changed.set(true); }}) ; @@ -57,7 +57,7 @@ void watchRecursiveDirectory() throws IOException, InterruptedException { .filter(p -> !p.getParent().equals(testDir.getTestDirectory())) .findFirst() .orElseThrow(); - var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) + var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.INCLUDING_ALL_DESCENDANTS) .onEvent(ev -> { if (ev.getKind() == MODIFIED && ev.calculateFullPath().equals(target)) { changed.set(true);}}) ; @@ -75,7 +75,7 @@ void watchSingleFile() throws IOException { .findFirst() .orElseThrow(); - var watchConfig = Watcher.single(target) + var watchConfig = Watcher.watch(target, WatchScope.SINGLE) .onEvent(ev -> { if (ev.calculateFullPath().equals(target)) { changed.set(true); diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 63682f51..3abfb4e6 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -119,7 +119,7 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO var seenCreates = ConcurrentHashMap.newKeySet(); - var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) + var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.INCLUDING_ALL_DESCENDANTS) .withExecutor(pool) .onEvent(ev -> { var fullPath = ev.calculateFullPath(); @@ -188,7 +188,7 @@ void pressureOnFSShouldNotMissDeletes() throws InterruptedException, IOException final var events = new AtomicInteger(0); final var happened = new Semaphore(0); - var watchConfig = Watcher.recursiveDirectory(testDir.getTestDirectory()) + var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.INCLUDING_ALL_DESCENDANTS) .withExecutor(pool) .onEvent(ev -> { events.getAndIncrement(); From 0d5a969bedadccd55a3b77c843c22fc00da3d039 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 24 Sep 2024 20:11:37 +0200 Subject: [PATCH 65/89] Nullable fix --- .../java/engineering/swat/watch/impl/SubscriptionKey.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java b/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java index 7a6401b6..8120791e 100644 --- a/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java +++ b/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java @@ -3,6 +3,8 @@ import java.nio.file.Path; import java.util.Objects; +import org.checkerframework.checker.nullness.qual.Nullable; + public class SubscriptionKey { private final Path path; private final boolean recursive; @@ -21,7 +23,7 @@ public boolean isRecursive() { } @Override - public boolean equals(Object obj) { + public boolean equals(@Nullable Object obj) { if (obj instanceof SubscriptionKey) { var other = (SubscriptionKey)obj; return (other.recursive == recursive) From 6eca5ffe696e6984598c6e2b50a2c9b7659e6fe7 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 24 Sep 2024 20:12:11 +0200 Subject: [PATCH 66/89] Increase the pressure a bit for the torture test --- src/test/java/engineering/swat/watch/TortureTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 3abfb4e6..ce7af909 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -113,7 +113,7 @@ Set stop() throws InterruptedException { @Test void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IOException { final var root = testDir.getTestDirectory(); - var pool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2); + var pool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 4); var io = new IOGenerator(THREADS, root, pool); From 3a106ff8efa881eb46006dcde415a6f1effa80b3 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 24 Sep 2024 20:17:48 +0200 Subject: [PATCH 67/89] Do not print the exception everytime --- src/main/java/engineering/swat/watch/Watcher.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 36158a2a..a891c695 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -113,7 +113,8 @@ public Closeable start() throws IOException, IllegalStateException { return result; } catch (Throwable ex) { // no native support, use the simulation - logger.info("Not possible to register the native watcher for {}", path, ex); + logger.debug("Not possible to register the native watcher, using fallback for {}", path); + logger.trace(ex); var result = new JDKRecursiveDirectoryWatcher(path, executor, this.eventHandler); result.start(); return result; From e4f87a95ed488d4c24dc62ecaa9148e14f538c55 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Wed, 25 Sep 2024 16:25:06 +0200 Subject: [PATCH 68/89] Applied all the comments from @sungshik --- .../engineering/swat/watch/WatchEvent.java | 2 +- .../java/engineering/swat/watch/Watcher.java | 8 +- .../swat/watch/impl/BundledSubscription.java | 73 ++++++++++--------- .../swat/watch/impl/ISubscribable.java | 4 +- .../swat/watch/impl/JDKDirectoryWatcher.java | 4 +- .../swat/watch/impl/JDKFileWatcher.java | 28 ++++--- .../swat/watch/impl/JDKPoller.java | 8 +- .../impl/JDKRecursiveDirectoryWatcher.java | 12 +-- 8 files changed, 75 insertions(+), 64 deletions(-) diff --git a/src/main/java/engineering/swat/watch/WatchEvent.java b/src/main/java/engineering/swat/watch/WatchEvent.java index 5c8cc54a..27c34534 100644 --- a/src/main/java/engineering/swat/watch/WatchEvent.java +++ b/src/main/java/engineering/swat/watch/WatchEvent.java @@ -32,7 +32,7 @@ public enum Kind { DELETED, /** * Rare event where there were so many file events, that the kernel lost a few. - * In that case you'll have to consider the whole directory (and it's sub directories) as modified. + * In that case you'll have to consider the whole directory (and its sub directories) as modified. * The library will try and send events for new and deleted files, but it won't be able to detect modified files. */ OVERFLOW diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index a891c695..f3831d13 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -26,10 +26,10 @@ public class Watcher { private final Logger logger = LogManager.getLogger(); private final WatchScope scope; private final Path path; - private Executor executor = CompletableFuture::runAsync; + private volatile Executor executor = CompletableFuture::runAsync; private static final Consumer NULL_HANDLER = p -> {}; - private Consumer eventHandler = NULL_HANDLER; + private volatile Consumer eventHandler = NULL_HANDLER; private Watcher(WatchScope scope, Path path) { @@ -94,9 +94,9 @@ public Watcher withExecutor(Executor callbackHandler) { * Start watch the path for events. * @return a subscription for the watch, when closed, new events will stop being registered to the worker pool. * @throws IOException in case the starting of the watcher caused an underlying IO exception - * @throws IllegalStateException the watchers is not configured correctly (for example, missing {@link #onEvent(Consumer)}) + * @throws IllegalStateException the watchers is not configured correctly (for example, missing {@link #onEvent(Consumer)}, or a watcher is started twice) */ - public Closeable start() throws IOException, IllegalStateException { + public Closeable start() throws IOException { if (this.eventHandler == NULL_HANDLER) { throw new IllegalStateException("There is no onEvent handler defined"); } diff --git a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java index 2680ccc9..771f4035 100644 --- a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java +++ b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java @@ -11,33 +11,35 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.NonNull; -public class BundledSubscription implements ISubscribable { - private final ISubscribable around; - private final ConcurrentMap> subscriptions = new ConcurrentHashMap<>(); +/** + * This is an internal class where we can join multiple subscriptions to the same target by only taking 1 actual subscription but forwarding them to all the interested parties. + * This is used (for example) to avoid multiple JDKPoller registries for the same path + */ +public class BundledSubscription implements ISubscribable { + private final ISubscribable wrapped; + private final ConcurrentMap> subscriptions = new ConcurrentHashMap<>(); - public BundledSubscription(ISubscribable around) { - this.around = around; + public BundledSubscription(ISubscribable wrapped) { + this.wrapped = wrapped; } private static class Subscription implements Consumer { private final List> consumers = new CopyOnWriteArrayList<>(); - private volatile @MonotonicNonNull Closeable closer; - Subscription(Consumer initialConsumer) { - consumers.add(initialConsumer); + private volatile @MonotonicNonNull Closeable toBeClosed; + Subscription() { } - public void setCloser(Closeable closer) { - this.closer = closer; + public void setToBeClosed(Closeable closer) { + this.toBeClosed = closer; } - void add(Consumer newConsumer) { + public void add(Consumer newConsumer) { consumers.add(newConsumer); } - synchronized boolean remove(Consumer existingConsumer) { + public void remove(Consumer existingConsumer) { consumers.remove(existingConsumer); - return consumers.isEmpty(); } @Override @@ -55,38 +57,41 @@ boolean hasActiveConsumers() { } @Override - public Closeable subscribe(A target, Consumer eventListener) throws IOException { - var active = this.subscriptions.get(target); - if (active == null) { - active = new Subscription<>(eventListener); - var newSubscriptions = around.subscribe(target, active); - active.setCloser(newSubscriptions); - var lostRace = this.subscriptions.putIfAbsent(target, active); - if (lostRace != null) { - try { - newSubscriptions.close(); - } catch (IOException _ignore) { - // ignore + public Closeable subscribe(Key target, Consumer eventListener) throws IOException { + var active = this.subscriptions.computeIfAbsent(target, t -> new Subscription<>()); + boolean first = false; + if (active.toBeClosed == null) { + // we just added a new one + // so lets take a lock on it, and try to be the one that gets to initialize it + synchronized(active) { + // now lock on it to make sure nobo + if (active.toBeClosed == null) { + first = true; + active.add(eventListener); // we know we already have the lock, and we need to do this before we register the watch + var newSubscriptions = wrapped.subscribe(target, active); + active.setToBeClosed(newSubscriptions); + } + else { } - lostRace.add(eventListener); - active = lostRace; } } - else { + // at this point we have to be sure that we're not the first to in the list + // since we might have won the race on the compute, but lost the race + if (!first) { active.add(eventListener); } - var finalActive = active; return () -> { - if (finalActive.remove(eventListener)) { + active.remove(eventListener); + if (!active.hasActiveConsumers()) { subscriptions.remove(target); - if (finalActive.hasActiveConsumers()) { + if (active.hasActiveConsumers()) { // we lost the race, someone else added something again // so we put it back in the list - subscriptions.put(target, finalActive); + subscriptions.put(target, active); } else { - if (finalActive.closer != null) { - finalActive.closer.close(); + if (active.toBeClosed != null) { + active.toBeClosed.close(); } } } diff --git a/src/main/java/engineering/swat/watch/impl/ISubscribable.java b/src/main/java/engineering/swat/watch/impl/ISubscribable.java index 01646ae5..4d5fe5b8 100644 --- a/src/main/java/engineering/swat/watch/impl/ISubscribable.java +++ b/src/main/java/engineering/swat/watch/impl/ISubscribable.java @@ -5,6 +5,6 @@ import java.util.function.Consumer; @FunctionalInterface -public interface ISubscribable { - Closeable subscribe(A target, Consumer eventListener) throws IOException; +public interface ISubscribable { + Closeable subscribe(Key target, Consumer eventListener) throws IOException; } diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java index a0b87c27..1357908d 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java @@ -49,7 +49,7 @@ synchronized boolean safeStart() throws IOException { public void start() throws IOException { try { if (!safeStart()) { - throw new IOException("Cannot start a watcher twice"); + throw new IllegalStateException("Cannot start a watcher twice"); } logger.debug("Started watch for: {}", directory); } catch (IOException e) { @@ -93,7 +93,7 @@ else if (ev.kind() == StandardWatchEventKinds.OVERFLOW) { } @Override - public void close() throws IOException { + public synchronized void close() throws IOException { if (activeWatch != null) { logger.debug("Closing watch for: {}", this.directory); activeWatch.close(); diff --git a/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java index 6306ae95..b1c66692 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java @@ -14,6 +14,8 @@ /** * It's not possible to monitor a single file (or directory), so we have to find a directory watcher, and connect to that + * + * Note that you should take care to call start only once. */ public class JDKFileWatcher implements Closeable { private final Logger logger = LogManager.getLogger(); @@ -34,23 +36,27 @@ public JDKFileWatcher(Path file, Executor exec, Consumer eventHandle this.eventHandler = eventHandler; } + /** + * Start the file watcher, but only do it once + * @throws IOException + */ public void start() throws IOException { try { + var dir = file.getParent(); + if (dir == null) { + throw new IllegalArgumentException("cannot watch a single entry that is on the root"); + + } + assert !dir.equals(file); + JDKDirectoryWatcher parentWatch; synchronized(this) { if (activeWatch != null) { throw new IOException("Cannot start an already started watch"); } - var dir = file.getParent(); - if (dir == null) { - throw new IllegalArgumentException("cannot watch a single entry that is on the root"); - - } - assert !dir.equals(file); - var parentWatch = new JDKDirectoryWatcher(dir, exec, this::filter); - activeWatch = parentWatch; + activeWatch = parentWatch = new JDKDirectoryWatcher(dir, exec, this::filter); parentWatch.start(); - logger.debug("Started file watch for {} (in reality a watch on {}): {}", file, dir, parentWatch); } + logger.debug("Started file watch for {} (in reality a watch on {}): {}", file, dir, parentWatch); } catch (IOException e) { throw new IOException("Could not register file watcher for: " + file, e); @@ -59,12 +65,12 @@ public void start() throws IOException { private void filter(WatchEvent event) { if (fileName.equals(event.getRelativePath())) { - exec.execute(() -> eventHandler.accept(event)); + eventHandler.accept(event); } } @Override - public void close() throws IOException { + public synchronized void close() throws IOException { if (activeWatch != null) { activeWatch.close(); } diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index aacfba5a..b799024e 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -84,7 +84,7 @@ private static void poll() { } - public static Closeable register(SubscriptionKey path, Consumer>> changes) throws IOException { + public static Closeable register(SubscriptionKey path, Consumer>> changesHandler) throws IOException { logger.debug("Register watch for: {}", path); try { @@ -102,7 +102,7 @@ public static Closeable register(SubscriptionKey path, Consumer { - watchers.put(key, changes); + watchers.put(key, changesHandler); return new Closeable() { @Override public void close() throws IOException { @@ -114,8 +114,8 @@ public void close() throws IOException { }) .get(); // we have to do a get here, to make sure the `register` function blocks } catch (ExecutionException e) { - if (e.getCause() instanceof IOException) { - throw (IOException)e.getCause(); + if (e.getCause() instanceof RuntimeException && e.getCause().getCause() instanceof IOException) { + throw (IOException)e.getCause().getCause(); } throw new IOException("Could not register path", e.getCause()); } catch (InterruptedException e) { diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java index aea8ae5e..a772daf5 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java @@ -9,7 +9,6 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Collections; import java.util.Deque; import java.util.HashSet; import java.util.List; @@ -20,8 +19,6 @@ import java.util.concurrent.Executor; import java.util.function.Consumer; -import javax.management.RuntimeErrorException; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -75,7 +72,8 @@ private void handleCreate(WatchEvent ev) { // we might have some nested directories we missed // so if we have a new directory, we have to go in and iterate over it // we also have to report all nested files & dirs as created paths - // but we don't want to burden ourselves with those events + // but we don't want to delay the publication of this + // create till after the processing is done, so we schedule it in the background var fullPath = ev.calculateFullPath(); if (!activeWatches.containsKey(fullPath)) { CompletableFuture @@ -114,7 +112,7 @@ private void handleDeleteDirectory(WatchEvent ev) { } } - /** Only register a watched for every sub directory */ + /** Only register a watch for every sub directory */ private class InitialDirectoryScan extends SimpleFileVisitor { protected final Path subRoot; @@ -145,7 +143,7 @@ private void addNewDirectory(Path dir) throws IOException { var watcher = activeWatches.computeIfAbsent(dir, d -> new JDKDirectoryWatcher(d, exec, relocater(dir))); try { if (!watcher.safeStart()) { - logger.debug("We lost the race on starting a nested watcher, that shouldn't be a problem, but its a very busy, so we might have lost a few events in {}", dir); + logger.debug("We lost the race on starting a nested watcher, that shouldn't be a problem, but it's a very busy, so we might have lost a few events in {}", dir); } } catch (IOException ex) { activeWatches.remove(dir); @@ -212,6 +210,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO if (attrs.size() > 0) { events.add(new WatchEvent(WatchEvent.Kind.MODIFIED, root, relative)); } + seenFiles.add(file); } return FileVisitResult.CONTINUE; } @@ -282,6 +281,7 @@ private void detectedMissingEntries(Path dir, ArrayList events, Hash // and when the watches are active. so after we know all the new watches have been registered // we do a second scan and make sure to find paths that weren't visible the first time // and emulate events for them (and register new watches) + // In essence this is the same as when an Overflow happened, so we can reuse that handler. int directoryCount = seenDirectories.size() - 1; while (directoryCount != seenDirectories.size()) { Files.walkFileTree(dir, new OverflowSyncScan(dir, events, seenFiles, seenDirectories)); From 538b0453875cadfe81d328687ee35b0c56bcd731 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Wed, 25 Sep 2024 16:33:41 +0200 Subject: [PATCH 69/89] Added a bit more comment --- .../swat/watch/impl/JDKPoller.java | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index b799024e..67fb274e 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -26,6 +26,9 @@ import com.sun.nio.file.ExtendedWatchEventModifier; +/** + * This class is a wrapper around the JDK WatchService, it takes care to poll the service for new events, and then distributes them to the right parties + */ class JDKPoller { private JDKPoller() {} @@ -70,7 +73,6 @@ private static void poll() { hit.reset(); } } - } finally { // schedule next run @@ -89,30 +91,30 @@ public static Closeable register(SubscriptionKey path, Consumer { - try { - WatchEvent.Kind[] kinds = new WatchEvent.Kind[]{ ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY, ENTRY_DELETE }; - if (path.isRecursive()) { - return path.getPath().register(service, kinds, ExtendedWatchEventModifier.FILE_TREE); - } - else { - return path.getPath().register(service, kinds); - } - } catch (IOException e) { - throw new RuntimeException(e); + try { + WatchEvent.Kind[] kinds = new WatchEvent.Kind[]{ ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY, ENTRY_DELETE }; + if (path.isRecursive()) { + return path.getPath().register(service, kinds, ExtendedWatchEventModifier.FILE_TREE); + } + else { + return path.getPath().register(service, kinds); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }, registerPool) // read registerPool why we have to add a limiter here + .thenApplyAsync(key -> { + watchers.put(key, changesHandler); + return new Closeable() { + @Override + public void close() throws IOException { + logger.debug("Closing watch for: {}", path); + key.cancel(); + watchers.remove(key); } - }, registerPool) // read registerPool why we have to add a limiter here - .thenApplyAsync(key -> { - watchers.put(key, changesHandler); - return new Closeable() { - @Override - public void close() throws IOException { - logger.debug("Closing watch for: {}", path); - key.cancel(); - watchers.remove(key); - } - }; - }) - .get(); // we have to do a get here, to make sure the `register` function blocks + }; + }) + .get(); // we have to do a get here, to make sure the `register` function blocks } catch (ExecutionException e) { if (e.getCause() instanceof RuntimeException && e.getCause().getCause() instanceof IOException) { throw (IOException)e.getCause().getCause(); From 39a8b7442a4d58935fb9ca6f334fe101d72144bf Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 1 Oct 2024 20:39:19 +0200 Subject: [PATCH 70/89] Added registration test --- .../engineering/swat/watch/TortureTests.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index ce7af909..c2c49b83 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -3,6 +3,7 @@ import static org.awaitility.Awaitility.doNotCatchUncaughtExceptionsByDefault; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.nio.file.Files; @@ -15,6 +16,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -169,6 +171,50 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO } } + @Test + void manyRegistrationsForSamePath() throws InterruptedException, IOException { + var startRegistering = new Semaphore(0); + var startDeregistring = new Semaphore(0); + var done = new Semaphore(0); + var seen = ConcurrentHashMap.newKeySet(); + var exceptions = new LinkedBlockingDeque(); + for (int t = 0; t < THREADS * 100; t++) { + var r = new Thread(() -> { + try { + var watcher = Watcher + .watch(testDir.getTestDirectory(), WatchScope.INCLUDING_CHILDREN) + .onEvent(e -> { if (e.getKind() == Kind.CREATED) seen.add(e.calculateFullPath()); }); + startRegistering.acquire(); + try (var c = watcher.start()) { + startDeregistring.acquire(); + } + catch(Exception e) { + exceptions.push(e); + } + } catch (InterruptedException e1) { + } + finally { + done.release(); + } + }); + r.setDaemon(true); + r.start(); + } + + startRegistering.release(THREADS * 100); + startDeregistring.release((THREADS * 100) - 1); + done.acquire((THREADS * 100) - 1); + assertTrue(seen.isEmpty(), "No events should have been sent"); + Files.writeString(testDir.getTestDirectory().resolve("test124.txt"), "Hello World"); + await("We should see only one event") + .during(Duration.ofMillis(500)) + .until(() -> seen.size() == 1); + if (!exceptions.isEmpty()) { + fail(exceptions.pop()); + } + startDeregistring.release(); + } + @Test From e1c7186ae9553f59fbf5aac8ceba420151dde0f0 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 1 Oct 2024 20:56:40 +0200 Subject: [PATCH 71/89] Extra test around registration --- .../engineering/swat/watch/TortureTests.java | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index c2c49b83..5f1402af 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -14,12 +14,14 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -215,6 +217,66 @@ void manyRegistrationsForSamePath() throws InterruptedException, IOException { startDeregistring.release(); } + @Test + void manyRegisterAndUnregisterSameTime() throws InterruptedException, IOException { + var startRegistering = new Semaphore(0); + var stopAll = new Semaphore(0); + var done = new Semaphore(0); + var seen = new ConcurrentLinkedDeque(); + var exceptions = new LinkedBlockingDeque(); + int amountOfWatchersActive = 0; + try { + for (int t = 0; t < THREADS; t++) { + final boolean finishWatching = t % 2 == 0; + if (finishWatching) { + amountOfWatchersActive++; + } + var r = new Thread(() -> { + try { + startRegistering.acquire(); + for (int k = 0; k < 1000; k++) { + var watcher = Watcher + .watch(testDir.getTestDirectory(), WatchScope.INCLUDING_CHILDREN) + .onEvent(e -> { if (e.getKind() == Kind.CREATED) seen.add(e.calculateFullPath()); }); + try (var c = watcher.start()) { + if (finishWatching && k + 1 == 1000) { + logger.info("Waiting on stop signal"); + stopAll.acquire(); + } + } + catch(Exception e) { + exceptions.push(e); + } + } + } catch (InterruptedException e1) { + } + finally { + done.release(); + } + }); + r.setDaemon(true); + r.start(); + } + + startRegistering.release(THREADS); + done.acquire(THREADS - amountOfWatchersActive); + assertTrue(seen.isEmpty(), "No events should have been sent"); + Files.writeString(testDir.getTestDirectory().resolve("test124.txt"), "Hello World"); + await("We should see only exactly the events we expect") + .failFast(() -> !exceptions.isEmpty()) + .pollDelay(TestHelper.NORMAL_WAIT.minusMillis(100)) + .until(seen::size, Predicate.isEqual(amountOfWatchersActive)) + ; + if (!exceptions.isEmpty()) { + fail(exceptions.pop()); + } + } + finally { + stopAll.release(amountOfWatchersActive); + } + + } + @Test From 096a6f9fab64d1c85bc8d528de9453054fbefd43 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 14 Oct 2024 11:01:10 +0200 Subject: [PATCH 72/89] Renamed enums and processes review comments --- .../engineering/swat/watch/WatchScope.java | 21 ++++++++++++++++--- .../java/engineering/swat/watch/Watcher.java | 14 ++++++------- .../swat/watch/impl/JDKPoller.java | 3 ++- .../swat/watch/DeleteLockTests.java | 6 +++--- .../swat/watch/RecursiveWatchTests.java | 6 +++--- .../swat/watch/SingleDirectoryTests.java | 2 +- .../swat/watch/SingleFileTests.java | 4 ++-- .../engineering/swat/watch/SmokeTests.java | 6 +++--- .../engineering/swat/watch/TortureTests.java | 8 +++---- 9 files changed, 43 insertions(+), 27 deletions(-) diff --git a/src/main/java/engineering/swat/watch/WatchScope.java b/src/main/java/engineering/swat/watch/WatchScope.java index 8b38c8d9..4ef8b0bf 100644 --- a/src/main/java/engineering/swat/watch/WatchScope.java +++ b/src/main/java/engineering/swat/watch/WatchScope.java @@ -1,7 +1,22 @@ package engineering.swat.watch; +/** + * Configure the depth of the events you want to receive for a given path + */ public enum WatchScope { - SINGLE, - INCLUDING_CHILDREN, - INCLUDING_ALL_DESCENDANTS + /** + * Watch changes to a single file or (metadata of) a directory + */ + PATH_ONLY, + /** + * Watch changes to (metadata of) a directory and its content, + * non-recursively. That is, changes to the content of nested directories + * are not watched. + */ + PATH_AND_CHILDREN, + /** + * Watch changes to (metadata of) a directory and its content, recursively. + * That is, changes to the content of nested directories are also watched. + */ + PATH_AND_ALL_DESCENDANTS } diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index f3831d13..0a511bde 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -49,13 +49,13 @@ public static Watcher watch(Path path, WatchScope scope) { throw new IllegalArgumentException("We can only watch absolute paths"); } switch (scope) { - case INCLUDING_CHILDREN: // intended fallthrough - case INCLUDING_ALL_DESCENDANTS: + case PATH_AND_CHILDREN: // intended fallthrough + case PATH_AND_ALL_DESCENDANTS: if (!Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { - throw new IllegalArgumentException("Only directories are supported for this scope"); + throw new IllegalArgumentException("Only directories are supported for this scope: " + scope); } break; - case SINGLE: + case PATH_ONLY: if (Files.isSymbolicLink(path)) { throw new IllegalArgumentException("Symlinks are not supported"); } @@ -101,12 +101,12 @@ public Closeable start() throws IOException { throw new IllegalStateException("There is no onEvent handler defined"); } switch (scope) { - case INCLUDING_CHILDREN: { + case PATH_AND_CHILDREN: { var result = new JDKDirectoryWatcher(path, executor, this.eventHandler, false); result.start(); return result; } - case INCLUDING_ALL_DESCENDANTS: { + case PATH_AND_ALL_DESCENDANTS: { try { var result = new JDKDirectoryWatcher(path, executor, this.eventHandler, true); result.start(); @@ -120,7 +120,7 @@ public Closeable start() throws IOException { return result; } } - case SINGLE: { + case PATH_ONLY: { var result = new JDKFileWatcher(path, executor, this.eventHandler); result.start(); return result; diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index 67fb274e..247271bc 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -3,6 +3,7 @@ import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; +import static java.nio.file.StandardWatchEventKinds.OVERFLOW; import java.io.Closeable; import java.io.IOException; @@ -92,7 +93,7 @@ public static Closeable register(SubscriptionKey path, Consumer { try { - WatchEvent.Kind[] kinds = new WatchEvent.Kind[]{ ENTRY_CREATE, ENTRY_MODIFY, ENTRY_MODIFY, ENTRY_DELETE }; + WatchEvent.Kind[] kinds = new WatchEvent.Kind[]{ ENTRY_CREATE, ENTRY_MODIFY, OVERFLOW, ENTRY_DELETE }; if (path.isRecursive()) { return path.getPath().register(service, kinds, ExtendedWatchEventModifier.FILE_TREE); } diff --git a/src/test/java/engineering/swat/watch/DeleteLockTests.java b/src/test/java/engineering/swat/watch/DeleteLockTests.java index 0d22cba3..44861bf0 100644 --- a/src/test/java/engineering/swat/watch/DeleteLockTests.java +++ b/src/test/java/engineering/swat/watch/DeleteLockTests.java @@ -61,7 +61,7 @@ private void deleteAndVerify(Path target, WatchScope scope) throws IOException { void watchedFileCanBeDeleted() throws IOException { deleteAndVerify( testDir.getTestFiles().get(0), - WatchScope.SINGLE + WatchScope.PATH_ONLY ); } @@ -70,7 +70,7 @@ void watchedFileCanBeDeleted() throws IOException { void watchedDirectoryCanBeDeleted() throws IOException { deleteAndVerify( testDir.getTestDirectory(), - WatchScope.INCLUDING_CHILDREN + WatchScope.PATH_AND_CHILDREN ); } @@ -79,7 +79,7 @@ void watchedDirectoryCanBeDeleted() throws IOException { void watchedRecursiveDirectoryCanBeDeleted() throws IOException { deleteAndVerify( testDir.getTestDirectory(), - WatchScope.INCLUDING_ALL_DESCENDANTS + WatchScope.PATH_AND_ALL_DESCENDANTS ); } } diff --git a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java index 3a9fbbc3..7c744fbd 100644 --- a/src/test/java/engineering/swat/watch/RecursiveWatchTests.java +++ b/src/test/java/engineering/swat/watch/RecursiveWatchTests.java @@ -47,7 +47,7 @@ void newDirectoryWithFilesChangesDetected() throws IOException { var target = new AtomicReference(); var created = new AtomicBoolean(false); var changed = new AtomicBoolean(false); - var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.INCLUDING_ALL_DESCENDANTS) + var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.PATH_AND_ALL_DESCENDANTS) .onEvent(ev -> { logger.debug("Event received: {}", ev); if (ev.calculateFullPath().equals(target.get())) { @@ -79,7 +79,7 @@ void newDirectoryWithFilesChangesDetected() throws IOException { void correctRelativePathIsReported() throws IOException { Path relative = Path.of("a","b", "c", "d.txt"); var seen = new AtomicBoolean(false); - var watcher = Watcher.watch(testDir.getTestDirectory(), WatchScope.INCLUDING_ALL_DESCENDANTS) + var watcher = Watcher.watch(testDir.getTestDirectory(), WatchScope.PATH_AND_ALL_DESCENDANTS) .onEvent(ev -> { logger.debug("Seen event: {}", ev); if (ev.getRelativePath().equals(relative)) { @@ -104,7 +104,7 @@ void deleteOfFileInDirectoryShouldBeVisible() throws IOException, InterruptedExc .findAny() .orElseThrow(); var seen = new AtomicBoolean(false); - var watchConfig = Watcher.watch(target.getParent(), WatchScope.INCLUDING_CHILDREN) + var watchConfig = Watcher.watch(target.getParent(), WatchScope.PATH_AND_CHILDREN) .onEvent(ev -> { if (ev.getKind() == Kind.DELETED && ev.calculateFullPath().equals(target)) { seen.set(true); diff --git a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java index 9b92b74f..fdc3411e 100644 --- a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java +++ b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java @@ -40,7 +40,7 @@ static void setupEverything() { void deleteOfFileInDirectoryShouldBeVisible() throws IOException, InterruptedException { var target = testDir.getTestFiles().get(0); var seen = new AtomicBoolean(false); - var watchConfig = Watcher.watch(target.getParent(), WatchScope.INCLUDING_CHILDREN) + var watchConfig = Watcher.watch(target.getParent(), WatchScope.PATH_AND_CHILDREN) .onEvent(ev -> { if (ev.getKind() == Kind.DELETED && ev.calculateFullPath().equals(target)) { seen.set(true); diff --git a/src/test/java/engineering/swat/watch/SingleFileTests.java b/src/test/java/engineering/swat/watch/SingleFileTests.java index 04aabe25..328c7c32 100644 --- a/src/test/java/engineering/swat/watch/SingleFileTests.java +++ b/src/test/java/engineering/swat/watch/SingleFileTests.java @@ -39,7 +39,7 @@ void singleFileShouldNotTriggerOnOtherFilesInSameDir() throws IOException, Inter var target = testDir.getTestFiles().get(0); var seen = new AtomicBoolean(false); var others = new AtomicBoolean(false); - var watchConfig = Watcher.watch(target, WatchScope.SINGLE) + var watchConfig = Watcher.watch(target, WatchScope.PATH_ONLY) .onEvent(ev -> { if (ev.calculateFullPath().equals(target)) { seen.set(true); @@ -68,7 +68,7 @@ void singleFileThatMonitorsOnlyADirectory() throws IOException, InterruptedExcep var target = testDir.getTestDirectory(); var seen = new AtomicBoolean(false); var others = new AtomicBoolean(false); - var watchConfig = Watcher.watch(target, WatchScope.SINGLE) + var watchConfig = Watcher.watch(target, WatchScope.PATH_ONLY) .onEvent(ev -> { if (ev.calculateFullPath().equals(target)) { seen.set(true); diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index c83a58a0..3f38e484 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -40,7 +40,7 @@ static void setupEverything() { void watchDirectory() throws IOException, InterruptedException { var changed = new AtomicBoolean(false); var target = testDir.getTestFiles().get(0); - var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.INCLUDING_CHILDREN) + var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.PATH_AND_CHILDREN) .onEvent(ev -> {if (ev.getKind() == MODIFIED && ev.calculateFullPath().equals(target)) { changed.set(true); }}) ; @@ -57,7 +57,7 @@ void watchRecursiveDirectory() throws IOException, InterruptedException { .filter(p -> !p.getParent().equals(testDir.getTestDirectory())) .findFirst() .orElseThrow(); - var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.INCLUDING_ALL_DESCENDANTS) + var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.PATH_AND_ALL_DESCENDANTS) .onEvent(ev -> { if (ev.getKind() == MODIFIED && ev.calculateFullPath().equals(target)) { changed.set(true);}}) ; @@ -75,7 +75,7 @@ void watchSingleFile() throws IOException { .findFirst() .orElseThrow(); - var watchConfig = Watcher.watch(target, WatchScope.SINGLE) + var watchConfig = Watcher.watch(target, WatchScope.PATH_ONLY) .onEvent(ev -> { if (ev.calculateFullPath().equals(target)) { changed.set(true); diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 5f1402af..9a411988 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -123,7 +123,7 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO var seenCreates = ConcurrentHashMap.newKeySet(); - var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.INCLUDING_ALL_DESCENDANTS) + var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.PATH_AND_ALL_DESCENDANTS) .withExecutor(pool) .onEvent(ev -> { var fullPath = ev.calculateFullPath(); @@ -184,7 +184,7 @@ void manyRegistrationsForSamePath() throws InterruptedException, IOException { var r = new Thread(() -> { try { var watcher = Watcher - .watch(testDir.getTestDirectory(), WatchScope.INCLUDING_CHILDREN) + .watch(testDir.getTestDirectory(), WatchScope.PATH_AND_CHILDREN) .onEvent(e -> { if (e.getKind() == Kind.CREATED) seen.add(e.calculateFullPath()); }); startRegistering.acquire(); try (var c = watcher.start()) { @@ -236,7 +236,7 @@ void manyRegisterAndUnregisterSameTime() throws InterruptedException, IOExceptio startRegistering.acquire(); for (int k = 0; k < 1000; k++) { var watcher = Watcher - .watch(testDir.getTestDirectory(), WatchScope.INCLUDING_CHILDREN) + .watch(testDir.getTestDirectory(), WatchScope.PATH_AND_CHILDREN) .onEvent(e -> { if (e.getKind() == Kind.CREATED) seen.add(e.calculateFullPath()); }); try (var c = watcher.start()) { if (finishWatching && k + 1 == 1000) { @@ -296,7 +296,7 @@ void pressureOnFSShouldNotMissDeletes() throws InterruptedException, IOException final var events = new AtomicInteger(0); final var happened = new Semaphore(0); - var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.INCLUDING_ALL_DESCENDANTS) + var watchConfig = Watcher.watch(testDir.getTestDirectory(), WatchScope.PATH_AND_ALL_DESCENDANTS) .withExecutor(pool) .onEvent(ev -> { events.getAndIncrement(); From de22a03d7e05be8f777b049407a966df1ebf739a Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 14 Oct 2024 11:01:41 +0200 Subject: [PATCH 73/89] Working on the fix for the race Sung found --- .../swat/watch/impl/BundledSubscription.java | 98 +++++++++++-------- 1 file changed, 57 insertions(+), 41 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java index 771f4035..625f4a6a 100644 --- a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java +++ b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java @@ -27,18 +27,16 @@ public BundledSubscription(ISubscribable wrapped) { private static class Subscription implements Consumer { private final List> consumers = new CopyOnWriteArrayList<>(); private volatile @MonotonicNonNull Closeable toBeClosed; - Subscription() { + private volatile boolean closed = false; + Subscription(Consumer initial) { + consumers.add(initial); } - public void setToBeClosed(Closeable closer) { - this.toBeClosed = closer; - } - - public void add(Consumer newConsumer) { + void add(Consumer newConsumer) { consumers.add(newConsumer); } - public void remove(Consumer existingConsumer) { + void remove(Consumer existingConsumer) { consumers.remove(existingConsumer); } @@ -53,49 +51,67 @@ boolean hasActiveConsumers() { return !consumers.isEmpty(); } + boolean firstIs(Consumer eventListener) { + return consumers.stream().findFirst().orElse(null) == eventListener; + } + } @Override public Closeable subscribe(Key target, Consumer eventListener) throws IOException { - var active = this.subscriptions.computeIfAbsent(target, t -> new Subscription<>()); - boolean first = false; - if (active.toBeClosed == null) { - // we just added a new one - // so lets take a lock on it, and try to be the one that gets to initialize it - synchronized(active) { - // now lock on it to make sure nobo - if (active.toBeClosed == null) { - first = true; - active.add(eventListener); // we know we already have the lock, and we need to do this before we register the watch - var newSubscriptions = wrapped.subscribe(target, active); - active.setToBeClosed(newSubscriptions); - } - else { + while (true) { + Subscription active = this.subscriptions.computeIfAbsent(target, t -> new Subscription<>(eventListener)); + // after this, there will only be 1 instance of active subscription in the map. + // but we might have a race with remove, which can close the subscript between our get and our addition + if (active.toBeClosed == null) { + // we might be the first + synchronized(active) { + if (active.toBeClosed == null) { + // we're the first here, so we can't be closed + // let's register ourselves + active.toBeClosed = (wrapped.subscribe(target, active)); + } } } - } - // at this point we have to be sure that we're not the first to in the list - // since we might have won the race on the compute, but lost the race - if (!first) { - active.add(eventListener); - } - return () -> { - active.remove(eventListener); - if (!active.hasActiveConsumers()) { - subscriptions.remove(target); - if (active.hasActiveConsumers()) { - // we lost the race, someone else added something again - // so we put it back in the list - subscriptions.put(target, active); - } - else { - if (active.toBeClosed != null) { - active.toBeClosed.close(); + if (!active.firstIs(eventListener)) { + // we weren't the one that got the compute action + // so we'll add ourselves to the list + // + active.add(active); + } + if (active.closed) { + // we tried, but we lost the race to add something to the list of subscriptions before we got closed + continue; + } + return () -> { + active.remove(eventListener); + if (!active.hasActiveConsumers()) { + // we might be able to close it + // let's try to lock us down. + // just so that there are no 2 threads closing it + // and a bit so that there is no thread also just starting to register it + synchronized (active) { + if (!active.hasActiveConsumers() && !active.closed) { + // okay, it's still legal to close this one + // and no other thread is closing it + // so we're going to remove it from the map + + + // TODO: still a race! since beween line 95 and the one below, the line 83 can have been checked. + // maybe use atomic boolean to get this logic better? + active.closed = true; + subscriptions.remove(target, active); + + if (active.toBeClosed != null) { + active.toBeClosed.close(); + } + } } } - } - }; + }; + } + } From 700654886aafb6205126962081a0276175e0fae7 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 14 Oct 2024 11:44:30 +0200 Subject: [PATCH 74/89] Removed races around registering and unregistering by adding a big old lock --- .../swat/watch/impl/BundledSubscription.java | 69 +++++-------------- .../engineering/swat/watch/TortureTests.java | 5 +- 2 files changed, 22 insertions(+), 52 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java index 625f4a6a..760ac9a6 100644 --- a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java +++ b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java @@ -28,8 +28,7 @@ private static class Subscription implements Consumer { private final List> consumers = new CopyOnWriteArrayList<>(); private volatile @MonotonicNonNull Closeable toBeClosed; private volatile boolean closed = false; - Subscription(Consumer initial) { - consumers.add(initial); + Subscription() { } void add(Consumer newConsumer) { @@ -50,69 +49,39 @@ public void accept(R t) { boolean hasActiveConsumers() { return !consumers.isEmpty(); } - - boolean firstIs(Consumer eventListener) { - return consumers.stream().findFirst().orElse(null) == eventListener; - } - - } @Override public Closeable subscribe(Key target, Consumer eventListener) throws IOException { while (true) { - Subscription active = this.subscriptions.computeIfAbsent(target, t -> new Subscription<>(eventListener)); + Subscription active = this.subscriptions.computeIfAbsent(target, t -> new Subscription<>()); // after this, there will only be 1 instance of active subscription in the map. // but we might have a race with remove, which can close the subscript between our get and our addition - if (active.toBeClosed == null) { - // we might be the first - synchronized(active) { - if (active.toBeClosed == null) { - // we're the first here, so we can't be closed - // let's register ourselves - active.toBeClosed = (wrapped.subscribe(target, active)); - } + synchronized(active) { + if (active.closed) { + // we lost the race with closing the subscription + // so we retry + continue; + } + active.add(eventListener); + if (active.toBeClosed == null) { + // the watch is not active yet, and we were the first to get the lock + active.toBeClosed = wrapped.subscribe(target, active); } - } - if (!active.firstIs(eventListener)) { - // we weren't the one that got the compute action - // so we'll add ourselves to the list - // - active.add(active); - } - if (active.closed) { - // we tried, but we lost the race to add something to the list of subscriptions before we got closed - continue; } return () -> { - active.remove(eventListener); - if (!active.hasActiveConsumers()) { - // we might be able to close it - // let's try to lock us down. - // just so that there are no 2 threads closing it - // and a bit so that there is no thread also just starting to register it - synchronized (active) { - if (!active.hasActiveConsumers() && !active.closed) { - // okay, it's still legal to close this one - // and no other thread is closing it - // so we're going to remove it from the map - - - // TODO: still a race! since beween line 95 and the one below, the line 83 can have been checked. - // maybe use atomic boolean to get this logic better? - active.closed = true; - subscriptions.remove(target, active); - - if (active.toBeClosed != null) { - active.toBeClosed.close(); - } + synchronized(active) { + active.remove(eventListener); + if (!active.hasActiveConsumers() && !active.closed) { + active.closed = true; + this.subscriptions.remove(target, active); + if (active.toBeClosed != null) { + active.toBeClosed.close(); } } } }; } - - } diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 9a411988..ee24546e 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -209,8 +209,9 @@ void manyRegistrationsForSamePath() throws InterruptedException, IOException { assertTrue(seen.isEmpty(), "No events should have been sent"); Files.writeString(testDir.getTestDirectory().resolve("test124.txt"), "Hello World"); await("We should see only one event") - .during(Duration.ofMillis(500)) - .until(() -> seen.size() == 1); + .timeout(TestHelper.LONG_WAIT) + .pollInterval(Duration.ofMillis(10)) + .until(seen::size, s -> s == 1); if (!exceptions.isEmpty()) { fail(exceptions.pop()); } From 80628ab6cf5a47dd7334cb776613433430582b30 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 14 Oct 2024 12:10:11 +0200 Subject: [PATCH 75/89] Tweaked the test a bit --- src/test/java/engineering/swat/watch/TortureTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index ee24546e..bebee146 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -209,6 +209,7 @@ void manyRegistrationsForSamePath() throws InterruptedException, IOException { assertTrue(seen.isEmpty(), "No events should have been sent"); Files.writeString(testDir.getTestDirectory().resolve("test124.txt"), "Hello World"); await("We should see only one event") + .failFast(() -> !exceptions.isEmpty()) .timeout(TestHelper.LONG_WAIT) .pollInterval(Duration.ofMillis(10)) .until(seen::size, s -> s == 1); From d6b1f2818942a25c4cd3e84c9c9ffe7f63982b67 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 14 Oct 2024 15:19:51 +0200 Subject: [PATCH 76/89] Improved torture tests --- .../swat/watch/impl/BundledSubscription.java | 5 +- .../swat/watch/impl/SubscriptionKey.java | 5 ++ .../engineering/swat/watch/TortureTests.java | 73 ++++++++++++------- .../swat/watch/impl/BundlingTests.java | 47 +++++++++++- 4 files changed, 99 insertions(+), 31 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java index 760ac9a6..4ff26d47 100644 --- a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java +++ b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java @@ -57,10 +57,11 @@ public Closeable subscribe(Key target, Consumer eventListener) throws IOE Subscription active = this.subscriptions.computeIfAbsent(target, t -> new Subscription<>()); // after this, there will only be 1 instance of active subscription in the map. // but we might have a race with remove, which can close the subscript between our get and our addition + // since this code is very hard to get right without locks, and shouldn't be run too often + // we take a big lock around the subscription management synchronized(active) { if (active.closed) { - // we lost the race with closing the subscription - // so we retry + // we lost the race with closing the subscription, so we retry continue; } active.add(eventListener); diff --git a/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java b/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java index 8120791e..698a9251 100644 --- a/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java +++ b/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java @@ -36,4 +36,9 @@ public boolean equals(@Nullable Object obj) { public int hashCode() { return Objects.hash(path, recursive); } + + @Override + public String toString() { + return path.toString() + (recursive ? "[recursive]" : ""); + } } diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index bebee146..8cee7212 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -1,6 +1,5 @@ package engineering.swat.watch; -import static org.awaitility.Awaitility.doNotCatchUncaughtExceptionsByDefault; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -11,8 +10,8 @@ import java.time.Duration; import java.time.LocalTime; import java.util.Random; +import java.util.Collections; import java.util.Set; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.Executor; @@ -27,10 +26,8 @@ import org.apache.logging.log4j.Logger; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable; -import org.junit.jupiter.api.condition.EnabledIf; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import engineering.swat.watch.WatchEvent.Kind; @@ -173,24 +170,30 @@ void pressureOnFSShouldNotMissNewFilesAnything() throws InterruptedException, IO } } - @Test + private final int TORTURE_REGISTRATION_THREADS = THREADS * 500; + + @RepeatedTest(failureThreshold=1, value = 20) void manyRegistrationsForSamePath() throws InterruptedException, IOException { var startRegistering = new Semaphore(0); + var startedWatching = new Semaphore(0); var startDeregistring = new Semaphore(0); var done = new Semaphore(0); var seen = ConcurrentHashMap.newKeySet(); var exceptions = new LinkedBlockingDeque(); - for (int t = 0; t < THREADS * 100; t++) { + + for (int t = 0; t < TORTURE_REGISTRATION_THREADS; t++) { var r = new Thread(() -> { try { var watcher = Watcher .watch(testDir.getTestDirectory(), WatchScope.PATH_AND_CHILDREN) - .onEvent(e -> { if (e.getKind() == Kind.CREATED) seen.add(e.calculateFullPath()); }); + .onEvent(e -> seen.add(e.calculateFullPath())); startRegistering.acquire(); try (var c = watcher.start()) { + startedWatching.release(); startDeregistring.acquire(); } catch(Exception e) { + startedWatching.release(); exceptions.push(e); } } catch (InterruptedException e1) { @@ -203,29 +206,39 @@ void manyRegistrationsForSamePath() throws InterruptedException, IOException { r.start(); } - startRegistering.release(THREADS * 100); - startDeregistring.release((THREADS * 100) - 1); - done.acquire((THREADS * 100) - 1); - assertTrue(seen.isEmpty(), "No events should have been sent"); - Files.writeString(testDir.getTestDirectory().resolve("test124.txt"), "Hello World"); - await("We should see only one event") - .failFast(() -> !exceptions.isEmpty()) - .timeout(TestHelper.LONG_WAIT) - .pollInterval(Duration.ofMillis(10)) - .until(seen::size, s -> s == 1); - if (!exceptions.isEmpty()) { - fail(exceptions.pop()); + try { + startRegistering.release(TORTURE_REGISTRATION_THREADS); + startDeregistring.release(TORTURE_REGISTRATION_THREADS - 1); + startedWatching.acquire(TORTURE_REGISTRATION_THREADS); // make sure they area ll started + done.acquire(TORTURE_REGISTRATION_THREADS - 1); + assertTrue(seen.isEmpty(), "No events should have been sent"); + var target = testDir.getTestDirectory().resolve("test124.txt"); + //logger.info("Writing: {}", target); + Files.writeString(target, "Hello World"); + var expected = Collections.singleton(target); + await("We should see only one event") + .failFast(() -> !exceptions.isEmpty()) + .timeout(TestHelper.LONG_WAIT) + .pollInterval(Duration.ofMillis(10)) + .until(() -> seen, expected::equals); + if (!exceptions.isEmpty()) { + fail(exceptions.pop()); + } + } + finally { + startDeregistring.release(TORTURE_REGISTRATION_THREADS); } - startDeregistring.release(); } - @Test + @RepeatedTest(failureThreshold=1, value = 20) void manyRegisterAndUnregisterSameTime() throws InterruptedException, IOException { var startRegistering = new Semaphore(0); + var startedWatching = new Semaphore(0); var stopAll = new Semaphore(0); var done = new Semaphore(0); - var seen = new ConcurrentLinkedDeque(); + var seen = ConcurrentHashMap.newKeySet(); var exceptions = new LinkedBlockingDeque(); + var target = testDir.getTestDirectory().resolve("test124.txt"); int amountOfWatchersActive = 0; try { for (int t = 0; t < THREADS; t++) { @@ -239,10 +252,14 @@ void manyRegisterAndUnregisterSameTime() throws InterruptedException, IOExceptio for (int k = 0; k < 1000; k++) { var watcher = Watcher .watch(testDir.getTestDirectory(), WatchScope.PATH_AND_CHILDREN) - .onEvent(e -> { if (e.getKind() == Kind.CREATED) seen.add(e.calculateFullPath()); }); + .onEvent(e -> { + if (e.calculateFullPath().equals(target)) { + seen.add(Thread.currentThread().getId()); + } + }); try (var c = watcher.start()) { if (finishWatching && k + 1 == 1000) { - logger.info("Waiting on stop signal"); + startedWatching.release(); stopAll.acquire(); } } @@ -253,6 +270,7 @@ void manyRegisterAndUnregisterSameTime() throws InterruptedException, IOExceptio } catch (InterruptedException e1) { } finally { + startedWatching.release(); done.release(); } }); @@ -262,9 +280,10 @@ void manyRegisterAndUnregisterSameTime() throws InterruptedException, IOExceptio startRegistering.release(THREADS); done.acquire(THREADS - amountOfWatchersActive); + startedWatching.acquire(THREADS); assertTrue(seen.isEmpty(), "No events should have been sent"); - Files.writeString(testDir.getTestDirectory().resolve("test124.txt"), "Hello World"); - await("We should see only exactly the events we expect") + Files.writeString(target, "Hello World"); + await("We should see only exactly the " + amountOfWatchersActive + " events we expect") .failFast(() -> !exceptions.isEmpty()) .pollDelay(TestHelper.NORMAL_WAIT.minusMillis(100)) .until(seen::size, Predicate.isEqual(amountOfWatchersActive)) diff --git a/src/test/java/engineering/swat/watch/impl/BundlingTests.java b/src/test/java/engineering/swat/watch/impl/BundlingTests.java index b7378195..2385c575 100644 --- a/src/test/java/engineering/swat/watch/impl/BundlingTests.java +++ b/src/test/java/engineering/swat/watch/impl/BundlingTests.java @@ -11,14 +11,17 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.awaitility.Awaitility; +import org.hamcrest.core.IsEqual; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import engineering.swat.watch.TestHelper; @@ -36,7 +39,7 @@ private static class FakeSubscribable implements ISubscribable { public Closeable subscribe(Long target, Consumer eventListener) throws IOException { subs.put(target, eventListener); return () -> { - subs.remove(target); + subs.remove(target, eventListener); }; } @@ -60,7 +63,7 @@ static void setupEverything() { Awaitility.setDefaultTimeout(TestHelper.LONG_WAIT.getSeconds(), TimeUnit.SECONDS); } - private static final long SUBs = 100; + private static final int SUBs = 100; private static final long MSGs = 100_000; @Test @@ -101,4 +104,44 @@ void manySubscriptions() throws IOException { } + + @RepeatedTest(failureThreshold = 1, value=50) + void parallelSubscriptions() throws IOException, InterruptedException { + var hits = new AtomicInteger(); + var endPointReached = new Semaphore(0); + var waitingForClose = new Semaphore(0); + var done = new Semaphore(0); + + int active = 0; + for (int j = 0; j < SUBs; j++) { + boolean keepAround = j % 2 == 0; + if (keepAround) { + active++; + } + var t = new Thread(() -> { + for (int k =0; k < 1000; k++) { + try (var c = target.subscribe(Long.valueOf(0), b -> hits.incrementAndGet())) { + if (keepAround && k + 1 == 1000) { + endPointReached.release(); + waitingForClose.acquire(); + } + } catch (Exception ignored) { + logger.catching(ignored); + } + } + done.release(); + }); + t.setDaemon(true); + t.start(); + } + + endPointReached.acquire(active); + done.acquire(SUBs - active); + fakeSubs.publish(Long.valueOf(0)); + + await("Subscriptions should have hit") + .untilAtomic(hits, IsEqual.equalTo(active)); + waitingForClose.release(active); + } + } From 4f8cf592342657c8e3e9b99739cbc535247ad3df Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 14 Oct 2024 15:20:23 +0200 Subject: [PATCH 77/89] Fixed bug around fast registration and unregistration in JDKPoller Looks like it might be a bug in the JDK registry --- src/main/java/engineering/swat/watch/Watcher.java | 1 - src/main/java/engineering/swat/watch/impl/JDKPoller.java | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 0a511bde..f05f5e84 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -35,7 +35,6 @@ public class Watcher { private Watcher(WatchScope scope, Path path) { this.scope = scope; this.path = path; - logger.info("Constructor logger for: {} at {} level", path, scope); } /** diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/JDKPoller.java index 247271bc..7324d5de 100644 --- a/src/main/java/engineering/swat/watch/impl/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/JDKPoller.java @@ -110,8 +110,9 @@ public static Closeable register(SubscriptionKey path, Consumer Date: Mon, 14 Oct 2024 16:27:58 +0200 Subject: [PATCH 78/89] Fixed test that would be triggered by events spread over different worker pools --- src/test/java/engineering/swat/watch/TortureTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 8cee7212..18ad5702 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -248,13 +248,14 @@ void manyRegisterAndUnregisterSameTime() throws InterruptedException, IOExceptio } var r = new Thread(() -> { try { + var id = Thread.currentThread().getId(); startRegistering.acquire(); for (int k = 0; k < 1000; k++) { var watcher = Watcher .watch(testDir.getTestDirectory(), WatchScope.PATH_AND_CHILDREN) .onEvent(e -> { if (e.calculateFullPath().equals(target)) { - seen.add(Thread.currentThread().getId()); + seen.add(id); } }); try (var c = watcher.start()) { @@ -270,7 +271,6 @@ void manyRegisterAndUnregisterSameTime() throws InterruptedException, IOExceptio } catch (InterruptedException e1) { } finally { - startedWatching.release(); done.release(); } }); @@ -280,7 +280,7 @@ void manyRegisterAndUnregisterSameTime() throws InterruptedException, IOExceptio startRegistering.release(THREADS); done.acquire(THREADS - amountOfWatchersActive); - startedWatching.acquire(THREADS); + startedWatching.acquire(amountOfWatchersActive); assertTrue(seen.isEmpty(), "No events should have been sent"); Files.writeString(target, "Hello World"); await("We should see only exactly the " + amountOfWatchersActive + " events we expect") From 5bcf1ceb0c531dc5c96f746011143dc0dbfe4a3e Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 14 Oct 2024 16:28:26 +0200 Subject: [PATCH 79/89] Remove JDK watch only after a delay, just to avoid hammering the registry api --- .../swat/watch/impl/BundledSubscription.java | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java index 4ff26d47..33666193 100644 --- a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java +++ b/src/main/java/engineering/swat/watch/impl/BundledSubscription.java @@ -3,11 +3,15 @@ import java.io.Closeable; import java.io.IOException; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.NonNull; @@ -16,6 +20,7 @@ * This is used (for example) to avoid multiple JDKPoller registries for the same path */ public class BundledSubscription implements ISubscribable { + private static final Logger logger = LogManager.getLogger(); private final ISubscribable wrapped; private final ConcurrentMap> subscriptions = new ConcurrentHashMap<>(); @@ -71,15 +76,33 @@ public Closeable subscribe(Key target, Consumer eventListener) throws IOE } } return () -> { + boolean scheduleClose = false; synchronized(active) { active.remove(eventListener); - if (!active.hasActiveConsumers() && !active.closed) { - active.closed = true; - this.subscriptions.remove(target, active); - if (active.toBeClosed != null) { - active.toBeClosed.close(); - } - } + scheduleClose = !active.hasActiveConsumers() && !active.closed; + } + if (scheduleClose) { + // to avoid hammering the system with closes & registers in a short periode + // we schedule the cleanup of watches in the background, when even after a small delay + // nobody is interested in a certain file anymore + CompletableFuture + .delayedExecutor(100, TimeUnit.MILLISECONDS) + .execute(() -> { + synchronized(active) { + if (!active.hasActiveConsumers() && !active.closed) { + // still ready to be closed + active.closed = true; + this.subscriptions.remove(target, active); + if (active.toBeClosed != null) { + try { + active.toBeClosed.close(); + } catch (IOException e) { + logger.error("Unhandled exception while closing the watcher for {} in the background", target, e); + } + } + } + } + }); } }; } From ac9c73e0fe929877ad001ba0a0262b5cb1f38b13 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 14 Oct 2024 17:10:01 +0200 Subject: [PATCH 80/89] Extended readme with example and description --- README.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bb944d1e..1f58af5e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,59 @@ # java-watch -a java file watcher that works across platforms and supports recursion and single file watches +a java file watcher that works across platforms and supports recursion, single file watches, and tries to make sure no events are missed. +## Features + +Currently working features in java-watch: + +- Recursive watches, even if platform doesn't support it natively. +- Recursive watches also work inside directories created after the watch started +- On overflow events no **new** directories (and it's recursive files) are missed, modification events will however not be simulated +- Single file watches +- Multiple watches for the same directory are merged to avoid overloading the kernel +- Events are process on a worker pool, which you can customize. + +Future features: + +- Avoid poll based watcher in macOS/OSX that only detects changes every 2 seconds +- Support file watches natively in linux +- Monitor only specific events (such as only CREATES) + +## Usage + +Import dependency in pom.xml: + +```xml + + engineering.swat + java-watch + ${java-watch-version} + +``` + +Start using java-watch: + +```java +var directory = Path.of("tmp", "test-dir"); +var watcherSetup = Watcher.watch(directory, WatchScope.PATH_AND_CHILDREN) + .withExecutor(Executors.newCachedThreadPool()) // optionally configure a custom thread pool + .onEvent(watchEvent -> { + System.err.println(watchEvent); + }); + +try(var active = watcherSetup.start()) { + System.out.println("Monitoring files, press any key to stop"); + System.in.read(); +} +// after active.close(), the watch is stopped and +// no new events will be scheduled on the threadpool +``` ## Related work + +Before starting this library, we wanted to use existing libraries, but they all lacked proper support for recursive file watches or lacked configurability. This library now has a growing collection of tests and a small API that should allow for future improvements without breaking compatibility. + +The following section describes the related work research on the libraries and underlying limitations. + After reading the documentation of the following discussion on file system watches: - [Paul Millr's nodejs chokidar](https://github.com/paulmillr/chokidar) From 69870e275e631a3ff2361fa7d4abb37b6b94fde1 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 14 Oct 2024 22:19:14 +0200 Subject: [PATCH 81/89] Slight wording fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1f58af5e..e74e0ace 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Currently working features in java-watch: - Recursive watches, even if platform doesn't support it natively. - Recursive watches also work inside directories created after the watch started -- On overflow events no **new** directories (and it's recursive files) are missed, modification events will however not be simulated +- Even in case of overflow you will get notifications of **new** directories (and it's recursive files), modification events will however not be simulated - Single file watches - Multiple watches for the same directory are merged to avoid overloading the kernel - Events are process on a worker pool, which you can customize. From 9095927225d039061bfd3d97c0c1b73dc5bad26e Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Tue, 15 Oct 2024 11:29:49 +0200 Subject: [PATCH 82/89] Apply suggestions from code review Co-authored-by: sungshik <16154899+sungshik@users.noreply.github.com> --- src/test/java/engineering/swat/watch/TestHelper.java | 2 +- src/test/java/engineering/swat/watch/TortureTests.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/engineering/swat/watch/TestHelper.java b/src/test/java/engineering/swat/watch/TestHelper.java index 17264b4c..717d5818 100644 --- a/src/test/java/engineering/swat/watch/TestHelper.java +++ b/src/test/java/engineering/swat/watch/TestHelper.java @@ -6,7 +6,7 @@ public class TestHelper { public static final Duration SHORT_WAIT; public static final Duration NORMAL_WAIT; - public final static Duration LONG_WAIT; + public static final Duration LONG_WAIT; static { var delayFactorConfig = System.getenv("DELAY_FACTOR"); diff --git a/src/test/java/engineering/swat/watch/TortureTests.java b/src/test/java/engineering/swat/watch/TortureTests.java index 18ad5702..c2812895 100644 --- a/src/test/java/engineering/swat/watch/TortureTests.java +++ b/src/test/java/engineering/swat/watch/TortureTests.java @@ -76,7 +76,7 @@ private void startJob(final Path root, Random r, Executor exec) { break; } try { - // burst a bunch of creates creates and then sleep a bit + // burst a bunch of creates and then sleep a bit for (int i = 0; i< BURST_SIZE; i++) { var file = root.resolve("l1-" + r.nextInt(1000)) .resolve("l2-" + r.nextInt(100)) @@ -209,7 +209,7 @@ void manyRegistrationsForSamePath() throws InterruptedException, IOException { try { startRegistering.release(TORTURE_REGISTRATION_THREADS); startDeregistring.release(TORTURE_REGISTRATION_THREADS - 1); - startedWatching.acquire(TORTURE_REGISTRATION_THREADS); // make sure they area ll started + startedWatching.acquire(TORTURE_REGISTRATION_THREADS); // make sure they are all started done.acquire(TORTURE_REGISTRATION_THREADS - 1); assertTrue(seen.isEmpty(), "No events should have been sent"); var target = testDir.getTestDirectory().resolve("test124.txt"); From fb01d397c0002e73f86b80420bb5d30db3fce3e1 Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Fri, 18 Oct 2024 13:05:53 +0200 Subject: [PATCH 83/89] Improve documentation of `PATH_ONLY` watch scopes --- src/main/java/engineering/swat/watch/WatchScope.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/engineering/swat/watch/WatchScope.java b/src/main/java/engineering/swat/watch/WatchScope.java index 4ef8b0bf..f1646a9d 100644 --- a/src/main/java/engineering/swat/watch/WatchScope.java +++ b/src/main/java/engineering/swat/watch/WatchScope.java @@ -5,7 +5,14 @@ */ public enum WatchScope { /** - * Watch changes to a single file or (metadata of) a directory + * Watch changes to a single file or (metadata of) a directory, but not its + * content. That is, the following changes to a directory are watched: + * - any modification caused by the creation of a nested file/directory; + * - any modification caused by the deletion of a nested file/directory; + * - any modification of its own metadata. + * + * When changes to nested files/directories should also be watched, use + * PATH_AND_CHILDREN or PATH_AND_ALL_DESCENDANTS. */ PATH_ONLY, /** From 23b239bba32b312d72df90d103dba030f62da0c4 Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Fri, 18 Oct 2024 13:41:54 +0200 Subject: [PATCH 84/89] Refine test (deleteOfFileInDirectoryShouldBeVisible) --- .../swat/watch/SingleDirectoryTests.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java index fdc3411e..9628b900 100644 --- a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java +++ b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java @@ -39,17 +39,28 @@ static void setupEverything() { @Test void deleteOfFileInDirectoryShouldBeVisible() throws IOException, InterruptedException { var target = testDir.getTestFiles().get(0); - var seen = new AtomicBoolean(false); + var seenDelete = new AtomicBoolean(false); + var seenCreate = new AtomicBoolean(false); var watchConfig = Watcher.watch(target.getParent(), WatchScope.PATH_AND_CHILDREN) .onEvent(ev -> { if (ev.getKind() == Kind.DELETED && ev.calculateFullPath().equals(target)) { - seen.set(true); + seenDelete.set(true); + } + if (ev.getKind() == Kind.CREATED && ev.calculateFullPath().equals(target)) { + seenCreate.set(true); } }); try (var watch = watchConfig.start()) { + + // Delete the file Files.delete(target); await("File deletion should generate delete event") - .untilTrue(seen); + .untilTrue(seenDelete); + + // Re-create it again + Files.writeString(target, "Hello World"); + await("File creation should generate create event") + .untilTrue(seenCreate); } } } From 88170a258e4dcd06a5ef8757504e3b67707c031b Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 11 Nov 2024 11:13:53 +0100 Subject: [PATCH 85/89] Tweaked documentation a bit --- .../java/engineering/swat/watch/WatchScope.java | 17 ++++++++++------- .../java/engineering/swat/watch/Watcher.java | 5 +++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/engineering/swat/watch/WatchScope.java b/src/main/java/engineering/swat/watch/WatchScope.java index f1646a9d..ec0e11b7 100644 --- a/src/main/java/engineering/swat/watch/WatchScope.java +++ b/src/main/java/engineering/swat/watch/WatchScope.java @@ -5,14 +5,17 @@ */ public enum WatchScope { /** - * Watch changes to a single file or (metadata of) a directory, but not its - * content. That is, the following changes to a directory are watched: - * - any modification caused by the creation of a nested file/directory; - * - any modification caused by the deletion of a nested file/directory; - * - any modification of its own metadata. + *

Watch changes to a single file or (metadata of) a single directory.

* - * When changes to nested files/directories should also be watched, use - * PATH_AND_CHILDREN or PATH_AND_ALL_DESCENDANTS. + *

Note, depending on the platform you can receive events for a directory + * in case of these events:

+ *
    + *
  • a MODIFIED caused by the creation of a nested file/directory
    • + *
  • a MODIFIED caused by the deletion of a nested file/directory
    • + *
  • a MODIFIED of its own metadata
    • + *
+ * + *

In most cases when Path is a Directory you're interested in which nested entries changes, in that case use {@link #PATH_AND_CHILDREN} or {@link #PATH_AND_ALL_DESCENDANTS}.

*/ PATH_ONLY, /** diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index f05f5e84..f0310463 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -17,9 +17,10 @@ import engineering.swat.watch.impl.JDKRecursiveDirectoryWatcher; /** - * Watch a path for changes. + *

Watch a path for changes.

* - * It will avoid common errors using the raw apis, and will try to use the most native api where possible. + * + *

It will avoid common errors using the raw apis, and will try to use the most native api where possible.

* Note, there are differences per platform that cannot be avoided, please review the readme of the library. */ public class Watcher { From 87c63e6c79535174bf4447bbea9ca4531be2681e Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 11 Nov 2024 11:15:25 +0100 Subject: [PATCH 86/89] Moved JDK classes to its own namespace --- .../swat/watch/impl/{ => jdk}/JDKDirectoryWatcher.java | 0 .../engineering/swat/watch/impl/{ => jdk}/JDKFileWatcher.java | 0 .../java/engineering/swat/watch/impl/{ => jdk}/JDKPoller.java | 0 .../swat/watch/impl/{ => jdk}/JDKRecursiveDirectoryWatcher.java | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename src/main/java/engineering/swat/watch/impl/{ => jdk}/JDKDirectoryWatcher.java (100%) rename src/main/java/engineering/swat/watch/impl/{ => jdk}/JDKFileWatcher.java (100%) rename src/main/java/engineering/swat/watch/impl/{ => jdk}/JDKPoller.java (100%) rename src/main/java/engineering/swat/watch/impl/{ => jdk}/JDKRecursiveDirectoryWatcher.java (100%) diff --git a/src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKDirectoryWatcher.java similarity index 100% rename from src/main/java/engineering/swat/watch/impl/JDKDirectoryWatcher.java rename to src/main/java/engineering/swat/watch/impl/jdk/JDKDirectoryWatcher.java diff --git a/src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKFileWatcher.java similarity index 100% rename from src/main/java/engineering/swat/watch/impl/JDKFileWatcher.java rename to src/main/java/engineering/swat/watch/impl/jdk/JDKFileWatcher.java diff --git a/src/main/java/engineering/swat/watch/impl/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKPoller.java similarity index 100% rename from src/main/java/engineering/swat/watch/impl/JDKPoller.java rename to src/main/java/engineering/swat/watch/impl/jdk/JDKPoller.java diff --git a/src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatcher.java similarity index 100% rename from src/main/java/engineering/swat/watch/impl/JDKRecursiveDirectoryWatcher.java rename to src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatcher.java From d392f1c0a68d164ea6ab33a0b7a1d1af721c5f12 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 11 Nov 2024 11:17:33 +0100 Subject: [PATCH 87/89] Improved impl directory --- src/main/java/engineering/swat/watch/Watcher.java | 6 +++--- .../java/engineering/swat/watch/impl/jdk/JDKPoller.java | 4 +++- .../swat/watch/impl/jdk/JDKRecursiveDirectoryWatcher.java | 2 +- .../swat/watch/impl/{ => util}/BundledSubscription.java | 2 +- .../swat/watch/impl/{ => util}/ISubscribable.java | 2 +- .../swat/watch/impl/{ => util}/SubscriptionKey.java | 2 +- 6 files changed, 10 insertions(+), 8 deletions(-) rename src/main/java/engineering/swat/watch/impl/{ => util}/BundledSubscription.java (99%) rename src/main/java/engineering/swat/watch/impl/{ => util}/ISubscribable.java (85%) rename src/main/java/engineering/swat/watch/impl/{ => util}/SubscriptionKey.java (95%) diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index f0310463..8b25549f 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -12,9 +12,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import engineering.swat.watch.impl.JDKDirectoryWatcher; -import engineering.swat.watch.impl.JDKFileWatcher; -import engineering.swat.watch.impl.JDKRecursiveDirectoryWatcher; +import engineering.swat.watch.impl.jdk.JDKDirectoryWatcher; +import engineering.swat.watch.impl.jdk.JDKFileWatcher; +import engineering.swat.watch.impl.jdk.JDKRecursiveDirectoryWatcher; /** *

Watch a path for changes.

diff --git a/src/main/java/engineering/swat/watch/impl/jdk/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKPoller.java index 7324d5de..2f0c9992 100644 --- a/src/main/java/engineering/swat/watch/impl/jdk/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/jdk/JDKPoller.java @@ -1,4 +1,4 @@ -package engineering.swat.watch.impl; +package engineering.swat.watch.impl.jdk; import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; @@ -27,6 +27,8 @@ import com.sun.nio.file.ExtendedWatchEventModifier; +import engineering.swat.watch.impl.util.SubscriptionKey; + /** * This class is a wrapper around the JDK WatchService, it takes care to poll the service for new events, and then distributes them to the right parties */ diff --git a/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatcher.java index a772daf5..245b3be2 100644 --- a/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatcher.java @@ -1,4 +1,4 @@ -package engineering.swat.watch.impl; +package engineering.swat.watch.impl.jdk; import java.io.Closeable; import java.io.IOException; diff --git a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java b/src/main/java/engineering/swat/watch/impl/util/BundledSubscription.java similarity index 99% rename from src/main/java/engineering/swat/watch/impl/BundledSubscription.java rename to src/main/java/engineering/swat/watch/impl/util/BundledSubscription.java index 33666193..ac352996 100644 --- a/src/main/java/engineering/swat/watch/impl/BundledSubscription.java +++ b/src/main/java/engineering/swat/watch/impl/util/BundledSubscription.java @@ -1,4 +1,4 @@ -package engineering.swat.watch.impl; +package engineering.swat.watch.impl.util; import java.io.Closeable; import java.io.IOException; diff --git a/src/main/java/engineering/swat/watch/impl/ISubscribable.java b/src/main/java/engineering/swat/watch/impl/util/ISubscribable.java similarity index 85% rename from src/main/java/engineering/swat/watch/impl/ISubscribable.java rename to src/main/java/engineering/swat/watch/impl/util/ISubscribable.java index 4d5fe5b8..5803c96a 100644 --- a/src/main/java/engineering/swat/watch/impl/ISubscribable.java +++ b/src/main/java/engineering/swat/watch/impl/util/ISubscribable.java @@ -1,4 +1,4 @@ -package engineering.swat.watch.impl; +package engineering.swat.watch.impl.util; import java.io.Closeable; import java.io.IOException; diff --git a/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java b/src/main/java/engineering/swat/watch/impl/util/SubscriptionKey.java similarity index 95% rename from src/main/java/engineering/swat/watch/impl/SubscriptionKey.java rename to src/main/java/engineering/swat/watch/impl/util/SubscriptionKey.java index 698a9251..4afa2ede 100644 --- a/src/main/java/engineering/swat/watch/impl/SubscriptionKey.java +++ b/src/main/java/engineering/swat/watch/impl/util/SubscriptionKey.java @@ -1,4 +1,4 @@ -package engineering.swat.watch.impl; +package engineering.swat.watch.impl.util; import java.nio.file.Path; import java.util.Objects; From 982ce65e2a01b975d336b554c544c95e5dbbd086 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 11 Nov 2024 11:21:58 +0100 Subject: [PATCH 88/89] Make sure to have a dedicated interface, to allow for future additions without breaking binary compatibility --- .../java/engineering/swat/watch/ActiveWatch.java | 12 ++++++++++++ src/main/java/engineering/swat/watch/Watcher.java | 3 +-- .../swat/watch/impl/jdk/JDKDirectoryWatcher.java | 9 ++++++--- .../swat/watch/impl/jdk/JDKFileWatcher.java | 5 +++-- .../watch/impl/jdk/JDKRecursiveDirectoryWatcher.java | 3 ++- 5 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 src/main/java/engineering/swat/watch/ActiveWatch.java diff --git a/src/main/java/engineering/swat/watch/ActiveWatch.java b/src/main/java/engineering/swat/watch/ActiveWatch.java new file mode 100644 index 00000000..a3a3dc10 --- /dev/null +++ b/src/main/java/engineering/swat/watch/ActiveWatch.java @@ -0,0 +1,12 @@ +package engineering.swat.watch; + +import java.io.Closeable; + +/** + *

Marker interface for an active watch, in the future might get properties you can inspect.

+ * + *

For now, make sure to close the watch when not interested in new events

+ */ +public interface ActiveWatch extends Closeable { + +} diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 8b25549f..91c1e8db 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -1,6 +1,5 @@ package engineering.swat.watch; -import java.io.Closeable; import java.io.IOException; import java.nio.file.Files; import java.nio.file.LinkOption; @@ -96,7 +95,7 @@ public Watcher withExecutor(Executor callbackHandler) { * @throws IOException in case the starting of the watcher caused an underlying IO exception * @throws IllegalStateException the watchers is not configured correctly (for example, missing {@link #onEvent(Consumer)}, or a watcher is started twice) */ - public Closeable start() throws IOException { + public ActiveWatch start() throws IOException { if (this.eventHandler == NULL_HANDLER) { throw new IllegalStateException("There is no onEvent handler defined"); } diff --git a/src/main/java/engineering/swat/watch/impl/jdk/JDKDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKDirectoryWatcher.java index 1357908d..a5c94dfd 100644 --- a/src/main/java/engineering/swat/watch/impl/jdk/JDKDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/jdk/JDKDirectoryWatcher.java @@ -1,4 +1,4 @@ -package engineering.swat.watch.impl; +package engineering.swat.watch.impl.jdk; import java.io.Closeable; import java.io.IOException; @@ -13,9 +13,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import engineering.swat.watch.ActiveWatch; import engineering.swat.watch.WatchEvent; +import engineering.swat.watch.impl.util.BundledSubscription; +import engineering.swat.watch.impl.util.SubscriptionKey; -public class JDKDirectoryWatcher implements Closeable { +public class JDKDirectoryWatcher implements ActiveWatch { private final Logger logger = LogManager.getLogger(); private final Path directory; private final Executor exec; @@ -95,7 +98,7 @@ else if (ev.kind() == StandardWatchEventKinds.OVERFLOW) { @Override public synchronized void close() throws IOException { if (activeWatch != null) { - logger.debug("Closing watch for: {}", this.directory); + logger.trace("Closing watch for: {}", this.directory); activeWatch.close(); } } diff --git a/src/main/java/engineering/swat/watch/impl/jdk/JDKFileWatcher.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKFileWatcher.java index b1c66692..535401a3 100644 --- a/src/main/java/engineering/swat/watch/impl/jdk/JDKFileWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/jdk/JDKFileWatcher.java @@ -1,4 +1,4 @@ -package engineering.swat.watch.impl; +package engineering.swat.watch.impl.jdk; import java.io.Closeable; import java.io.IOException; @@ -10,6 +10,7 @@ import org.apache.logging.log4j.Logger; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import engineering.swat.watch.ActiveWatch; import engineering.swat.watch.WatchEvent; /** @@ -17,7 +18,7 @@ * * Note that you should take care to call start only once. */ -public class JDKFileWatcher implements Closeable { +public class JDKFileWatcher implements ActiveWatch { private final Logger logger = LogManager.getLogger(); private final Path file; private final Path fileName; diff --git a/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatcher.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatcher.java index 245b3be2..64e15c6f 100644 --- a/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatcher.java +++ b/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatcher.java @@ -22,9 +22,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import engineering.swat.watch.ActiveWatch; import engineering.swat.watch.WatchEvent; -public class JDKRecursiveDirectoryWatcher implements Closeable { +public class JDKRecursiveDirectoryWatcher implements ActiveWatch { private final Logger logger = LogManager.getLogger(); private final Path root; private final Executor exec; From d1b6e9e518d48affdddcdab44004b2abf3ef9355 Mon Sep 17 00:00:00 2001 From: Davy Landman Date: Mon, 11 Nov 2024 11:29:38 +0100 Subject: [PATCH 89/89] Missed refactor of test --- src/test/java/engineering/swat/watch/impl/BundlingTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/engineering/swat/watch/impl/BundlingTests.java b/src/test/java/engineering/swat/watch/impl/BundlingTests.java index 2385c575..916a6401 100644 --- a/src/test/java/engineering/swat/watch/impl/BundlingTests.java +++ b/src/test/java/engineering/swat/watch/impl/BundlingTests.java @@ -25,6 +25,8 @@ import org.junit.jupiter.api.Test; import engineering.swat.watch.TestHelper; +import engineering.swat.watch.impl.util.BundledSubscription; +import engineering.swat.watch.impl.util.ISubscribable; public class BundlingTests {