diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 469bf14d..fda45120 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -12,10 +12,17 @@ jobs: test: strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: + - image: ubuntu-latest + - image: macos-latest + mac-backend: jdk + - image: macos-latest + mac-backend: jna + - image: windows-latest jdk: [11, 17, 21] + fail-fast: false - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os.image }} steps: - uses: actions/checkout@v4 - run: echo " " >> pom.xml # make sure the cache is slightly different for these runners @@ -27,7 +34,7 @@ jobs: cache: 'maven' - name: test - run: mvn -B clean test + run: mvn -B clean test -DargLine="-Dengineering.swat.java-watch.mac=${{ matrix.os.mac-backend }}" env: DELAY_FACTOR: 3 diff --git a/README.md b/README.md index bbd246b6..bbd96f67 100644 --- a/README.md +++ b/README.md @@ -57,13 +57,13 @@ try(var active = watcherSetup.start()) { // no new events will be scheduled on the threadpool ``` -## Internals +## Backends On all platforms except macOS, the library internally uses the JDK default implementation of the Java NIO [`WatchService`](https://docs.oracle.com/javase/8/docs/api/java/nio/file/WatchService.html) API. On macOS, the library internally uses our custom `WatchService` implementation based on macOS's native [file system event streams](https://developer.apple.com/documentation/coreservices/file_system_events?language=objc) (using JNA). Generally, it offers better performance than the JDK default implementation (because the latter uses a polling loop to detect changes only once every two seconds). -To force the library to use the JDK default implementation on macOS, set system property `engineering.swat.watch.impl` to `default`. +To force the library to use the JDK default implementation on macOS, set system property `engineering.swat.java-watch.mac` to `jdk`. ## Related work diff --git a/pom.xml b/pom.xml index 245fa629..7cb9a2ec 100644 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,7 @@ 3.49.2 5.12.2 2.24.3 + 5.16.0 11 11 @@ -231,7 +232,7 @@ net.java.dev.jna jna-platform - 5.16.0 + ${jna.version} diff --git a/src/main/java/engineering/swat/watch/impl/jdk/JDKPoller.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKPoller.java index 2e3137d4..237c0753 100644 --- a/src/main/java/engineering/swat/watch/impl/jdk/JDKPoller.java +++ b/src/main/java/engineering/swat/watch/impl/jdk/JDKPoller.java @@ -190,20 +190,22 @@ public Watchable newWatchable(Path path) { static final Platform CURRENT = current(); // Assumption: the platform doesn't change private static Platform current() { - var key = "engineering.swat.watch.impl"; - var val = System.getProperty(key); - if (val != null) { - if (val.equals("mac")) { - return MAC; - } else if (val.equals("default")) { - return DEFAULT; - } else { - logger.warn("Unexpected value \"{}\" for system property \"{}\". Using value \"default\" instead.", val, key); - return DEFAULT; + if (com.sun.jna.Platform.isMac()) { + var key = "engineering.swat.java-watch.mac"; + var val = System.getProperty(key); + if (val != null) { + if (val.equals("jna")) { + return MAC; + } else if (val.equals("jdk")) { + return DEFAULT; + } else { + logger.warn("Unexpected value \"{}\" for system property \"{}\". Using value \"jdk\" instead.", val, key); + return DEFAULT; + } } } - return com.sun.jna.Platform.isMac() ? MAC : DEFAULT; + return DEFAULT; } static Platform get() { diff --git a/src/main/java/engineering/swat/watch/impl/mac/NativeEventStream.java b/src/main/java/engineering/swat/watch/impl/mac/NativeEventStream.java index 22cad3a1..58856731 100644 --- a/src/main/java/engineering/swat/watch/impl/mac/NativeEventStream.java +++ b/src/main/java/engineering/swat/watch/impl/mac/NativeEventStream.java @@ -33,6 +33,7 @@ import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamEventFlag.ITEM_INODE_META_MOD; import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamEventFlag.ITEM_MODIFIED; import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamEventFlag.ITEM_REMOVED; +import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamEventFlag.ITEM_RENAMED; import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamEventFlag.MUST_SCAN_SUB_DIRS; import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; @@ -41,6 +42,7 @@ import java.io.Closeable; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -153,6 +155,19 @@ public void callback(Pointer streamRef, Pointer clientCallBackInfo, if (any(flags[i], MUST_SCAN_SUB_DIRS.mask)) { handler.handle(OVERFLOW, null); } + if (any(flags[i], ITEM_RENAMED.mask)) { + // For now, check if the file exists to determine if the + // event pertains to the target of the rename (if it + // exists) or to the source (else). This is an + // approximation. It might be more accurate to maintain + // an internal index (but getting the concurrency right + // requires care). + if (Files.exists(Path.of(paths[i]))) { + handler.handle(ENTRY_CREATE, context); + } else { + handler.handle(ENTRY_DELETE, context); + } + } } } diff --git a/src/test/java/engineering/swat/watch/SingleFileTests.java b/src/test/java/engineering/swat/watch/SingleFileTests.java index 7fe04a2b..26c0f015 100644 --- a/src/test/java/engineering/swat/watch/SingleFileTests.java +++ b/src/test/java/engineering/swat/watch/SingleFileTests.java @@ -130,7 +130,7 @@ void noRescanOnOverflow() throws IOException, InterruptedException { try (var watch = startWatchAndTriggerOverflow(Approximation.NONE, bookkeeper)) { Thread.sleep(TestHelper.SHORT_WAIT.toMillis()); - await("Overflow shouldn't trigger created, modified, or deleted events") + await("Overflow shouldn't trigger created, modified, or deleted events: " + bookkeeper) .until(() -> bookkeeper.events().kind(CREATED, MODIFIED, DELETED).none()); await("Overflow should be visible to user-defined event handler") .until(() -> bookkeeper.events().kind(OVERFLOW).any()); diff --git a/src/test/java/engineering/swat/watch/SmokeTests.java b/src/test/java/engineering/swat/watch/SmokeTests.java index 3e2dc5fe..44cdd119 100644 --- a/src/test/java/engineering/swat/watch/SmokeTests.java +++ b/src/test/java/engineering/swat/watch/SmokeTests.java @@ -38,7 +38,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -115,12 +114,11 @@ void watchSingleFile() throws IOException { } @Test - @Disabled - void moveRegularFileBetweenNestedDirectories() throws IOException { + void moveRegularFile() throws IOException { var parent = testDir.getTestDirectory(); var child1 = Files.createDirectories(parent.resolve("from")); var child2 = Files.createDirectories(parent.resolve("to")); - var file = Files.createFile(child1.resolve("file.txt")); + var regularFile = Files.createFile(child1.resolve("file.txt")); var parentWatchBookkeeper = new TestHelper.Bookkeeper(); var parentWatchConfig = Watch @@ -139,7 +137,7 @@ void moveRegularFileBetweenNestedDirectories() throws IOException { var fileWatchBookkeeper = new TestHelper.Bookkeeper(); var fileWatchConfig = Watch - .build(file, WatchScope.PATH_ONLY) + .build(regularFile, WatchScope.PATH_ONLY) .on(fileWatchBookkeeper); try (var parentWatch = parentWatchConfig.start(); @@ -147,17 +145,17 @@ void moveRegularFileBetweenNestedDirectories() throws IOException { var child2Watch = child2WatchConfig.start(); var fileWatch = fileWatchConfig.start()) { - var source = child1.resolve(file.getFileName()); - var target = child2.resolve(file.getFileName()); + var source = child1.resolve(regularFile.getFileName()); + var target = child2.resolve(regularFile.getFileName()); Files.move(source, target); - await("Move should be observed as delete by `parent` watch (file tree)") - .until(() -> parentWatchBookkeeper - .events().kind(DELETED).rootPath(parent).relativePath(parent.relativize(source)).any()); - - await("Move should be observed as create by `parent` watch (file tree)") - .until(() -> parentWatchBookkeeper - .events().kind(CREATED).rootPath(parent).relativePath(parent.relativize(target)).any()); + for (var e : new WatchEvent[] { + new WatchEvent(DELETED, parent, parent.relativize(source)), + new WatchEvent(CREATED, parent, parent.relativize(target)) + }) { + await("Move should be observed as delete/create by `parent` watch (file tree): " + e) + .until(() -> parentWatchBookkeeper.events().any(e)); + } await("Move should be observed as delete by `child1` watch (single directory)") .until(() -> child1WatchBookkeeper @@ -172,4 +170,82 @@ void moveRegularFileBetweenNestedDirectories() throws IOException { .events().kind(DELETED).rootPath(source).any()); } } + + @Test + void moveDirectory() throws IOException { + var parent = testDir.getTestDirectory(); + var child1 = Files.createDirectories(parent.resolve("from")); + var child2 = Files.createDirectories(parent.resolve("to")); + + var directory = Files.createDirectory(child1.resolve("directory")); + var regularFile1 = Files.createFile(directory.resolve("file1.txt")); + var regularFile2 = Files.createFile(directory.resolve("file2.txt")); + + var parentWatchBookkeeper = new TestHelper.Bookkeeper(); + var parentWatchConfig = Watch + .build(parent, WatchScope.PATH_AND_ALL_DESCENDANTS) + .on(parentWatchBookkeeper); + + var child1WatchBookkeeper = new TestHelper.Bookkeeper(); + var child1WatchConfig = Watch + .build(child1, WatchScope.PATH_AND_CHILDREN) + .on(child1WatchBookkeeper); + + var child2WatchBookkeeper = new TestHelper.Bookkeeper(); + var child2WatchConfig = Watch + .build(child2, WatchScope.PATH_AND_CHILDREN) + .on(child2WatchBookkeeper); + + var directoryWatchBookkeeper = new TestHelper.Bookkeeper(); + var directoryWatchConfig = Watch + .build(directory, WatchScope.PATH_ONLY) + .on(directoryWatchBookkeeper); + + try (var parentWatch = parentWatchConfig.start(); + var child1Watch = child1WatchConfig.start(); + var child2Watch = child2WatchConfig.start(); + var fileWatch = directoryWatchConfig.start()) { + + var sourceDirectory = child1.resolve(directory.getFileName()); + var sourceRegularFile1 = sourceDirectory.resolve(regularFile1.getFileName()); + var sourceRegularFile2 = sourceDirectory.resolve(regularFile2.getFileName()); + + var targetDirectory = child2.resolve(directory.getFileName()); + var targetRegularFile1 = targetDirectory.resolve(regularFile1.getFileName()); + var targetRegularFile2 = targetDirectory.resolve(regularFile2.getFileName()); + + Files.move(sourceDirectory, targetDirectory); + + for (var e : new WatchEvent[] { + new WatchEvent(DELETED, parent, parent.relativize(sourceDirectory)), + new WatchEvent(CREATED, parent, parent.relativize(targetDirectory)), + // The following events currently *aren't* observed by the + // `parent` watch for the whole file tree: moving a directory + // doesn't trigger events for the deletion/creation of the files + // contained in it (neither using the general default/JDK + // implementation of Watch Service, nor using our special macOS + // implementation). + // + // new WatchEvent(DELETED, parent, parent.relativize(sourceRegularFile1)), + // new WatchEvent(DELETED, parent, parent.relativize(sourceRegularFile2)), + // new WatchEvent(CREATED, parent, parent.relativize(targetRegularFile1)), + // new WatchEvent(CREATED, parent, parent.relativize(targetRegularFile2)) + }) { + await("Move should be observed as delete/create by `parent` watch (file tree): " + e) + .until(() -> parentWatchBookkeeper.events().any(e)); + } + + await("Move should be observed as delete by `child1` watch (single directory)") + .until(() -> child1WatchBookkeeper + .events().kind(DELETED).rootPath(child1).relativePath(child1.relativize(sourceDirectory)).any()); + + await("Move should be observed as create by `child2` watch (single directory)") + .until(() -> child2WatchBookkeeper + .events().kind(CREATED).rootPath(child2).relativePath(child2.relativize(targetDirectory)).any()); + + await("Move should be observed as delete by `directory` watch") + .until(() -> directoryWatchBookkeeper + .events().kind(DELETED).rootPath(sourceDirectory).any()); + } + } } diff --git a/src/test/java/engineering/swat/watch/impl/mac/APIs.java b/src/test/java/engineering/swat/watch/impl/mac/APIs.java index acab53c5..4b408b1a 100644 --- a/src/test/java/engineering/swat/watch/impl/mac/APIs.java +++ b/src/test/java/engineering/swat/watch/impl/mac/APIs.java @@ -28,6 +28,8 @@ import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamCreateFlag.FILE_EVENTS; import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamCreateFlag.NO_DEFER; +import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamCreateFlag.USE_CF_TYPES; +import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamCreateFlag.USE_EXTENDED_DATA; import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamCreateFlag.WATCH_ROOT; import static org.awaitility.Awaitility.await; @@ -52,7 +54,9 @@ import com.sun.jna.Pointer; import com.sun.jna.platform.mac.CoreFoundation; import com.sun.jna.platform.mac.CoreFoundation.CFArrayRef; +import com.sun.jna.platform.mac.CoreFoundation.CFDictionaryRef; import com.sun.jna.platform.mac.CoreFoundation.CFIndex; +import com.sun.jna.platform.mac.CoreFoundation.CFNumberRef; import com.sun.jna.platform.mac.CoreFoundation.CFStringRef; import engineering.swat.watch.TestDirectory; @@ -78,13 +82,13 @@ void smokeTest() throws IOException { var paths = ConcurrentHashMap. newKeySet(); var s = test.getTestDirectory().toString(); - var handler = (MinimalWorkingExample.EventHandler) (path, flags, id) -> { + var handler = (MinimalWorkingExample.EventHandler) (path, inode, flags, id) -> { synchronized (ready) { while (!ready.get()) { try { ready.wait(); } catch (InterruptedException e) { - LOGGER.error("Unexpected interrupt. Test likely to fail. Event ignored ({}).", prettyPrint(path, flags, id)); + LOGGER.error("Unexpected interrupt. Test likely to fail. Event ignored ({}).", prettyPrint(path, inode, flags, id)); Thread.currentThread().interrupt(); return; } @@ -93,7 +97,7 @@ void smokeTest() throws IOException { paths.remove(path); }; - try (var mwe = new MinimalWorkingExample(s, handler)) { + try (var mwe = new MinimalWorkingExample(s, handler, true)) { var dir = test.getTestDirectory().toRealPath(); paths.add(Files.writeString(dir.resolve("a.txt"), "foo").toString()); paths.add(Files.writeString(dir.resolve("b.txt"), "bar").toString()); @@ -111,25 +115,26 @@ void smokeTest() throws IOException { public static void main(String[] args) throws IOException { var s = args[0]; - var handler = (MinimalWorkingExample.EventHandler) (path, flags, id) -> { - LOGGER.info(prettyPrint(path, flags, id)); + var handler = (MinimalWorkingExample.EventHandler) (path, inode, flags, id) -> { + LOGGER.info(prettyPrint(path, inode, flags, id)); }; + var useExtendedData = args.length >= 2 && Boolean.parseBoolean(args[1]); - try (var mwe = new MinimalWorkingExample(s, handler)) { + try (var mwe = new MinimalWorkingExample(s, handler, useExtendedData)) { // Block the program from terminating until `ENTER` is pressed new BufferedReader(new InputStreamReader(System.in)).readLine(); } } - private static String prettyPrint(String path, int flags, long id) { + private static String prettyPrint(String path, long inode, int flags, long id) { var flagsPrettyPrinted = Stream .of(FSEventStreamEventFlag.values()) .filter(f -> (f.mask & flags) == f.mask) .map(Object::toString) .collect(Collectors.joining(", ")); - var format = "path: \"%s\", flags: [%s], id: %s"; - return String.format(format, path, flagsPrettyPrinted, id); + var format = "path: \"%s\", inode: %s, flags: [%s], id: %s"; + return String.format(format, path, inode, flagsPrettyPrinted, id); } private static class MinimalWorkingExample implements Closeable { @@ -137,7 +142,7 @@ private static class MinimalWorkingExample implements Closeable { private Pointer stream; private Pointer queue; - public MinimalWorkingExample(String s, EventHandler handler) { + public MinimalWorkingExample(String s, EventHandler handler, boolean useExtendedData) { // Allocate singleton array of paths CFStringRef pathToWatch = CFStringRef.createCFString(s); @@ -154,11 +159,24 @@ public MinimalWorkingExample(String s, EventHandler handler) { // Allocate callback this.callback = (x1, x2, x3, x4, x5, x6) -> { - var paths = x4.getStringArray(0, (int) x3); - var flags = x5.getIntArray(0, (int) x3); - var ids = x6.getLongArray(0, (int) x3); + var paths = x4.getStringArray(0, (int) x3); + var inodes = new long[(int) x3]; + var flags = x5.getIntArray(0, (int) x3); + var ids = x6.getLongArray(0, (int) x3); + + if (useExtendedData) { + var extendedData = new CFArrayRef(x4); + for (int i = 0; i < x3; i++) { + var dictionary = new CFDictionaryRef(extendedData.getValueAtIndex(i)); + var dictionaryPath = dictionary.getValue(FileSystemEvents.kFSEventStreamEventExtendedDataPathKey); + var dictionaryInode = dictionary.getValue(FileSystemEvents.kFSEventStreamEventExtendedFileIDKey); + paths[i] = dictionaryPath == null ? null : new CFStringRef(dictionaryPath).stringValue(); + inodes[i] = dictionaryInode == null ? 0 : new CFNumberRef(dictionaryInode).longValue(); + } + } + for (int i = 0; i < x3; i++) { - handler.handle(paths[i], flags[i], ids[i]); + handler.handle(paths[i], inodes[i], flags[i], ids[i]); } }; @@ -170,7 +188,8 @@ public MinimalWorkingExample(String s, EventHandler handler) { pathsToWatch, FSE.FSEventsGetCurrentEventId(), 0.15, - NO_DEFER.mask | WATCH_ROOT.mask | FILE_EVENTS.mask); + NO_DEFER.mask | WATCH_ROOT.mask | FILE_EVENTS.mask | + (useExtendedData ? USE_EXTENDED_DATA.mask | USE_CF_TYPES.mask : 0)); // Deallocate array of paths pathsToWatch.release(); @@ -203,7 +222,7 @@ public void close() throws IOException { @FunctionalInterface private static interface EventHandler { - void handle(String path, int flags, long id); + void handle(String path, long inode, int flags, long id); } } }