WIP: Using JNI+Rust instead of JNA on macOS

.editorconfig

@@ -15,3 +15,6 @@ end_of_line = lf
 [*.java]
 indent_size = 4
 max_line_length = 120
+
+[*.rs]
+indent_size = 4

.github/workflows/build.yaml

@@ -13,12 +13,12 @@ jobs:
     strategy:
       matrix:
         os:
-          - image: ubuntu-latest
-          - image: macos-latest
-            mac-backend: jdk
+          #- image: ubuntu-latest
+          #- image: macos-latest
+            #mac-backend: jdk
           - image: macos-latest
             mac-backend: fsevents
-          - image: windows-latest
+          #- image: windows-latest
         jdk: [11, 17, 21]
 
       fail-fast: false
@@ -32,6 +32,9 @@ jobs:
           java-version: ${{ matrix.jdk }}
           distribution: 'temurin'
           cache: 'maven'
+      #- uses: actions-rust-lang/setup-rust-toolchain@v1
+      - run: ./update-rust-jni-libs.sh
+        if: ${{ matrix.os.image == 'macos-latest' }}
 
       - name: test
         run: mvn -B clean test "-Dwatch.mac.backend=${{ matrix.os.mac-backend }}"

.gitignore

@@ -27,4 +27,4 @@ replay_pid*
 
 # release plugin state files
 /pom.xml.releaseBackup
-/release.properties
+/release.properties

pom.xml

@@ -81,6 +81,14 @@
   </properties>
 
   <build>
+    <resources>
+      <resource>
+        <directory>src/main/resources</directory>
+        <excludes>
+          <exclude>src/main/rust/**/*.*</exclude>
+        </excludes>
+      </resource>
+    </resources>
     <plugins>
       <plugin> <!-- configure java compiler -->
         <groupId>org.apache.maven.plugins</groupId>
@@ -170,6 +178,11 @@
             </goals>
           </execution>
         </executions>
+        <configuration>
+          <excludes>
+            <exclude>src/main/resources/**</exclude>
+          </excludes>
+        </configuration>
       </plugin>
       <plugin> <!-- use a new version of maven  -->
         <groupId>org.apache.maven.plugins</groupId>

src/main/java/engineering/swat/watch/impl/mac/NativeEventHandler.java

@@ -26,8 +26,13 @@
  */
 package engineering.swat.watch.impl.mac;
 
+import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
+import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
+import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
+import static java.nio.file.StandardWatchEventKinds.OVERFLOW;
+
+import java.nio.file.Path;
 import java.nio.file.WatchEvent;
-import java.nio.file.WatchEvent.Kind;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 
@@ -51,5 +56,28 @@
  */
 @FunctionalInterface
 interface NativeEventHandler {
-    <T> void handle(Kind<T> kind, @Nullable T context);
+    <T> void handle(java.nio.file.WatchEvent.Kind<T> kind, @Nullable T context);
+
+    default void handle(int kindOrdinal, String rootPath, String relativePath) {
+        if (kindOrdinal == Kind.OVERFLOW.ordinal()) {
+            handle(OVERFLOW, null);
+        } else {
+            var context = Path.of(rootPath).relativize(Path.of(relativePath));
+            var kind =
+                kindOrdinal == Kind.CREATE.ordinal() ? ENTRY_CREATE :
+                kindOrdinal == Kind.MODIFY.ordinal() ? ENTRY_MODIFY :
+                kindOrdinal == Kind.DELETE.ordinal() ? ENTRY_DELETE : null;
+
+            if (kind != null) {
+                handle(kind, context);
+            }
+        }
+    }
+}
+
+enum Kind {
+    OVERFLOW,
+    CREATE,
+    DELETE,
+    MODIFY;
 }

src/main/java/engineering/swat/watch/impl/mac/NativeEventStream.java

@@ -26,40 +26,9 @@
  */
 package engineering.swat.watch.impl.mac;
 
-import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamCreateFlag.FILE_EVENTS;
-import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamCreateFlag.NO_DEFER;
-import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamCreateFlag.WATCH_ROOT;
-import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamEventFlag.ITEM_CREATED;
-import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamEventFlag.ITEM_INODE_META_MOD;
-import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamEventFlag.ITEM_MODIFIED;
-import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamEventFlag.ITEM_REMOVED;
-import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamEventFlag.ITEM_RENAMED;
-import static engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamEventFlag.MUST_SCAN_SUB_DIRS;
-import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
-import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
-import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
-import static java.nio.file.StandardWatchEventKinds.OVERFLOW;
-
 import java.io.Closeable;
 import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
-
-import org.checkerframework.checker.nullness.qual.Nullable;
-
-import com.sun.jna.Memory;
-import com.sun.jna.Native;
-import com.sun.jna.Pointer;
-import com.sun.jna.platform.mac.CoreFoundation;
-import com.sun.jna.platform.mac.CoreFoundation.CFArrayRef;
-import com.sun.jna.platform.mac.CoreFoundation.CFIndex;
-import com.sun.jna.platform.mac.CoreFoundation.CFStringRef;
-
-import engineering.swat.watch.impl.mac.apis.DispatchObjects;
-import engineering.swat.watch.impl.mac.apis.DispatchQueue;
-import engineering.swat.watch.impl.mac.apis.FileSystemEvents;
-import engineering.swat.watch.impl.mac.apis.FileSystemEvents.FSEventStreamCallback;
 
 // Note: This file is designed to be the only place in this package where JNA is
 // used and/or the native APIs are invoked. If the need to do so arises outside
@@ -79,23 +48,14 @@
  * </p>
  */
 class NativeEventStream implements Closeable {
-
-    // Native APIs
-    private static final CoreFoundation   CF  = CoreFoundation.INSTANCE;
-    private static final DispatchObjects  DO  = DispatchObjects.INSTANCE;
-    private static final DispatchQueue    DQ  = DispatchQueue.INSTANCE;
-    private static final FileSystemEvents FSE = FileSystemEvents.INSTANCE;
-
-    // Native memory
-    private @Nullable FSEventStreamCallback callback; // Keep reference to avoid premature GC'ing
-    private @Nullable Pointer stream;
-    private @Nullable Pointer queue;
-    // Note: These fields aren't volatile, as all reads/write from/to them are
-    // inside synchronized blocks. Be careful to not break this invariant.
+    static {
+        NativeLibrary.load();
+    }
 
     private final Path path;
     private final NativeEventHandler handler;
     private volatile boolean closed;
+    private volatile long nativeWatch;
 
     public NativeEventStream(Path path, NativeEventHandler handler) throws IOException {
         this.path = path.toRealPath(); // Resolve symbolic links
@@ -110,88 +70,7 @@ public synchronized void open() {
             closed = false;
         }
 
-        // Allocate native memory
-        callback = createCallback();
-        stream = createFSEventStream(callback);
-        queue = createDispatchQueue();
-
-        // Start the stream
-        var streamNonNull = stream;
-        if (streamNonNull != null) {
-            FSE.FSEventStreamSetDispatchQueue(streamNonNull, queue);
-            FSE.FSEventStreamStart(streamNonNull);
-        }
-    }
-
-    private FSEventStreamCallback createCallback() {
-        return new FSEventStreamCallback() {
-            @Override
-            public void callback(Pointer streamRef, Pointer clientCallBackInfo,
-                    long numEvents, Pointer eventPaths, Pointer eventFlags, Pointer eventIds) {
-                // This function is called each time native events are issued by
-                // macOS. The purpose of this function is to perform the minimal
-                // amount of processing to hide the native APIs from downstream
-                // consumers, who are offered native events via `handler`.
-
-                var paths = eventPaths.getStringArray(0, (int) numEvents);
-                var flags = eventFlags.getIntArray(0, (int) numEvents);
-
-                for (var i = 0; i < numEvents; i++) {
-                    var context = path.relativize(Path.of(paths[i]));
-
-                    // Note: Multiple "physical" native events might be
-                    // coalesced into a single "logical" native event, so the
-                    // following series of checks should be if-statements
-                    // (instead of if/else-statements).
-                    if (any(flags[i], ITEM_CREATED.mask)) {
-                        handler.handle(ENTRY_CREATE, context);
-                    }
-                    if (any(flags[i], ITEM_REMOVED.mask)) {
-                        handler.handle(ENTRY_DELETE, context);
-                    }
-                    if (any(flags[i], ITEM_MODIFIED.mask | ITEM_INODE_META_MOD.mask)) {
-                        handler.handle(ENTRY_MODIFY, context);
-                    }
-                    if (any(flags[i], MUST_SCAN_SUB_DIRS.mask)) {
-                        handler.handle(OVERFLOW, null);
-                    }
-                    if (any(flags[i], ITEM_RENAMED.mask)) {
-                        // For now, check if the file exists to determine if the
-                        // event pertains to the target of the rename (if it
-                        // exists) or to the source (else). This is an
-                        // approximation. It might be more accurate to maintain
-                        // an internal index (but getting the concurrency right
-                        // requires care).
-                        if (Files.exists(Path.of(paths[i]))) {
-                            handler.handle(ENTRY_CREATE, context);
-                        } else {
-                            handler.handle(ENTRY_DELETE, context);
-                        }
-                    }
-                }
-            }
-
-            private boolean any(int bits, int mask) {
-                return (bits & mask) != 0;
-            }
-        };
-    }
-
-    private Pointer createFSEventStream(FSEventStreamCallback callback) {
-        try (var pathsToWatch = new Strings(path.toString())) {
-            var allocator = CF.CFAllocatorGetDefault();
-            var context   = Pointer.NULL;
-            var sinceWhen = FSE.FSEventsGetCurrentEventId();
-            var latency   = 0.15;
-            var flags     = NO_DEFER.mask | WATCH_ROOT.mask | FILE_EVENTS.mask;
-            return FSE.FSEventStreamCreate(allocator, callback, context, pathsToWatch.toCFArray(), sinceWhen, latency, flags);
-        }
-    }
-
-    private Pointer createDispatchQueue() {
-        var label = "engineering.swat.watch";
-        var attr  = Pointer.NULL;
-        return DQ.dispatch_queue_create(label, attr);
+        nativeWatch = NativeLibrary.start(path.toString(), handler);
     }
 
     // -- Closeable --
@@ -204,97 +83,6 @@ public synchronized void close() {
             closed = true;
         }
 
-        var streamNonNull = stream;
-        var queueNonNull = queue;
-        if (streamNonNull != null && queueNonNull != null) {
-
-            // Stop the stream
-            FSE.FSEventStreamStop(streamNonNull);
-            FSE.FSEventStreamSetDispatchQueue(streamNonNull, Pointer.NULL);
-            FSE.FSEventStreamInvalidate(streamNonNull);
-
-            // Deallocate native memory
-            DO.dispatch_release(queueNonNull);
-            FSE.FSEventStreamRelease(streamNonNull);
-            queue = null;
-            stream = null;
-            callback = null;
-        }
-    }
-}
-
-/**
- * Array of strings in native memory, needed to create a new native event stream
- * (i.e., the {@code pathsToWatch} argument of {@code FSEventStreamCreate} is an
- * array of strings).
- */
-class Strings implements AutoCloseable {
-
-    // Native APIs
-    private static final CoreFoundation CF = CoreFoundation.INSTANCE;
-
-    // Native memory
-    private final CFStringRef[] strings;
-    private final CFArrayRef array;
-
-    private volatile boolean closed = false;
-
-    public Strings(String... strings) {
-        // Allocate native memory
-        this.strings = createCFStrings(strings);
-        this.array = createCFArray(this.strings);
-    }
-
-    public CFArrayRef toCFArray() {
-        if (closed) {
-            throw new IllegalStateException("Strings are already deallocated");
-        } else {
-            return array;
-        }
-    }
-
-    private static CFStringRef[] createCFStrings(String[] pathsToWatch) {
-        return Arrays.stream(pathsToWatch)
-            .map(CFStringRef::createCFString)
-            .toArray(CFStringRef[]::new);
-    }
-
-    private static CFArrayRef createCFArray(CFStringRef[] strings) {
-        var n = strings.length;
-        var size = Native.getNativeSize(CFStringRef.class);
-
-        // Create a temporary array of pointers to the strings (automatically
-        // freed when `values` goes out of scope)
-        var values = new Memory(n * size);
-        for (int i = 0; i < n; i++) {
-            values.setPointer(i * size, strings[i].getPointer());
-        }
-
-        // Create a permanent array based on the temporary array
-        var alloc = CF.CFAllocatorGetDefault();
-        var numValues = new CFIndex(n);
-        var callBacks = Pointer.NULL;
-        return CF.CFArrayCreate(alloc, values, numValues, callBacks);
-    }
-
-    // -- AutoCloseable --
-
-    @Override
-    public void close() {
-        if (closed) {
-            throw new IllegalStateException("Strings are already deallocated");
-        } else {
-            closed = true;
-        }
-
-        // Deallocate native memory
-        for (var s : strings) {
-            if (s != null) {
-                s.release();
-            }
-        }
-        if (array != null) {
-            array.release();
-        }
+        NativeLibrary.stop(nativeWatch);
     }
 }