watch: use sinceWhen and HistoryDone to avoid spurious events (docker#557)

Here's our new watch strategy on Darwin in a nutshell:

1. Create an fsevents stream for events "since" the last event ID that
we saw globally.
2. Add a path that we want to watch
3. Add that path to a map of paths that we're watching _directly_.
4. Restart the event stream to pick up the new path.
5. Ignore all events for all watches until we've seen a `HistoryDone`
event.
6. Ignore the first `ItemCreated` event for paths we're watching that
are also directories
7. Otherwise, forward along all events.

Author: Dan Miller

pkg/watch/notify_test.go

@@ -303,6 +303,40 @@ func TestWatchBrokenLink(t *testing.T) {
 	f.assertEvents(link)
 }
 
+func TestMoveAndReplace(t *testing.T) {
+	f := newNotifyFixture(t)
+	defer f.tearDown()
+
+	root, err := f.root.NewDir("root")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	file := filepath.Join(root.Path(), "myfile")
+	err = ioutil.WriteFile(file, []byte("hello"), 0777)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = f.notify.Add(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tmpFile := filepath.Join(root.Path(), ".myfile.swp")
+	err = ioutil.WriteFile(tmpFile, []byte("world"), 0777)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = os.Rename(tmpFile, file)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f.assertEvents(file)
+}
+
 type notifyFixture struct {
 	t       *testing.T
 	root    *TempDir

pkg/watch/watcher_darwin.go

@@ -21,9 +21,8 @@ type darwinNotify struct {
 	// change.
 	sm *sync.Mutex
 
-	// When a watch is created for a directory, we've seen fsevents non-determistically
-	// fire 0-3 CREATE events for that directory. We want to ignore these.
-	ignoreCreatedEvents map[string]bool
+	pathsWereWatching map[string]interface{}
+	sawAnyHistoryDone bool
 }
 
 func (d *darwinNotify) loop() {
@@ -39,20 +38,24 @@ func (d *darwinNotify) loop() {
 			for _, e := range events {
 				e.Path = filepath.Join("/", e.Path)
 
-				if e.Flags&fsevents.ItemCreated == fsevents.ItemCreated {
+				if e.Flags&fsevents.HistoryDone == fsevents.HistoryDone {
 					d.sm.Lock()
-					shouldIgnore := d.ignoreCreatedEvents[e.Path]
-					if !shouldIgnore {
-						// If we got a created event for something
-						// that's not on the ignore list, we assume
-						// we're done with the spurious events.
-						d.ignoreCreatedEvents = nil
-					}
+					d.sawAnyHistoryDone = true
 					d.sm.Unlock()
+					continue
+				}
+
+				// We wait until we've seen the HistoryDone event for this watcher before processing any events
+				// so that we skip all of the "spurious" events that precede it.
+				if !d.sawAnyHistoryDone {
+					continue
+				}
 
-					if shouldIgnore {
-						continue
-					}
+				_, isPathWereWatching := d.pathsWereWatching[e.Path]
+				if e.Flags&fsevents.ItemIsDir == fsevents.ItemIsDir && e.Flags&fsevents.ItemCreated == fsevents.ItemCreated && isPathWereWatching {
+					// This is the first create for the path that we're watching. We always get exactly one of these
+					// even after we get the HistoryDone event. Skip it.
+					continue
 				}
 
 				d.events <- FileEvent{
@@ -80,10 +83,10 @@ func (d *darwinNotify) Add(name string) error {
 
 	es.Paths = append(es.Paths, name)
 
-	if d.ignoreCreatedEvents == nil {
-		d.ignoreCreatedEvents = make(map[string]bool, 1)
+	if d.pathsWereWatching == nil {
+		d.pathsWereWatching = make(map[string]interface{})
 	}
-	d.ignoreCreatedEvents[name] = true
+	d.pathsWereWatching[name] = struct{}{}
 
 	if len(es.Paths) == 1 {
 		es.Start()
@@ -119,6 +122,9 @@ func NewWatcher() (Notify, error) {
 		stream: &fsevents.EventStream{
 			Latency: 1 * time.Millisecond,
 			Flags:   fsevents.FileEvents,
+			// NOTE(dmiller): this corresponds to the `sinceWhen` parameter in FSEventStreamCreate
+			// https://developer.apple.com/documentation/coreservices/1443980-fseventstreamcreate
+			EventID: fsevents.LatestEventID(),
 		},
 		sm:     &sync.Mutex{},
 		events: make(chan FileEvent),