fsmonitor: wait for watchman full crawl and show progress

Summary:
If watchman is in full crawl, it probably won't respond in time, wait for it
instead of falling back to a non-watchman crawl.

As we're here, show a progress bar so users have some expectation about what's
going on.

Reviewed By: jordanwebster

Differential Revision: D37473141

fbshipit-source-id: 979760d3bd9ab5d8a222ade5555acdcf66995b10

Author: quark-zju

eden/scm/edenscm/hgext/extlib/watchmanclient/__init__.py

@@ -11,9 +11,11 @@
 import getpass
 import os
 import sys
+import time
 
 from bindings import tracing
 from edenscm.mercurial import blackbox, encoding, json, progress, pycompat, util
+from edenscm.mercurial.i18n import _
 from edenscm.mercurial.node import hex
 
 from .. import pywatchman
@@ -101,6 +103,7 @@ def __init__(self, repo, timeout=1.0):
         self._resolved_root = getcanonicalpath(self._root)
         self._ui = repo.ui
         self._firsttime = True
+        self._approx_total_file_count = len(repo.dirstate._map)
 
     def settimeout(self, timeout):
         self._timeout = timeout
@@ -197,9 +200,97 @@ def _retrycommand(self, span, retry, *args):
         if needretry:
             return self._retrycommand(span, retry + 1, *args)
 
+    def debug_status(self):
+        """Return the RootDebugStatus, which might look like:
+
+            {
+                "recrawl_info": {
+                    "count": 0,
+                    "should-recrawl": true,
+                    "warning": null,
+                    "reason": "startup",
+                    "completed": null,
+                    "started": -20162,
+                    "stats": 295541
+                }
+                "crawl-status": "crawling for ...",
+                "enable_parallel_crawl": false,
+                "cookie_list": [],
+                "path": "...",
+                "queries": [],
+                "fstype": "btrfs",
+                "cookie_prefix": ["..."],
+                "watcher": "inotify",
+                "uptime": ...,
+                "done_initial": false,
+                "cookie_dir": [".../.hg"],
+                "case_sensitive": true,
+                "cancelled": false,
+            }
+
+        Return an empty dict if watchman does not support debug-root-status.
+        """
+        try:
+            # use _command to bypass progress and util.timefuntion
+            root_status = self._command("debug-root-status")["root_status"]
+        except (Unavailable, AttributeError):
+            # watchman does not support this command
+            root_status = {}
+        return root_status
+
+    def recrawl_info(self):
+        """Return the RootRecrawlInfo in the RootDebugStatus.
+
+        Return an empty dict if watchman does not support getting the recrawl
+        info.
+        """
+        debug_status = self.debug_status()
+        info = {}
+        try:
+            info = debug_status["recrawl_info"]
+        except AttributeError:
+            pass
+        return info
+
+    def recrawl_stat_count(self):
+        """Return the count of files stat()-ed by watchman during a full crawl
+        Return None if watchman does not provide the information, or watchman
+        is not in a full crawl state.
+        """
+        stats = None
+        try:
+            stats = self.recrawl_info()["stats"]
+        except AttributeError:
+            pass
+        return stats
+
+    def wait_for_full_crawl(self):
+        """Wait for watchman to complete a full recrawl. Blocking.
+        Show a progress bar.
+        """
+        ui = self._ui
+        if not ui.configbool("fsmonitor", "wait-full-crawl"):
+            return
+
+        stats = self.recrawl_stat_count()
+        if stats is None:
+            # Not in full recrawl
+            return
+
+        # Show progress bar.
+        total = self._approx_total_file_count
+        with progress.bar(ui, _("crawling"), _("files (approx)"), total) as prog:
+            while stats is not None:
+                prog.value = stats
+                stats = self.recrawl_stat_count()
+                if stats is not None:
+                    time.sleep(0.1)
+
     @util.timefunction("watchmanquery", 0, "_ui")
     def command(self, *args, **kwargs):
         ignoreerrors = kwargs.get("ignoreerrors", False)
+        if args and args[0] in {"clock", "query"}:
+            self.wait_for_full_crawl()
         with progress.spinner(self._ui, "querying watchman"):
             try:
                 try:

eden/scm/edenscm/hgext/fsmonitor/__init__.py

@@ -152,6 +152,12 @@
 
 If set to true then take a lock when running watchman queries to avoid
 overloading watchman.
+
+::
+    [fsmonitor]
+    wait-full-crawl = true
+
+If set, wait for watchman to complete a full crawl before performing queries.
 """
 
 # Platforms Supported
@@ -243,6 +249,7 @@
 configitem("fsmonitor", "tcp-host", default="::1")
 configitem("fsmonitor", "tcp-port", default=12300)
 configitem("fsmonitor", "watchman-query-lock", default=False)
+configitem("fsmonitor", "wait-full-crawl", default=True)
 
 # This extension is incompatible with the following incompatible extensions
 # and will disable itself when encountering one of these: