Streamline mount/unmount for dehydrating folders

In microsoft#1890 I demonstrated that it's possible to dehydrate folders
without unmounting at all. Unfortunately that requires deleting all the
placeholders and hydrated files which is too slow for dehydrating
anything substantial. The current implementation avoids this by moving
instead of deleting (which has the additional benefit of providing a
backup) but ProjFS doesn't support moving or renaming folders, so we
can't do that while mounted.

This pull request takes a different approach to reducing the overhead
of unmounting for dehydration. Instead of unmounting, moving, and
remounting from the dehydrate verb, those steps are moved into the
mount process under its dehydrate message handler. The mount process
only disposes and recreates the components required for virtualization,
avoiding several costly steps (eg authentication with Azure DevOps,
verification of the cache server, verification of ProjFS installation).
For the repo I work in, dehydrating a top-level directory is reduced
from 33 seconds to 11 seconds with this change.

Specific changes:
* Backup of non-src folders (.git, .gvfs) is added to dehydrate folders.
Previously it was only done for full dehydrate.
* Unmount, move/backup of folders, and mount are moved from
DehydrateVerb to InProcessMount. To support this, the DehydrateFolders
message has the backup folder added to its fields.
* The core methods of Mount and Unmount have a parameter added to skip
disposing (on unmount) and initialization (on mount) of certain
components which are ok to leave alive during the temporary unmount.
* Ownership of GVFSContext disposal fixed - FileSystemCallbacks was
disposing it despite not owning it.
* Missing disposal of a file stream in BackgroundFileSystemTaskRunner
is fixed.
* WindowsFileSystemVirtualizer.DehydrateFolder will now delete a
tombstone file for the directory if present. This allows us to support
fixing a directory that the user manually deleted while mounted (perhaps
in a misguided attempt to dehydrate it), though that would typically
require running 'gvfs dehydrate --no-status' to skip verifying that the
working directory matches the index.
* '--no-status' is now supported with '--folders'

Author: tyrielv

GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs

@@ -49,6 +49,11 @@ public virtual void DeleteDirectory(string path, bool recursive = true, bool ign
             }
         }
 
+        public virtual void MoveDirectory(string sourceDirName, string destDirName)
+        {
+            Directory.Move(sourceDirName, destDirName);
+        }
+
         public virtual void CopyDirectoryRecursive(
             string srcDirectoryPath,
             string dstDirectoryPath,

GVFS/GVFS.Common/NamedPipes/NamedPipeMessages.cs

@@ -203,21 +203,24 @@ public static class DehydrateFolders
 
             public class Request
             {
-                public Request(string folders)
+                public Request(string backupFolderPath, string folders)
                 {
                     this.Folders = folders;
+                    this.BackupFolderPath = backupFolderPath;
                 }
 
-                public Request(Message message)
+                public static Request FromMessage(Message message)
                 {
-                    this.Folders = message.Body;
+                    return JsonConvert.DeserializeObject<Request>(message.Body);
                 }
 
                 public string Folders { get; }
 
+                public string BackupFolderPath { get; }
+
                 public Message CreateMessage()
                 {
-                    return new Message(Dehydrate, this.Folders);
+                    return new Message(Dehydrate, JsonConvert.SerializeObject(this));
                 }
             }
 

GVFS/GVFS.Mount/InProcessMount.cs

@@ -295,7 +295,7 @@ private void HandleRequest(ITracer tracer, string request, NamedPipeServer.Conne
 
         private void HandleDehydrateFolders(NamedPipeMessages.Message message, NamedPipeServer.Connection connection)
         {
-            NamedPipeMessages.DehydrateFolders.Request request = new NamedPipeMessages.DehydrateFolders.Request(message);
+            NamedPipeMessages.DehydrateFolders.Request request = NamedPipeMessages.DehydrateFolders.Request.FromMessage(message);
 
             EventMetadata metadata = new EventMetadata();
             metadata.Add(nameof(request.Folders), request.Folders);
@@ -308,7 +308,9 @@ private void HandleDehydrateFolders(NamedPipeMessages.Message message, NamedPipe
                 response = new NamedPipeMessages.DehydrateFolders.Response(NamedPipeMessages.DehydrateFolders.DehydratedResult);
                 string[] folders = request.Folders.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
                 StringBuilder resetFolderPaths = new StringBuilder();
-                foreach (string folder in folders)
+                List<string> movedFolders = BackupFoldersWhileUnmounted(request, response, folders);
+
+                foreach (string folder in movedFolders)
                 {
                     if (this.fileSystemCallbacks.TryDehydrateFolder(folder, out string errorMessage))
                     {
@@ -357,6 +359,50 @@ private void HandleDehydrateFolders(NamedPipeMessages.Message message, NamedPipe
             connection.TrySendResponse(response.CreateMessage());
         }
 
+        private List<string> BackupFoldersWhileUnmounted(NamedPipeMessages.DehydrateFolders.Request request, NamedPipeMessages.DehydrateFolders.Response response, string[] folders)
+        {
+            /* We can't move folders while the virtual file system is mounted, so unmount it first.
+             * After moving the folders, remount the virtual file system.
+             */
+
+            var movedFolders = new List<string>();
+            try
+            {
+                /* Set to "Mounting" instead of "Unmounting" so that incoming requests
+                 * that are rejected will know they can try again soon.
+                 */
+                this.currentState = MountState.Mounting;
+                this.UnmountAndStopWorkingDirectoryCallbacks(willRemountInSameProcess: true);
+                foreach (string folder in folders)
+                {
+                    try
+                    {
+                        var source = Path.Combine(this.enlistment.WorkingDirectoryBackingRoot, folder);
+                        var destination = Path.Combine(request.BackupFolderPath, folder);
+                        var destinationParent = Path.GetDirectoryName(destination);
+                        this.context.FileSystem.CreateDirectory(destinationParent);
+                        if (this.context.FileSystem.DirectoryExists(source))
+                        {
+                            this.context.FileSystem.MoveDirectory(source, destination);
+                        }
+                        movedFolders.Add(folder);
+                    }
+                    catch (Exception ex)
+                    {
+                        response.FailedFolders.Add($"{folder}\0{ex.Message}");
+                        continue;
+                    }
+                }
+            }
+            finally
+            {
+                this.MountAndStartWorkingDirectoryCallbacks(this.cacheServer, alreadyInitialized: true);
+                this.currentState = MountState.Ready;
+            }
+
+            return movedFolders;
+        }
+
         private void HandleLockRequest(string messageBody, NamedPipeServer.Connection connection)
         {
             NamedPipeMessages.AcquireLock.Response response;
@@ -551,9 +597,9 @@ private void HandleDownloadObjectRequest(NamedPipeMessages.Message message, Name
                         /* If a commit is downloaded, it wasn't prefetched.
                          * If any prefetch has been done, there is probably a commit in the prefetch packs that is close enough that
                          * loose object download of missing trees will be faster than downloading a pack of all the trees for the commit.
-                         * Otherwise, the trees for the commit may be needed soon depending on the context. 
+                         * Otherwise, the trees for the commit may be needed soon depending on the context.
                          * e.g. git log (without a pathspec) doesn't need trees, but git checkout does.
-                         * 
+                         *
                          * Save the tree/commit so if more trees are requested we can download all the trees for the commit in a batch.
                          */
                         this.treesWithDownloadedCommits[treeSha] = objectSha;
@@ -596,7 +642,7 @@ private bool ShouldDownloadCommitPack(string objectSha, out string commitSha)
         private void UpdateTreesForDownloadedCommits(string objectSha)
         {
             /* If we are downloading missing trees, we probably are missing more trees for the commit.
-             * Update our list of trees associated with the commit so we can use the # of missing trees 
+             * Update our list of trees associated with the commit so we can use the # of missing trees
              * as a heuristic to decide whether to batch download all the trees for the commit the
              * next time a missing one is requested.
              */
@@ -723,12 +769,15 @@ private void HandleUnmountRequest(NamedPipeServer.Connection connection)
             }
         }
 
-        private void MountAndStartWorkingDirectoryCallbacks(CacheServerInfo cache)
+        private void MountAndStartWorkingDirectoryCallbacks(CacheServerInfo cache, bool alreadyInitialized = false)
         {
             string error;
-            if (!this.context.Enlistment.Authentication.TryInitialize(this.context.Tracer, this.context.Enlistment, out error))
+            if (!alreadyInitialized)
             {
-                this.FailMountAndExit("Failed to obtain git credentials: " + error);
+                if (!this.context.Enlistment.Authentication.TryInitialize(this.context.Tracer, this.context.Enlistment, out error))
+                {
+                    this.FailMountAndExit("Failed to obtain git credentials: " + error);
+                }
             }
 
             GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(this.context.Tracer, this.context.Enlistment, cache, this.retryConfig);
@@ -763,19 +812,22 @@ private void MountAndStartWorkingDirectoryCallbacks(CacheServerInfo cache)
                 }, "Failed to create src folder callback listener");
             this.maintenanceScheduler = this.CreateOrReportAndExit(() => new GitMaintenanceScheduler(this.context, this.gitObjects), "Failed to start maintenance scheduler");
 
-            int majorVersion;
-            int minorVersion;
-            if (!RepoMetadata.Instance.TryGetOnDiskLayoutVersion(out majorVersion, out minorVersion, out error))
+            if (!alreadyInitialized)
             {
-                this.FailMountAndExit("Error: {0}", error);
-            }
+                int majorVersion;
+                int minorVersion;
+                if (!RepoMetadata.Instance.TryGetOnDiskLayoutVersion(out majorVersion, out minorVersion, out error))
+                {
+                    this.FailMountAndExit("Error: {0}", error);
+                }
 
-            if (majorVersion != GVFSPlatform.Instance.DiskLayoutUpgrade.Version.CurrentMajorVersion)
-            {
-                this.FailMountAndExit(
-                    "Error: On disk version ({0}) does not match current version ({1})",
-                    majorVersion,
-                    GVFSPlatform.Instance.DiskLayoutUpgrade.Version.CurrentMajorVersion);
+                if (majorVersion != GVFSPlatform.Instance.DiskLayoutUpgrade.Version.CurrentMajorVersion)
+                {
+                    this.FailMountAndExit(
+                        "Error: On disk version ({0}) does not match current version ({1})",
+                        majorVersion,
+                        GVFSPlatform.Instance.DiskLayoutUpgrade.Version.CurrentMajorVersion);
+                }
             }
 
             try
@@ -794,7 +846,7 @@ private void MountAndStartWorkingDirectoryCallbacks(CacheServerInfo cache)
             this.heartbeat.Start();
         }
 
-        private void UnmountAndStopWorkingDirectoryCallbacks()
+        private void UnmountAndStopWorkingDirectoryCallbacks(bool willRemountInSameProcess = false)
         {
             if (this.maintenanceScheduler != null)
             {
@@ -817,6 +869,12 @@ private void UnmountAndStopWorkingDirectoryCallbacks()
 
             this.gvfsDatabase?.Dispose();
             this.gvfsDatabase = null;
+
+            if (!willRemountInSameProcess)
+            {
+                this.context?.Dispose();
+                this.context = null;
+            }
         }
     }
 }

GVFS/GVFS.Virtualization/Background/BackgroundFileSystemTaskRunner.cs

@@ -131,6 +131,11 @@ protected void Dispose(bool disposing)
                     this.backgroundThread.Dispose();
                     this.backgroundThread = null;
                 }
+                if (this.backgroundTasks != null)
+                {
+                    this.backgroundTasks.Dispose();
+                    this.backgroundTasks = null;
+                }
             }
         }
 

GVFS/GVFS.Virtualization/FileSystemCallbacks.cs

@@ -227,12 +227,6 @@ public void Dispose()
                 this.backgroundFileSystemTaskRunner.Dispose();
                 this.backgroundFileSystemTaskRunner = null;
             }
-
-            if (this.context != null)
-            {
-                this.context.Dispose();
-                this.context = null;
-            }
         }
 
         public bool IsReadyForExternalAcquireLockRequests(NamedPipeMessages.LockData requester, out string denyMessage)