Fix extension comparison (#43877)

Author: tmat

src/BuiltInTools/DotNetDeltaApplier/StartupHook.cs

@@ -23,11 +23,9 @@ public static void Initialize()
         // When launching the application process dotnet-watch sets Hot Reload environment variables via CLI environment directives (dotnet [env:X=Y] run).
         // Currently, the CLI parser sets the env variables to the dotnet.exe process itself, rather then to the target process.
         // This may cause the dotnet.exe process to connect to the named pipe and break it for the target process.
-        var processExe = Path.ChangeExtension(processPath, ".exe");
-        var expectedExe = Path.ChangeExtension(s_targetProcessPath, ".exe");
-        if (!string.Equals(processExe, expectedExe, Path.DirectorySeparatorChar == '\\' ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal))
+        if (!IsMatchingProcess(processPath, s_targetProcessPath))
         {
-            Log($"Ignoring process '{processExe}', expecting '{expectedExe}'");
+            Log($"Ignoring process '{processPath}', expecting '{s_targetProcessPath}'");
             return;
         }
 
@@ -67,6 +65,24 @@ public static void Initialize()
         });
     }
 
+    public static bool IsMatchingProcess(string processPath, string targetProcessPath)
+    {
+        var comparison = Path.DirectorySeparatorChar == '\\' ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
+        var (shorter, longer) = (processPath.Length > targetProcessPath.Length) ? (targetProcessPath, processPath) : (processPath, targetProcessPath);
+
+        // one or both have no extension, or they have the same extension
+        if (longer.StartsWith(shorter, comparison))
+        {
+            var suffix = longer[shorter.Length..];
+            return suffix is "" || suffix.Equals(".exe", comparison) || suffix.Equals(".dll", comparison);
+        }
+
+        // different extension:
+        return (processPath.EndsWith(".exe", comparison) || processPath.EndsWith(".dll", comparison)) &&
+               (targetProcessPath.EndsWith(".exe", comparison) || targetProcessPath.EndsWith(".dll", comparison)) &&
+               string.Equals(processPath[..^4], targetProcessPath[..^4], comparison);
+    }
+
     internal static void ClearHotReloadEnvironmentVariables()
     {
         // Clear any hot-reload specific environment variables. This prevents child processes from being

test/Microsoft.Extensions.DotNetDeltaApplier.Tests/Microsoft.Extensions.DotNetDeltaApplier.Tests.csproj

@@ -11,6 +11,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\src\BuiltInTools\DotNetDeltaApplier\Microsoft.Extensions.DotNetDeltaApplier.csproj" />
     <ProjectReference Include="..\Microsoft.NET.TestFramework\Microsoft.NET.TestFramework.csproj" />
+    <PackageReference Include="Xunit.Combinatorial" />
     <PackageReference Include="Moq" />
   </ItemGroup>
 

test/Microsoft.Extensions.DotNetDeltaApplier.Tests/StartupHookTests.cs

@@ -24,5 +24,60 @@ public void ClearHotReloadEnvironmentVariables_RemovesHotReloadStartup_InCaseInv
             var customStartupHook = "/path/mycoolstartup.dll";
             Assert.Equal(customStartupHook, StartupHook.RemoveCurrentAssembly(customStartupHook + Path.PathSeparator + typeof(StartupHook).Assembly.Location.ToUpperInvariant()));
         }
+
+        [Theory]
+        [CombinatorialData]
+        public void IsMatchingProcess_Matching_SimpleName(
+            [CombinatorialValues("", ".dll", ".exe")] string extension,
+            [CombinatorialValues("", ".dll", ".exe")] string targetExtension)
+        {
+            var dir = Path.GetDirectoryName(typeof(StartupHookTests).Assembly.Location)!;
+            var name = "a";
+            var processPath = Path.Combine(dir, name + extension);
+            var targetProcessPath = Path.Combine(dir, "a" + targetExtension);
+
+            Assert.True(StartupHook.IsMatchingProcess(processPath, targetProcessPath));
+        }
+
+        [Theory]
+        [CombinatorialData]
+        public void IsMatchingProcess_Matching_DotInName(
+            [CombinatorialValues("", ".dll", ".exe")] string extension,
+            [CombinatorialValues("", ".dll", ".exe")] string targetExtension)
+        {
+            var dir = Path.GetDirectoryName(typeof(StartupHookTests).Assembly.Location)!;
+            var name = "a.b";
+            var processPath = Path.Combine(dir, name + extension);
+            var targetProcessPath = Path.Combine(dir, name + targetExtension);
+
+            Assert.True(StartupHook.IsMatchingProcess(processPath, targetProcessPath));
+        }
+
+        [Theory]
+        [CombinatorialData]
+        public void IsMatchingProcess_Matching_DotDllInName(
+            [CombinatorialValues("", ".dll", ".exe")] string extension,
+            [CombinatorialValues("", ".dll", ".exe")] string targetExtension)
+        {
+            var dir = Path.GetDirectoryName(typeof(StartupHookTests).Assembly.Location)!;
+            var name = "a.dll";
+            var processPath = Path.Combine(dir, name + extension);
+            var targetProcessPath = Path.Combine(dir, name + targetExtension);
+
+            Assert.True(StartupHook.IsMatchingProcess(processPath, targetProcessPath));
+        }
+
+        [Theory]
+        [CombinatorialData]
+        public void IsMatchingProcess_NotMatching(
+            [CombinatorialValues("", ".dll", ".exe")] string extension,
+            [CombinatorialValues("", ".dll", ".exe")] string targetExtension)
+        {
+            var dir = Path.GetDirectoryName(typeof(StartupHookTests).Assembly.Location)!;
+            var processPath = Path.Combine(dir, "a" + extension);
+            var targetProcessPath = Path.Combine(dir, "b" + targetExtension);
+
+            Assert.False(StartupHook.IsMatchingProcess(processPath, targetProcessPath));
+        }
     }
 }