Allow live debugging tasks with actual sources (#19978)

This change makes it much easier to troubleshoot problems with tasks or develop new features. It makes it possible to connect to a specific task which is being executed by the agent and debug this in Visual Studio Code.

Author: kboom

BuildConfigGen/Debugging/DebugConfigGenerator.cs

@@ -0,0 +1,29 @@
+namespace BuildConfigGen.Debugging
+{
+    internal interface IDebugConfigGenerator
+    {
+        void WriteTypescriptConfig(string taskOutput);
+
+        void AddForTask(string taskConfigPath);
+
+        void WriteLaunchConfigurations();
+    }
+
+    sealed internal class NoDebugConfigGenerator : IDebugConfigGenerator
+    {
+        public void AddForTask(string taskConfigPath)
+        {
+            // noop
+        }
+
+        public void WriteLaunchConfigurations()
+        {
+            // noop
+        }
+
+        public void WriteTypescriptConfig(string taskOutput)
+        {
+            // noop
+        }
+    }
+}

BuildConfigGen/Debugging/VsCodeLaunchConfigGenerator.cs

@@ -0,0 +1,87 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+namespace BuildConfigGen.Debugging
+{
+    internal class VsCodeLaunchConfigGenerator : IDebugConfigGenerator
+    {
+        private string GitRootPath { get; }
+
+        private string AgentPath { get; }
+
+        private string LaunchConfigPath => Path.Combine(GitRootPath, ".vscode", "launch.json");
+
+        private VsCodeLaunchConfiguration LaunchConfig { get; }
+
+        public VsCodeLaunchConfigGenerator(string gitRootPath, string agentPath)
+        {
+            ArgumentException.ThrowIfNullOrEmpty(agentPath, nameof(agentPath));
+            ArgumentException.ThrowIfNullOrEmpty(gitRootPath, nameof(gitRootPath));
+
+            if (!Directory.Exists(agentPath))
+            {
+                throw new ArgumentException($"Agent directory used for debugging could not be found at {Path.GetFullPath(agentPath)}!");
+            }
+
+            AgentPath = agentPath;
+            GitRootPath = gitRootPath;
+            LaunchConfig = VsCodeLaunchConfiguration.ReadFromFileIfPresentOrDefault(LaunchConfigPath);
+        }
+
+        public void AddForTask(string taskConfigPath)
+        {
+            if (!File.Exists(taskConfigPath))
+            {
+                throw new ArgumentException($"Task configuration (task.json) does not exist at path {taskConfigPath}!");
+            }
+
+            var taskContent = File.ReadAllText(taskConfigPath);
+            var taskConfig = JsonNode.Parse(taskContent)!;
+
+            JsonNode versionNode = taskConfig["version"]!;
+            int major = versionNode["Major"]!.GetValue<int>();
+            int minor = versionNode["Minor"]!.GetValue<int>();
+            int patch = versionNode["Patch"]!.GetValue<int>();
+
+            var version = new TaskVersion(major, minor, patch);
+
+            LaunchConfig.AddConfigForTask(
+                taskId: taskConfig["id"]!.GetValue<string>(),
+                taskName: taskConfig["name"]!.GetValue<string>(),
+                taskVersion: version.ToString(),
+                agentPath: AgentPath
+            );
+        }
+
+        public void WriteLaunchConfigurations()
+        {
+            var launchConfigString = LaunchConfig.ToJsonString();
+            File.WriteAllText(LaunchConfigPath, launchConfigString);
+        }
+
+        public void WriteTypescriptConfig(string taskOutput)
+        {
+            var tsconfigPath = Path.Combine(taskOutput, "tsconfig.json");
+            if (!File.Exists(tsconfigPath))
+            {
+                return;
+            }
+
+            var tsConfigContent = File.ReadAllText(tsconfigPath);
+            var tsConfigObject = JsonNode.Parse(tsConfigContent)?.AsObject();
+
+            if (tsConfigObject == null)
+            {
+                return;
+            }
+
+            var compilerOptionsObject = tsConfigObject["compilerOptions"]?.AsObject();
+            compilerOptionsObject?.Add("inlineSourceMap", true);
+            compilerOptionsObject?.Add("inlineSources", true);
+
+            JsonSerializerOptions options = new() { WriteIndented = true };
+            var outputTsConfigString = JsonSerializer.Serialize(tsConfigObject, options);
+            File.WriteAllText(tsconfigPath, outputTsConfigString);
+        }
+    }
+}

BuildConfigGen/Debugging/VsCodeLaunchConfiguration.cs

@@ -0,0 +1,119 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.RegularExpressions;
+
+namespace BuildConfigGen
+{
+    internal partial class VsCodeLaunchConfiguration
+    {
+        private JsonObject LaunchConfiguration { get; }
+
+        private JsonArray ConfigurationsList => _configurationsList.Value;
+
+        private readonly Lazy<JsonArray> _configurationsList;
+
+        public VsCodeLaunchConfiguration(JsonObject launchConfiguration)
+        {
+            ArgumentNullException.ThrowIfNull(launchConfiguration);
+            LaunchConfiguration = launchConfiguration;
+
+            _configurationsList = new(() =>
+            {
+                if (!LaunchConfiguration.TryGetPropertyValue("configurations", out JsonNode? configurationsNode))
+                {
+                    configurationsNode = new JsonArray();
+                    LaunchConfiguration["configurations"] = configurationsNode;
+                }
+                return configurationsNode!.AsArray();
+            });
+        }
+
+        public static VsCodeLaunchConfiguration ReadFromFileIfPresentOrDefault(string configPath)
+        {
+            ArgumentException.ThrowIfNullOrEmpty(configPath);
+
+            JsonObject launchConfiguration;
+            if (File.Exists(configPath))
+            {
+                var rawConfigurationsString = File.ReadAllText(configPath);
+                var safeConfigurationsString = RemoveJsonComments(rawConfigurationsString);
+
+                launchConfiguration = JsonNode.Parse(safeConfigurationsString)?.AsObject() ?? throw new ArgumentException($"Provided configuration file at {Path.GetFullPath(configPath)} is not a valid JSON file!");
+            } else
+            {
+                launchConfiguration = new JsonObject
+                {
+                    ["version"] = "0.2.0",
+                    ["configurations"] = new JsonArray()
+                };
+            }
+
+            return new VsCodeLaunchConfiguration(launchConfiguration);
+        }
+
+        public void AddConfigForTask(
+            string taskName,
+            string taskVersion,
+            string taskId,
+            string agentPath)
+        {
+            ArgumentException.ThrowIfNullOrEmpty(taskName);
+            ArgumentException.ThrowIfNullOrEmpty(taskVersion);
+            ArgumentException.ThrowIfNullOrEmpty(taskId);
+            ArgumentException.ThrowIfNullOrEmpty(agentPath);
+
+            var launchConfigName = GetLaunchConfigurationName(taskName, taskVersion);
+
+            var existingLaunchConfig = ConfigurationsList.FirstOrDefault(x =>
+            {
+                var name = x?[c_taskName]?.GetValue<string>();
+
+                return string.Equals(name, launchConfigName, StringComparison.OrdinalIgnoreCase);
+            });
+
+            ConfigurationsList.Remove(existingLaunchConfig);
+
+            var launchConfig = new JsonObject
+            {
+                [c_taskName] = launchConfigName,
+                ["type"] = "node",
+                ["request"] = "attach",
+                ["address"] = "localhost",
+                ["port"] = 9229,
+                ["autoAttachChildProcesses"] = true,
+                ["skipFiles"] = new JsonArray("<node_internals>/**"),
+                ["sourceMaps"] = true,
+                ["remoteRoot"] = GetRemoteSourcesPath(taskName, taskVersion, taskId, agentPath)
+            };
+
+            ConfigurationsList.Add(launchConfig);
+        }
+
+        public string ToJsonString()
+        {
+            var options = new JsonSerializerOptions { WriteIndented = true };
+            return JsonSerializer.Serialize(LaunchConfiguration, options);
+        }
+
+        private static string GetLaunchConfigurationName(string task, string version) =>
+            $"Attach to {task} ({version})";
+
+        private static string GetRemoteSourcesPath(string taskName, string taskVersion, string taskId, string agentPath) =>
+            @$"{agentPath}\_work\_tasks\{taskName}_{taskId.ToLower()}\{taskVersion}";
+
+        private static string RemoveJsonComments(string jsonString)
+        {
+            jsonString = SingleLineCommentsRegex().Replace(jsonString, string.Empty);
+            jsonString = MultiLineCommentsRegex().Replace(jsonString, string.Empty);
+            return jsonString;
+        }
+
+        [GeneratedRegex(@"//.*(?=\r?\n|$)")]
+        private static partial Regex SingleLineCommentsRegex();
+
+        [GeneratedRegex(@"/\*.*?\*/", RegexOptions.Singleline)]
+        private static partial Regex MultiLineCommentsRegex();
+
+        private const string c_taskName = "name";
+    }
+}

BuildConfigGen/Program.cs

@@ -1,8 +1,10 @@
 using System.Diagnostics.CodeAnalysis;
+using System.Runtime.InteropServices.JavaScript;
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using System.Text.RegularExpressions;
+using BuildConfigGen.Debugging;
 
 namespace BuildConfigGen
 {
@@ -63,11 +65,12 @@ public record ConfigRecord(string name, string constMappingKey, bool isDefault,
         /// <param name="writeUpdates">Write updates if true, else validate that the output is up-to-date</param>
         /// <param name="allTasks"></param>
         /// <param name="getTaskVersionTable"></param>
-        static void Main(string? task = null, string? configs = null, int? currentSprint = null, bool writeUpdates = false, bool allTasks = false, bool getTaskVersionTable = false)
+        /// <param name="debugAgentDir">When set to the local pipeline agent directory, this tool will produce tasks in debug mode with the corresponding visual studio launch configurations that can be used to attach to built tasks running on this agent</param>
+        static void Main(string? task = null, string? configs = null, int? currentSprint = null, bool writeUpdates = false, bool allTasks = false, bool getTaskVersionTable = false, string? debugAgentDir = null)
         {
             try
             {
-                MainInner(task, configs, currentSprint, writeUpdates, allTasks, getTaskVersionTable);
+                MainInner(task, configs, currentSprint, writeUpdates, allTasks, getTaskVersionTable, debugAgentDir);
             }
             catch (Exception e2)
             {
@@ -85,7 +88,7 @@ static void Main(string? task = null, string? configs = null, int? currentSprint
             }
         }
 
-        private static void MainInner(string? task, string? configs, int? currentSprint, bool writeUpdates, bool allTasks, bool getTaskVersionTable)
+        private static void MainInner(string? task, string? configs, int? currentSprint, bool writeUpdates, bool allTasks, bool getTaskVersionTable, string? debugAgentDir)
         {
             if (allTasks)
             {
@@ -98,11 +101,10 @@ private static void MainInner(string? task, string? configs, int? currentSprint,
                 NotNullOrThrow(configs, "Configs is required");
             }
 
+            string currentDir = Environment.CurrentDirectory;
+            string gitRootPath = GitUtil.GetGitRootPath(currentDir);
             if (getTaskVersionTable)
             {
-                string currentDir = Environment.CurrentDirectory;
-                string gitRootPath = GitUtil.GetGitRootPath(currentDir);
-
                 var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath);
 
                 Console.WriteLine("config\ttask\tversion");
@@ -120,15 +122,16 @@ private static void MainInner(string? task, string? configs, int? currentSprint,
                 return;
             }
 
+            IDebugConfigGenerator debugConfGen = string.IsNullOrEmpty(debugAgentDir)
+                ? new NoDebugConfigGenerator()
+                : new VsCodeLaunchConfigGenerator(gitRootPath, debugAgentDir);
+
             if (allTasks)
             {
-                string currentDir = Environment.CurrentDirectory;
-                string gitRootPath = GitUtil.GetGitRootPath(currentDir);
-
                 var tasks = MakeOptionsReader.ReadMakeOptions(gitRootPath);
                 foreach (var t in tasks.Values)
                 {
-                    MainUpdateTask(t.Name, string.Join('|', t.Configs), writeUpdates, currentSprint);
+                    MainUpdateTask(t.Name, string.Join('|', t.Configs), writeUpdates, currentSprint, debugConfGen);
                 }
             }
             else
@@ -139,10 +142,12 @@ private static void MainInner(string? task, string? configs, int? currentSprint,
                 // 3. Ideally default windows exception will occur and errors reported to WER/watson.  I'm not sure this is happening, perhaps DragonFruit is handling the exception
                 foreach (var t in task!.Split(',', '|'))
                 {
-                    MainUpdateTask(t, configs!, writeUpdates, currentSprint);
+                    MainUpdateTask(t, configs!, writeUpdates, currentSprint, debugConfGen);
                 }
             }
 
+            debugConfGen.WriteLaunchConfigurations();
+
             if (notSyncronizedDependencies.Count > 0)
             {
                 notSyncronizedDependencies.Insert(0, $"##vso[task.logissue type=error]There are problems with the dependencies in the buildConfig's package.json files. Please fix the following issues:");
@@ -225,7 +230,12 @@ private static void GetVersions(string task, string configsString, out List<(str
             }
         }
 
-        private static void MainUpdateTask(string task, string configsString, bool writeUpdates, int? currentSprint)
+        private static void MainUpdateTask(
+            string task,
+            string configsString,
+            bool writeUpdates,
+            int? currentSprint,
+            IDebugConfigGenerator debugConfigGen)
         {
             if (string.IsNullOrEmpty(task))
             {
@@ -265,7 +275,7 @@ private static void MainUpdateTask(string task, string configsString, bool write
             {
                 ensureUpdateModeVerifier = new EnsureUpdateModeVerifier(!writeUpdates);
 
-                MainUpdateTaskInner(task, currentSprint, targetConfigs);
+                MainUpdateTaskInner(task, currentSprint, targetConfigs, debugConfigGen);
 
                 ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferError(task, skipContentCheck: false);
             }
@@ -309,7 +319,11 @@ private static void ThrowWithUserFriendlyErrorToRerunWithWriteUpdatesIfVeriferEr
             }
         }
 
-        private static void MainUpdateTaskInner(string task, int? currentSprint, HashSet<Config.ConfigRecord> targetConfigs)
+        private static void MainUpdateTaskInner(
+            string task,
+            int? currentSprint,
+            HashSet<Config.ConfigRecord> targetConfigs,
+            IDebugConfigGenerator debugConfigGen)
         {
             if (!currentSprint.HasValue)
             {
@@ -387,7 +401,8 @@ private static void MainUpdateTaskInner(string task, int? currentSprint, HashSet
                     EnsureBuildConfigFileOverrides(config, taskTargetPath);
                 }
 
-                var taskConfigExists = File.Exists(Path.Combine(taskOutput, "task.json"));
+                var taskConfigPath = Path.Combine(taskOutput, "task.json");
+                var taskConfigExists = File.Exists(taskConfigPath);
 
                 // only update task output if a new version was added, the config exists, or the task contains preprocessor instructions
                 // Note: CheckTaskInputContainsPreprocessorInstructions is expensive, so only call if needed
@@ -423,6 +438,9 @@ private static void MainUpdateTaskInner(string task, int? currentSprint, HashSet
                         Path.Combine(taskTargetPath, buildConfigs, configTaskPath, "package.json"));
                     WriteNodePackageJson(taskOutput, config.nodePackageVersion, config.shouldUpdateTypescript);
                 }
+
+                debugConfigGen.WriteTypescriptConfig(taskOutput);
+                debugConfigGen.AddForTask(taskConfigPath);
             }
 
             // delay updating version map file until after buildconfigs generated

BuildConfigGen/TaskVersion.cs

@@ -1,4 +1,5 @@
-using System.Globalization;
+using System;
+using System.Globalization;
 
 internal class TaskVersion : IComparable<TaskVersion>, IEquatable<TaskVersion>
 {

Tasks/BashV3/task.json

@@ -17,7 +17,7 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 3,
-    "Minor": 239,
+    "Minor": 241,
     "Patch": 0
   },
   "releaseNotes": "Script task consistency. Added support for multiple lines and added support for Windows.",