Merge branch 'master' into Localization

Author: vmapetr

.vsts.ci.yml

@@ -33,7 +33,7 @@ parameters:
 - name: alpine_arm64
   type: boolean
   displayName: Alpine (ARM64)
-  default: false
+  default: true
 - name: macOS_x64
   type: boolean
   displayName: macOS (x64)

.vsts.release.yml

@@ -75,10 +75,6 @@ extends:
       - job: Set_variables
         displayName: Set release-specific variables
         steps:
-        # There's no need to clone the repo since no sources are used.
-        - checkout: none
-          displayName: Skip repo clone
-
         - pwsh: |
             $isBuildStageOnly = [System.Convert]::ToBoolean('${{ parameters.buildStageOnly }}')
             $buildReason = '$(Build.Reason)'

src/Agent.Sdk/Knob/AgentKnobs.cs

@@ -633,6 +633,12 @@ public class AgentKnobs
             new RuntimeKnobSource("AZP_AGENT_CHECK_FOR_TASK_DEPRECATION"),
             new BuiltInDefaultKnobSource("false"));
 
+        public static readonly Knob CheckIfTaskNodeRunnerIsDeprecated = new Knob(
+            nameof(CheckIfTaskNodeRunnerIsDeprecated),
+            "If true, the agent will check in the 'Initialize job' step each task used in the job if this task has node handlers, and all of them are deprecated.",
+            new RuntimeKnobSource("AZP_AGENT_CHECK_IF_TASK_NODE_RUNNER_IS_DEPRECATED"),
+            new BuiltInDefaultKnobSource("false"));
+
         public static readonly Knob MountWorkspace = new Knob(
             nameof(MountWorkspace),
             "If true, the agent will mount the Pipeline.Workspace directory instead of the Working directory for steps which target a Docker container.",

src/Agent.Worker/TaskManager.cs

@@ -76,6 +76,8 @@ into taskGrouping
                 return;
             }
 
+            HashSet<Guid> exceptionList = GetTaskExceptionSet();
+
             foreach (var task in uniqueTasks.Select(x => x.Reference))
             {
                 if (task.Id == Pipelines.PipelineConstants.CheckoutTask.Id && task.Version == Pipelines.PipelineConstants.CheckoutTask.Version)
@@ -90,6 +92,14 @@ into taskGrouping
                 {
                     CheckForTaskDeprecation(executionContext, task);
                 }
+
+                if (AgentKnobs.CheckIfTaskNodeRunnerIsDeprecated.GetValue(executionContext).AsBoolean())
+                {
+                    if (!exceptionList.Contains(task.Id))
+                    {
+                        CheckIfTaskNodeRunnerIsDeprecated(executionContext, task);
+                    }
+                }
             }
         }
 
@@ -319,9 +329,7 @@ private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.T
 
         private void CheckForTaskDeprecation(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task)
         {
-            string taskJsonPath = Path.Combine(GetDirectory(task), "task.json");
-            string taskJsonText = File.ReadAllText(taskJsonPath);
-            JObject taskJson = JObject.Parse(taskJsonText);
+            JObject taskJson = GetTaskJson(task);
             var deprecated = taskJson["deprecated"];
 
             if (deprecated != null && deprecated.Value<bool>())
@@ -363,6 +371,80 @@ private void CheckForTaskDeprecation(IExecutionContext executionContext, Pipelin
             }
         }
 
+        private void CheckIfTaskNodeRunnerIsDeprecated(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task)
+        {
+            string[] deprecatedNodeRunners = { "Node", "Node10" };
+            string[] approvedNodeRunners = { "Node16", "Node20_1" }; // Node runners which are not considered as deprecated
+
+            JObject taskJson = GetTaskJson(task);
+            var taskRunners = (JObject)taskJson["execution"];
+
+            foreach (string runner in approvedNodeRunners)
+            {
+                if (taskRunners.ContainsKey(runner))
+                {
+                    return; // Agent never uses deprecated Node runners if there are approved Node runners
+                }
+            }
+
+            List<string> taskNodeRunners = new(); // If we are here and task has Node runners, all of them are deprecated
+
+            foreach (string runner in deprecatedNodeRunners)
+            {
+                if (taskRunners.ContainsKey(runner))
+                {
+                    switch (runner)
+                    {
+                        case "Node":
+                            taskNodeRunners.Add("6"); // Just "Node" is Node version 6
+                            break;
+                        default:
+                            taskNodeRunners.Add(runner[4..]); // Postfix after "Node"
+                            break;
+                    }
+                }
+            }
+
+            if (taskNodeRunners.Count > 0) // Tasks may have only PowerShell runners and don't have Node runners at all
+            {
+                string friendlyName = taskJson["friendlyName"].Value<string>();
+                int majorVersion = new Version(task.Version).Major;
+                executionContext.Warning(StringUtil.Loc("DeprecatedNodeRunner", friendlyName, majorVersion, task.Name, taskNodeRunners.Last()));
+            }
+        }
+
+        /// <summary> 
+        /// This method provides a set of in-the-box pipeline tasks for which we don't want to display Node deprecation warnings. 
+        /// </summary>
+        /// <returns> Set of tasks ID </returns>
+        private HashSet<Guid> GetTaskExceptionSet()
+        {
+            string exceptionListFile = HostContext.GetConfigFile(WellKnownConfigFile.TaskExceptionList);
+            var exceptionList = new List<Guid>();
+
+            if (File.Exists(exceptionListFile))
+            {
+                try
+                {
+                    exceptionList = IOUtil.LoadObject<List<Guid>>(exceptionListFile);
+                }
+                catch (Exception ex)
+                {
+                    Trace.Info($"Unable to deserialize exception list {ex}");
+                    exceptionList = new List<Guid>();
+                }
+            }
+
+            return exceptionList.ToHashSet();
+        }
+
+        private JObject GetTaskJson(Pipelines.TaskStepDefinitionReference task)
+        {
+            string taskJsonPath = Path.Combine(GetDirectory(task), "task.json");
+            string taskJsonText = File.ReadAllText(taskJsonPath);
+            return JObject.Parse(taskJsonText);
+        }
+
         private void ExtractZip(String zipFile, String destinationDirectory)
         {
             ZipFile.ExtractToDirectory(zipFile, destinationDirectory);

src/Misc/externals.sh

@@ -36,15 +36,20 @@ else
 fi
 
 function failed() {
-   local error=${1:-Undefined error}
-   echo "Failed: $error" >&2
-   exit 1
+    local error=${1:-Undefined error}
+    local command_identifier=$2
+    echo "Failed: $error" >&2
+    if [[ $command_identifier == 'download_node_alpine_arm64' ]]; then
+        echo "Node for Alpine ARM64 not found in blob storage. If the version of Node (for tasks execution) has been updated, it should be built for Alpine ARM64 and uploaded to blob storage. Read documentation about the agent release for more info."
+    fi
+    exit 1
 }
 
 function checkRC() {
     local rc=$?
+    local command_identifier=$2
     if [ $rc -ne 0 ]; then
-        failed "${1} failed with return code $rc"
+        failed "${1} failed with return code $rc" $command_identifier
     fi
 }
 
@@ -56,6 +61,7 @@ function acquireExternalTool() {
                             # of the nested TEE-CLC-14.0.4 directory are moved up one directory, and then the empty directory
                             # TEE-CLC-14.0.4 is removed.
     local dont_uncompress=$4
+    local tool_name=$5
 
     # Extract the portion of the URL after the protocol. E.g. vstsagenttools.blob.core.windows.net/tools/pdbstr/1/pdbstr.zip
     local relative_url="${download_source#*://}"
@@ -84,7 +90,7 @@ function acquireExternalTool() {
             #      -S Show error. With -s, make curl show errors when they occur
             #      -L Follow redirects (H)
             #      -o FILE    Write to FILE instead of stdout
-            curl --retry 10 -fkSL -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl'
+            curl --retry 10 -fkSL -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl' "download_${tool_name}"
 
             # Move the partial file to the download target.
             mv "$partial_target" "$download_target" || checkRC 'mv'
@@ -205,11 +211,11 @@ else
         ARCH="linux-arm64-musl"
 
         if [[ "$INCLUDE_NODE10" == "true" ]]; then
-            acquireExternalTool "${CONTAINER_URL}/nodejs/${ARCH}/node-v${NODE10_VERSION}-${ARCH}.tar.gz" node10/bin fix_nested_dir
+            acquireExternalTool "${CONTAINER_URL}/nodejs/${ARCH}/node-v${NODE10_VERSION}-${ARCH}.tar.gz" node10/bin fix_nested_dir false node_alpine_arm64
         fi
 
-        acquireExternalTool "${CONTAINER_URL}/nodejs/${ARCH}/node-v${NODE16_VERSION}-${ARCH}.tar.gz" node16/bin fix_nested_dir
-        acquireExternalTool "${CONTAINER_URL}/nodejs/${ARCH}/node-v${NODE20_VERSION}-${ARCH}.tar.gz" node20_1/bin fix_nested_dir
+        acquireExternalTool "${CONTAINER_URL}/nodejs/${ARCH}/node-v${NODE16_VERSION}-${ARCH}.tar.gz" node16/bin fix_nested_dir false node_alpine_arm64
+        acquireExternalTool "${CONTAINER_URL}/nodejs/${ARCH}/node-v${NODE20_VERSION}-${ARCH}.tar.gz" node20_1/bin fix_nested_dir false node_alpine_arm64
     else
         case $PACKAGERUNTIME in
             "linux-musl-x64") ARCH="linux-x64-musl";;

src/Misc/layoutbin/en-US/strings.json

@@ -262,6 +262,7 @@
   "DeploymentPoolName": "Deployment Pool name",
   "DeploymentPoolNotFound": "Deployment pool not found: '{0}'",
   "DeprecatedNode6": "This task uses Node 6 execution handler, which will be removed March 31st 2022. If you are the developer of the task - please consider the migration guideline to Node 10 handler - https://aka.ms/migrateTaskNode10 (check this page also if you would like to disable Node 6 deprecation warnings). If you are the user - feel free to reach out to the owners of this task to proceed on migration.",
+  "DeprecatedNodeRunner": "Task '{0}' version {1} ({2}@{1}) is dependent on a Node version ({3}) that is end-of-life. Contact the extension owner for an updated version of the task. Task maintainers should review Node upgrade guidance: https://aka.ms/node-runner-guidance",
   "DeprecatedRunner": "Task '{0}' is dependent on a task runner that is end-of-life and will be removed in the future. Authors should review Node upgrade guidance: https://aka.ms/node-runner-guidance.",
   "DeprecationMessage": "Task '{0}' version {1} ({2}@{1}) is deprecated.",
   "DeprecationMessageHelpUrl": "Please see {0} for more information about this task.",

src/Test/L0/TestHostContext.cs

@@ -359,6 +359,12 @@ public string GetConfigFile(WellKnownConfigFile configFile)
                         ".setup_info");
                     break;
 
+                case WellKnownConfigFile.TaskExceptionList:
+                    path = Path.Combine(
+                        GetDirectory(WellKnownDirectory.Bin),
+                        "tasks-exception-list.json");
+                    break;
+
                 default:
                     throw new NotSupportedException($"Unexpected well known config file: '{configFile}'");
             }