Fix inputs trimming for variables (#4940)

* Testing fix for input trimming

* Added runtime knob source for DisableInputTrimming knob and cleaned up the code

* Add knob for variables trimming and refactor the logic to use it

* Fix bug where the variable input trimming logic was placed incorrectly in the ExpandValues method

---------

Co-authored-by: v-levockina <undefined>

Author: aleksandrlevochkin

src/Agent.Sdk/Knob/AgentKnobs.cs

@@ -403,8 +403,16 @@ public class AgentKnobs
             nameof(DisableInputTrimming),
             "By default, the agent trims whitespace and new line characters from all task inputs. Setting this to true disables this behavior.",
             new EnvironmentKnobSource("DISABLE_INPUT_TRIMMING"),
+            new RuntimeKnobSource("DISABLE_INPUT_TRIMMING"),
             new BuiltInDefaultKnobSource("false"));
 
+        public static readonly Knob EnableVariableInputTrimming = new Knob(
+           nameof(EnableVariableInputTrimming),
+           "By default, the agent does not trim whitespace and new line characters if an input comes from a variable. Setting this to true enables this behavior.",
+           new EnvironmentKnobSource("AGENT_ENABLE_VARIABLE_INPUT_TRIMMING"),
+           new RuntimeKnobSource("AGENT_ENABLE_VARIABLE_INPUT_TRIMMING"),
+           new BuiltInDefaultKnobSource("false"));
+
         public static readonly Knob DecodePercents = new Knob(
             nameof(DecodePercents),
             "By default, the agent does not decodes %AZP25 as % which may be needed to allow users to work around reserved values. Setting this to true enables this behavior.",

src/Agent.Worker/TaskRunner.cs

@@ -230,7 +230,8 @@ private async Task RunAsyncInternal()
 
                 // Expand the inputs.
                 Trace.Verbose("Expanding inputs.");
-                runtimeVariables.ExpandValues(target: inputs);
+                bool enableVariableInputTrimmingKnob = AgentKnobs.EnableVariableInputTrimming.GetValue(ExecutionContext).AsBoolean();
+                runtimeVariables.ExpandValues(target: inputs, enableVariableInputTrimmingKnob);
 
                 // We need to verify inputs of the tasks that were injected by decorators, to check if they contain secrets,
                 // for security reasons execution of tasks in this case should be skipped.

src/Agent.Worker/Variables.cs

@@ -281,7 +281,7 @@ public bool Retain_Default_Encoding
             Constants.Variables.Agent.Name,
         };
 
-        public void ExpandValues(IDictionary<string, string> target)
+        public void ExpandValues(IDictionary<string, string> target, bool enableVariableInputTrimming = false)
         {
             ArgUtil.NotNull(target, nameof(target));
             _trace.Entering();
@@ -292,7 +292,7 @@ public void ExpandValues(IDictionary<string, string> target)
                 source[variable.Name] = value;
             }
 
-            VarUtil.ExpandValues(_hostContext, source, target);
+            VarUtil.ExpandValues(_hostContext, source, target, enableVariableInputTrimming);
         }
 
         public string ExpandValue(string name, string value)

src/Microsoft.VisualStudio.Services.Agent/Util/VarUtil.cs

@@ -12,7 +12,7 @@
 
 namespace Microsoft.VisualStudio.Services.Agent.Util
 {
-  public static class VarUtil
+    public static class VarUtil
     {
         public static StringComparer EnvironmentVariableKeyComparer
         {
@@ -142,7 +142,7 @@ public static JToken ExpandValues(IHostContext context, IDictionary<string, stri
             return target.Map(mapFuncs);
         }
 
-        public static void ExpandValues(IHostContext context, IDictionary<string, string> source, IDictionary<string, string> target)
+        public static void ExpandValues(IHostContext context, IDictionary<string, string> source, IDictionary<string, string> target, bool enableVariableInputTrimming = false)
         {
             ArgUtil.NotNull(context, nameof(context));
             ArgUtil.NotNull(source, nameof(source));
@@ -183,6 +183,10 @@ public static void ExpandValues(IHostContext context, IDictionary<string, string
                             variableValue ?? string.Empty,
                             targetValue.Substring(suffixIndex + Constants.Variables.MacroSuffix.Length));
 
+                        targetValue = enableVariableInputTrimming
+                            ? targetValue?.Trim() ?? string.Empty
+                            : targetValue ?? string.Empty;
+
                         // Bump the start index to prevent recursive replacement.
                         startIndex = prefixIndex + (variableValue ?? string.Empty).Length;
                     }