Enable stream and file logs output (#4920)

* Adding cmd option to enable both stream and log file output

* Add timestamp to log line for local logs

* Fixing date format for log lines

* fixing the date formate to universal time for log lines

* Renaming enableLogOutput to reStreamLogsToFiles and fixing error handling

* Adding strings for different locals.

* Removing string

* Adding loc key for error message

* Reuse variables

* Simplify conditions by enum flags

* Add trace with current logs streaming mode

---------

Co-authored-by: Kidd, Steve <[email protected]>
Co-authored-by: srkidd <[email protected]>
Co-authored-by: MerlynOMsft <[email protected]>

Author: 3 people

src/Agent.Listener/CommandLine/ConfigureAgent.cs

@@ -60,6 +60,9 @@ public class ConfigureAgent : ConfigureOrRemoveBase
         [Option(Constants.Agent.CommandLine.Flags.DisableLogUploads)]
         public bool DisableLogUploads { get; set; }
 
+        [Option(Constants.Agent.CommandLine.Flags.ReStreamLogsToFiles)]
+        public bool ReStreamLogsToFiles { get; set; }
+
         [Option(Constants.Agent.CommandLine.Flags.MachineGroup)]
         public bool MachineGroup { get; set; }
 

src/Agent.Listener/CommandSettings.cs

@@ -571,6 +571,11 @@ public bool GetDisableLogUploads()
             return TestFlag(Configure?.DisableLogUploads, Constants.Agent.CommandLine.Flags.DisableLogUploads);
         }
 
+        public bool GetReStreamLogsToFiles()
+        {
+            return TestFlag(Configure?.ReStreamLogsToFiles, Constants.Agent.CommandLine.Flags.ReStreamLogsToFiles);
+        }
+
         public bool Unattended()
         {
             if (TestFlag(GetConfigureOrRemoveBase()?.Unattended, Constants.Agent.CommandLine.Flags.Unattended))

src/Agent.Listener/Configuration/ConfigurationManager.cs

@@ -398,6 +398,12 @@ public async Task ConfigureAsync(CommandSettings command)
             agentSettings.NotificationSocketAddress = command.GetNotificationSocketAddress();
 
             agentSettings.DisableLogUploads = command.GetDisableLogUploads();
+            agentSettings.ReStreamLogsToFiles = command.GetReStreamLogsToFiles();
+
+            if (agentSettings.DisableLogUploads && agentSettings.ReStreamLogsToFiles)
+            {
+                throw new NotSupportedException(StringUtil.Loc("ReStreamLogsToFilesError"));
+            }
 
             agentSettings.AlwaysExtractTask = command.GetAlwaysExtractTask();
 
@@ -736,7 +742,7 @@ private void CheckAgentRootDirectorySecure()
                     // Get info about root folder
                     DirectoryInfo dirInfo = new DirectoryInfo(rootDirPath);
 
-                    // Get directory access control list 
+                    // Get directory access control list
                     DirectorySecurity directorySecurityInfo = dirInfo.GetAccessControl();
                     AuthorizationRuleCollection dirAccessRules = directorySecurityInfo.GetAccessRules(true, true, typeof(NTAccount));
 

src/Agent.Worker/ExecutionContext.cs

@@ -117,7 +117,7 @@ public sealed class ExecutionContext : AgentService, IExecutionContext, IDisposa
         private bool _throttlingReported = false;
         private ExecutionTargetInfo _defaultStepTarget;
         private ExecutionTargetInfo _currentStepTarget;
-        private bool _disableLogUploads;
+        private LogsStreamingOptions _logsStreamingOptions;
         private string _buildLogsFolderPath;
         private string _buildLogsFile;
         private FileStream _buildLogsData;
@@ -179,9 +179,21 @@ public override void Initialize(IHostContext hostContext)
         {
             base.Initialize(hostContext);
 
-            _disableLogUploads = HostContext.GetService<IConfigurationStore>().GetSettings().DisableLogUploads;
+            var agentSettings = HostContext.GetService<IConfigurationStore>().GetSettings();
 
-            if (_disableLogUploads)
+
+            _logsStreamingOptions = LogsStreamingOptions.StreamToServer;
+            if (agentSettings.ReStreamLogsToFiles)
+            {
+                _logsStreamingOptions |= LogsStreamingOptions.StreamToFiles;
+            }
+            else if (agentSettings.DisableLogUploads)
+            {
+                _logsStreamingOptions = LogsStreamingOptions.StreamToFiles;
+            }
+            Trace.Info($"Logs streaming mode: {_logsStreamingOptions}");
+
+            if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToFiles))
             {
                 _buildLogsFolderPath = Path.Combine(hostContext.GetDiagDirectory(), _buildLogsFolderName);
                 Directory.CreateDirectory(_buildLogsFolderPath);
@@ -264,7 +276,7 @@ public void Start(string currentOperation = null)
 
             _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
 
-            if (_disableLogUploads)
+            if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToFiles))
             {
                 var buildLogsJobFolder = Path.Combine(_buildLogsFolderPath, _mainTimelineId.ToString());
                 Directory.CreateDirectory(buildLogsJobFolder);
@@ -276,7 +288,14 @@ public void Start(string currentOperation = null)
                 _buildLogsData = new FileStream(_buildLogsFile, FileMode.CreateNew);
                 _buildLogsWriter = new StreamWriter(_buildLogsData, System.Text.Encoding.UTF8);
 
-                _logger.Write(StringUtil.Loc("BuildLogsMessage", _buildLogsFile));
+                if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToServerAndFiles))
+                {
+                    _logger.Write(StringUtil.Loc("LogOutputMessage", _buildLogsFile));
+                }
+                else
+                {
+                    _logger.Write(StringUtil.Loc("BuildLogsMessage", _buildLogsFile));
+                }
             }
         }
 
@@ -287,7 +306,7 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation =
                 Result = result;
             }
 
-            if (_disableLogUploads)
+            if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToFiles))
             {
                 _buildLogsWriter.Flush();
                 _buildLogsData.Flush();
@@ -717,21 +736,21 @@ public long Write(string tag, string inputMessage, bool canMaskSecrets = true)
             {
                 totalLines = _logger.TotalLines + 1;
 
-                if (_disableLogUploads)
+                if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToServer))
                 {
-                    _buildLogsWriter.WriteLine(message);
+                    _logger.Write(message);
                 }
-                else
+                if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToFiles))
                 {
-                    _logger.Write(message);
+                    //Add date time stamp to log line
+                    _buildLogsWriter.WriteLine("{0:O} {1}", DateTime.UtcNow, message);
                 }
             }
 
-            if (!_disableLogUploads)
+            if (_logsStreamingOptions.HasFlag(LogsStreamingOptions.StreamToServer))
             {
                 // write to job level execution context's log file.
-                var parentContext = _parentExecutionContext as ExecutionContext;
-                if (parentContext != null)
+                if (_parentExecutionContext is ExecutionContext parentContext)
                 {
                     lock (parentContext._loggerLock)
                     {
@@ -951,7 +970,7 @@ private void PublishTelemetry(
             publishTelemetryCmd.ProcessCommand(this, cmd);
         }
 
-        public void PublishTaskRunnerTelemetry(Dictionary<string,string> taskRunnerData)
+        public void PublishTaskRunnerTelemetry(Dictionary<string, string> taskRunnerData)
         {
             PublishTelemetry(taskRunnerData, IsAgentTelemetry: true);
         }
@@ -966,6 +985,15 @@ public void Dispose()
             _buildLogsData?.Dispose();
             _buildLogsData = null;
         }
+
+        [Flags]
+        private enum LogsStreamingOptions
+        {
+            None = 0,
+            StreamToServer = 1,
+            StreamToFiles = 2,
+            StreamToServerAndFiles = StreamToServer | StreamToFiles
+        }
     }
 
     // The Error/Warning/etc methods are created as extension methods to simplify unit testing.

src/Microsoft.VisualStudio.Services.Agent/ConfigurationStore.cs

@@ -89,6 +89,9 @@ public string Fingerprint
         [DataMember(EmitDefaultValue = false)]
         public bool DisableLogUploads { get; set; }
 
+        [DataMember(EmitDefaultValue = false)]
+        public bool ReStreamLogsToFiles { get; set; }
+
         [DataMember(EmitDefaultValue = false)]
         public int PoolId { get; set; }
 

src/Microsoft.VisualStudio.Services.Agent/Constants.cs

@@ -220,6 +220,7 @@ public static class Flags
                     public const string GitUseSChannel = "gituseschannel";
                     public const string Help = "help";
                     public const string DisableLogUploads = "disableloguploads";
+                    public const string ReStreamLogsToFiles = "restreamlogstofiles";
                     public const string MachineGroup = "machinegroup";
                     public const string Replace = "replace";
                     public const string NoRestart = "norestart";

src/Misc/layoutbin/en-US/strings.json

@@ -137,6 +137,8 @@
     " --acceptTeeEula                   macOS and Linux only. Accept the TEE end user license agreement.",
     " --gitUseSChannel                  Windows only. Tell Git to use Windows' native cert store.",
     " --alwaysExtractTask               Perform an unzip for tasks for each pipeline step.",
+    " --disableLogUploads               Don't stream or send console log output to the server. Instead, you may retrieve them from the agent host's filesystem after the job completes. NOTE: Cannot be used with --reStreamLogsToFiles, it will cause an error.",
+    " --reStreamLogsToFiles             Stream or send console log output to the server as well as a log file on the agent host's filesystem. NOTE: Cannot be used with --disableLogUploads, it will cause an error.",
     "",
     "CLI-WIDTH-OPTIONS-(35-CHARS)-------CLI-WIDTH-DESCRIPTION-(70-CHARS)--------------------------------------",
     "Startup options (Windows only):",
@@ -394,6 +396,7 @@
   "ListenForJobs": "{0:u}: Listening for Jobs",
   "LocalClockSkewed": "The local machine's clock may be out of sync with the server time by more than five minutes. Please sync your clock with your domain or internet time and try again.",
   "LocalSystemAccountNotFound": "Cannot find local system account",
+  "LogOutputMessage": "The agent has enabled uploading logs as well as saving log to file. After the job completes, you can retrieve this step's logs at {0} on the agent.",
   "Maintenance": "Maintenance",
   "MaxHierarchyLevelReached": "Hierarchy level is more than supported limit {0}, truncating lower hierarchy.",
   "MaxSubResultLimitReached": "Number of subresults in test case '{0}' is more than the supported limit of {1}, truncating remaining ones.",
@@ -503,6 +506,7 @@
   "ResourceUtilizationWarningsIsDisabled": "Resource Utilization warnings is disabled, switch \"DISABLE_RESOURCE_UTILIZATION_WARNINGS\" variable to \"true\" if you want to enable it",
   "RestartIn15SecMessage": "Restarting the machine in 15 seconds...",
   "RestartMessage": "Restart the machine to launch agent and for autologon settings to take effect.",
+  "ReStreamLogsToFilesError": "You cannot use --disableloguploads and --reStreamLogsToFiles at the same time!",
   "RetryCountLimitExceeded": "The maximum allowed number of attempts is {0} but got {1}. Retry attempts count will be decreased to {0}.",
   "RMApiFailure": "Api {0} failed with an error code {1}",
   "RMArtifactContainerDetailsInvalidError": "The artifact does not have valid container details: {0}",