Modifications in the Script based Executors (#536)

* scriptExecChanges

* add doc

* nit

* resolve comments

* deserialize to Metrics

---------

Co-authored-by: Aditya Abhishek <[email protected]>

Author: imadityaa

src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/ScriptExecutor/validJsonExample_array.json

@@ -0,0 +1,46 @@
+[
+	{
+		"Name": "metric1",
+		"Value": 0,
+		"Unit": "unit1",
+		"MetaData": {
+			"metadata1": 0.1,
+			"metadata2": "m2"
+		}
+	},
+	{
+		"Name": "metric2",
+		"Value": -1,
+		"Unit": "unit2",
+		"MetaData": {
+			"metadata1": "m3",
+			"metadata2": true
+		}
+	},
+	{
+		"Name": "metric3",
+		"Value": 1.2,
+		"Unit": "unit3",
+		"MetaData": {
+			"metadata1": "m5",
+			"metadata2": -2
+		}
+	},
+	{
+		"Name": "metric4",
+		"Value": 1.0,
+		"MetaData": {
+			"metadata1": "m7",
+			"metadata2": "m8"
+		}
+	},
+	{
+		"Name": "metric5",
+		"Value": "1.24",
+		"Unit": "unit5"
+	},
+	{
+		"Name": "metric6",
+		"Value": "-5.8"
+	}
+]

src/VirtualClient/VirtualClient.Actions.UnitTests/ScriptExecutor/JsonMetricsParserTests.cs

@@ -20,7 +20,7 @@ class JsonMetricsParserTests
         private JsonMetricsParser testParser;
 
         [Test]
-        public void JsonMetricsParserVerifyMetricsForPassResults()
+        public void JsonMetricsParserVerifyMetricsForPassResults_Format1()
         {
             string resultsPath = MockFixture.GetDirectory(typeof(JsonMetricsParserTests), "Examples", "ScriptExecutor", "validJsonExample.json");
             string rawText = File.ReadAllText(resultsPath);
@@ -33,6 +33,41 @@ public void JsonMetricsParserVerifyMetricsForPassResults()
             MetricAssert.Exists(metrics, "metric3", 1279854282929.09);
         }
 
+        [Test]
+        public void JsonMetricsParserVerifyMetricsForPassResults_ArrayFormat()
+        {
+            string resultsPath = MockFixture.GetDirectory(typeof(JsonMetricsParserTests), "Examples", "ScriptExecutor", "validJsonExample_array.json");
+            string rawText = File.ReadAllText(resultsPath);
+            this.testParser = new JsonMetricsParser(rawText, new InMemoryLogger(), EventContext.None);
+            IList<Metric> metrics = this.testParser.Parse();
+
+            Assert.AreEqual(6, metrics.Count);
+            MetricAssert.Exists(metrics, "metric1", 0);
+            MetricAssert.Exists(metrics, "metric2", -1);
+            MetricAssert.Exists(metrics, "metric3", 1.2);
+            MetricAssert.Exists(metrics, "metric4", 1);
+            MetricAssert.Exists(metrics, "metric5", 1.24);
+            MetricAssert.Exists(metrics, "metric6", -5.8);
+        }
+
+        [Test]
+        public void JsonMetricsParserThrowsIfTheJsonArrayResultsHaveInvalidMetrics()
+        {
+            string rawText = "[\r\n\t{\r\n\t\t\"Name\": \"metric3\",\r\n\t\t\"Value\": \"a1\",\r\n\t\t\"Unit\": \"unit3\",\r\n\t\t\"MetaData\": {\r\n\t\t\t\"metadata1\": \"m5\",\r\n\t\t\t\"metadata2\": \"m6\"\r\n\t\t}\r\n\t},\r\n\t{\r\n\t\t\"Name\": \"metric4\",\r\n\t\t\"Value\": 1.0,\r\n\t\t\"MetaData\": {\r\n\t\t\t\"metadata1\": \"m7\"\r\n\t\t}\r\n\t}\r\n]";
+
+            this.testParser = new JsonMetricsParser(rawText, new InMemoryLogger(), EventContext.None);
+            Assert.Throws<WorkloadResultsException>(() => this.testParser.Parse());
+        }
+
+        [Test]
+        public void JsonMetricsParserThrowsIfTheJsonArrayResultsHaveMisingMetricName()
+        {
+            string rawText = "[\r\n\t{\r\n\t\t\"Value\": 0,\r\n\t\t\"Unit\": \"unit3\",\r\n\t\t\"MetaData\": {\r\n\t\t\t\"metadata1\": \"m5\",\r\n\t\t\t\"metadata2\": \"m6\"\r\n\t\t}\r\n\t},\r\n\t{\r\n\t\t\"Name\": \"metric4\",\r\n\t\t\"Value\": 1.0,\r\n\t\t\"MetaData\": {\r\n\t\t\t\"metadata1\": \"m7\"\r\n\t\t}\r\n\t}\r\n]";
+
+            this.testParser = new JsonMetricsParser(rawText, new InMemoryLogger(), EventContext.None);
+            Assert.Throws<WorkloadResultsException>(() => this.testParser.Parse());
+        }
+
         [Test]
         public void JsonMetricsParserThrowsIfTheJsonResultsHaveInvalidMetrics()
         {

src/VirtualClient/VirtualClient.Actions/ScriptExecutor/JsonMetricsParser.cs

@@ -5,9 +5,7 @@ namespace VirtualClient.Actions
 {
     using System;
     using System.Collections.Generic;
-    using System.ComponentModel;
     using System.Linq;
-    using System.Text.RegularExpressions;
     using global::VirtualClient;
     using global::VirtualClient.Actions;
     using global::VirtualClient.Contracts;
@@ -46,20 +44,64 @@ public override IList<Metric> Parse()
 
             try
             {
-                JObject keyValuePairs = JObject.Parse(this.RawText);
-                foreach (var keyValuePair in keyValuePairs.Properties())
+                JToken token = JToken.Parse(this.RawText);
+
+                if (token.Type == JTokenType.Object)
+                {
+                    // Format 1: { "metric1": 123, "metric2": 2.5, ... }
+                    JObject keyValuePairs = (JObject)token;
+                    foreach (var keyValuePair in keyValuePairs.Properties())
+                    {
+                        if (double.TryParse(keyValuePair.Value.ToString(), out var value))
+                        {
+                            metrics.Add(new Metric(keyValuePair.Name, value, MetricRelativity.Undefined));
+                        }
+                        else
+                        {
+                            throw new WorkloadResultsException(
+                                $"Invalid JSON metrics content formatting. The metric value for '{keyValuePair.Name}' is not a valid numeric data type. Provided metric value is '{keyValuePair.Value}'",
+                                ErrorReason.InvalidResults);
+                        }
+                    }
+                }
+                else if (token.Type == JTokenType.Array)
                 {
-                    if (double.TryParse(keyValuePair.Value.ToString(), out var value))
+                    /* Format 2:
+                        [
+                            {
+                                "Nmae": "metric1",
+                                "Value": 0,
+                                "Unit": "unit1",
+                                "MetaData": {
+                                    "metadata1": "m1",
+                                    "metadata2": "m2"
+                                }
+                            }
+                        ]
+                    */
+                    try
                     {
-                        metrics.Add(new Metric(keyValuePair.Name, value, MetricRelativity.Undefined));
+                        metrics = JsonConvert.DeserializeObject<List<Metric>>(this.RawText);
                     }
-                    else
+                    catch (Exception exc)
                     {
                         throw new WorkloadResultsException(
-                            $"Invalid JSON metrics content formatting. The metric value for '{keyValuePair.Name}' is not a valid numeric data type.",
+                            "Invalid JSON metrics content formatting. Failed to deserialze the Array JSON Contents into Metrics format.", exc, ErrorReason.InvalidResults);
+                    }
+
+                    if (metrics.Any(m => string.IsNullOrWhiteSpace(m.Name)))
+                    {
+                        throw new WorkloadResultsException(
+                            "Invalid JSON metrics content formatting. 'Name' is a required property for each metric, it can't be null or whitespace.",
                             ErrorReason.InvalidResults);
                     }
                 }
+                else
+                {
+                    throw new WorkloadResultsException(
+                        "Invalid JSON metrics content formatting. The root element must be either an object or an array.",
+                        ErrorReason.InvalidResults);
+                }
             }
             catch (WorkloadResultsException)
             {
@@ -68,8 +110,8 @@ public override IList<Metric> Parse()
             catch (Exception exc)
             {
                 throw new WorkloadResultsException(
-                    $"Invalid JSON metrics content formatting. The metrics content must be in a valid JSON key/value pair JSON-format " +
-                    $"(e.g. {{ \"metric1\": 1234, \"metric2\": 987.65, \"metric3\": 32.0023481 }} )",
+                    $"Invalid JSON metrics content formatting. The metrics content must be in a valid JSON key/value pair format " +
+                    $"(e.g. {{ \"metric1\": 1234, \"metric2\": 987.65, \"metric3\": 32.0023481 }} ) or an array of Json objects.",
                     exc,
                     ErrorReason.InvalidResults);
             }

src/VirtualClient/VirtualClient.Actions/ScriptExecutor/ScriptExecutor.cs

@@ -261,7 +261,7 @@ protected async Task CaptureLogsAsync(CancellationToken cancellationToken)
                 {
                     foreach (string logFilePath in this.fileSystem.Directory.GetFiles(fullLogPath, "*", SearchOption.AllDirectories))
                     {
-                        this.RequestUploadAndMoveToLogsDirectory(logFilePath, destinitionLogsDir, cancellationToken);
+                        this.RequestUploadAndMoveToLogsDirectory(logFilePath, destinitionLogsDir, cancellationToken, sourceRoot: fullLogPath);
                     }
                 }
 
@@ -301,9 +301,13 @@ protected Task RequestUpload(string logPath)
         }
 
         /// <summary>
-        /// Move the log files to central logs directory and Upload to Content Store
+        /// Move the log files to central logs directory (retaining source directory structure) and Upload to Content Store.
         /// </summary>
-        protected async Task RequestUploadAndMoveToLogsDirectory(string sourcePath, string destinitionDirectory, CancellationToken cancellationToken)
+        private async Task RequestUploadAndMoveToLogsDirectory(
+            string sourcePath,
+            string destinitionDirectory,
+            CancellationToken cancellationToken,
+            string sourceRoot = null)
         {
             if (string.Equals(sourcePath, this.ExecutablePath))
             {
@@ -317,13 +321,27 @@ protected async Task RequestUploadAndMoveToLogsDirectory(string sourcePath, stri
 
             await (this.FileOperationsRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(() =>
             {
-                // e.g.
-                // /logs/anytool/executecustomscript1/2023-06-27T21-13-12-51001Z-CustomScript.sh
-                // /logs/anytool/executecustomscript1/2023-06-27T21-15-36-12018Z-CustomScript.sh
                 string fileName = Path.GetFileName(sourcePath);
-                string destinitionPath = this.Combine(destinitionDirectory, BlobDescriptor.SanitizeBlobPath($"{DateTime.UtcNow.ToString("o").Replace('.', '-')}-{fileName}"));
-                this.fileSystem.File.Move(sourcePath, destinitionPath, true);
+                string destPath;
 
+                if (!string.IsNullOrEmpty(sourceRoot))
+                {
+                    // Compute relative path from sourceRoot to sourcePath
+                    string relativePath = this.fileSystem.Path.GetRelativePath(sourceRoot, sourcePath);
+                    string destDir = this.fileSystem.Path.Combine(destinitionDirectory, this.fileSystem.Path.GetDirectoryName(relativePath));
+                    if (!this.fileSystem.Directory.Exists(destDir))
+                    {
+                        this.fileSystem.Directory.CreateDirectory(destDir);
+                    }
+
+                    destPath = this.fileSystem.Path.Combine(destDir, BlobDescriptor.SanitizeBlobPath($"{DateTime.UtcNow:O}".Replace('.', '-') + "-" + fileName));
+                }
+                else
+                {
+                    destPath = this.Combine(destinitionDirectory, BlobDescriptor.SanitizeBlobPath($"{DateTime.UtcNow:O}".Replace('.', '-') + "-" + fileName));
+                }
+
+                this.fileSystem.File.Move(sourcePath, destPath, true);
                 return Task.CompletedTask;
             });
         }

src/VirtualClient/VirtualClient.Contracts/Metric.cs

@@ -8,6 +8,8 @@ namespace VirtualClient.Contracts
     using System.Diagnostics;
     using System.Linq;
     using System.Text;
+    using Newtonsoft.Json;
+    using VirtualClient.Common.Contracts;
 
     /// <summary>
     /// Represents the result of a single metric
@@ -18,6 +20,7 @@ public class Metric : IEquatable<Metric>
         /// <summary>
         /// Creates a metric
         /// </summary>
+        [JsonConstructor]
         public Metric(string name, double value)
         {
             this.Name = name;
@@ -133,6 +136,7 @@ public Metric(string name, double value, string unit, MetricRelativity relativit
         /// <summary>
         /// Telemetry context for metric.
         /// </summary>
+        [JsonConverter(typeof(ParameterDictionaryJsonConverter))]
         public IDictionary<string, IConvertible> Metadata { get; }
 
         /// <summary>

website/docs/guides/0221-usage-extensions.md

@@ -471,8 +471,9 @@ and requirements.
     * [Geekbench Workload Metrics](https://microsoft.github.io/VirtualClient/docs/workloads/geekbench/)
 
     Virtual Client provides a facility for script-based automation to emit metrics for capture as well. To enable metrics capture, scripts emit the metrics to a single/central file on the file
-    system. The file should be named ```test-metrics.json``` and should exist in the same directory as the script that generated it. The file contents should be a simple JSON-formatted structure 
-    as illustrated below. Virtual Client will read this file and upload the metrics defined within alongside any out-of-box metrics already captured.
+    system. The file should be named ```test-metrics.json``` and should exist in the same directory as the script that generated it. There are two acceptable formats for test-metrics.json. 
+  
+    * The file contents are a simple JSON-formatted structure as illustrated below. Virtual Client will read this file and upload the metrics defined within, alongside any out-of-box metrics already captured.
 
       ``` json
       # Example contents of the 'test-metrics.json' file. Simple key/value pairs. This file should
@@ -489,6 +490,64 @@ and requirements.
       }
       ```
 
+    * The file contents are formatted as a JSON array as illustrated below. Virtual Client will read this file and upload the metrics defined within, alongside any out-of-box metrics already captured.
+
+      ``` json
+      # Example contents of the 'test-metrics.json' file. JSON Array based structure. This file should
+      # be written to the same directory where the script that generated it exists. Here, 'metricName' and 'metricValue' are mandatory fields, while 'metricUnit' and 'metricMetadata' are optional. The field 'metricMetadata' can be used to provide additional information about the metric, such as the source of the metric or any other relevant context. It is a key-value pair structure.
+      #
+      # e.g.
+      # Given a script /any.script.extensions.1.0.0/linux-x64/install.py, the file should be
+      # written to /any.script.extensions.1.0.0/linux-x64/test-metrics.json
+
+      [
+          {
+              "Name": "metric1",
+              "Value": 0,
+              "Unit": "unit1",
+              "MetaData": {
+                  "metadata1": "m1",
+                  "metadata2": "m2"
+              }
+          },
+          {
+              "Name": "metric2",
+              "Value": -1,
+              "Unit": "unit2",
+              "MetaData": {
+                  "metadata1": "m3",
+                  "metadata2": "m4"
+              }
+          },
+          {
+              "Name": "metric3",
+              "Value": 1.2,
+              "Unit": "unit3",
+              "MetaData": {
+                  "metadata1": "m5",
+                  "metadata2": "m6"
+              }
+          },
+          {
+              "Name": "metric4",
+              "Value": 1.0,
+              "MetaData": {
+                  "metadata1": "m7",
+                  "metadata2": "m8"
+              }
+          },
+          {
+              "Name": "metric5",
+              "Value": "1.24",
+              "Unit": "unit5"
+          },
+          {
+              "Name": "metric6",
+              "Value": "-5.8"
+          }
+      ]
+      ```
+
 ### Out-of-Box Components for Execution of Script-Based Automation
 The following section provides examples of how scripts can be integrated into the Virtual Client using out-of-box features to integrate script-based
 extensions.