diff --git a/Scripts/common.psm1 b/Scripts/common.psm1 index dbff411ea..da75adda0 100644 --- a/Scripts/common.psm1 +++ b/Scripts/common.psm1 @@ -38,19 +38,21 @@ function Invoke-DotnetBuild([String]$dotnet, [String]$solution, [String]$config, Write-Comment -prefix "..." -text "Building $solution" $platform = "/p:Platform=`"Any CPU`"" - $restore_command = "restore $solution" - $build_command = "build -c $config $solution --no-restore" + $restore_command = "restore $solution $platform" + $build_command = "build -c $config $solution --no-restore $platform" if ($local -and $nuget) { $nuget_config_file = "$PSScriptRoot/../NuGet.config" - $restore_command = "$restore_command --configfile $nuget_config_file /p:UseLocalNugetPackages=true $platform" - $build_command = "$build_command /p:UseLocalNugetPackages=true $platform" + $restore_command = "$restore_command --configfile $nuget_config_file /p:UseLocalNugetPackages=true" + $build_command = "$build_command /p:UseLocalNugetPackages=true" } elseif ($local) { $nuget_config_file = "$PSScriptRoot/../Samples/NuGet.config" - $restore_command = "$restore_command --configfile $nuget_config_file /p:UseLocalCoyote=true $platform" - $build_command = "$build_command /p:UseLocalCoyote=true $platform" + $restore_command = "$restore_command --configfile $nuget_config_file /p:UseLocalCoyote=true" + $build_command = "$build_command /p:UseLocalCoyote=true" } + Write-Host $restore_command Invoke-ToolCommand -tool $dotnet -cmd $restore_command -error_msg "Failed to restore $solution" + Write-Host $build_command Invoke-ToolCommand -tool $dotnet -cmd $build_command -error_msg "Failed to build $solution" } diff --git a/Source/Test/Rewriting/RewritingEngine.cs b/Source/Test/Rewriting/RewritingEngine.cs index 7d6a20bd1..15bcd3375 100644 --- a/Source/Test/Rewriting/RewritingEngine.cs +++ b/Source/Test/Rewriting/RewritingEngine.cs @@ -7,10 +7,6 @@ using System.Reflection; using System.Runtime.ExceptionServices; using System.Threading.Tasks; -using Microsoft.ApplicationInsights; -using Microsoft.ApplicationInsights.Channel; -using Microsoft.ApplicationInsights.DataContracts; -using Microsoft.ApplicationInsights.Extensibility; using Microsoft.Coyote.Logging; using Microsoft.Coyote.Runtime; using Mono.Cecil; @@ -311,11 +307,7 @@ private string CreateOutputDirectoryAndCopyFiles() foreach (var type in new Type[] { typeof(CoyoteRuntime), - typeof(RewritingEngine), - typeof(TelemetryConfiguration), - typeof(EventTelemetry), - typeof(ITelemetry), - typeof(TelemetryClient) + typeof(RewritingEngine) }) { string assemblyPath = type.Assembly.Location; diff --git a/Source/Test/SystematicTesting/TestingEngine.cs b/Source/Test/SystematicTesting/TestingEngine.cs index 1ce409322..57822682c 100644 --- a/Source/Test/SystematicTesting/TestingEngine.cs +++ b/Source/Test/SystematicTesting/TestingEngine.cs @@ -267,7 +267,8 @@ public void Run() if (this.Configuration.IsTelemetryEnabled) { - this.TrackTelemetry(); + // 10 seconds should be enough, any more than that and too bad. + this.TrackTelemetry().Wait(10000, this.CancellationTokenSource.Token); } } @@ -591,25 +592,16 @@ private void GatherTestingStatistics(CoyoteRuntime runtime) /// /// Tracks anonymized telemetry data. /// - private void TrackTelemetry() + private async Task TrackTelemetry() { bool isReplaying = this.Scheduler.IsReplaying; - TelemetryClient.TrackEvent(isReplaying ? "replay" : "test"); - if (Debugger.IsAttached) - { - TelemetryClient.TrackEvent(isReplaying ? "replay-debug" : "test-debug"); - } - else - { - TelemetryClient.TrackMetric(isReplaying ? "replay-time" : "test-time", this.Profiler.Results()); - } - - if (this.TestReport != null && this.TestReport.NumOfFoundBugs > 0) - { - TelemetryClient.TrackMetric(isReplaying ? "replay-bugs" : "test-bugs", this.TestReport.NumOfFoundBugs); - } - - TelemetryClient.Flush(); + await TelemetryClient.TrackEvent( + action: Debugger.IsAttached ? + (isReplaying ? "debug-replay" : "debug-test") : + (isReplaying ? "replay" : "test"), + result: (this.TestReport.NumOfFoundBugs > 0) ? "failed" : "passed", + bugsFound: this.TestReport?.NumOfFoundBugs, + testTime: this.Profiler?.Results()); } /// diff --git a/Source/Test/Telemetry/HttpProtocol.cs b/Source/Test/Telemetry/HttpProtocol.cs new file mode 100644 index 000000000..2e42427c4 --- /dev/null +++ b/Source/Test/Telemetry/HttpProtocol.cs @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Net.Http; +using System.Runtime.InteropServices; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Json; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Coyote.Telemetry +{ + internal static class HttpProtocol + { + private const string BaseUrl = "https://www.google-analytics.com/mp/collect"; + private const string DebugBaseUrl = "https://www.google-analytics.com/debug/mp/collect"; + + public static Task PostMeasurements(Analytics a) + { + return Post(BaseUrl, a); + } + + public static async Task ValidateMeasurements(Analytics a) + { + var response = await Post(DebugBaseUrl, a); + if (response.Content != null) + { + using (var stream = await response.Content.ReadAsStreamAsync()) + { + var responseSerializer = new DataContractJsonSerializer(typeof(ValidationResponse)); + return (ValidationResponse)responseSerializer.ReadObject(stream); + } + } + + throw new Exception("No validation response"); + } + + private static async Task Post(string baseUri, Analytics a) + { + const string guide = "\r\nSee https://developers.google.com/analytics/devguides/collection/protocol/ga4"; + + if (a.Events.Count > 25) + { + throw new Exception("A maximum of 25 events can be specified per request." + guide); + } + + if (!System.Net.NetworkInformation.NetworkInterface.GetIsNetworkAvailable()) + { + throw new Exception("A network intercace is not available"); + } + + string query = a.ToQueryString(); + string url = baseUri + "?" + query; + + if (string.IsNullOrEmpty(a.UserId)) + { + a.UserId = a.ClientId; + } + + HttpClient client = new HttpClient(); + AddUserProperties(client, a); + + DataContractJsonSerializerSettings settings = new DataContractJsonSerializerSettings(); + settings.EmitTypeInformation = EmitTypeInformation.Never; + settings.UseSimpleDictionaryFormat = true; + settings.KnownTypes = GetKnownTypes(a); + var serializer = new DataContractJsonSerializer(typeof(Analytics), settings); + var ms = new MemoryStream(); + serializer.WriteObject(ms, a); + var bytes = ms.GetBuffer(); + if (bytes.Length > 130000) + { + throw new Exception("The total size of analytics payloads cannot be greater than 130kb bytes" + guide); + } + + var json = Encoding.UTF8.GetString(bytes); + json = new StreamReader("d:\\temp\\json2.json").ReadToEnd(); + var jsonContent = new StringContent(json, Encoding.UTF8, "application/json"); + var response = await client.PostAsync(new Uri(url), jsonContent); + response.EnsureSuccessStatusCode(); + return response; + } + + private static void AddUserProperties(HttpClient client, Analytics a) + { + string platform = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "Windows" : + (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? "Linux" : "OSX"); + var arch = RuntimeInformation.OSArchitecture.ToString(); + client.DefaultRequestHeaders.Add("User-Agent", string.Format("Mozilla/5.0 ({0}; {1})", platform, arch)); + client.DefaultRequestHeaders.Add("Accept-Language", CultureInfo.CurrentCulture.Name); + a.UserProperties = new UserProperties() + { + FrameworkVersion = new UserPropertyValue(RuntimeInformation.FrameworkDescription), + Platform = new UserPropertyValue(platform), + PlatformVersion = new UserPropertyValue(RuntimeInformation.OSDescription), + Language = new UserPropertyValue(CultureInfo.CurrentCulture.Name) + }; + } + + private static Type[] GetKnownTypes(Analytics a) + { + HashSet types = new HashSet(); + foreach (var e in a.Events) + { + types.Add(e.GetType()); + } + + return new List(types).ToArray(); + } + } + + [DataContract] + internal class ValidationResponse + { + [DataMember(Name = "validationMessages")] + public ValidationMessage[] ValidationMessages; + } + + [DataContract] + internal class ValidationMessage + { + [DataMember(Name = "description")] + public string Description; + [DataMember(Name = "fieldPath")] + public string InvalidFieldPath; + [DataMember(Name = "validationCode")] + public string ValidationCode; + } +} diff --git a/Source/Test/Telemetry/Measurement.cs b/Source/Test/Telemetry/Measurement.cs new file mode 100644 index 000000000..a8d730f6d --- /dev/null +++ b/Source/Test/Telemetry/Measurement.cs @@ -0,0 +1,306 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace Microsoft.Coyote.Telemetry +{ + /// + /// This class wraps the GA4 protocol. + /// See https://developers.google.com/analytics/devguides/collection/protocol/ga4/reference. + /// + [KnownType(typeof(PageMeasurement))] + [KnownType(typeof(EventMeasurement))] + [KnownType(typeof(ExceptionMeasurement))] + [KnownType(typeof(UserTimingMeasurement))] + [KnownType(typeof(TestEventMeasurement))] + [KnownType(typeof(UserProperties))] + [KnownType(typeof(UserPropertyValue))] + [DataContract] + internal class Analytics + { + public Analytics() + { + this.TimeStamp = DateTimeOffset.Now.ToUnixTimeMilliseconds() * 1000; + } + + [IgnoreDataMember] + public string ApiSecret { get; set; } + + [IgnoreDataMember] + public string MeasurementId { get; set; } + + [DataMember(Name = "client_id", Order = 1)] + public string ClientId { get; set; } + + [DataMember(Name = "user_id", Order = 2)] + public string UserId { get; set; } + + [DataMember(Name = "timestamp_micros", Order = 3)] + public long TimeStamp { get; set; } + + [DataMember(Name = "non_personalized_ads", Order = 3)] + public bool NonPersonalizedAds { get; set; } + + [DataMember(Name = "user_properties", Order = 4)] + public UserProperties UserProperties { get; set; } + + [DataMember(Name = "events", Order = 5)] + public List Events = new List(); + + public string ToQueryString() + { + Required(this.MeasurementId, "MeasurementId"); + Required(this.ClientId, "ClientId"); + Required(this.ApiSecret, "ApiSecret"); + return string.Format("api_secret={0}&measurement_id={1}", this.ApiSecret, this.MeasurementId); + } + + protected static void Required(string value, string name) + { + if (string.IsNullOrEmpty(value)) + { + throw new ArgumentNullException(name); + } + } + } + + [DataContract] + internal class UserProperties + { + [DataMember(Name = "platform")] + public UserPropertyValue Platform { get; set; } + [DataMember(Name = "platform_version")] + public UserPropertyValue PlatformVersion { get; set; } + [DataMember(Name = "dotnet")] + public UserPropertyValue FrameworkVersion { get; set; } + [DataMember(Name = "language")] + public UserPropertyValue Language { get; set; } + } + + [DataContract] + internal class UserPropertyValue + { + public UserPropertyValue(string value) + { + this.Value = value; + } + + [DataMember(Name = "value")] + public string Value { get; set; } + } + + [DataContract] + internal abstract class Measurement + { + public Measurement() + { + this.Version = 1; + } + + protected int Version { get; set; } + + [DataMember(Name = "name")] + protected string Name { get; set; } + + [DataMember(Name = "params")] + public Dictionary Params = new Dictionary(); + + protected string GetParam(string name) + { + if (this.Params.TryGetValue(name, out object value) && value is string s) + { + return s; + } + + return null; + } + + protected void SetParam(string name, string value) + { + this.Params[name] = value; + } + + protected double GetDoubleParam(string name) + { + if (this.Params.TryGetValue(name, out object value) && value is double d) + { + return d; + } + + return 0; + } + + protected void SetDoubleParam(string name, double value) + { + this.Params[name] = value; + } + } + + [DataContract] + internal class PageMeasurement : Measurement + { + public PageMeasurement() + { + this.Name = "page_view"; + } + + public string HostName + { + get => this.GetParam("host_name"); + set => this.SetParam("host_name", value); + } + + public string Path + { + get => this.GetParam("page_location"); + set => this.SetParam("page_location", value); + } + + public string Title + { + get => this.GetParam("page_title"); + set => this.SetParam("page_title", value); + } + + public string Referrer + { + get => this.GetParam("page_referrer"); + set => this.SetParam("page_referrer", value); + } + + public string UserAgent + { + get => this.GetParam("user_agent"); + set => this.SetParam("user_agent", value); + } + } + + [DataContract] + internal class EventMeasurement : Measurement + { + public EventMeasurement() + { + this.Name = "event"; + } + + public string Category + { + get => this.GetParam("category"); + set => this.SetParam("category", value); + } + + public string Action + { + get => this.GetParam("action"); + set => this.SetParam("action", value); + } + + public string Label + { + get => this.GetParam("label"); + set => this.SetParam("label", value); + } + + public string Value + { + get => this.GetParam("value"); + set => this.SetParam("value", value); + } + } + + [DataContract] + internal class ExceptionMeasurement : Measurement + { + public ExceptionMeasurement() + { + this.Name = "exception"; + } + + public string Description + { + get => this.GetParam("description"); + set => this.SetParam("description", value); + } + + public string Fatal + { + get => this.GetParam("fatal"); + set => this.SetParam("fatal", value); + } + } + + [DataContract] + internal class UserTimingMeasurement : Measurement + { + public UserTimingMeasurement() + { + this.Name = "timing"; + } + + public string Category + { + get => this.GetParam("category"); + set => this.SetParam("category", value); + } + + public string Variable + { + get => this.GetParam("variable"); + set => this.SetParam("variable", value); + } + + public string Time + { + get => this.GetParam("time"); + set => this.SetParam("time", value); + } + + public string Label + { + get => this.GetParam("label"); + set => this.SetParam("label", value); + } + } + + [DataContract] + internal class TestEventMeasurement : Measurement + { + public TestEventMeasurement() + { + this.Name = "event"; + } + + public string Coyote + { + get => this.GetParam("coyote"); + set => this.SetParam("coyote", value); + } + + public string Action + { + get => this.GetParam("action"); + set => this.SetParam("action", value); + } + + public string Result + { + get => this.GetParam("value"); + set => this.SetParam("value", value); + } + + public int Bugs + { + get => (int)this.GetDoubleParam("bugs"); + set => this.SetDoubleParam("bugs", value); + } + + public double TestTime + { + get => (int)this.GetDoubleParam("test_time"); + set => this.SetDoubleParam("test_time", value); + } + } +} diff --git a/Source/Test/Telemetry/TelemetryClient.cs b/Source/Test/Telemetry/TelemetryClient.cs index 562d1022e..9bebb9857 100644 --- a/Source/Test/Telemetry/TelemetryClient.cs +++ b/Source/Test/Telemetry/TelemetryClient.cs @@ -3,12 +3,9 @@ using System; using System.IO; -using System.Runtime.InteropServices; using System.Threading; -using Microsoft.ApplicationInsights.DataContracts; -using Microsoft.ApplicationInsights.Extensibility; +using System.Threading.Tasks; using Microsoft.Coyote.Logging; -using AppInsightsClient = Microsoft.ApplicationInsights.TelemetryClient; namespace Microsoft.Coyote.Telemetry { @@ -20,6 +17,9 @@ namespace Microsoft.Coyote.Telemetry /// internal class TelemetryClient { + private const string ApiSecret = "eOkUmW73T9Wv5TUynCLAEA"; + private const string MeasurementId = "G-JS8YSYVDQX"; + /// /// Path to the Coyote home directory where the UUID is stored. /// @@ -42,11 +42,6 @@ internal class TelemetryClient /// private static TelemetryClient Current; - /// - /// The App Insights client. - /// - private readonly AppInsightsClient Client; - /// /// Responsible for writing to the installed . /// @@ -57,36 +52,27 @@ internal class TelemetryClient /// private readonly bool IsEnabled; + /// + /// A unique id for this user on this device. + /// + private string DeviceId; + + /// + /// The coyote version. + /// + private readonly string Version; + /// /// Initializes a new instance of the class. /// private TelemetryClient(LogWriter logWriter, bool isEnabled) { + this.IsEnabled = isEnabled; if (isEnabled) { - TelemetryConfiguration configuration = TelemetryConfiguration.CreateDefault(); - configuration.InstrumentationKey = "17a6badb-bf2d-4f5d-959b-6843b8bb1f7f"; - this.Client = new AppInsightsClient(configuration); + this.Version = typeof(Runtime.CoyoteRuntime).Assembly.GetName().Version.ToString(); this.LogWriter = logWriter; - - string version = typeof(Runtime.CoyoteRuntime).Assembly.GetName().Version.ToString(); - this.Client.Context.GlobalProperties["coyote"] = version; -#if NETFRAMEWORK - this.Client.Context.GlobalProperties["dotnet"] = ".NET Framework"; -#else - this.Client.Context.GlobalProperties["dotnet"] = RuntimeInformation.FrameworkDescription; -#endif - this.Client.Context.Device.Id = GetOrCreateDeviceId(out bool isFirstTime); - this.Client.Context.Device.OperatingSystem = Environment.OSVersion.Platform.ToString(); - this.Client.Context.Session.Id = Guid.NewGuid().ToString(); - - if (isFirstTime) - { - this.TrackEvent("welcome"); - } } - - this.IsEnabled = isEnabled; } /// @@ -105,66 +91,64 @@ internal static TelemetryClient GetOrCreate(Configuration configuration, LogWrit /// /// Tracks the specified telemetry event. /// - internal void TrackEvent(string name) + internal Task TrackEvent(string action, string result, int? bugsFound, double? testTime) { if (this.IsEnabled) { - lock (SyncObject) + Task task = Task.CompletedTask; + + if (string.IsNullOrEmpty(this.DeviceId)) { - try - { - this.LogWriter.LogDebug("[coyote::telemetry] Tracking event: {0}.", name); - this.Client.TrackEvent(new EventTelemetry(name)); - } - catch (Exception ex) + this.DeviceId = GetOrCreateDeviceId(out bool isFirstTime); + if (isFirstTime) { - this.LogWriter.LogDebug("[coyote::telemetry] Unable to send event: {0}", ex.Message); + task = this.TrackEvent("welcome", null, null, null); } } - } - } - /// - /// Tracks the specified telemetry metric. - /// - internal void TrackMetric(string name, double value) - { - if (this.IsEnabled) - { - lock (SyncObject) + try { - try + var analytics = new Analytics() { - this.LogWriter.LogDebug("[coyote::telemetry] Tracking metric: {0}={1}.", name, value); - this.Client.TrackMetric(new MetricTelemetry(name, value)); - } - catch (Exception ex) + ApiSecret = ApiSecret, + MeasurementId = MeasurementId, + ClientId = this.DeviceId + }; + + var m = new TestEventMeasurement() { - this.LogWriter.LogDebug("[coyote::telemetry] Unable to send metric: {0}", ex.Message); - } - } - } - } + Action = action, + Result = result + }; - /// - /// Flushes any buffered in-memory telemetry data. - /// - internal void Flush() - { - if (this.IsEnabled) - { - lock (SyncObject) - { - try + m.Coyote = this.Version; + + if (bugsFound.HasValue) { - this.Client.Flush(); + m.Bugs = bugsFound.Value; } - catch (Exception ex) + + if (testTime.HasValue) { - this.LogWriter.LogDebug("[coyote::telemetry] Error flushing: {0}", ex.Message); + m.TestTime = testTime.Value; } + + analytics.Events.Add(m); + + this.LogWriter.LogDebug("[coyote::telemetry] Tracking event: {0}.", action); + + // handy for debugging errors from google. + // var response = HttpProtocol.ValidateMeasurements(analytics).Result; + + return Task.WhenAll(task, HttpProtocol.PostMeasurements(analytics)); + } + catch (Exception ex) + { + this.LogWriter.LogDebug("[coyote::telemetry] Unable to send event: {0}", ex.Message); } } + + return Task.CompletedTask; } /// diff --git a/Source/Test/Test.csproj b/Source/Test/Test.csproj index 731a417de..86fd78005 100644 --- a/Source/Test/Test.csproj +++ b/Source/Test/Test.csproj @@ -10,7 +10,6 @@ -