diff --git a/Scripts/common.psm1 b/Scripts/common.psm1
index dbff411ea..da75adda0 100644
--- a/Scripts/common.psm1
+++ b/Scripts/common.psm1
@@ -38,19 +38,21 @@ function Invoke-DotnetBuild([String]$dotnet, [String]$solution, [String]$config,
Write-Comment -prefix "..." -text "Building $solution"
$platform = "/p:Platform=`"Any CPU`""
- $restore_command = "restore $solution"
- $build_command = "build -c $config $solution --no-restore"
+ $restore_command = "restore $solution $platform"
+ $build_command = "build -c $config $solution --no-restore $platform"
if ($local -and $nuget) {
$nuget_config_file = "$PSScriptRoot/../NuGet.config"
- $restore_command = "$restore_command --configfile $nuget_config_file /p:UseLocalNugetPackages=true $platform"
- $build_command = "$build_command /p:UseLocalNugetPackages=true $platform"
+ $restore_command = "$restore_command --configfile $nuget_config_file /p:UseLocalNugetPackages=true"
+ $build_command = "$build_command /p:UseLocalNugetPackages=true"
} elseif ($local) {
$nuget_config_file = "$PSScriptRoot/../Samples/NuGet.config"
- $restore_command = "$restore_command --configfile $nuget_config_file /p:UseLocalCoyote=true $platform"
- $build_command = "$build_command /p:UseLocalCoyote=true $platform"
+ $restore_command = "$restore_command --configfile $nuget_config_file /p:UseLocalCoyote=true"
+ $build_command = "$build_command /p:UseLocalCoyote=true"
}
+ Write-Host $restore_command
Invoke-ToolCommand -tool $dotnet -cmd $restore_command -error_msg "Failed to restore $solution"
+ Write-Host $build_command
Invoke-ToolCommand -tool $dotnet -cmd $build_command -error_msg "Failed to build $solution"
}
diff --git a/Source/Test/Rewriting/RewritingEngine.cs b/Source/Test/Rewriting/RewritingEngine.cs
index 7d6a20bd1..15bcd3375 100644
--- a/Source/Test/Rewriting/RewritingEngine.cs
+++ b/Source/Test/Rewriting/RewritingEngine.cs
@@ -7,10 +7,6 @@
using System.Reflection;
using System.Runtime.ExceptionServices;
using System.Threading.Tasks;
-using Microsoft.ApplicationInsights;
-using Microsoft.ApplicationInsights.Channel;
-using Microsoft.ApplicationInsights.DataContracts;
-using Microsoft.ApplicationInsights.Extensibility;
using Microsoft.Coyote.Logging;
using Microsoft.Coyote.Runtime;
using Mono.Cecil;
@@ -311,11 +307,7 @@ private string CreateOutputDirectoryAndCopyFiles()
foreach (var type in new Type[]
{
typeof(CoyoteRuntime),
- typeof(RewritingEngine),
- typeof(TelemetryConfiguration),
- typeof(EventTelemetry),
- typeof(ITelemetry),
- typeof(TelemetryClient)
+ typeof(RewritingEngine)
})
{
string assemblyPath = type.Assembly.Location;
diff --git a/Source/Test/SystematicTesting/TestingEngine.cs b/Source/Test/SystematicTesting/TestingEngine.cs
index 1ce409322..57822682c 100644
--- a/Source/Test/SystematicTesting/TestingEngine.cs
+++ b/Source/Test/SystematicTesting/TestingEngine.cs
@@ -267,7 +267,8 @@ public void Run()
if (this.Configuration.IsTelemetryEnabled)
{
- this.TrackTelemetry();
+ // 10 seconds should be enough, any more than that and too bad.
+ this.TrackTelemetry().Wait(10000, this.CancellationTokenSource.Token);
}
}
@@ -591,25 +592,16 @@ private void GatherTestingStatistics(CoyoteRuntime runtime)
///
/// Tracks anonymized telemetry data.
///
- private void TrackTelemetry()
+ private async Task TrackTelemetry()
{
bool isReplaying = this.Scheduler.IsReplaying;
- TelemetryClient.TrackEvent(isReplaying ? "replay" : "test");
- if (Debugger.IsAttached)
- {
- TelemetryClient.TrackEvent(isReplaying ? "replay-debug" : "test-debug");
- }
- else
- {
- TelemetryClient.TrackMetric(isReplaying ? "replay-time" : "test-time", this.Profiler.Results());
- }
-
- if (this.TestReport != null && this.TestReport.NumOfFoundBugs > 0)
- {
- TelemetryClient.TrackMetric(isReplaying ? "replay-bugs" : "test-bugs", this.TestReport.NumOfFoundBugs);
- }
-
- TelemetryClient.Flush();
+ await TelemetryClient.TrackEvent(
+ action: Debugger.IsAttached ?
+ (isReplaying ? "debug-replay" : "debug-test") :
+ (isReplaying ? "replay" : "test"),
+ result: (this.TestReport.NumOfFoundBugs > 0) ? "failed" : "passed",
+ bugsFound: this.TestReport?.NumOfFoundBugs,
+ testTime: this.Profiler?.Results());
}
///
diff --git a/Source/Test/Telemetry/HttpProtocol.cs b/Source/Test/Telemetry/HttpProtocol.cs
new file mode 100644
index 000000000..2e42427c4
--- /dev/null
+++ b/Source/Test/Telemetry/HttpProtocol.cs
@@ -0,0 +1,133 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Net.Http;
+using System.Runtime.InteropServices;
+using System.Runtime.Serialization;
+using System.Runtime.Serialization.Json;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Microsoft.Coyote.Telemetry
+{
+ internal static class HttpProtocol
+ {
+ private const string BaseUrl = "https://www.google-analytics.com/mp/collect";
+ private const string DebugBaseUrl = "https://www.google-analytics.com/debug/mp/collect";
+
+ public static Task PostMeasurements(Analytics a)
+ {
+ return Post(BaseUrl, a);
+ }
+
+ public static async Task ValidateMeasurements(Analytics a)
+ {
+ var response = await Post(DebugBaseUrl, a);
+ if (response.Content != null)
+ {
+ using (var stream = await response.Content.ReadAsStreamAsync())
+ {
+ var responseSerializer = new DataContractJsonSerializer(typeof(ValidationResponse));
+ return (ValidationResponse)responseSerializer.ReadObject(stream);
+ }
+ }
+
+ throw new Exception("No validation response");
+ }
+
+ private static async Task Post(string baseUri, Analytics a)
+ {
+ const string guide = "\r\nSee https://developers.google.com/analytics/devguides/collection/protocol/ga4";
+
+ if (a.Events.Count > 25)
+ {
+ throw new Exception("A maximum of 25 events can be specified per request." + guide);
+ }
+
+ if (!System.Net.NetworkInformation.NetworkInterface.GetIsNetworkAvailable())
+ {
+ throw new Exception("A network intercace is not available");
+ }
+
+ string query = a.ToQueryString();
+ string url = baseUri + "?" + query;
+
+ if (string.IsNullOrEmpty(a.UserId))
+ {
+ a.UserId = a.ClientId;
+ }
+
+ HttpClient client = new HttpClient();
+ AddUserProperties(client, a);
+
+ DataContractJsonSerializerSettings settings = new DataContractJsonSerializerSettings();
+ settings.EmitTypeInformation = EmitTypeInformation.Never;
+ settings.UseSimpleDictionaryFormat = true;
+ settings.KnownTypes = GetKnownTypes(a);
+ var serializer = new DataContractJsonSerializer(typeof(Analytics), settings);
+ var ms = new MemoryStream();
+ serializer.WriteObject(ms, a);
+ var bytes = ms.GetBuffer();
+ if (bytes.Length > 130000)
+ {
+ throw new Exception("The total size of analytics payloads cannot be greater than 130kb bytes" + guide);
+ }
+
+ var json = Encoding.UTF8.GetString(bytes);
+ json = new StreamReader("d:\\temp\\json2.json").ReadToEnd();
+ var jsonContent = new StringContent(json, Encoding.UTF8, "application/json");
+ var response = await client.PostAsync(new Uri(url), jsonContent);
+ response.EnsureSuccessStatusCode();
+ return response;
+ }
+
+ private static void AddUserProperties(HttpClient client, Analytics a)
+ {
+ string platform = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "Windows" :
+ (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? "Linux" : "OSX");
+ var arch = RuntimeInformation.OSArchitecture.ToString();
+ client.DefaultRequestHeaders.Add("User-Agent", string.Format("Mozilla/5.0 ({0}; {1})", platform, arch));
+ client.DefaultRequestHeaders.Add("Accept-Language", CultureInfo.CurrentCulture.Name);
+ a.UserProperties = new UserProperties()
+ {
+ FrameworkVersion = new UserPropertyValue(RuntimeInformation.FrameworkDescription),
+ Platform = new UserPropertyValue(platform),
+ PlatformVersion = new UserPropertyValue(RuntimeInformation.OSDescription),
+ Language = new UserPropertyValue(CultureInfo.CurrentCulture.Name)
+ };
+ }
+
+ private static Type[] GetKnownTypes(Analytics a)
+ {
+ HashSet types = new HashSet();
+ foreach (var e in a.Events)
+ {
+ types.Add(e.GetType());
+ }
+
+ return new List(types).ToArray();
+ }
+ }
+
+ [DataContract]
+ internal class ValidationResponse
+ {
+ [DataMember(Name = "validationMessages")]
+ public ValidationMessage[] ValidationMessages;
+ }
+
+ [DataContract]
+ internal class ValidationMessage
+ {
+ [DataMember(Name = "description")]
+ public string Description;
+ [DataMember(Name = "fieldPath")]
+ public string InvalidFieldPath;
+ [DataMember(Name = "validationCode")]
+ public string ValidationCode;
+ }
+}
diff --git a/Source/Test/Telemetry/Measurement.cs b/Source/Test/Telemetry/Measurement.cs
new file mode 100644
index 000000000..a8d730f6d
--- /dev/null
+++ b/Source/Test/Telemetry/Measurement.cs
@@ -0,0 +1,306 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
+
+namespace Microsoft.Coyote.Telemetry
+{
+ ///
+ /// This class wraps the GA4 protocol.
+ /// See https://developers.google.com/analytics/devguides/collection/protocol/ga4/reference.
+ ///
+ [KnownType(typeof(PageMeasurement))]
+ [KnownType(typeof(EventMeasurement))]
+ [KnownType(typeof(ExceptionMeasurement))]
+ [KnownType(typeof(UserTimingMeasurement))]
+ [KnownType(typeof(TestEventMeasurement))]
+ [KnownType(typeof(UserProperties))]
+ [KnownType(typeof(UserPropertyValue))]
+ [DataContract]
+ internal class Analytics
+ {
+ public Analytics()
+ {
+ this.TimeStamp = DateTimeOffset.Now.ToUnixTimeMilliseconds() * 1000;
+ }
+
+ [IgnoreDataMember]
+ public string ApiSecret { get; set; }
+
+ [IgnoreDataMember]
+ public string MeasurementId { get; set; }
+
+ [DataMember(Name = "client_id", Order = 1)]
+ public string ClientId { get; set; }
+
+ [DataMember(Name = "user_id", Order = 2)]
+ public string UserId { get; set; }
+
+ [DataMember(Name = "timestamp_micros", Order = 3)]
+ public long TimeStamp { get; set; }
+
+ [DataMember(Name = "non_personalized_ads", Order = 3)]
+ public bool NonPersonalizedAds { get; set; }
+
+ [DataMember(Name = "user_properties", Order = 4)]
+ public UserProperties UserProperties { get; set; }
+
+ [DataMember(Name = "events", Order = 5)]
+ public List Events = new List();
+
+ public string ToQueryString()
+ {
+ Required(this.MeasurementId, "MeasurementId");
+ Required(this.ClientId, "ClientId");
+ Required(this.ApiSecret, "ApiSecret");
+ return string.Format("api_secret={0}&measurement_id={1}", this.ApiSecret, this.MeasurementId);
+ }
+
+ protected static void Required(string value, string name)
+ {
+ if (string.IsNullOrEmpty(value))
+ {
+ throw new ArgumentNullException(name);
+ }
+ }
+ }
+
+ [DataContract]
+ internal class UserProperties
+ {
+ [DataMember(Name = "platform")]
+ public UserPropertyValue Platform { get; set; }
+ [DataMember(Name = "platform_version")]
+ public UserPropertyValue PlatformVersion { get; set; }
+ [DataMember(Name = "dotnet")]
+ public UserPropertyValue FrameworkVersion { get; set; }
+ [DataMember(Name = "language")]
+ public UserPropertyValue Language { get; set; }
+ }
+
+ [DataContract]
+ internal class UserPropertyValue
+ {
+ public UserPropertyValue(string value)
+ {
+ this.Value = value;
+ }
+
+ [DataMember(Name = "value")]
+ public string Value { get; set; }
+ }
+
+ [DataContract]
+ internal abstract class Measurement
+ {
+ public Measurement()
+ {
+ this.Version = 1;
+ }
+
+ protected int Version { get; set; }
+
+ [DataMember(Name = "name")]
+ protected string Name { get; set; }
+
+ [DataMember(Name = "params")]
+ public Dictionary Params = new Dictionary();
+
+ protected string GetParam(string name)
+ {
+ if (this.Params.TryGetValue(name, out object value) && value is string s)
+ {
+ return s;
+ }
+
+ return null;
+ }
+
+ protected void SetParam(string name, string value)
+ {
+ this.Params[name] = value;
+ }
+
+ protected double GetDoubleParam(string name)
+ {
+ if (this.Params.TryGetValue(name, out object value) && value is double d)
+ {
+ return d;
+ }
+
+ return 0;
+ }
+
+ protected void SetDoubleParam(string name, double value)
+ {
+ this.Params[name] = value;
+ }
+ }
+
+ [DataContract]
+ internal class PageMeasurement : Measurement
+ {
+ public PageMeasurement()
+ {
+ this.Name = "page_view";
+ }
+
+ public string HostName
+ {
+ get => this.GetParam("host_name");
+ set => this.SetParam("host_name", value);
+ }
+
+ public string Path
+ {
+ get => this.GetParam("page_location");
+ set => this.SetParam("page_location", value);
+ }
+
+ public string Title
+ {
+ get => this.GetParam("page_title");
+ set => this.SetParam("page_title", value);
+ }
+
+ public string Referrer
+ {
+ get => this.GetParam("page_referrer");
+ set => this.SetParam("page_referrer", value);
+ }
+
+ public string UserAgent
+ {
+ get => this.GetParam("user_agent");
+ set => this.SetParam("user_agent", value);
+ }
+ }
+
+ [DataContract]
+ internal class EventMeasurement : Measurement
+ {
+ public EventMeasurement()
+ {
+ this.Name = "event";
+ }
+
+ public string Category
+ {
+ get => this.GetParam("category");
+ set => this.SetParam("category", value);
+ }
+
+ public string Action
+ {
+ get => this.GetParam("action");
+ set => this.SetParam("action", value);
+ }
+
+ public string Label
+ {
+ get => this.GetParam("label");
+ set => this.SetParam("label", value);
+ }
+
+ public string Value
+ {
+ get => this.GetParam("value");
+ set => this.SetParam("value", value);
+ }
+ }
+
+ [DataContract]
+ internal class ExceptionMeasurement : Measurement
+ {
+ public ExceptionMeasurement()
+ {
+ this.Name = "exception";
+ }
+
+ public string Description
+ {
+ get => this.GetParam("description");
+ set => this.SetParam("description", value);
+ }
+
+ public string Fatal
+ {
+ get => this.GetParam("fatal");
+ set => this.SetParam("fatal", value);
+ }
+ }
+
+ [DataContract]
+ internal class UserTimingMeasurement : Measurement
+ {
+ public UserTimingMeasurement()
+ {
+ this.Name = "timing";
+ }
+
+ public string Category
+ {
+ get => this.GetParam("category");
+ set => this.SetParam("category", value);
+ }
+
+ public string Variable
+ {
+ get => this.GetParam("variable");
+ set => this.SetParam("variable", value);
+ }
+
+ public string Time
+ {
+ get => this.GetParam("time");
+ set => this.SetParam("time", value);
+ }
+
+ public string Label
+ {
+ get => this.GetParam("label");
+ set => this.SetParam("label", value);
+ }
+ }
+
+ [DataContract]
+ internal class TestEventMeasurement : Measurement
+ {
+ public TestEventMeasurement()
+ {
+ this.Name = "event";
+ }
+
+ public string Coyote
+ {
+ get => this.GetParam("coyote");
+ set => this.SetParam("coyote", value);
+ }
+
+ public string Action
+ {
+ get => this.GetParam("action");
+ set => this.SetParam("action", value);
+ }
+
+ public string Result
+ {
+ get => this.GetParam("value");
+ set => this.SetParam("value", value);
+ }
+
+ public int Bugs
+ {
+ get => (int)this.GetDoubleParam("bugs");
+ set => this.SetDoubleParam("bugs", value);
+ }
+
+ public double TestTime
+ {
+ get => (int)this.GetDoubleParam("test_time");
+ set => this.SetDoubleParam("test_time", value);
+ }
+ }
+}
diff --git a/Source/Test/Telemetry/TelemetryClient.cs b/Source/Test/Telemetry/TelemetryClient.cs
index 562d1022e..9bebb9857 100644
--- a/Source/Test/Telemetry/TelemetryClient.cs
+++ b/Source/Test/Telemetry/TelemetryClient.cs
@@ -3,12 +3,9 @@
using System;
using System.IO;
-using System.Runtime.InteropServices;
using System.Threading;
-using Microsoft.ApplicationInsights.DataContracts;
-using Microsoft.ApplicationInsights.Extensibility;
+using System.Threading.Tasks;
using Microsoft.Coyote.Logging;
-using AppInsightsClient = Microsoft.ApplicationInsights.TelemetryClient;
namespace Microsoft.Coyote.Telemetry
{
@@ -20,6 +17,9 @@ namespace Microsoft.Coyote.Telemetry
///
internal class TelemetryClient
{
+ private const string ApiSecret = "eOkUmW73T9Wv5TUynCLAEA";
+ private const string MeasurementId = "G-JS8YSYVDQX";
+
///
/// Path to the Coyote home directory where the UUID is stored.
///
@@ -42,11 +42,6 @@ internal class TelemetryClient
///
private static TelemetryClient Current;
- ///
- /// The App Insights client.
- ///
- private readonly AppInsightsClient Client;
-
///
/// Responsible for writing to the installed .
///
@@ -57,36 +52,27 @@ internal class TelemetryClient
///
private readonly bool IsEnabled;
+ ///
+ /// A unique id for this user on this device.
+ ///
+ private string DeviceId;
+
+ ///
+ /// The coyote version.
+ ///
+ private readonly string Version;
+
///
/// Initializes a new instance of the class.
///
private TelemetryClient(LogWriter logWriter, bool isEnabled)
{
+ this.IsEnabled = isEnabled;
if (isEnabled)
{
- TelemetryConfiguration configuration = TelemetryConfiguration.CreateDefault();
- configuration.InstrumentationKey = "17a6badb-bf2d-4f5d-959b-6843b8bb1f7f";
- this.Client = new AppInsightsClient(configuration);
+ this.Version = typeof(Runtime.CoyoteRuntime).Assembly.GetName().Version.ToString();
this.LogWriter = logWriter;
-
- string version = typeof(Runtime.CoyoteRuntime).Assembly.GetName().Version.ToString();
- this.Client.Context.GlobalProperties["coyote"] = version;
-#if NETFRAMEWORK
- this.Client.Context.GlobalProperties["dotnet"] = ".NET Framework";
-#else
- this.Client.Context.GlobalProperties["dotnet"] = RuntimeInformation.FrameworkDescription;
-#endif
- this.Client.Context.Device.Id = GetOrCreateDeviceId(out bool isFirstTime);
- this.Client.Context.Device.OperatingSystem = Environment.OSVersion.Platform.ToString();
- this.Client.Context.Session.Id = Guid.NewGuid().ToString();
-
- if (isFirstTime)
- {
- this.TrackEvent("welcome");
- }
}
-
- this.IsEnabled = isEnabled;
}
///
@@ -105,66 +91,64 @@ internal static TelemetryClient GetOrCreate(Configuration configuration, LogWrit
///
/// Tracks the specified telemetry event.
///
- internal void TrackEvent(string name)
+ internal Task TrackEvent(string action, string result, int? bugsFound, double? testTime)
{
if (this.IsEnabled)
{
- lock (SyncObject)
+ Task task = Task.CompletedTask;
+
+ if (string.IsNullOrEmpty(this.DeviceId))
{
- try
- {
- this.LogWriter.LogDebug("[coyote::telemetry] Tracking event: {0}.", name);
- this.Client.TrackEvent(new EventTelemetry(name));
- }
- catch (Exception ex)
+ this.DeviceId = GetOrCreateDeviceId(out bool isFirstTime);
+ if (isFirstTime)
{
- this.LogWriter.LogDebug("[coyote::telemetry] Unable to send event: {0}", ex.Message);
+ task = this.TrackEvent("welcome", null, null, null);
}
}
- }
- }
- ///
- /// Tracks the specified telemetry metric.
- ///
- internal void TrackMetric(string name, double value)
- {
- if (this.IsEnabled)
- {
- lock (SyncObject)
+ try
{
- try
+ var analytics = new Analytics()
{
- this.LogWriter.LogDebug("[coyote::telemetry] Tracking metric: {0}={1}.", name, value);
- this.Client.TrackMetric(new MetricTelemetry(name, value));
- }
- catch (Exception ex)
+ ApiSecret = ApiSecret,
+ MeasurementId = MeasurementId,
+ ClientId = this.DeviceId
+ };
+
+ var m = new TestEventMeasurement()
{
- this.LogWriter.LogDebug("[coyote::telemetry] Unable to send metric: {0}", ex.Message);
- }
- }
- }
- }
+ Action = action,
+ Result = result
+ };
- ///
- /// Flushes any buffered in-memory telemetry data.
- ///
- internal void Flush()
- {
- if (this.IsEnabled)
- {
- lock (SyncObject)
- {
- try
+ m.Coyote = this.Version;
+
+ if (bugsFound.HasValue)
{
- this.Client.Flush();
+ m.Bugs = bugsFound.Value;
}
- catch (Exception ex)
+
+ if (testTime.HasValue)
{
- this.LogWriter.LogDebug("[coyote::telemetry] Error flushing: {0}", ex.Message);
+ m.TestTime = testTime.Value;
}
+
+ analytics.Events.Add(m);
+
+ this.LogWriter.LogDebug("[coyote::telemetry] Tracking event: {0}.", action);
+
+ // handy for debugging errors from google.
+ // var response = HttpProtocol.ValidateMeasurements(analytics).Result;
+
+ return Task.WhenAll(task, HttpProtocol.PostMeasurements(analytics));
+ }
+ catch (Exception ex)
+ {
+ this.LogWriter.LogDebug("[coyote::telemetry] Unable to send event: {0}", ex.Message);
}
}
+
+ return Task.CompletedTask;
}
///
diff --git a/Source/Test/Test.csproj b/Source/Test/Test.csproj
index 731a417de..86fd78005 100644
--- a/Source/Test/Test.csproj
+++ b/Source/Test/Test.csproj
@@ -10,7 +10,6 @@
-