Merge pull request #34 from microsoft/alzollin/telemetry

Enhance telemetry and logging functionality.

Author: azchohfi

src/winsdk-CLI/Winsdk.Cli/Program.cs

@@ -6,7 +6,9 @@
 using System.Text;
 using Winsdk.Cli.Commands;
 using Winsdk.Cli.Helpers;
+using Winsdk.Cli.Telemetry;
 using Winsdk.Cli.Telemetry.Events;
+using LogLevel = Microsoft.Extensions.Logging.LogLevel;
 
 namespace Winsdk.Cli;
 
@@ -62,9 +64,22 @@ static async Task<int> Main(string[] args)
 
         var parseResult = rootCommand.Parse(args);
 
-        CommandExecutedEvent.Log(parseResult.CommandResult.Command.GetType().FullName!);
+        try
+        {
+            CommandInvokedEvent.Log(parseResult.CommandResult);
+
+            var returnCode = await parseResult.InvokeAsync();
+
+            CommandCompletedEvent.Log(parseResult.CommandResult, returnCode);
 
-        return await parseResult.InvokeAsync();
+            return returnCode;
+        }
+        catch (Exception ex)
+        {
+            TelemetryFactory.Get<ITelemetry>().LogException(parseResult.CommandResult.Command.Name, ex);
+            Console.Error.WriteLine($"An unexpected error occurred: {ex.Message}");
+            return 1;
+        }
     }
 
     internal static bool PromptYesNo(string message)

src/winsdk-CLI/Winsdk.Cli/Services/CertificateService.cs

@@ -431,7 +431,7 @@ public static async Task ValidatePublisherMatchAsync(string certificatePath, str
                     $"Error: Publisher in {manifestPath} (CN={normalizedManifestPublisher}) does not match the publisher in the certificate {certificatePath} (CN={normalizedCertPublisher}).");
             }
         }
-        catch (Exception ex) when (!(ex is InvalidOperationException))
+        catch (Exception ex) when (ex is not InvalidOperationException)
         {
             throw new InvalidOperationException($"Failed to validate publisher match: {ex.Message}", ex);
         }
@@ -494,6 +494,6 @@ private async Task<string> InferPublisherAsync(
         return defaultPublisher;
     }
 
-    [GeneratedRegexAttribute(@"CN=([^,]+)", RegexOptions.IgnoreCase, "en-US")]
+    [GeneratedRegex(@"CN=([^,]+)", RegexOptions.IgnoreCase, "en-US")]
     private static partial Regex CnFieldRegex();
 }

src/winsdk-CLI/Winsdk.Cli/Telemetry/CIEnvironmentDetectorForTelemetry.cs

@@ -0,0 +1,84 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+namespace Winsdk.Cli.Telemetry;
+
+// Detection of CI: https://learn.microsoft.com/dotnet/core/tools/telemetry#continuous-integration-detection
+// From: https://github.com/microsoft/testfx/blob/main/src/Platform/Microsoft.Testing.Extensions.Telemetry/CIEnvironmentDetectorForTelemetry.cs
+internal sealed class CIEnvironmentDetectorForTelemetry
+{
+    // Systems that provide boolean values only, so we can simply parse and check for true
+    private static readonly string[] BooleanVariables =
+    [
+        // Azure Pipelines - https://docs.microsoft.com/azure/devops/pipelines/build/variables#system-variables-devops-services
+        "TF_BUILD",
+
+        // GitHub Actions - https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
+        "GITHUB_ACTIONS",
+
+        // AppVeyor - https://www.appveyor.com/docs/environment-variables/
+        "APPVEYOR",
+
+        // A general-use flag - Many of the major players support this: AzDo, GitHub, GitLab, AppVeyor, Travis CI, CircleCI.
+        // Given this, we could potentially remove all of these other options?
+        "CI",
+
+        // Travis CI - https://docs.travis-ci.com/user/environment-variables/#default-environment-variables
+        "TRAVIS",
+
+        // CircleCI - https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
+        "CIRCLECI"
+    ];
+
+    // Systems where every variable must be present and not-null before returning true
+    private static readonly string[][] AllNotNullVariables =
+    [
+        // AWS CodeBuild - https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
+        ["CODEBUILD_BUILD_ID", "AWS_REGION"],
+
+        // Jenkins - https://github.com/jenkinsci/jenkins/blob/master/core/src/main/resources/jenkins/model/CoreEnvironmentContributor/buildEnv.groovy
+        ["BUILD_ID", "BUILD_URL"],
+
+        // Google Cloud Build - https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values#using_default_substitutions
+        ["BUILD_ID", "PROJECT_ID"],
+    ];
+
+    // Systems where the variable must be present and not-null
+    private static readonly string[] IfNonNullVariables =
+    [
+        // TeamCity - https://www.jetbrains.com/help/teamcity/predefined-build-parameters.html#Predefined+Server+Build+Parameters
+        "TEAMCITY_VERSION",
+
+        // JetBrains Space - https://www.jetbrains.com/help/space/automation-environment-variables.html#general
+        "JB_SPACE_API_URL"
+    ];
+
+    public static bool IsCIEnvironment()
+    {
+        foreach (string booleanVariable in BooleanVariables)
+        {
+            if (bool.TryParse(Environment.GetEnvironmentVariable(booleanVariable), out bool envVar) && envVar)
+            {
+                return true;
+            }
+        }
+
+        foreach (string[] variables in AllNotNullVariables)
+        {
+            if (variables.All(variable => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(variable))))
+            {
+                return true;
+            }
+        }
+
+        foreach (string variable in IfNonNullVariables)
+        {
+            if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable(variable)))
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+}

src/winsdk-CLI/Winsdk.Cli/Telemetry/Events/CommandCompletedEvent.cs

@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Microsoft.Diagnostics.Telemetry;
+using Microsoft.Diagnostics.Telemetry.Internal;
+using System.CommandLine.Parsing;
+using System.Diagnostics.Tracing;
+
+namespace Winsdk.Cli.Telemetry.Events;
+
+[EventData]
+internal class CommandCompletedEvent : EventBase
+{
+    internal CommandCompletedEvent(CommandResult commandResult, DateTime finishedTime, int exitCode)
+    {
+        CommandName = commandResult.Command.GetType().FullName!;
+        FinishedTime = finishedTime;
+        ExitCode = exitCode;
+    }
+
+    public string CommandName { get; private set; }
+
+    public DateTime FinishedTime { get; private set; }
+
+    public int ExitCode { get; }
+
+    public override PartA_PrivTags PartA_PrivTags => PrivTags.ProductAndServiceUsage;
+
+    public override void ReplaceSensitiveStrings(Func<string, string> replaceSensitiveStrings)
+    {
+        CommandName = replaceSensitiveStrings(CommandName);
+    }
+
+    public static void Log(CommandResult commandResult, int exitCode)
+    {
+        TelemetryFactory.Get<ITelemetry>().Log("CommandCompleted_Event", LogLevel.Critical, new CommandCompletedEvent(commandResult, DateTime.Now, exitCode));
+    }
+}

src/winsdk-CLI/Winsdk.Cli/Telemetry/Events/CommandExecutedEvent.cs

@@ -0,0 +1,65 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Microsoft.Diagnostics.Telemetry;
+using Microsoft.Diagnostics.Telemetry.Internal;
+using System.CommandLine;
+using System.CommandLine.Parsing;
+using System.Diagnostics.Tracing;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Winsdk.Cli.Telemetry.Events;
+
+internal record CommandExecutionContext(Dictionary<string, string?> Arguments, Dictionary<string, string?> Options);
+
+[JsonSerializable(typeof(CommandExecutionContext))]
+[JsonSourceGenerationOptions]
+internal partial class CommandInvokedEventJsonContext : JsonSerializerContext
+{
+}
+
+[EventData]
+internal class CommandInvokedEvent : EventBase
+{
+    internal CommandInvokedEvent(CommandResult commandResult, DateTime startedTime)
+    {
+        CommandName = commandResult.Command.GetType().FullName!;
+        var argumentsDict = commandResult.Children
+            .OfType<ArgumentResult>()
+            .ToDictionary(a => a.Argument.Name, GetValue);
+        var optionsDict = commandResult.Children
+            .OfType<OptionResult>()
+            .ToDictionary(o => o.Option.Name, GetValue);
+        var commandExecutionContext = new CommandExecutionContext(argumentsDict, optionsDict);
+        Context = JsonSerializer.Serialize(commandExecutionContext, CommandInvokedEventJsonContext.Default.CommandExecutionContext);
+        StartedTime = startedTime;
+    }
+
+    private string? GetValue(OptionResult o) => GetValue(o.Option.ValueType, o.Implicit, o.GetValueOrDefault<object?>());
+    private string? GetValue(ArgumentResult a) => GetValue(a.Argument.ValueType, a.Implicit, a.GetValueOrDefault<object?>());
+
+    private static string? GetValue(Type valueType, bool isImplicit, object? value)
+    {
+        return isImplicit ? null : (valueType == typeof(string) ? "[string]" : value)?.ToString();
+    }
+
+    public string CommandName { get; private set; }
+
+    public string Context { get; private set; }
+
+    public DateTime StartedTime { get; private set; }
+
+    public override PartA_PrivTags PartA_PrivTags => PrivTags.ProductAndServiceUsage;
+
+    public override void ReplaceSensitiveStrings(Func<string, string> replaceSensitiveStrings)
+    {
+        CommandName = replaceSensitiveStrings(CommandName);
+        Context = replaceSensitiveStrings(Context);
+    }
+
+    public static void Log(CommandResult commandResult)
+    {
+        TelemetryFactory.Get<ITelemetry>().Log("CommandInvoked_Event", LogLevel.Critical, new CommandInvokedEvent(commandResult, DateTime.Now));
+    }
+}

src/winsdk-CLI/Winsdk.Cli/Telemetry/Events/CommandInvokedEvent.cs

@@ -4,19 +4,14 @@
 using Microsoft.Diagnostics.Telemetry.Internal;
 using System.Diagnostics.Tracing;
 
-namespace Winsdk.Cli.Telemetry;
+namespace Winsdk.Cli.Telemetry.Events;
 
 [EventData]
-internal class EmptyEvent : EventBase
+internal class EmptyEvent(PartA_PrivTags tags) : EventBase
 {
-    public override PartA_PrivTags PartA_PrivTags { get; }
+    public override PartA_PrivTags PartA_PrivTags { get; } = tags;
 
-    public EmptyEvent(PartA_PrivTags tags)
-    {
-        PartA_PrivTags = tags;
-    }
-
-    public override void ReplaceSensitiveStrings(Func<string?, string?> replaceSensitiveStrings)
+    public override void ReplaceSensitiveStrings(Func<string, string> replaceSensitiveStrings)
     {
         // No sensitive string
     }

src/winsdk-CLI/Winsdk.Cli/Telemetry/Events/EmptyEvent.cs

@@ -4,7 +4,7 @@
 using Microsoft.Diagnostics.Telemetry.Internal;
 using System.Diagnostics.Tracing;
 
-namespace Winsdk.Cli.Telemetry;
+namespace Winsdk.Cli.Telemetry.Events;
 
 /// <summary>
 /// Base class for all telemetry events to ensure they are properly tagged.
@@ -26,6 +26,13 @@ public abstract PartA_PrivTags PartA_PrivTags
         get;
     }
 
+    /// <summary>
+    /// Gets the app version from the assembly.
+    /// </summary>
+    public string AppVersion { get; } = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "Unknown";
+
+    public bool CI { get; } = CIEnvironmentDetectorForTelemetry.IsCIEnvironment();
+
     /// <summary>
     /// Replaces all the strings in this event that may contain PII using the provided function.
     /// </summary>
@@ -37,5 +44,5 @@ public abstract PartA_PrivTags PartA_PrivTags
     /// <param name="replaceSensitiveStrings">
     /// A function that replaces all the sensitive strings in a given string with tokens
     /// </param>
-    public abstract void ReplaceSensitiveStrings(Func<string?, string?> replaceSensitiveStrings);
+    public abstract void ReplaceSensitiveStrings(Func<string, string> replaceSensitiveStrings);
 }

src/winsdk-CLI/Winsdk.Cli/Telemetry/Events/EventBase.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using System.Diagnostics.CodeAnalysis;
+using Winsdk.Cli.Telemetry.Events;
 
 namespace Winsdk.Cli.Telemetry;